health/web: cluster-level semantics — one dead node or switch off = cluster OFF

Shared rule (cluster_state::verdict + home_assistant::extra_entities_off):
a cluster is UP only when ALL nodes are reachable AND its HA
extra_entities report 'on'. Down clusters report no containers/free
anywhere — even on nodes that are still alive.

- health: down clusters print the reason (switch off / N offline) +
  '⏻ cluster offline'; live nodes show as 'online (cluster offline)'
  with no containers; running-recipes summary skips containers whose
  labeled cluster is down
- web: ClusterSnap carries up/switch_off; chips show ⏻ OFF with a
  reason tooltip; down clusters offer only power on/switch; node rows
  say 'online — cluster offline'; jobs window gains a 'running
  (docker labels)' section fed by the sm121.cluster/recipe container
  labels — running serves show up without starting from the dashboard
- checkbox: accent-color styling (was drifting)
- i18n: health-cluster-down / health-free-count /
  health-node-online-cluster-down in all 6 locales
- cluster_state tests split to src/cluster_state/tests.rs (500-line
  cap); new verdict + down-cluster snapshot tests (190 total)

AGENTS.md

@@ -48,6 +48,7 @@ the smallest cluster, so pass `--cluster <name> --fit` to scale onto a
 specific larger cluster. Incompatible with `--solo`.
 
 `serve --mirror` (optional value) mounts a **per-node local mirror** of the HF
+
 cache instead of the shared one at `/root/.cache/huggingface`. Bare `--mirror`
 uses `~/.cache/hf-models`; `--mirror <path>` uses that exact path (on each
 node). The mirror keeps the HF hub layout (`<mirror>/hub/models--org--model/...`),
@@ -58,6 +59,13 @@ from the shared cache (`cp -a`, one-time per-node cost); a repo absent from
 BOTH locations aborts with a `huggingface-cli download` hint. Later serves read
 the model locally — no streaming weights over slow shared storage at load time.
 
+**Cluster health is cluster-level** (`health`, web dashboard): a cluster is
+UP only when *all* its nodes are reachable AND its `extra_entities` (HA
+switches) report `on`. One offline node — or a switch off — takes the whole
+cluster down: containers on surviving nodes are not reported (serve/stop/
+jobs views all ignore them; `power on` stays available). Shared rule:
+`cluster_state::verdict` + `home_assistant::extra_entities_off`.
+
 `serve --clear` deletes the **host-side `extra_mounts` folders** (alias/host
 — repo-based path refs are never touched) on the launch nodes with
 `sudo rm -rf` before containers start. The containers run as root, so the

locales/de/main.ftl

@@ -35,6 +35,9 @@ health-switch-off = 🚫 Switch aus
 health-interface-down = 🌐 Schnittstelle down
 health-all-online = ✅ alle online
 health-partial-online =  ({ $online } online, { $offline } offline)
+health-node-online-cluster-down =   ◍ { $name } ({ $ip }){ $net } — online (Cluster offline)
+health-free-count = — { $free } frei
+health-cluster-down = ⏻ Cluster offline
 health-net-status =  [Netz: { $details }]
 health-no-containers = (keine Container)
 health-no-containers-inline = , keine Container

locales/en/main.ftl

@@ -35,6 +35,9 @@ health-switch-off = 🚫 switch off
 health-interface-down = 🌐 interface down
 health-all-online = ✅ all online
 health-partial-online =  ({ $online } online, { $offline } offline)
+health-node-online-cluster-down =   ◍ { $name } ({ $ip }){ $net } — online (cluster offline)
+health-free-count = — { $free } free
+health-cluster-down = ⏻ cluster offline
 health-net-status =  [net: { $details }]
 health-no-containers = (no containers)
 health-no-containers-inline = , no containers

locales/es/main.ftl

@@ -35,6 +35,9 @@ health-switch-off = 🚫 switch apagado
 health-interface-down = 🌐 interfaz caída
 health-all-online = ✅ todos en línea
 health-partial-online =  ({ $online } en línea, { $offline } desconectados)
+health-node-online-cluster-down =   ◍ { $name } ({ $ip }){ $net } — en línea (cluster fuera de línea)
+health-free-count = — { $free } libres
+health-cluster-down = ⏻ cluster fuera de línea
 health-net-status =  [red: { $details }]
 health-no-containers = (sin contenedores)
 health-no-containers-inline = , sin contenedores

locales/fr/main.ftl

@@ -35,6 +35,9 @@ health-switch-off = 🚫 switch éteint
 health-interface-down = 🌐 interface indisponible
 health-all-online = ✅ tous en ligne
 health-partial-online =  ({ $online } en ligne, { $offline } hors ligne)
+health-node-online-cluster-down =   ◍ { $name } ({ $ip }){ $net } — en ligne (cluster hors ligne)
+health-free-count = — { $free } libres
+health-cluster-down = ⏻ cluster hors ligne
 health-net-status =  [réseau : { $details }]
 health-no-containers = (aucun conteneur)
 health-no-containers-inline = , aucun conteneur

locales/it/main.ftl

@@ -35,6 +35,9 @@ health-switch-off = 🚫 switch spento
 health-interface-down = 🌐 interfaccia down
 health-all-online = ✅ tutti online
 health-partial-online =  ({ $online } online, { $offline } offline)
+health-node-online-cluster-down =   ◍ { $name } ({ $ip }){ $net } — online (cluster offline)
+health-free-count = — { $free } liberi
+health-cluster-down = ⏻ cluster offline
 health-net-status =  [rete: { $details }]
 health-no-containers = (nessun container)
 health-no-containers-inline = , nessun container

locales/ru/main.ftl

@@ -37,6 +37,9 @@ health-switch-off = 🚫 переключатель выключен
 health-interface-down = 🌐 интерфейс недоступен
 health-all-online = ✅ все онлайн
 health-partial-online =  ({ $online } онлайн, { $offline } офлайн)
+health-node-online-cluster-down =   ◍ { $name } ({ $ip }){ $net } — онлайн (кластер офлайн)
+health-free-count = — { $free } свободно
+health-cluster-down = ⏻ кластер офлайн
 health-net-status =  [сеть: { $details }]
 health-no-containers = (нет контейнеров)
 health-no-containers-inline = , нет контейнеров

src/cluster_state.rs

@@ -28,6 +28,39 @@ pub struct ClusterState {
     pub nodes: HashMap<String, NodeState>,
 }
 
+/// Cluster-level health verdict.
+///
+/// A cluster is **UP** only when *all* of its nodes are reachable AND its
+/// HA switch entities (`extra_entities`, when HA is configured) report
+/// "on". One offline node — or a switch off — takes the whole cluster
+/// down: containers on surviving nodes are ignored, `free` is 0.
+#[derive(Clone, Debug, PartialEq, serde::Serialize)]
+pub struct ClusterVerdict {
+    pub up: bool,
+    /// HA extra entities report anything other than "on".
+    pub switch_off: bool,
+    /// Free nodes; 0 when the cluster is down.
+    pub free: u32,
+}
+
+/// Derive the verdict from per-node `(reachable, occupied)` pairs plus the
+/// HA switch state. The single source of truth for cluster health — both
+/// the `health` command and the web dashboard route through this.
+pub fn verdict(reachables: &[(bool, bool)], switch_off: bool) -> ClusterVerdict {
+    let all_reachable = reachables.iter().all(|(r, _)| *r);
+    let up = all_reachable && !switch_off;
+    let free = if up {
+        reachables.iter().filter(|(_, o)| !o).count() as u32
+    } else {
+        0
+    };
+    ClusterVerdict {
+        up,
+        switch_off,
+        free,
+    }
+}
+
 impl ClusterState {
     /// Probe all unique nodes across the config and collect their state.
     /// This is the only function that touches SSH (the "control" side).
@@ -158,325 +191,4 @@ pub fn select_cluster<'c>(
 }
 
 #[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::config::{Config, NcclConfig, Node};
-
-    fn test_node(name: &str) -> Node {
-        Node {
-            name: name.to_string(),
-            ssh_ip: format!("10.0.0.{}", name.chars().last().unwrap_or('0')),
-            ib_ip: Some("10.0.1.1".to_string()),
-            rank: 0,
-            host_key: None,
-            entity: None,
-        }
-    }
-
-    fn test_cluster(name: &str, node_names: &[&str]) -> Cluster {
-        Cluster {
-            name: name.to_string(),
-            master_ib_ip: Some("10.0.1.1".to_string()),
-            nodes: node_names.iter().map(|n| test_node(n)).collect(),
-            extra_entities: vec![],
-        }
-    }
-
-    fn test_nccl() -> NcclConfig {
-        NcclConfig {
-            ib_hca: "r".to_string(),
-            socket_ifname: "e".to_string(),
-            ib_gid_index: Some(3),
-            ib_disable: 0,
-            gloo_socket_ifname: None,
-            ucx_net_devices: None,
-            nccl_cross_nic: None,
-            nccl_proto: None,
-            nccl_p2p_level: None,
-        }
-    }
-
-    #[test]
-    fn free_count_all_reachable() {
-        let mut state = ClusterState::default();
-        state.nodes.insert(
-            "n1".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        state.nodes.insert(
-            "n2".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        let c = test_cluster("c", &["n1", "n2"]);
-        assert_eq!(state.free_count(&c), 2);
-    }
-
-    #[test]
-    fn free_count_occupied_excluded() {
-        let mut state = ClusterState::default();
-        state.nodes.insert(
-            "n1".into(),
-            NodeState {
-                reachable: true,
-                occupied: true,
-            },
-        );
-        state.nodes.insert(
-            "n2".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        let c = test_cluster("c", &["n1", "n2"]);
-        assert_eq!(state.free_count(&c), 1);
-    }
-
-    #[test]
-    fn free_count_unreachable_excluded() {
-        let mut state = ClusterState::default();
-        state.nodes.insert(
-            "n1".into(),
-            NodeState {
-                reachable: false,
-                occupied: false,
-            },
-        );
-        state.nodes.insert(
-            "n2".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        let c = test_cluster("c", &["n1", "n2"]);
-        assert_eq!(state.free_count(&c), 1);
-    }
-
-    #[test]
-    fn free_count_missing_node_not_free() {
-        let mut state = ClusterState::default();
-        state.nodes.insert(
-            "n1".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        // n2 not probed — absent is treated as not free
-        let c = test_cluster("c", &["n1", "n2"]);
-        assert_eq!(state.free_count(&c), 1);
-    }
-
-    #[test]
-    fn select_smallest_with_enough_free() {
-        let mut state = ClusterState::default();
-        // big cluster: 4 nodes, 2 free
-        state.nodes.insert(
-            "n1".into(),
-            NodeState {
-                reachable: true,
-                occupied: true,
-            },
-        );
-        state.nodes.insert(
-            "n2".into(),
-            NodeState {
-                reachable: true,
-                occupied: true,
-            },
-        );
-        state.nodes.insert(
-            "n3".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        state.nodes.insert(
-            "n4".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        // small cluster: 2 nodes, 1 free
-        state.nodes.insert(
-            "n5".into(),
-            NodeState {
-                reachable: true,
-                occupied: true,
-            },
-        );
-        state.nodes.insert(
-            "n6".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-
-        let cfg = Config {
-            ssh_user: "u".to_string(),
-            ssh_key_path: "/k".into(),
-            clusters: vec![
-                test_cluster("big", &["n1", "n2", "n3", "n4"]),
-                test_cluster("small", &["n5", "n6"]),
-            ],
-            nccl: {
-                let mut m = HashMap::new();
-                m.insert("big".into(), test_nccl());
-                m.insert("small".into(), test_nccl());
-                m
-            },
-            hf_cache_dir: None,
-            init_port: 25000,
-            before_serve: HashMap::new(),
-            exec_aliases: HashMap::new(),
-            home_assistant: None,
-            drop_caches: true,
-            mount_aliases: HashMap::new(),
-            recipe_sources: Vec::new(),
-            cluster_sources: Vec::new(),
-            global_mods_dir: None,
-        };
-
-        // Request 1 node — small cluster is smallest with free >= 1
-        let (c, _) = select_cluster(&cfg, "auto", Some(1), &state).unwrap();
-        assert_eq!(c.name, "small");
-
-        // Request 2 nodes — only big cluster has 2 free
-        let (c, _) = select_cluster(&cfg, "auto", Some(2), &state).unwrap();
-        assert_eq!(c.name, "big");
-    }
-
-    #[test]
-    fn select_cluster_not_found_errors() {
-        let state = ClusterState::default();
-        let cfg = Config {
-            ssh_user: "u".to_string(),
-            ssh_key_path: "/k".into(),
-            clusters: vec![],
-            nccl: HashMap::new(),
-            hf_cache_dir: None,
-            init_port: 25000,
-            before_serve: HashMap::new(),
-            exec_aliases: HashMap::new(),
-            home_assistant: None,
-            drop_caches: true,
-            mount_aliases: HashMap::new(),
-            recipe_sources: Vec::new(),
-            cluster_sources: Vec::new(),
-            global_mods_dir: None,
-        };
-        assert!(select_cluster(&cfg, "nonexistent", None, &state).is_err());
-    }
-
-    #[test]
-    fn no_cluster_has_enough_free_errors() {
-        let mut state = ClusterState::default();
-        state.nodes.insert(
-            "n1".into(),
-            NodeState {
-                reachable: true,
-                occupied: true,
-            },
-        );
-        let cfg = Config {
-            ssh_user: "u".to_string(),
-            ssh_key_path: "/k".into(),
-            clusters: vec![test_cluster("c", &["n1"])],
-            nccl: {
-                let mut m = HashMap::new();
-                m.insert("c".into(), test_nccl());
-                m
-            },
-            hf_cache_dir: None,
-            init_port: 25000,
-            before_serve: HashMap::new(),
-            exec_aliases: HashMap::new(),
-            home_assistant: None,
-            drop_caches: true,
-            mount_aliases: HashMap::new(),
-            recipe_sources: Vec::new(),
-            cluster_sources: Vec::new(),
-            global_mods_dir: None,
-        };
-        assert!(select_cluster(&cfg, "auto", Some(1), &state).is_err());
-    }
-
-    #[test]
-    fn single_node_allows_cluster_without_nccl() {
-        let mut state = ClusterState::default();
-        state.nodes.insert(
-            "n1".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        let cfg = Config {
-            ssh_user: "u".to_string(),
-            ssh_key_path: "/k".into(),
-            clusters: vec![test_cluster("c", &["n1"])],
-            nccl: HashMap::new(), // no NCCL config
-            hf_cache_dir: None,
-            init_port: 25000,
-            before_serve: HashMap::new(),
-            exec_aliases: HashMap::new(),
-            home_assistant: None,
-            drop_caches: true,
-            mount_aliases: HashMap::new(),
-            recipe_sources: Vec::new(),
-            cluster_sources: Vec::new(),
-            global_mods_dir: None,
-        };
-        // Single-node recipe on a cluster without NCCL config → OK
-        let (c, nccl) = select_cluster(&cfg, "auto", Some(1), &state).unwrap();
-        assert_eq!(c.name, "c");
-        assert!(nccl.is_none());
-    }
-
-    #[test]
-    fn multi_node_requires_nccl() {
-        let mut state = ClusterState::default();
-        state.nodes.insert(
-            "n1".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        state.nodes.insert(
-            "n2".into(),
-            NodeState {
-                reachable: true,
-                occupied: false,
-            },
-        );
-        let cfg = Config {
-            ssh_user: "u".to_string(),
-            ssh_key_path: "/k".into(),
-            clusters: vec![test_cluster("c", &["n1", "n2"])],
-            nccl: HashMap::new(), // no NCCL config
-            hf_cache_dir: None,
-            init_port: 25000,
-            before_serve: HashMap::new(),
-            exec_aliases: HashMap::new(),
-            home_assistant: None,
-            drop_caches: true,
-            mount_aliases: HashMap::new(),
-            recipe_sources: Vec::new(),
-            cluster_sources: Vec::new(),
-            global_mods_dir: None,
-        };
-        // Multi-node recipe on a cluster without NCCL config → error
-        assert!(select_cluster(&cfg, "auto", Some(2), &state).is_err());
-    }
-}
+mod tests;

src/cluster_state/tests.rs

@@ -0,0 +1,348 @@
+//! Tests for [`crate::cluster_state`] (separate file — the main
+//! module sits at the 500-line cap).
+
+use crate::cluster_state::*;
+#[cfg(test)]
+use crate::config::{Config, NcclConfig, Node};
+use std::collections::HashMap;
+
+fn test_node(name: &str) -> Node {
+    Node {
+        name: name.to_string(),
+        ssh_ip: format!("10.0.0.{}", name.chars().last().unwrap_or('0')),
+        ib_ip: Some("10.0.1.1".to_string()),
+        rank: 0,
+        host_key: None,
+        entity: None,
+    }
+}
+
+fn test_cluster(name: &str, node_names: &[&str]) -> Cluster {
+    Cluster {
+        name: name.to_string(),
+        master_ib_ip: Some("10.0.1.1".to_string()),
+        nodes: node_names.iter().map(|n| test_node(n)).collect(),
+        extra_entities: vec![],
+    }
+}
+
+fn test_nccl() -> NcclConfig {
+    NcclConfig {
+        ib_hca: "r".to_string(),
+        socket_ifname: "e".to_string(),
+        ib_gid_index: Some(3),
+        ib_disable: 0,
+        gloo_socket_ifname: None,
+        ucx_net_devices: None,
+        nccl_cross_nic: None,
+        nccl_proto: None,
+        nccl_p2p_level: None,
+    }
+}
+
+#[test]
+fn verdict_all_up() {
+    let v = verdict(&[(true, false), (true, true)], false);
+    assert!(v.up);
+    assert!(!v.switch_off);
+    assert_eq!(v.free, 1);
+}
+
+#[test]
+fn verdict_one_offline_node_takes_cluster_down() {
+    let v = verdict(&[(true, false), (false, false)], false);
+    assert!(!v.up);
+    assert_eq!(v.free, 0);
+}
+
+#[test]
+fn verdict_switch_off_takes_cluster_down() {
+    let v = verdict(&[(true, false), (true, false)], true);
+    assert!(!v.up);
+    assert!(v.switch_off);
+    assert_eq!(v.free, 0);
+}
+
+#[test]
+fn free_count_all_reachable() {
+    let mut state = ClusterState::default();
+    state.nodes.insert(
+        "n1".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    state.nodes.insert(
+        "n2".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    let c = test_cluster("c", &["n1", "n2"]);
+    assert_eq!(state.free_count(&c), 2);
+}
+
+#[test]
+fn free_count_occupied_excluded() {
+    let mut state = ClusterState::default();
+    state.nodes.insert(
+        "n1".into(),
+        NodeState {
+            reachable: true,
+            occupied: true,
+        },
+    );
+    state.nodes.insert(
+        "n2".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    let c = test_cluster("c", &["n1", "n2"]);
+    assert_eq!(state.free_count(&c), 1);
+}
+
+#[test]
+fn free_count_unreachable_excluded() {
+    let mut state = ClusterState::default();
+    state.nodes.insert(
+        "n1".into(),
+        NodeState {
+            reachable: false,
+            occupied: false,
+        },
+    );
+    state.nodes.insert(
+        "n2".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    let c = test_cluster("c", &["n1", "n2"]);
+    assert_eq!(state.free_count(&c), 1);
+}
+
+#[test]
+fn free_count_missing_node_not_free() {
+    let mut state = ClusterState::default();
+    state.nodes.insert(
+        "n1".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    // n2 not probed — absent is treated as not free
+    let c = test_cluster("c", &["n1", "n2"]);
+    assert_eq!(state.free_count(&c), 1);
+}
+
+#[test]
+fn select_smallest_with_enough_free() {
+    let mut state = ClusterState::default();
+    // big cluster: 4 nodes, 2 free
+    state.nodes.insert(
+        "n1".into(),
+        NodeState {
+            reachable: true,
+            occupied: true,
+        },
+    );
+    state.nodes.insert(
+        "n2".into(),
+        NodeState {
+            reachable: true,
+            occupied: true,
+        },
+    );
+    state.nodes.insert(
+        "n3".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    state.nodes.insert(
+        "n4".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    // small cluster: 2 nodes, 1 free
+    state.nodes.insert(
+        "n5".into(),
+        NodeState {
+            reachable: true,
+            occupied: true,
+        },
+    );
+    state.nodes.insert(
+        "n6".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+
+    let cfg = Config {
+        ssh_user: "u".to_string(),
+        ssh_key_path: "/k".into(),
+        clusters: vec![
+            test_cluster("big", &["n1", "n2", "n3", "n4"]),
+            test_cluster("small", &["n5", "n6"]),
+        ],
+        nccl: {
+            let mut m = HashMap::new();
+            m.insert("big".into(), test_nccl());
+            m.insert("small".into(), test_nccl());
+            m
+        },
+        hf_cache_dir: None,
+        init_port: 25000,
+        before_serve: HashMap::new(),
+        exec_aliases: HashMap::new(),
+        home_assistant: None,
+        drop_caches: true,
+        mount_aliases: HashMap::new(),
+        recipe_sources: Vec::new(),
+        cluster_sources: Vec::new(),
+        global_mods_dir: None,
+    };
+
+    // Request 1 node — small cluster is smallest with free >= 1
+    let (c, _) = select_cluster(&cfg, "auto", Some(1), &state).unwrap();
+    assert_eq!(c.name, "small");
+
+    // Request 2 nodes — only big cluster has 2 free
+    let (c, _) = select_cluster(&cfg, "auto", Some(2), &state).unwrap();
+    assert_eq!(c.name, "big");
+}
+
+#[test]
+fn select_cluster_not_found_errors() {
+    let state = ClusterState::default();
+    let cfg = Config {
+        ssh_user: "u".to_string(),
+        ssh_key_path: "/k".into(),
+        clusters: vec![],
+        nccl: HashMap::new(),
+        hf_cache_dir: None,
+        init_port: 25000,
+        before_serve: HashMap::new(),
+        exec_aliases: HashMap::new(),
+        home_assistant: None,
+        drop_caches: true,
+        mount_aliases: HashMap::new(),
+        recipe_sources: Vec::new(),
+        cluster_sources: Vec::new(),
+        global_mods_dir: None,
+    };
+    assert!(select_cluster(&cfg, "nonexistent", None, &state).is_err());
+}
+
+#[test]
+fn no_cluster_has_enough_free_errors() {
+    let mut state = ClusterState::default();
+    state.nodes.insert(
+        "n1".into(),
+        NodeState {
+            reachable: true,
+            occupied: true,
+        },
+    );
+    let cfg = Config {
+        ssh_user: "u".to_string(),
+        ssh_key_path: "/k".into(),
+        clusters: vec![test_cluster("c", &["n1"])],
+        nccl: {
+            let mut m = HashMap::new();
+            m.insert("c".into(), test_nccl());
+            m
+        },
+        hf_cache_dir: None,
+        init_port: 25000,
+        before_serve: HashMap::new(),
+        exec_aliases: HashMap::new(),
+        home_assistant: None,
+        drop_caches: true,
+        mount_aliases: HashMap::new(),
+        recipe_sources: Vec::new(),
+        cluster_sources: Vec::new(),
+        global_mods_dir: None,
+    };
+    assert!(select_cluster(&cfg, "auto", Some(1), &state).is_err());
+}
+
+#[test]
+fn single_node_allows_cluster_without_nccl() {
+    let mut state = ClusterState::default();
+    state.nodes.insert(
+        "n1".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    let cfg = Config {
+        ssh_user: "u".to_string(),
+        ssh_key_path: "/k".into(),
+        clusters: vec![test_cluster("c", &["n1"])],
+        nccl: HashMap::new(), // no NCCL config
+        hf_cache_dir: None,
+        init_port: 25000,
+        before_serve: HashMap::new(),
+        exec_aliases: HashMap::new(),
+        home_assistant: None,
+        drop_caches: true,
+        mount_aliases: HashMap::new(),
+        recipe_sources: Vec::new(),
+        cluster_sources: Vec::new(),
+        global_mods_dir: None,
+    };
+    // Single-node recipe on a cluster without NCCL config → OK
+    let (c, nccl) = select_cluster(&cfg, "auto", Some(1), &state).unwrap();
+    assert_eq!(c.name, "c");
+    assert!(nccl.is_none());
+}
+
+#[test]
+fn multi_node_requires_nccl() {
+    let mut state = ClusterState::default();
+    state.nodes.insert(
+        "n1".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    state.nodes.insert(
+        "n2".into(),
+        NodeState {
+            reachable: true,
+            occupied: false,
+        },
+    );
+    let cfg = Config {
+        ssh_user: "u".to_string(),
+        ssh_key_path: "/k".into(),
+        clusters: vec![test_cluster("c", &["n1", "n2"])],
+        nccl: HashMap::new(), // no NCCL config
+        hf_cache_dir: None,
+        init_port: 25000,
+        before_serve: HashMap::new(),
+        exec_aliases: HashMap::new(),
+        home_assistant: None,
+        drop_caches: true,
+        mount_aliases: HashMap::new(),
+        recipe_sources: Vec::new(),
+        cluster_sources: Vec::new(),
+        global_mods_dir: None,
+    };
+    // Multi-node recipe on a cluster without NCCL config → error
+    assert!(select_cluster(&cfg, "auto", Some(2), &state).is_err());
+}

src/docker/health.rs

@@ -2,7 +2,6 @@
 //! deployments across all clusters.
 
 use crate::config::{Cluster, Config};
-use crate::home_assistant::Client as HaClient;
 use crate::ssh::run_ssh;
 use anyhow::Result;
 use std::collections::HashMap;
@@ -19,28 +18,22 @@ struct NodeHealth {
 }
 
 /// Checks Docker health — shows node status and containers belonging to this cluster.
+///
+/// Cluster-level semantics (shared with the web dashboard via
+/// [`crate::cluster_state::verdict`]): the cluster is UP only when every
+/// node is reachable and the HA switch entities (if configured) are on.
+/// A down cluster prints no containers, even on nodes that are alive.
 pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) -> Result<()> {
     let nodes = &cluster.nodes;
     let cluster_label = &cluster.name;
     let nccl = config.nccl.get(cluster_label);
 
     // Check HA cluster-level entities (switch, etc.) in parallel.
-    let ha_handle = if let Some(ref ha_cfg) = config.home_assistant {
-        let ha = HaClient::new(ha_cfg).ok();
-        let entities = cluster.extra_entities.clone();
-        Some(crate::locale::spawn_with_locale(async move {
-            let client = ha.as_ref()?;
-            for e in &entities {
-                match client.get_state(e).await {
-                    Some(ref s) if s == "on" => continue,
-                    _ => return Some(false),
-                }
-            }
-            Some(true)
-        }))
-    } else {
-        None
-    };
+    let config_ha = config.clone();
+    let cluster_ha = cluster.clone();
+    let ha_handle = crate::locale::spawn_with_locale(async move {
+        crate::home_assistant::extra_entities_off(&config_ha, &cluster_ha).await
+    });
 
     // Build the network check command snippet for this cluster's NCCL interfaces.
     let net_script = nccl.as_ref().and_then(|n| {
@@ -143,41 +136,61 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
         }
     }
 
-    // Resolve HA result.
-    let ha_ok = match ha_handle {
-        Some(h) => h.await.unwrap_or(Some(true)).unwrap_or(true),
-        None => true,
-    };
-
+    // Resolve HA result + cluster verdict.
+    let switch_off = ha_handle.await.unwrap_or(false);
     let total = nodes.len();
     let online = node_health.iter().filter(|n| n.online).count();
     let offline = total - online;
-    let container_count: usize = node_health.iter().map(|n| n.containers.len()).sum();
     let net_ok = node_health.iter().all(|n| !n.online || n.net_up);
+    let reachables: Vec<(bool, bool)> = node_health
+        .iter()
+        .map(|n| (n.online, !n.containers.is_empty()))
+        .collect();
+    let v = crate::cluster_state::verdict(&reachables, switch_off);
+    let container_count: usize = if v.up {
+        node_health.iter().map(|n| n.containers.len()).sum()
+    } else {
+        0
+    };
 
     print!(
         "{}",
         fluent_i18n::t!("health-cluster-header", { "name" => &cluster_label, "nodes" => total })
     );
-    if !ha_ok && !cluster.extra_entities.is_empty() {
-        print!(" {}", fluent_i18n::t!("health-switch-off"));
-    } else if !net_ok {
-        print!(" {}", fluent_i18n::t!("health-interface-down"));
-    } else if online == total {
-        print!(" {}", fluent_i18n::t!("health-all-online"));
+    if !v.up {
+        // Cluster is down as a whole — say why, nothing else to report.
+        if v.switch_off {
+            print!(" {}", fluent_i18n::t!("health-switch-off"));
+        } else {
+            print!(
+                "{}",
+                fluent_i18n::t!("health-partial-online", { "online" => online, "offline" => offline })
+            );
+        }
+        println!(" {}", fluent_i18n::t!("health-cluster-down"));
     } else {
+        // `v.up` ⇒ every node is reachable.
+        print!(" {}", fluent_i18n::t!("health-all-online"));
+        if !net_ok {
+            print!(" {}", fluent_i18n::t!("health-interface-down"));
+        }
         print!(
-            "{}",
-            fluent_i18n::t!("health-partial-online", { "online" => online, "offline" => offline })
+            " {}",
+            fluent_i18n::t!("health-free-count", { "free" => v.free })
         );
     }
-    if container_count > 0 {
-        println!(
-            "{}",
-            fluent_i18n::t!("health-container-count", { "count" => container_count })
-        );
-    } else {
-        print!("{}", fluent_i18n::t!("health-no-containers-inline"));
+    // Container count: only for up clusters — the down path already
+    // printed its header line (with newline), and an offline cluster
+    // reports no containers by definition.
+    if v.up {
+        if container_count > 0 {
+            println!(
+                "{}",
+                fluent_i18n::t!("health-container-count", { "count" => container_count })
+            );
+        } else {
+            print!("{}", fluent_i18n::t!("health-no-containers-inline"));
+        }
     }
 
     for nh in &node_health {
@@ -193,6 +206,15 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
         } else {
             String::new()
         };
+        if !v.up {
+            // Cluster is down as a whole: the node's containers are not
+            // ours to report — print the node as alive-but-ignored.
+            println!(
+                "{}",
+                fluent_i18n::t!("health-node-online-cluster-down", { "name" => &nh.name, "ip" => &nh.ib_ip, "net" => &net_status })
+            );
+            continue;
+        }
         if nh.containers.is_empty() {
             println!(
                 "{}",
@@ -212,6 +234,12 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
 }
 
 /// Quick overview — recipes running on each cluster.
+///
+/// Cluster-level semantics: a container is only reported when the cluster
+/// it is labeled with (`sm121.cluster`) is UP — all nodes reachable and
+/// HA switches on (same [`crate::cluster_state::verdict`] rule as
+/// `health_check` and the web dashboard). Containers on live nodes of a
+/// down cluster are ignored.
 pub async fn health_summary(config: &Config) -> Result<()> {
     let mut seen = std::collections::HashSet::new();
     let mut handles = Vec::new();
@@ -228,17 +256,48 @@ pub async fn health_summary(config: &Config) -> Result<()> {
                        --format '{{.Label \"sm121.recipe\"}}|{{.Label \"sm121.cluster\"}}'"
                 .to_string();
             handles.push(crate::locale::spawn_with_locale(async move {
-                let out = run_ssh(&config, &node, cmd, false)
-                    .await
-                    .unwrap_or_default();
+                // Ok/Err doubles as the node's reachability signal.
+                let out = run_ssh(&config, &node, cmd, false).await;
                 (name, out)
             }));
         }
     }
 
-    let mut recipe_nodes: HashMap<String, std::collections::HashSet<String>> = HashMap::new();
+    // HA switch state for every cluster, in parallel.
+    let ha_handles: Vec<(String, _)> = config
+        .clusters
+        .iter()
+        .filter(|c| !c.extra_entities.is_empty())
+        .map(|c| {
+            let config = config.clone();
+            let cluster = c.clone();
+            (
+                c.name.clone(),
+                crate::locale::spawn_with_locale(async move {
+                    crate::home_assistant::extra_entities_off(&config, &cluster).await
+                }),
+            )
+        })
+        .collect();
+
+    let mut reachable: HashMap<String, bool> = HashMap::new();
+    let mut recipe_nodes: HashMap<String, HashMap<String, std::collections::HashSet<String>>> =
+        HashMap::new();
     for h in futures::future::join_all(handles).await {
-        let (node_name, output) = h.unwrap_or_default();
+        let (node_name, output) = match h {
+            Ok((node_name, Ok(output))) => {
+                reachable.insert(node_name.clone(), true);
+                (node_name, output)
+            }
+            Ok((node_name, Err(_))) => {
+                reachable.insert(node_name.clone(), false);
+                continue;
+            }
+            Err(e) => {
+                tracing::warn!("health summary task panicked: {e}");
+                continue;
+            }
+        };
         for line in output.lines() {
             let line = line.trim();
             if line.is_empty() {
@@ -252,24 +311,50 @@ pub async fn health_summary(config: &Config) -> Result<()> {
             recipe_nodes
                 .entry(recipe)
                 .or_default()
+                .entry(parts.next().unwrap_or("").to_string())
+                .or_default()
                 .insert(node_name.clone());
         }
     }
 
-    if recipe_nodes.is_empty() {
+    let mut cluster_up: HashMap<String, bool> = HashMap::new();
+    let ha_results: HashMap<String, bool> = futures::future::join_all(
+        ha_handles
+            .into_iter()
+            .map(|(name, h)| async move { (name, h.await.unwrap_or(false)) }),
+    )
+    .await
+    .into_iter()
+    .collect();
+    for cluster in &config.clusters {
+        let all_reachable = cluster
+            .nodes
+            .iter()
+            .all(|n| reachable.get(&n.name).copied().unwrap_or(false));
+        let switch_off = ha_results.get(&cluster.name).copied().unwrap_or(false);
+        cluster_up.insert(cluster.name.clone(), all_reachable && !switch_off);
+    }
+    let mut lines: Vec<String> = Vec::new();
+    for (recipe, by_cluster) in &recipe_nodes {
+        for (cluster, nodes) in by_cluster {
+            if !cluster_up.get(cluster).copied().unwrap_or(false) {
+                continue; // down cluster — its containers are not ours to report
+            }
+            let mut sorted: Vec<_> = nodes.iter().collect();
+            sorted.sort();
+            let node_list: Vec<&str> = sorted.iter().map(|s| s.as_str()).collect();
+            lines.push(fluent_i18n::t!("health-recipe-line", { "recipe" => recipe, "nodes" => node_list.len(), "list" => &node_list.join(", ") }));
+        }
+    }
+
+    if lines.is_empty() {
         println!("\n{}", fluent_i18n::t!("health-no-recipes"));
         return Ok(());
     }
 
     println!("\n{}", fluent_i18n::t!("health-running-recipes"));
-    for (recipe, nodes) in &recipe_nodes {
-        let mut sorted: Vec<_> = nodes.iter().collect();
-        sorted.sort();
-        let node_list: Vec<&str> = sorted.iter().map(|s| s.as_str()).collect();
-        println!(
-            "{}",
-            fluent_i18n::t!("health-recipe-line", { "recipe" => recipe, "nodes" => node_list.len(), "list" => &node_list.join(", ") })
-        );
+    for line in &lines {
+        println!("{line}");
     }
     Ok(())
 }

src/home_assistant.rs

@@ -10,12 +10,13 @@ use serde_json::json;
 use std::time::Duration;
 use tracing::{debug, info, warn};
 
-use crate::config::HomeAssistantConfig;
+use crate::config::{Cluster, Config, HomeAssistantConfig};
 
 /// Thin async wrapper around the HA REST API.
 ///
 /// `base_url` is the HA root URL (`http://homeassistant.local:8123`),
 /// `token` is the long-lived access token used as bearer auth.
+#[derive(Clone)]
 pub struct Client {
     base_url: String,
     token: String,
@@ -129,6 +130,38 @@ impl Client {
     }
 }
 
+/// True when HA is configured and any of the cluster's `extra_entities`
+/// (power switches) reports anything other than "on" — including HA being
+/// unreachable (unknown is treated as off, same as the `health` command
+/// has always done). No HA config, no entities, or an unparseable HA URL
+/// → `false` (nothing to check ≠ off).
+pub async fn extra_entities_off(config: &Config, cluster: &Cluster) -> bool {
+    let Some(ha_cfg) = &config.home_assistant else {
+        return false;
+    };
+    if cluster.extra_entities.is_empty() {
+        return false;
+    }
+    let Ok(client) = Client::new(ha_cfg) else {
+        return false;
+    };
+    let mut handles = Vec::new();
+    for e in &cluster.extra_entities {
+        let client = client.clone();
+        let e = e.clone();
+        handles.push(crate::locale::spawn_with_locale(async move {
+            client.get_state(&e).await
+        }));
+    }
+    for h in futures::future::join_all(handles).await {
+        match h {
+            Ok(Some(s)) if s == "on" => {}
+            _ => return true,
+        }
+    }
+    false
+}
+
 /// Parse `switch.dgx1` into `("switch", "dgx1")` so callers can build
 /// their own endpoints if needed.  Exposed for the `power` subcommand
 /// and for tests.

src/web/events.rs

@@ -44,6 +44,11 @@ pub struct NodeSnap {
 pub struct ClusterSnap {
     pub name: String,
     pub nodes: Vec<NodeSnap>,
+    /// All nodes reachable AND HA switches on (cluster-level verdict).
+    pub up: bool,
+    /// HA `extra_entities` report anything but "on".
+    pub switch_off: bool,
+    /// Free nodes; 0 when the cluster is down.
     pub free: u32,
 }
 
@@ -63,23 +68,39 @@ fn now_ts() -> u64 {
 }
 
 /// Pure snapshot builder — unit-testable without SSH.
+///
+/// Cluster-level semantics via [`crate::cluster_state::verdict`]: one
+/// offline node or a switch off takes the whole cluster down, and a down
+/// cluster reports no containers (even on nodes that are alive).
 pub fn build_snapshot(
     config: &Config,
     nodes: &HashMap<String, crate::web::probe::WebNodeState>,
+    switch_off: &HashMap<String, bool>,
     jobs: &[crate::web::jobs::Job],
 ) -> Snapshot {
     let mut clusters = Vec::with_capacity(config.clusters.len());
     for c in &config.clusters {
         let mut node_snaps = Vec::with_capacity(c.nodes.len());
-        let mut free = 0u32;
+        let mut reachables: Vec<(bool, bool)> = Vec::with_capacity(c.nodes.len());
         for n in &c.nodes {
             let s = nodes.get(&n.name);
             let reachable = s.is_some_and(|x| x.reachable);
             let occupied = s.is_some_and(|x| x.occupied);
-            let containers = s.map(|x| x.containers.clone()).unwrap_or_default();
-            if reachable && !occupied {
-                free += 1;
-            }
+            reachables.push((reachable, occupied));
+        }
+        let so = switch_off.get(&c.name).copied().unwrap_or(false);
+        let v = crate::cluster_state::verdict(&reachables, so);
+        for n in &c.nodes {
+            let s = nodes.get(&n.name);
+            let reachable = s.is_some_and(|x| x.reachable);
+            let occupied = s.is_some_and(|x| x.occupied);
+            // A down cluster reports no containers — the cluster is off as
+            // a whole, whatever individual nodes still run.
+            let containers = if v.up {
+                s.map(|x| x.containers.clone()).unwrap_or_default()
+            } else {
+                Vec::new()
+            };
             node_snaps.push(NodeSnap {
                 name: n.name.clone(),
                 ssh_ip: n.ssh_ip.clone(),
@@ -91,7 +112,9 @@ pub fn build_snapshot(
         clusters.push(ClusterSnap {
             name: c.name.clone(),
             nodes: node_snaps,
-            free,
+            up: v.up,
+            switch_off: so,
+            free: v.free,
         });
     }
     Snapshot {
@@ -187,19 +210,78 @@ mod tests {
             },
         );
         // n2 absent → unreachable.
-        let snap = build_snapshot(&config, &nodes, &[]);
+        let snap = build_snapshot(&config, &nodes, &std::collections::HashMap::new(), &[]);
         assert_eq!(snap.clusters.len(), 2);
         let a = &snap.clusters[0];
+        assert!(a.up);
         assert!(a.nodes[0].reachable && a.nodes[0].occupied);
         assert_eq!(a.nodes[0].containers[0].recipe, "r");
         assert!(a.nodes[1].reachable && !a.nodes[1].occupied);
         assert_eq!(a.free, 1);
+        // cluster b has one offline node → down as a whole
         let b = &snap.clusters[1];
+        assert!(!b.up);
         assert!(!b.nodes[0].reachable);
         assert_eq!(b.free, 0);
         assert!(snap.jobs.is_empty());
     }
 
+    #[test]
+    fn down_cluster_hides_containers_on_live_nodes() {
+        // Both nodes of cluster a are live; n1 runs a container but n0 is
+        // offline → cluster a is down as a whole and the live node's
+        // container must not be reported.
+        let config = config();
+        let mut nodes = HashMap::new();
+        nodes.insert("n0".into(), crate::web::probe::WebNodeState::default());
+        nodes.insert(
+            "n1".into(),
+            crate::web::probe::WebNodeState {
+                reachable: true,
+                occupied: true,
+                containers: vec![ContainerSnap {
+                    name: "c2".into(),
+                    status: "Up".into(),
+                    cluster: "a".into(),
+                    recipe: "r".into(),
+                }],
+            },
+        );
+        let snap = build_snapshot(&config, &nodes, &std::collections::HashMap::new(), &[]);
+        let a = &snap.clusters[0];
+        assert!(!a.up, "one offline node takes the cluster down");
+        assert_eq!(a.free, 0);
+        assert!(
+            a.nodes[1].containers.is_empty(),
+            "containers hidden on a down cluster"
+        );
+        assert!(a.nodes[1].reachable, "reachability is still shown");
+    }
+
+    #[test]
+    fn switch_off_takes_cluster_down() {
+        let config = config();
+        let mut nodes = HashMap::new();
+        for n in ["n0", "n1", "n2"] {
+            nodes.insert(
+                n.into(),
+                crate::web::probe::WebNodeState {
+                    reachable: true,
+                    occupied: false,
+                    containers: vec![],
+                },
+            );
+        }
+        let mut so = HashMap::new();
+        so.insert("a".into(), true);
+        let snap = build_snapshot(&config, &nodes, &so, &[]);
+        assert!(!snap.clusters[0].up);
+        assert!(snap.clusters[0].switch_off);
+        assert_eq!(snap.clusters[0].free, 0);
+        assert!(snap.clusters[1].up);
+        assert_eq!(snap.clusters[1].free, 1);
+    }
+
     #[test]
     fn job_event_serializes_with_type_tag() {
         let ev = Event::Job {

src/web/mod.rs

@@ -48,9 +48,9 @@ pub struct AppState {
 /// Probe the fleet and broadcast a fresh state snapshot (supervisor tick
 /// and job-finish both route through here).
 pub(crate) async fn push_state(app: &AppState) {
-    let st = probe::probe(&app.config).await;
+    let fleet = probe::probe(&app.config).await;
     let jobs = app.jobs.lock().unwrap().clone();
-    let snap = build_snapshot(&app.config, &st, &jobs);
+    let snap = build_snapshot(&app.config, &fleet.nodes, &fleet.switch_off, &jobs);
     let _ = app.bus.send(Event::State {
         ts: snap.ts,
         snapshot: snap.clone(),

src/web/probe.rs

@@ -22,9 +22,17 @@ pub struct WebNodeState {
     pub containers: Vec<ContainerSnap>,
 }
 
-/// One `docker ps` over SSH per unique node, parallel. Unreachable nodes
-/// come back as the default state (reachable: false) — probe never fails.
-pub async fn probe(config: &Config) -> HashMap<String, WebNodeState> {
+/// Fleet-wide probe result: per-node state + per-cluster HA switch state.
+pub struct Fleet {
+    pub nodes: HashMap<String, WebNodeState>,
+    /// cluster name → its `extra_entities` report anything but "on".
+    pub switch_off: HashMap<String, bool>,
+}
+
+/// One `docker ps` over SSH per unique node, parallel + one HA check per
+/// cluster with `extra_entities`. Unreachable nodes come back as the
+/// default state (reachable: false) — probe never fails.
+pub async fn probe(config: &Config) -> Fleet {
     const CMD: &str = r#"docker ps --filter label=sm121.cluster --format '{{.Names}}\t{{.Status}}\t{{.Label "sm121.cluster"}}\t{{.Label "sm121.recipe"}}' 2>/dev/null"#;
     let mut seen = std::collections::HashSet::new();
     let mut handles = Vec::new();
@@ -42,6 +50,21 @@ pub async fn probe(config: &Config) -> HashMap<String, WebNodeState> {
             }));
         }
     }
+    let mut ha_handles = Vec::new();
+    for cluster in &config.clusters {
+        if cluster.extra_entities.is_empty() {
+            continue;
+        }
+        let config = config.clone();
+        let cluster = cluster.clone();
+        let name = cluster.name.clone();
+        ha_handles.push((
+            name,
+            crate::locale::spawn_with_locale(async move {
+                crate::home_assistant::extra_entities_off(&config, &cluster).await
+            }),
+        ));
+    }
     let mut out: HashMap<String, WebNodeState> = HashMap::new();
     for h in futures::future::join_all(handles).await {
         match h {
@@ -77,5 +100,16 @@ pub async fn probe(config: &Config) -> HashMap<String, WebNodeState> {
             }
         }
     }
-    out
+    let switch_off: HashMap<String, bool> = futures::future::join_all(
+        ha_handles
+            .into_iter()
+            .map(|(name, h)| async move { (name, h.await.unwrap_or(false)) }),
+    )
+    .await
+    .into_iter()
+    .collect();
+    Fleet {
+        nodes: out,
+        switch_off,
+    }
 }

src/web/tests.rs

@@ -142,7 +142,12 @@ async fn bus_delivers_state_events() {
 async fn snapshot_serializes_with_type_tag() {
     let ev = Event::State {
         ts: 7,
-        snapshot: build_snapshot(&fake_config(), &std::collections::HashMap::new(), &[]),
+        snapshot: build_snapshot(
+            &fake_config(),
+            &std::collections::HashMap::new(),
+            &std::collections::HashMap::new(),
+            &[],
+        ),
     };
     let json = serde_json::to_value(&ev).unwrap();
     assert_eq!(json["type"], "state");

src/web/ui.html

@@ -52,6 +52,7 @@
   .cluster-head { display: flex; align-items: baseline; gap: 8px; margin-bottom: 6px; }
   .cluster-head b { font-size: 13px; }
   .cluster-head .free { color: var(--muted); font-size: 12px; }
+  .cluster-head .down { color: var(--led-off); font-size: 12px; }
   .node { display: flex; align-items: center; gap: 10px; padding: 5px 0; border-bottom: 1px solid var(--border); }
   .node:last-child { border-bottom: none; }
   .led {
@@ -70,6 +71,7 @@
   .cname { font-family: ui-monospace, monospace; color: var(--fg); }
   .actions { display: flex; gap: 8px; align-items: center; margin: 6px 0 0 21px; flex-wrap: wrap; }
   .actions select, .actions input[type=checkbox] { font: inherit; }
+  .actions input[type=checkbox] { accent-color: var(--accent); width: 14px; height: 14px; }
   .actions select {
     background: var(--btn); color: var(--fg); border: 1px solid var(--border);
     border-radius: 6px; padding: 2px 6px; font-size: 12.5px; max-width: 260px; height: 26px;
@@ -185,15 +187,26 @@ function connectSSE() {
 function ledState(n) { return n.reachable ? (n.occupied ? "b" : "on") : "off"; }
 function meta(n, c) {
   if (!n.reachable) return "offline";
+  if (!c.up) return "online — cluster offline";
   if (n.occupied) return n.containers.map((x) => x.status).join(", ");
-  return c.free ? "free" : "no free nodes in cluster";
+  return "free";
+}
+function clusterReason(c) {
+  if (c.up) return "";
+  if (c.switch_off) return "switch off";
+  const off = c.nodes.filter((n) => !n.reachable).map((n) => n.name);
+  return off.length + " node" + (off.length === 1 ? "" : "s") + " offline" + (off.length ? " (" + off.join(", ") + ")" : "");
 }
 function renderCluster(c, prev) {
   const el = document.createElement("div");
   el.className = "cluster";
   const head = document.createElement("div");
   head.className = "cluster-head";
-  head.innerHTML = `<b>${esc(c.name)}</b><span class="free">${c.free} free</span>`;
+  if (c.up) {
+    head.innerHTML = `<b>${esc(c.name)}</b><span class="free">${c.free} free</span>`;
+  } else {
+    head.innerHTML = `<b>${esc(c.name)}</b><span class="down">⏻ offline — ${esc(clusterReason(c))}</span>`;
+  }
   el.appendChild(head);
   const prevNodes = prev ? Object.fromEntries(prev.nodes.map((n) => [n.name, n])) : {};
   for (const n of c.nodes) {
@@ -223,6 +236,12 @@ function renderCluster(c, prev) {
 function actionBar(c, prevNodes) {
   const bar = document.createElement("div");
   bar.className = "actions";
+  if (!c.up) {
+    // Down cluster: serve/stop are meaningless — only power actions apply.
+    bar.appendChild(abtn("power-" + c.name, "power on", () => doPower("on", c.name), ""));
+    bar.appendChild(abtn("pwswitch-" + c.name, "switch", () => doPower("switch", c.name), ""));
+    return bar;
+  }
   const sel = document.createElement("select");
   sel.id = "recipe-" + c.name;
   if (!state.recipes.length) {
@@ -272,8 +291,10 @@ function onState(snap) {
   for (const c of snap.clusters) {
     const chip = document.createElement("div");
     chip.className = "chip" + (c.name === state.active ? " active" : "");
-    const dot = c.free > 0 ? "on" : c.nodes.some((n) => n.occupied) ? "b" : "off";
-    chip.innerHTML = `<span class="led ${dot}"></span>${esc(c.name)}<span class="m">${c.free} free</span>`;
+    const dot = c.up ? (c.free > 0 ? "on" : "b") : "off";
+    const m = c.up ? c.free + " free" : "⏻ OFF";
+    chip.title = c.up ? "" : clusterReason(c);
+    chip.innerHTML = `<span class="led ${dot}"></span>${esc(c.name)}<span class="m">${m}</span>`;
     chip.onclick = () => { state.active = c.name; localStorage.setItem("sm121-active", c.name); onState(state.snap); };
     strip.appendChild(chip);
   }
@@ -282,11 +303,42 @@ function onState(snap) {
 }
 
 // --- jobs ---------------------------------------------------------------------
+// "Running" entries come from the docker labels (sm121.recipe / sm121.cluster)
+// on live containers — jobs don't need to have started from this dashboard.
+function runningFromLabels() {
+  const out = [];
+  if (!state.snap) return out;
+  for (const c of state.snap.clusters) {
+    if (!c.up) continue; // down clusters report no containers
+    for (const n of c.nodes) {
+      for (const x of n.containers) out.push({ cluster: c.name, node: n.name, name: x.name, recipe: x.recipe, status: x.status });
+    }
+  }
+  return out;
+}
 function renderJobs() {
   const box = $("#jobs");
   box.innerHTML = "";
+  const run = runningFromLabels();
+  if (run.length) {
+    const h = document.createElement("div");
+    h.className = "jobs-empty"; h.textContent = "running (docker labels)";
+    box.appendChild(h);
+    for (const r of run) {
+      const el = document.createElement("div");
+      el.className = "job jjob";
+      el.innerHTML = `<span class="jdot"></span><span class="jkind">serve</span>
+        <span class="jmsg">${esc(r.cluster)}:${esc(r.recipe)} — ${esc(r.name)} on ${esc(r.node)} · ${esc(r.status)}</span>`;
+      box.appendChild(el);
+    }
+  }
   const js = [...state.jobs.values()].sort((a, b) => b.started - a.started);
-  if (!js.length) { box.innerHTML = '<div class="jobs-empty">no actions yet — serve / stop / power started from this dashboard appear here</div>'; return; }
+  if (!run.length && !js.length) { box.innerHTML = '<div class="jobs-empty">no sm121 containers running, no actions yet — serve / stop / power started from this dashboard appear here</div>'; return; }
+  if (run.length) {
+    const sep = document.createElement("div");
+    sep.className = "jobs-empty"; sep.textContent = "actions";
+    box.appendChild(sep);
+  }
   for (const j of js) {
     const el = document.createElement("div");
     el.className = "job " + (j.status === "ok" ? "ok" : j.status === "failed" ? "failed" : "jjob");