web: FLARE controller is discovered (not configured), dead groups are not running

- probe_order: pinned node -> last winner -> every cluster's rank-0 node; the
  first whose 'orbit status' lists groups becomes the controller and is
  remembered, so later ticks ask one node.
- A group that exited/still prints its row is no longer counted as running (the
  dashboard showed a dead lane as a live container on dgx5 — exactly the state
  left by the 08:04 RoCE 'two-shot op abandoned' crash).
- The pseudo-container stays on the controller node only: our clusters overlap
  (dgx5 belongs to six of them), so marking "the controller's cluster" lights
  up several clusters for one lane.

src/web/flare_nodes.rs

@@ -1,25 +1,59 @@
 //! Which of our nodes runs the `flare orbit` CLI.
 //!
-//! The orbit cluster is FLARE's own naming (`gx10-dgx5`…), so the controller
-//! is configured explicitly (`[flare].controller_node = "dgx5.lan"`); without
-//! it we fall back to the first cluster's rank-0 node — right for a single
-//! cluster, wrong for a multi-cluster config, which is why the explicit field
-//! exists.
+//! FLARE names its peers itself (`gx10-dgx5`…), so the launcher *finds* the
+//! controller: every cluster's rank-0 node is a candidate and the first one
+//! whose `orbit status` answers is it. The winner is remembered, and
+//! `[flare].controller_node` remains an explicit override.
 
 use crate::config::{Config, Node};
 
-/// The node that runs `flare orbit`.
-pub fn controller_node(config: &Config) -> Option<&Node> {
+/// `[flare].controller_node`, when the operator pinned it.
+pub fn pinned(config: &Config) -> Option<&Node> {
     let want = config
         .flare
         .as_ref()
-        .and_then(|f| f.controller_node.clone());
-    let all = config.clusters.iter().flat_map(|c| c.nodes.iter());
-    match want {
-        Some(name) => all.clone().find(|n| n.name == name),
-        None => all
-            .clone()
-            .find(|n| n.rank == 0)
-            .or_else(|| all.into_iter().next()),
+        .and_then(|f| f.controller_node.clone())?;
+    find(config, &want)
+}
+
+fn find<'a>(config: &'a Config, name: &str) -> Option<&'a Node> {
+    config
+        .clusters
+        .iter()
+        .flat_map(|c| c.nodes.iter())
+        .find(|n| n.name == name)
+}
+
+/// Candidate controllers: the rank-0 node of every cluster (deduplicated by
+/// ssh address), the pinned one first.
+pub fn candidates(config: &Config) -> Vec<&Node> {
+    let mut out: Vec<&Node> = Vec::new();
+    if let Some(n) = pinned(config) {
+        out.push(n);
+    }
+    for c in &config.clusters {
+        if let Some(n) = c.nodes.iter().find(|n| n.rank == 0)
+            && !out.iter().any(|x| x.ssh_ip == n.ssh_ip)
+        {
+            out.push(n);
+        }
+    }
+    out
+}
+
+/// Probe order for one tick: pinned → last winner → the rest of the candidates.
+pub fn probe_order(config: &Config, remembered: Option<&str>) -> Vec<String> {
+    let mut order: Vec<String> = Vec::new();
+    if let Some(n) = pinned(config) {
+        order.push(n.ssh_ip.clone());
+    }
+    if let Some(ip) = remembered {
+        order.push(ip.to_string());
+    }
+    for n in candidates(config) {
+        if !order.contains(&n.ssh_ip) {
+            order.push(n.ssh_ip.clone());
+        }
     }
+    order
 }

src/web/probe.rs

@@ -83,7 +83,9 @@ pub fn parse_orbit_groups(stdout: &str) -> Vec<(String, Option<u64>)> {
     // in the `held by <group>-<stamp>-<hex>` detail line, so collect names
     // first and attach a timestamp when one is found.
     for line in stdout.lines() {
-        if let Some((before, _)) = line.split_once(" rank ")
+        // A group that exited still prints its row: never count it as running.
+        if !(line.contains("exited") || line.contains("stopped"))
+            && let Some((before, _)) = line.split_once(" rank ")
             && let Some(name) = before.split_whitespace().last()
             && !name.starts_with("gx10")
             && !name.contains("GiB")
@@ -95,7 +97,12 @@ pub fn parse_orbit_groups(stdout: &str) -> Vec<(String, Option<u64>)> {
             out.push((name.to_string(), None));
         }
     }
-    for tok in stdout.split(|c: char| c.is_whitespace() || c == ',') {
+    let live: String = stdout
+        .lines()
+        .filter(|l| !(l.contains("exited") || l.contains("stopped")))
+        .collect::<Vec<_>>()
+        .join("\n");
+    for tok in live.split(|c: char| c.is_whitespace() || c == ',') {
         let t = tok.trim_matches(|c: char| c == '`' || c == '"' || c == '\'');
         let parts: Vec<&str> = t.split('-').collect();
         if parts.len() < 3 {
@@ -168,27 +175,43 @@ pub struct WebNodeState {
 /// a FLARE serve shows up in the dashboard exactly like a docker container
 /// (green "up (fleet)" + uptime). The recipe column carries the orbit group's
 /// base name (`[flare].name`), which the report maps back to a recipe.
-pub async fn flare_groups(config: &crate::config::Config) -> Vec<(String, Option<u64>)> {
+pub async fn flare_groups(
+    config: &crate::config::Config,
+) -> (Option<String>, Vec<(String, Option<u64>)>) {
     let Some(f) = config.flare.as_ref() else {
-        return Vec::new();
+        return (None, Vec::new());
     };
     let binary = f.binary.clone().unwrap_or_else(|| "flare".to_string());
     let orbit_cluster = f
         .orbit_cluster
         .clone()
         .unwrap_or_else(|| "sparks".to_string());
-    let Some(controller) = crate::web::flare_nodes::controller_node(config) else {
-        return Vec::new();
-    };
     let cmd = format!("{binary} orbit status --cluster {orbit_cluster} 2>/dev/null");
-    match crate::ssh::run_ssh(config, controller, cmd, false).await {
-        Ok(out) => parse_orbit_groups(&out),
-        Err(e) => {
-            tracing::debug!("flare orbit status failed: {e}");
-            Vec::new()
+    let remembered = LAST_CONTROLLER.lock().await.clone();
+    for ip in crate::web::flare_nodes::probe_order(config, remembered.as_deref()) {
+        let Some(node) = config
+            .clusters
+            .iter()
+            .flat_map(|c| c.nodes.iter())
+            .find(|n| n.ssh_ip == ip)
+        else {
+            continue;
+        };
+        if let Ok(out) = crate::ssh::run_ssh(config, node, cmd.clone(), false).await {
+            let groups = parse_orbit_groups(&out);
+            if !groups.is_empty() {
+                *LAST_CONTROLLER.lock().await = Some(ip.clone());
+                return (Some(ip), groups);
+            }
         }
     }
+    (None, Vec::new())
 }
+
+/// Controller address remembered between probe ticks.
+static LAST_CONTROLLER: std::sync::LazyLock<tokio::sync::Mutex<Option<String>>> =
+    std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
+
 pub struct Fleet {
     pub nodes: HashMap<String, WebNodeState>,
     /// cluster name → a blocking `extra_entity` reports anything but "on".
@@ -283,13 +306,14 @@ pub async fn probe(config: &Config) -> Fleet {
     // as pseudo-containers so the dashboard shows them like any other lane.
     // They carry the controller's cluster label — the snapshot builder drops
     // containers that belong to no known cluster.
-    let flare = flare_groups(config).await;
+    let (flare_controller, flare) = flare_groups(config).await;
     let controller_cluster = config
         .clusters
         .iter()
         .find(|c| {
-            crate::web::flare_nodes::controller_node(config)
-                .is_some_and(|n| c.nodes.iter().any(|x| x.name == n.name))
+            c.nodes
+                .iter()
+                .any(|x| Some(&x.ssh_ip) == flare_controller.as_ref())
         })
         .map(|c| c.name.clone())
         .unwrap_or_default();
@@ -314,8 +338,13 @@ pub async fn probe(config: &Config) -> Fleet {
                 // Only the controller node hosts the orbit groups (the fold
                 // below runs per node — without this guard every node would
                 // report the same group).
-                let is_controller = crate::web::flare_nodes::controller_node(config)
-                    .is_some_and(|c| c.ssh_ip == *node_ip_of(config, &name));
+                // The group is reported once, on the node that runs `orbit`:
+                // clusters overlap here (dgx5 sits in six of them), so folding
+                // it onto "the controller's cluster" would light up several
+                // clusters for one lane.
+                let is_controller = flare_controller
+                    .as_deref()
+                    .is_some_and(|ip| ip == node_ip_of(config, &name));
                 for (group, created) in flare.iter().filter(|_| is_controller) {
                     containers.push(ContainerSnap {
                         name: group.clone(),
@@ -416,3 +445,19 @@ mod orbit_tests {
         assert!(parse_orbit_groups("cluster sparks\nNODE FINGERPRINT FREE GiB REVISION GROUPS\ngx10-dgx5 ab12 562.3 1 -\n").is_empty());
     }
 }
+
+#[cfg(test)]
+mod orbit_dead_tests {
+    use super::parse_orbit_groups;
+
+    #[test]
+    fn dead_groups_are_not_running() {
+        let out =
+            "gx10-dgx5  ab12  86.1  1  grpA-20261024T062753Z-3dc3 rank 0 exited with status 1\n";
+        assert!(
+            parse_orbit_groups(out).is_empty(),
+            "{:?}",
+            parse_orbit_groups(out)
+        );
+    }
+}