Sfoglia il codice sorgente

flare on the prod cluster: fabric ring, ledger, per-cluster probe

Switching 4nodes-switch (dgx1-4) to FLARE turned up four real gaps:

- 'flare download' rejects --dangerously-approve-key ('unexpected argument'),
  so passing the configured accept_key there broke every serve. The approval
  belongs to 'orbit run' only (downloads were never gated).
- fabric shape is a cluster property: 4nodes-switch is a star through SW2-MUC
  and FLARE refuses with 'no fabric link from rank 0 to rank 2'. Per-cluster
  extra_flags in [flare.clusters.<x>] now carry --roce-ring (emulated ring over
  the switch) without touching the sparks lane.
- the run ledger was never written for FLARE, so a second launcher saw no open
  run; native serve now records it (runlock open adopts a lane started before).
- the fleet/health probe asked the FLAT orbit cluster ('sparks') for every
  cluster, so a serving prod4 lane looked empty. It now asks each cluster with
  its own orbit name and its own controller, so 'Up (flare)' shows on dgx1.
Alexander Korolev 21 ore fa
parent
commit
5b019795f9
7 ha cambiato i file con 292 aggiunte e 191 eliminazioni
  1. 8 0
      src/config/flare_cluster.rs
  2. 32 155
      src/native/flare.rs
  3. 186 0
      src/native/flare_tests.rs
  4. 6 0
      src/native/mod.rs
  5. 21 0
      src/runlock.rs
  6. 27 12
      src/web/flare_nodes.rs
  7. 12 24
      src/web/probe.rs

+ 8 - 0
src/config/flare_cluster.rs

@@ -18,6 +18,11 @@ pub struct FlareCluster {
     /// Local package mirror dir for this cluster's controller.
     #[serde(default)]
     pub local_dir: Option<String>,
+    /// Extra `orbit run` flags for this cluster — fabric shape differs per
+    /// cluster (a switch-emulated ring needs `--roce-ring`, a DAC pair wants
+    /// `--roce-chain`), so the shape belongs to the cluster, not the recipe.
+    #[serde(default)]
+    pub extra_flags: Vec<String>,
 }
 
 /// `[flare]` in the private config: how the launcher talks to FLARE.
@@ -57,4 +62,7 @@ pub struct FlareSettings {
     /// orbit cluster was paired under the same names.
     #[serde(default)]
     pub orbit_nodes: Vec<String>,
+    /// Flat fallback for the per-cluster extra `orbit run` flags.
+    #[serde(default)]
+    pub extra_flags: Vec<String>,
 }

+ 32 - 155
src/native/flare.rs

@@ -49,6 +49,26 @@ pub fn orbit_cluster_for(config: &Config, cluster: &str) -> String {
         .unwrap_or_else(|| "sparks".to_string())
 }
 
+/// Extra `orbit run` flags for a cluster (per-cluster map first, then flat).
+/// Fabric shape is a cluster property: `--roce-ring` emulates a ring on a
+/// switch (our 4nodes-switch) where FLARE cannot find direct links, while the
+/// DAC/switch pair clusters take `--roce-chain`/nothing.
+pub fn extra_flags_for(config: &Config, cluster: &str) -> Vec<String> {
+    let Some(f) = config.flare.as_ref() else {
+        return Vec::new();
+    };
+    let per = f
+        .clusters
+        .get(cluster)
+        .map(|c| c.extra_flags.clone())
+        .unwrap_or_default();
+    if per.is_empty() {
+        f.extra_flags.clone()
+    } else {
+        per
+    }
+}
+
 /// Local package mirror dir for a cluster (per-cluster map first).
 fn local_dir_for(config: &Config, cluster: &str) -> Option<String> {
     let f = config.flare.as_ref()?;
@@ -90,9 +110,10 @@ pub fn download_cmd(config: &Config, recipe: &Recipe) -> Result<String> {
     if let Some(rev) = &f.revision {
         cmd.push_str(&format!(" --revision {}", shq(rev)));
     }
-    if let Some(key) = config.flare.as_ref().and_then(|c| c.accept_key.clone()) {
-        cmd.push_str(&format!(" --dangerously-approve-key {}", shq(&key)));
-    }
+    // NOTE: `flare download` does NOT accept the approval flags (it fails with
+    // "unexpected argument") — the signature gate lives on `orbit run`, which
+    // is where `run_cmd` passes them. A download has always been accepted
+    // unattended, so approving a publisher here would break the serve.
     Ok(cmd)
 }
 
@@ -100,6 +121,7 @@ pub fn download_cmd(config: &Config, recipe: &Recipe) -> Result<String> {
 pub fn run_cmd(
     config: &Config,
     recipe: &Recipe,
+    our_cluster: &str,
     orbit_cluster: &str,
     controller: &str,
     package: &str,
@@ -130,6 +152,10 @@ pub fn run_cmd(
     for flag in &f.serve_flags {
         cmd.push_str(&format!(" {}", shq(flag)));
     }
+    // fabric shape is a cluster property (a switch-emulated ring, a DAC chain)
+    for flag in extra_flags_for(config, our_cluster) {
+        cmd.push_str(&format!(" {}", shq(&flag)));
+    }
     Ok(cmd)
 }
 
@@ -283,7 +309,7 @@ pub async fn serve(
         }
     }
     let orbit_name = orbit_node_name(config, &cluster.name, controller);
-    let run = run_cmd(config, recipe, &orbit, &orbit_name, &package)?;
+    let run = run_cmd(config, recipe, &cluster.name, &orbit, &orbit_name, &package)?;
     crate::ssh::run_ssh(config, controller, run.clone(), dry_run).await?;
     if dry_run {
         println!("  [dry-run] {}: {run}", controller.name);
@@ -335,154 +361,5 @@ pub async fn stop(
 }
 
 #[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::config::FlareRecipe;
-
-    fn cfg() -> Config {
-        let c: Config = toml::from_str(
-            "ssh_user = \"u\"\nssh_key_path = \"/k\"\n\n[flare]\naccept_key = \"key-123\"\naccept_publisher = \"local-development\"\norbit_cluster = \"sparks\"\n",
-        )
-        .expect("test config");
-        c
-    }
-
-    fn recipe() -> Recipe {
-        let mut r = crate::engine::tests::base_recipe();
-        r.engine = "flare".into();
-        r.model = "deepseek-ai/DeepSeek-V4.1-Flash-1017".into();
-        r.flare = Some(FlareRecipe {
-            package: "local-inference-lab/DeepSeek-V4.1-Flash-1017-FLARE".into(),
-            revision: Some("gb10".into()),
-            name: Some("deepseek1017".into()),
-            http_port: Some(8000),
-            serve_flags: vec![
-                "--max-concurrency".into(),
-                "16".into(),
-                "--mtp-adaptive-verification".into(),
-            ],
-        });
-        r
-    }
-
-    #[test]
-    fn download_carries_revision_and_key() {
-        let cmd = download_cmd(&cfg(), &recipe()).unwrap();
-        assert!(cmd.starts_with("flare download "), "{cmd}");
-        // shq() shell-quotes every value, so compare the quoted forms
-        assert!(cmd.contains("--revision 'gb10'"), "{cmd}");
-        assert!(cmd.contains("--dangerously-approve-key 'key-123'"), "{cmd}");
-    }
-
-    #[test]
-    fn run_passes_flags_verbatim_and_the_controller_package() {
-        let cmd = run_cmd(
-            &cfg(),
-            &recipe(),
-            "sparks",
-            "dgx5",
-            "/pkgs/dsv41.safetensors",
-        )
-        .unwrap();
-        assert!(
-            cmd.contains("orbit run 'dgx5:/pkgs/dsv41.safetensors'"),
-            "{cmd}"
-        );
-        assert!(cmd.contains("--cluster 'sparks'"), "{cmd}");
-        assert!(cmd.contains("--name 'deepseek1017'"), "{cmd}");
-        assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
-        assert!(cmd.contains("--detach"), "{cmd}");
-        assert!(cmd.contains("'--mtp-adaptive-verification'"), "{cmd}");
-        assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
-        assert!(
-            cmd.contains("--dangerously-accept-publisher 'local-development'"),
-            "{cmd}"
-        );
-    }
-
-    #[test]
-    fn orbit_node_names_follow_rank_order() {
-        let mut c = cfg();
-        c.flare.as_mut().unwrap().orbit_nodes = vec!["spark1".into()];
-        let node = Node {
-            name: "dgx5.lan".into(),
-            ssh_ip: "10.0.0.1".into(),
-            ib_ip: None,
-            rank: 0,
-            host_key: None,
-            entity: None,
-        };
-        assert_eq!(orbit_node_name(&c, "dgx58-switch", &node), "spark1");
-        let node1 = Node { rank: 1, ..node };
-        // no entry for rank 1 → the cluster's own name is used
-        assert_eq!(orbit_node_name(&c, "dgx58-switch", &node1), "dgx5.lan");
-    }
-
-    #[test]
-    fn per_cluster_mapping_wins_over_the_flat_block() {
-        // The same launcher config must serve FLARE on another cluster: the
-        // per-cluster map supplies that cluster's orbit name and node list.
-        let mut c = cfg();
-        c.flare.as_mut().unwrap().clusters.insert(
-            "prod4".into(),
-            crate::config::flare_cluster::FlareCluster {
-                orbit_cluster: Some("prod4".into()),
-                orbit_nodes: vec!["gx10-dgx1".into()],
-                controller_node: None,
-                local_dir: Some("/data/flare".into()),
-            },
-        );
-        assert_eq!(orbit_cluster_for(&c, "prod4"), "prod4");
-        assert_eq!(orbit_cluster_for(&c, "dgx58-switch"), "sparks");
-        assert_eq!(local_dir_for(&c, "prod4").as_deref(), Some("/data/flare"));
-        let node = Node {
-            name: "dgx1.lan".into(),
-            ssh_ip: "10.0.0.1".into(),
-            ib_ip: None,
-            rank: 0,
-            host_key: None,
-            entity: None,
-        };
-        assert_eq!(orbit_node_name(&c, "prod4", &node), "gx10-dgx1");
-        let cmd = stop_cmd(&c, &recipe(), &orbit_cluster_for(&c, "prod4")).unwrap();
-        assert!(cmd.contains("--cluster 'prod4'"), "{cmd}");
-    }
-
-    #[test]
-    fn stop_names_the_group_and_cluster() {
-        let cmd = stop_cmd(&cfg(), &recipe(), "sparks").unwrap();
-        assert_eq!(cmd, "flare orbit stop 'deepseek1017' --cluster 'sparks'");
-    }
-
-    #[test]
-    fn preflight_checks_agent_and_pairing() {
-        let cmd = preflight_cmd(&cfg(), "sparks", true);
-        assert!(cmd.contains("is-active flare-agent"), "{cmd}");
-        assert!(cmd.contains("orbit status --cluster 'sparks'"), "{cmd}");
-        // a peer has no orbit file: binary + agent only
-        assert!(!preflight_cmd(&cfg(), "sparks", false).contains("orbit status"));
-    }
-
-    #[test]
-    fn package_path_is_taken_from_the_download_output() {
-        let out = "Resolved package: org/repo@abc / pkg.safetensors\n\
-                   Downloading model file: aa.safetensors...\n\
-                   Downloading model file: bb.safetensors...\n\
-                   /mnt/nas/hub/snapshots/abc/pkg.safetensors\n";
-        assert_eq!(
-            parse_package_path(out).as_deref(),
-            Some("/mnt/nas/hub/snapshots/abc/pkg.safetensors")
-        );
-        assert_eq!(
-            parse_package_path("Downloading model file: x.safetensors..."),
-            None
-        );
-    }
-
-    #[test]
-    fn missing_block_is_an_error() {
-        let mut r = recipe();
-        r.flare = None;
-        assert!(download_cmd(&cfg(), &r).is_err());
-    }
-}
+#[path = "flare_tests.rs"]
+mod tests;

+ 186 - 0
src/native/flare_tests.rs

@@ -0,0 +1,186 @@
+//! Tests for the FLARE driver (`native::flare`).
+
+use super::*;
+use crate::config::{Config, Recipe};
+
+use crate::config::FlareRecipe;
+
+fn cfg() -> Config {
+    let c: Config = toml::from_str(
+        "ssh_user = \"u\"\nssh_key_path = \"/k\"\n\n[flare]\naccept_key = \"key-123\"\naccept_publisher = \"local-development\"\norbit_cluster = \"sparks\"\n",
+    )
+    .expect("test config");
+    c
+}
+
+fn recipe() -> Recipe {
+    let mut r = crate::engine::tests::base_recipe();
+    r.engine = "flare".into();
+    r.model = "deepseek-ai/DeepSeek-V4.1-Flash-1017".into();
+    r.flare = Some(FlareRecipe {
+        package: "local-inference-lab/DeepSeek-V4.1-Flash-1017-FLARE".into(),
+        revision: Some("gb10".into()),
+        name: Some("deepseek1017".into()),
+        http_port: Some(8000),
+        serve_flags: vec![
+            "--max-concurrency".into(),
+            "16".into(),
+            "--mtp-adaptive-verification".into(),
+        ],
+    });
+    r
+}
+
+#[test]
+fn download_carries_revision_but_no_approval_flags() {
+    let cmd = download_cmd(&cfg(), &recipe()).unwrap();
+    assert!(cmd.starts_with("flare download "), "{cmd}");
+    // shq() shell-quotes every value, so compare the quoted forms
+    assert!(cmd.contains("--revision 'gb10'"), "{cmd}");
+    // `flare download` rejects them ("unexpected argument") — they belong
+    // to `orbit run` only
+    assert!(!cmd.contains("--dangerously-approve-key"), "{cmd}");
+}
+
+#[test]
+fn cluster_extra_flags_reach_orbit_run() {
+    // a switch-emulated ring is a cluster property, not a recipe flag
+    let mut c = cfg();
+    if let Some(f) = c.flare.as_mut() {
+        f.clusters.insert(
+            "4nodes-switch".into(),
+            crate::config::flare_cluster::FlareCluster {
+                extra_flags: vec!["--roce-ring".into()],
+                ..Default::default()
+            },
+        );
+    }
+    let cmd = run_cmd(
+        &c,
+        &recipe(),
+        "4nodes-switch",
+        "prod4",
+        "gx10-dgx1",
+        "/pkgs/x.safetensors",
+    )
+    .unwrap();
+    assert!(cmd.contains("'--roce-ring'"), "{cmd}");
+    assert!(!cmd.contains("--roce-ring") || cmd.matches("--roce-ring").count() == 1);
+    // another cluster does not inherit it
+    let other = run_cmd(&c, &recipe(), "dgx58-switch", "sparks", "dgx5", "/p").unwrap();
+    assert!(!other.contains("--roce-ring"), "{other}");
+}
+
+#[test]
+fn run_passes_flags_verbatim_and_the_controller_package() {
+    let cmd = run_cmd(
+        &cfg(),
+        &recipe(),
+        "dgx58-switch",
+        "sparks",
+        "dgx5",
+        "/pkgs/dsv41.safetensors",
+    )
+    .unwrap();
+    assert!(
+        cmd.contains("orbit run 'dgx5:/pkgs/dsv41.safetensors'"),
+        "{cmd}"
+    );
+    assert!(cmd.contains("--cluster 'sparks'"), "{cmd}");
+    assert!(cmd.contains("--name 'deepseek1017'"), "{cmd}");
+    assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
+    assert!(cmd.contains("--detach"), "{cmd}");
+    assert!(cmd.contains("'--mtp-adaptive-verification'"), "{cmd}");
+    assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
+    assert!(
+        cmd.contains("--dangerously-accept-publisher 'local-development'"),
+        "{cmd}"
+    );
+}
+
+#[test]
+fn orbit_node_names_follow_rank_order() {
+    let mut c = cfg();
+    c.flare.as_mut().unwrap().orbit_nodes = vec!["spark1".into()];
+    let node = Node {
+        name: "dgx5.lan".into(),
+        ssh_ip: "10.0.0.1".into(),
+        ib_ip: None,
+        rank: 0,
+        host_key: None,
+        entity: None,
+    };
+    assert_eq!(orbit_node_name(&c, "dgx58-switch", &node), "spark1");
+    let node1 = Node { rank: 1, ..node };
+    // no entry for rank 1 → the cluster's own name is used
+    assert_eq!(orbit_node_name(&c, "dgx58-switch", &node1), "dgx5.lan");
+}
+
+#[test]
+fn per_cluster_mapping_wins_over_the_flat_block() {
+    // The same launcher config must serve FLARE on another cluster: the
+    // per-cluster map supplies that cluster's orbit name and node list.
+    let mut c = cfg();
+    c.flare.as_mut().unwrap().clusters.insert(
+        "prod4".into(),
+        crate::config::flare_cluster::FlareCluster {
+            orbit_cluster: Some("prod4".into()),
+            orbit_nodes: vec!["gx10-dgx1".into()],
+            controller_node: None,
+            local_dir: Some("/data/flare".into()),
+            extra_flags: Vec::new(),
+        },
+    );
+    assert_eq!(orbit_cluster_for(&c, "prod4"), "prod4");
+    assert_eq!(orbit_cluster_for(&c, "dgx58-switch"), "sparks");
+    assert_eq!(local_dir_for(&c, "prod4").as_deref(), Some("/data/flare"));
+    let node = Node {
+        name: "dgx1.lan".into(),
+        ssh_ip: "10.0.0.1".into(),
+        ib_ip: None,
+        rank: 0,
+        host_key: None,
+        entity: None,
+    };
+    assert_eq!(orbit_node_name(&c, "prod4", &node), "gx10-dgx1");
+    let cmd = stop_cmd(&c, &recipe(), &orbit_cluster_for(&c, "prod4")).unwrap();
+    assert!(cmd.contains("--cluster 'prod4'"), "{cmd}");
+}
+
+#[test]
+fn stop_names_the_group_and_cluster() {
+    let cmd = stop_cmd(&cfg(), &recipe(), "sparks").unwrap();
+    assert_eq!(cmd, "flare orbit stop 'deepseek1017' --cluster 'sparks'");
+}
+
+#[test]
+fn preflight_checks_agent_and_pairing() {
+    let cmd = preflight_cmd(&cfg(), "sparks", true);
+    assert!(cmd.contains("is-active flare-agent"), "{cmd}");
+    assert!(cmd.contains("orbit status --cluster 'sparks'"), "{cmd}");
+    // a peer has no orbit file: binary + agent only
+    assert!(!preflight_cmd(&cfg(), "sparks", false).contains("orbit status"));
+}
+
+#[test]
+fn package_path_is_taken_from_the_download_output() {
+    let out = "Resolved package: org/repo@abc / pkg.safetensors\n\
+               Downloading model file: aa.safetensors...\n\
+               Downloading model file: bb.safetensors...\n\
+               /mnt/nas/hub/snapshots/abc/pkg.safetensors\n";
+    assert_eq!(
+        parse_package_path(out).as_deref(),
+        Some("/mnt/nas/hub/snapshots/abc/pkg.safetensors")
+    );
+    assert_eq!(
+        parse_package_path("Downloading model file: x.safetensors..."),
+        None
+    );
+}
+
+#[test]
+fn missing_block_is_an_error() {
+    let mut r = recipe();
+    r.flare = None;
+    assert!(download_cmd(&cfg(), &r).is_err());
+}

+ 6 - 0
src/native/mod.rs

@@ -310,6 +310,12 @@ async fn flare_serve(
     )
     .await?;
     flare::serve(config, recipe, &cluster, opts.dry_run).await?;
+    // The run ledger is the single-launcher serialization point: a FLARE group
+    // is as real as a container, so it must be recorded (otherwise a second
+    // launcher sees "no open run" and starts a second lane on the same nodes).
+    if !opts.dry_run {
+        crate::runlock::mark_start(&recipe.name, &config.recipe_sources, Some(&cluster.name))?;
+    }
     Ok(cluster)
 }
 

+ 21 - 0
src/runlock.rs

@@ -25,6 +25,16 @@ pub enum RunlockCommands {
         /// Recipe name (its directory under the recipe sources)
         recipe: String,
     },
+    /// Record a run that is already serving (adopt it into the ledger) — for
+    /// a lane started before the launcher recorded it (an earlier release, a
+    /// manual `flare orbit run`), so the single-launcher guard sees it
+    Open {
+        /// Recipe name (its directory under the recipe sources)
+        recipe: String,
+        /// Cluster the lane runs on
+        #[arg(long)]
+        cluster: Option<String>,
+    },
     /// List every recipe with an open (running) ledger entry
     List,
 }
@@ -174,6 +184,17 @@ pub(crate) fn run(config: &crate::config::Config, cmd: &RunlockCommands) -> Resu
                 }
             }
         }
+        RunlockCommands::Open { recipe, cluster } => {
+            if open_runs(config).iter().any(|(n, _, _)| n == recipe) {
+                println!("'{recipe}' already has an open run (nothing to do)");
+                return Ok(());
+            }
+            mark_start(recipe, &config.recipe_sources, cluster.as_deref())?;
+            println!(
+                "'{recipe}' recorded as running on {}",
+                cluster.as_deref().unwrap_or("(cluster unknown)")
+            );
+        }
         RunlockCommands::List => {
             let open = open_runs(config);
             if open.is_empty() {

+ 27 - 12
src/web/flare_nodes.rs

@@ -44,19 +44,34 @@ pub fn candidates(config: &Config) -> Vec<&Node> {
     out
 }
 
-/// Probe order for one tick: pinned → last winner → the rest of the candidates.
-pub fn probe_order(config: &Config, remembered: Option<&str>) -> Vec<String> {
-    let mut order: Vec<String> = Vec::new();
-    if let Some(n) = pinned(config) {
-        order.push(n.ssh_ip.clone());
-    }
-    if let Some(ip) = remembered {
-        order.push(ip.to_string());
+/// Rank-0 candidates of ONE cluster (pinned controller first) — the fleet
+/// probe must ask each cluster with its own orbit name, so `prod4` groups are
+/// not looked for in the `sparks` orbit (that silently showed nothing).
+pub fn candidates_for<'a>(
+    config: &'a Config,
+    cluster: &'a crate::config::Cluster,
+) -> Vec<&'a Node> {
+    let pinned_ip = config
+        .flare
+        .as_ref()
+        .and_then(|f| f.clusters.get(&cluster.name))
+        .and_then(|c| c.controller_node.clone())
+        .or_else(|| {
+            config
+                .flare
+                .as_ref()
+                .and_then(|f| f.controller_node.clone())
+        });
+    let mut out: Vec<&Node> = Vec::new();
+    if let Some(ip) = pinned_ip
+        && let Some(n) = cluster.nodes.iter().find(|n| n.name == ip)
+    {
+        out.push(n);
     }
-    for n in candidates(config) {
-        if !order.contains(&n.ssh_ip) {
-            order.push(n.ssh_ip.clone());
+    for n in &cluster.nodes {
+        if n.rank == 0 && !out.iter().any(|x| x.ssh_ip == n.ssh_ip) {
+            out.push(n);
         }
     }
-    order
+    out
 }

+ 12 - 24
src/web/probe.rs

@@ -182,36 +182,24 @@ pub async fn flare_groups(
         return (None, Vec::new());
     };
     let binary = f.binary.clone().unwrap_or_else(|| "flare".to_string());
-    let orbit_cluster = f
-        .orbit_cluster
-        .clone()
-        .unwrap_or_else(|| "sparks".to_string());
-    let cmd = format!("{binary} orbit status --cluster {orbit_cluster} 2>/dev/null");
-    let remembered = LAST_CONTROLLER.lock().await.clone();
-    for ip in crate::web::flare_nodes::probe_order(config, remembered.as_deref()) {
-        let Some(node) = config
-            .clusters
-            .iter()
-            .flat_map(|c| c.nodes.iter())
-            .find(|n| n.ssh_ip == ip)
-        else {
-            continue;
-        };
-        if let Ok(out) = crate::ssh::run_ssh(config, node, cmd.clone(), false).await {
-            let groups = parse_orbit_groups(&out);
-            if !groups.is_empty() {
-                *LAST_CONTROLLER.lock().await = Some(ip.clone());
-                return (Some(ip), groups);
+    // Per cluster, with ITS orbit name: `sparks` groups do not exist in the
+    // `prod4` orbit, and asking the wrong one made a serving prod lane look
+    // empty (health said "no containers", the dashboard showed no rows).
+    for cluster in &config.clusters {
+        let orbit_cluster = crate::native::flare::orbit_cluster_for(config, &cluster.name);
+        let cmd = format!("{binary} orbit status --cluster {orbit_cluster} 2>/dev/null");
+        for node in crate::web::flare_nodes::candidates_for(config, cluster) {
+            if let Ok(out) = crate::ssh::run_ssh(config, node, cmd.clone(), false).await {
+                let groups = parse_orbit_groups(&out);
+                if !groups.is_empty() {
+                    return (Some(node.ssh_ip.clone()), groups);
+                }
             }
         }
     }
     (None, Vec::new())
 }
 
-/// Controller address remembered between probe ticks.
-static LAST_CONTROLLER: std::sync::LazyLock<tokio::sync::Mutex<Option<String>>> =
-    std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
-
 pub struct Fleet {
     pub nodes: HashMap<String, WebNodeState>,
     /// cluster name → a blocking `extra_entity` reports anything but "on".