Pārlūkot izejas kodu

flare on the prod cluster: fabric ring, ledger, per-cluster probe

Switching 4nodes-switch (dgx1-4) to FLARE turned up four real gaps:

- 'flare download' rejects --dangerously-approve-key ('unexpected argument'),
  so passing the configured accept_key there broke every serve. The approval
  belongs to 'orbit run' only (downloads were never gated).
- fabric shape is a cluster property: 4nodes-switch is a star through SW2-MUC
  and FLARE refuses with 'no fabric link from rank 0 to rank 2'. Per-cluster
  extra_flags in [flare.clusters.<x>] now carry --roce-ring (emulated ring over
  the switch) without touching the sparks lane.
- the run ledger was never written for FLARE, so a second launcher saw no open
  run; native serve now records it (runlock open adopts a lane started before).
- the fleet/health probe asked the FLAT orbit cluster ('sparks') for every
  cluster, so a serving prod4 lane looked empty. It now asks each cluster with
  its own orbit name and its own controller, so 'Up (flare)' shows on dgx1.
Alexander Korolev 1 dienu atpakaļ
vecāks
revīzija
5b019795f9
7 mainītis faili ar 292 papildinājumiem un 191 dzēšanām
  1. 8 0
      src/config/flare_cluster.rs
  2. 32 155
      src/native/flare.rs
  3. 186 0
      src/native/flare_tests.rs
  4. 6 0
      src/native/mod.rs
  5. 21 0
      src/runlock.rs
  6. 27 12
      src/web/flare_nodes.rs
  7. 12 24
      src/web/probe.rs

+ 8 - 0
src/config/flare_cluster.rs

@@ -18,6 +18,11 @@ pub struct FlareCluster {
     /// Local package mirror dir for this cluster's controller.
     #[serde(default)]
     pub local_dir: Option<String>,
+    /// Extra `orbit run` flags for this cluster — fabric shape differs per
+    /// cluster (a switch-emulated ring needs `--roce-ring`, a DAC pair wants
+    /// `--roce-chain`), so the shape belongs to the cluster, not the recipe.
+    #[serde(default)]
+    pub extra_flags: Vec<String>,
 }
 
 /// `[flare]` in the private config: how the launcher talks to FLARE.
@@ -57,4 +62,7 @@ pub struct FlareSettings {
     /// orbit cluster was paired under the same names.
     #[serde(default)]
     pub orbit_nodes: Vec<String>,
+    /// Flat fallback for the per-cluster extra `orbit run` flags.
+    #[serde(default)]
+    pub extra_flags: Vec<String>,
 }

+ 32 - 155
src/native/flare.rs

@@ -49,6 +49,26 @@ pub fn orbit_cluster_for(config: &Config, cluster: &str) -> String {
         .unwrap_or_else(|| "sparks".to_string())
 }
 
+/// Extra `orbit run` flags for a cluster (per-cluster map first, then flat).
+/// Fabric shape is a cluster property: `--roce-ring` emulates a ring on a
+/// switch (our 4nodes-switch) where FLARE cannot find direct links, while the
+/// DAC/switch pair clusters take `--roce-chain`/nothing.
+pub fn extra_flags_for(config: &Config, cluster: &str) -> Vec<String> {
+    let Some(f) = config.flare.as_ref() else {
+        return Vec::new();
+    };
+    let per = f
+        .clusters
+        .get(cluster)
+        .map(|c| c.extra_flags.clone())
+        .unwrap_or_default();
+    if per.is_empty() {
+        f.extra_flags.clone()
+    } else {
+        per
+    }
+}
+
 /// Local package mirror dir for a cluster (per-cluster map first).
 fn local_dir_for(config: &Config, cluster: &str) -> Option<String> {
     let f = config.flare.as_ref()?;
@@ -90,9 +110,10 @@ pub fn download_cmd(config: &Config, recipe: &Recipe) -> Result<String> {
     if let Some(rev) = &f.revision {
         cmd.push_str(&format!(" --revision {}", shq(rev)));
     }
-    if let Some(key) = config.flare.as_ref().and_then(|c| c.accept_key.clone()) {
-        cmd.push_str(&format!(" --dangerously-approve-key {}", shq(&key)));
-    }
+    // NOTE: `flare download` does NOT accept the approval flags (it fails with
+    // "unexpected argument") — the signature gate lives on `orbit run`, which
+    // is where `run_cmd` passes them. A download has always been accepted
+    // unattended, so approving a publisher here would break the serve.
     Ok(cmd)
 }
 
@@ -100,6 +121,7 @@ pub fn download_cmd(config: &Config, recipe: &Recipe) -> Result<String> {
 pub fn run_cmd(
     config: &Config,
     recipe: &Recipe,
+    our_cluster: &str,
     orbit_cluster: &str,
     controller: &str,
     package: &str,
@@ -130,6 +152,10 @@ pub fn run_cmd(
     for flag in &f.serve_flags {
         cmd.push_str(&format!(" {}", shq(flag)));
     }
+    // fabric shape is a cluster property (a switch-emulated ring, a DAC chain)
+    for flag in extra_flags_for(config, our_cluster) {
+        cmd.push_str(&format!(" {}", shq(&flag)));
+    }
     Ok(cmd)
 }
 
@@ -283,7 +309,7 @@ pub async fn serve(
         }
     }
     let orbit_name = orbit_node_name(config, &cluster.name, controller);
-    let run = run_cmd(config, recipe, &orbit, &orbit_name, &package)?;
+    let run = run_cmd(config, recipe, &cluster.name, &orbit, &orbit_name, &package)?;
     crate::ssh::run_ssh(config, controller, run.clone(), dry_run).await?;
     if dry_run {
         println!("  [dry-run] {}: {run}", controller.name);
@@ -335,154 +361,5 @@ pub async fn stop(
 }
 
 #[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::config::FlareRecipe;
-
-    fn cfg() -> Config {
-        let c: Config = toml::from_str(
-            "ssh_user = \"u\"\nssh_key_path = \"/k\"\n\n[flare]\naccept_key = \"key-123\"\naccept_publisher = \"local-development\"\norbit_cluster = \"sparks\"\n",
-        )
-        .expect("test config");
-        c
-    }
-
-    fn recipe() -> Recipe {
-        let mut r = crate::engine::tests::base_recipe();
-        r.engine = "flare".into();
-        r.model = "deepseek-ai/DeepSeek-V4.1-Flash-1017".into();
-        r.flare = Some(FlareRecipe {
-            package: "local-inference-lab/DeepSeek-V4.1-Flash-1017-FLARE".into(),
-            revision: Some("gb10".into()),
-            name: Some("deepseek1017".into()),
-            http_port: Some(8000),
-            serve_flags: vec![
-                "--max-concurrency".into(),
-                "16".into(),
-                "--mtp-adaptive-verification".into(),
-            ],
-        });
-        r
-    }
-
-    #[test]
-    fn download_carries_revision_and_key() {
-        let cmd = download_cmd(&cfg(), &recipe()).unwrap();
-        assert!(cmd.starts_with("flare download "), "{cmd}");
-        // shq() shell-quotes every value, so compare the quoted forms
-        assert!(cmd.contains("--revision 'gb10'"), "{cmd}");
-        assert!(cmd.contains("--dangerously-approve-key 'key-123'"), "{cmd}");
-    }
-
-    #[test]
-    fn run_passes_flags_verbatim_and_the_controller_package() {
-        let cmd = run_cmd(
-            &cfg(),
-            &recipe(),
-            "sparks",
-            "dgx5",
-            "/pkgs/dsv41.safetensors",
-        )
-        .unwrap();
-        assert!(
-            cmd.contains("orbit run 'dgx5:/pkgs/dsv41.safetensors'"),
-            "{cmd}"
-        );
-        assert!(cmd.contains("--cluster 'sparks'"), "{cmd}");
-        assert!(cmd.contains("--name 'deepseek1017'"), "{cmd}");
-        assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
-        assert!(cmd.contains("--detach"), "{cmd}");
-        assert!(cmd.contains("'--mtp-adaptive-verification'"), "{cmd}");
-        assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
-        assert!(
-            cmd.contains("--dangerously-accept-publisher 'local-development'"),
-            "{cmd}"
-        );
-    }
-
-    #[test]
-    fn orbit_node_names_follow_rank_order() {
-        let mut c = cfg();
-        c.flare.as_mut().unwrap().orbit_nodes = vec!["spark1".into()];
-        let node = Node {
-            name: "dgx5.lan".into(),
-            ssh_ip: "10.0.0.1".into(),
-            ib_ip: None,
-            rank: 0,
-            host_key: None,
-            entity: None,
-        };
-        assert_eq!(orbit_node_name(&c, "dgx58-switch", &node), "spark1");
-        let node1 = Node { rank: 1, ..node };
-        // no entry for rank 1 → the cluster's own name is used
-        assert_eq!(orbit_node_name(&c, "dgx58-switch", &node1), "dgx5.lan");
-    }
-
-    #[test]
-    fn per_cluster_mapping_wins_over_the_flat_block() {
-        // The same launcher config must serve FLARE on another cluster: the
-        // per-cluster map supplies that cluster's orbit name and node list.
-        let mut c = cfg();
-        c.flare.as_mut().unwrap().clusters.insert(
-            "prod4".into(),
-            crate::config::flare_cluster::FlareCluster {
-                orbit_cluster: Some("prod4".into()),
-                orbit_nodes: vec!["gx10-dgx1".into()],
-                controller_node: None,
-                local_dir: Some("/data/flare".into()),
-            },
-        );
-        assert_eq!(orbit_cluster_for(&c, "prod4"), "prod4");
-        assert_eq!(orbit_cluster_for(&c, "dgx58-switch"), "sparks");
-        assert_eq!(local_dir_for(&c, "prod4").as_deref(), Some("/data/flare"));
-        let node = Node {
-            name: "dgx1.lan".into(),
-            ssh_ip: "10.0.0.1".into(),
-            ib_ip: None,
-            rank: 0,
-            host_key: None,
-            entity: None,
-        };
-        assert_eq!(orbit_node_name(&c, "prod4", &node), "gx10-dgx1");
-        let cmd = stop_cmd(&c, &recipe(), &orbit_cluster_for(&c, "prod4")).unwrap();
-        assert!(cmd.contains("--cluster 'prod4'"), "{cmd}");
-    }
-
-    #[test]
-    fn stop_names_the_group_and_cluster() {
-        let cmd = stop_cmd(&cfg(), &recipe(), "sparks").unwrap();
-        assert_eq!(cmd, "flare orbit stop 'deepseek1017' --cluster 'sparks'");
-    }
-
-    #[test]
-    fn preflight_checks_agent_and_pairing() {
-        let cmd = preflight_cmd(&cfg(), "sparks", true);
-        assert!(cmd.contains("is-active flare-agent"), "{cmd}");
-        assert!(cmd.contains("orbit status --cluster 'sparks'"), "{cmd}");
-        // a peer has no orbit file: binary + agent only
-        assert!(!preflight_cmd(&cfg(), "sparks", false).contains("orbit status"));
-    }
-
-    #[test]
-    fn package_path_is_taken_from_the_download_output() {
-        let out = "Resolved package: org/repo@abc / pkg.safetensors\n\
-                   Downloading model file: aa.safetensors...\n\
-                   Downloading model file: bb.safetensors...\n\
-                   /mnt/nas/hub/snapshots/abc/pkg.safetensors\n";
-        assert_eq!(
-            parse_package_path(out).as_deref(),
-            Some("/mnt/nas/hub/snapshots/abc/pkg.safetensors")
-        );
-        assert_eq!(
-            parse_package_path("Downloading model file: x.safetensors..."),
-            None
-        );
-    }
-
-    #[test]
-    fn missing_block_is_an_error() {
-        let mut r = recipe();
-        r.flare = None;
-        assert!(download_cmd(&cfg(), &r).is_err());
-    }
-}
+#[path = "flare_tests.rs"]
+mod tests;

+ 186 - 0
src/native/flare_tests.rs

@@ -0,0 +1,186 @@
+//! Tests for the FLARE driver (`native::flare`).
+
+use super::*;
+use crate::config::{Config, Recipe};
+
+use crate::config::FlareRecipe;
+
+fn cfg() -> Config {
+    let c: Config = toml::from_str(
+        "ssh_user = \"u\"\nssh_key_path = \"/k\"\n\n[flare]\naccept_key = \"key-123\"\naccept_publisher = \"local-development\"\norbit_cluster = \"sparks\"\n",
+    )
+    .expect("test config");
+    c
+}
+
+fn recipe() -> Recipe {
+    let mut r = crate::engine::tests::base_recipe();
+    r.engine = "flare".into();
+    r.model = "deepseek-ai/DeepSeek-V4.1-Flash-1017".into();
+    r.flare = Some(FlareRecipe {
+        package: "local-inference-lab/DeepSeek-V4.1-Flash-1017-FLARE".into(),
+        revision: Some("gb10".into()),
+        name: Some("deepseek1017".into()),
+        http_port: Some(8000),
+        serve_flags: vec![
+            "--max-concurrency".into(),
+            "16".into(),
+            "--mtp-adaptive-verification".into(),
+        ],
+    });
+    r
+}
+
+#[test]
+fn download_carries_revision_but_no_approval_flags() {
+    let cmd = download_cmd(&cfg(), &recipe()).unwrap();
+    assert!(cmd.starts_with("flare download "), "{cmd}");
+    // shq() shell-quotes every value, so compare the quoted forms
+    assert!(cmd.contains("--revision 'gb10'"), "{cmd}");
+    // `flare download` rejects them ("unexpected argument") — they belong
+    // to `orbit run` only
+    assert!(!cmd.contains("--dangerously-approve-key"), "{cmd}");
+}
+
+#[test]
+fn cluster_extra_flags_reach_orbit_run() {
+    // a switch-emulated ring is a cluster property, not a recipe flag
+    let mut c = cfg();
+    if let Some(f) = c.flare.as_mut() {
+        f.clusters.insert(
+            "4nodes-switch".into(),
+            crate::config::flare_cluster::FlareCluster {
+                extra_flags: vec!["--roce-ring".into()],
+                ..Default::default()
+            },
+        );
+    }
+    let cmd = run_cmd(
+        &c,
+        &recipe(),
+        "4nodes-switch",
+        "prod4",
+        "gx10-dgx1",
+        "/pkgs/x.safetensors",
+    )
+    .unwrap();
+    assert!(cmd.contains("'--roce-ring'"), "{cmd}");
+    assert!(!cmd.contains("--roce-ring") || cmd.matches("--roce-ring").count() == 1);
+    // another cluster does not inherit it
+    let other = run_cmd(&c, &recipe(), "dgx58-switch", "sparks", "dgx5", "/p").unwrap();
+    assert!(!other.contains("--roce-ring"), "{other}");
+}
+
+#[test]
+fn run_passes_flags_verbatim_and_the_controller_package() {
+    let cmd = run_cmd(
+        &cfg(),
+        &recipe(),
+        "dgx58-switch",
+        "sparks",
+        "dgx5",
+        "/pkgs/dsv41.safetensors",
+    )
+    .unwrap();
+    assert!(
+        cmd.contains("orbit run 'dgx5:/pkgs/dsv41.safetensors'"),
+        "{cmd}"
+    );
+    assert!(cmd.contains("--cluster 'sparks'"), "{cmd}");
+    assert!(cmd.contains("--name 'deepseek1017'"), "{cmd}");
+    assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
+    assert!(cmd.contains("--detach"), "{cmd}");
+    assert!(cmd.contains("'--mtp-adaptive-verification'"), "{cmd}");
+    assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
+    assert!(
+        cmd.contains("--dangerously-accept-publisher 'local-development'"),
+        "{cmd}"
+    );
+}
+
+#[test]
+fn orbit_node_names_follow_rank_order() {
+    let mut c = cfg();
+    c.flare.as_mut().unwrap().orbit_nodes = vec!["spark1".into()];
+    let node = Node {
+        name: "dgx5.lan".into(),
+        ssh_ip: "10.0.0.1".into(),
+        ib_ip: None,
+        rank: 0,
+        host_key: None,
+        entity: None,
+    };
+    assert_eq!(orbit_node_name(&c, "dgx58-switch", &node), "spark1");
+    let node1 = Node { rank: 1, ..node };
+    // no entry for rank 1 → the cluster's own name is used
+    assert_eq!(orbit_node_name(&c, "dgx58-switch", &node1), "dgx5.lan");
+}
+
+#[test]
+fn per_cluster_mapping_wins_over_the_flat_block() {
+    // The same launcher config must serve FLARE on another cluster: the
+    // per-cluster map supplies that cluster's orbit name and node list.
+    let mut c = cfg();
+    c.flare.as_mut().unwrap().clusters.insert(
+        "prod4".into(),
+        crate::config::flare_cluster::FlareCluster {
+            orbit_cluster: Some("prod4".into()),
+            orbit_nodes: vec!["gx10-dgx1".into()],
+            controller_node: None,
+            local_dir: Some("/data/flare".into()),
+            extra_flags: Vec::new(),
+        },
+    );
+    assert_eq!(orbit_cluster_for(&c, "prod4"), "prod4");
+    assert_eq!(orbit_cluster_for(&c, "dgx58-switch"), "sparks");
+    assert_eq!(local_dir_for(&c, "prod4").as_deref(), Some("/data/flare"));
+    let node = Node {
+        name: "dgx1.lan".into(),
+        ssh_ip: "10.0.0.1".into(),
+        ib_ip: None,
+        rank: 0,
+        host_key: None,
+        entity: None,
+    };
+    assert_eq!(orbit_node_name(&c, "prod4", &node), "gx10-dgx1");
+    let cmd = stop_cmd(&c, &recipe(), &orbit_cluster_for(&c, "prod4")).unwrap();
+    assert!(cmd.contains("--cluster 'prod4'"), "{cmd}");
+}
+
+#[test]
+fn stop_names_the_group_and_cluster() {
+    let cmd = stop_cmd(&cfg(), &recipe(), "sparks").unwrap();
+    assert_eq!(cmd, "flare orbit stop 'deepseek1017' --cluster 'sparks'");
+}
+
+#[test]
+fn preflight_checks_agent_and_pairing() {
+    let cmd = preflight_cmd(&cfg(), "sparks", true);
+    assert!(cmd.contains("is-active flare-agent"), "{cmd}");
+    assert!(cmd.contains("orbit status --cluster 'sparks'"), "{cmd}");
+    // a peer has no orbit file: binary + agent only
+    assert!(!preflight_cmd(&cfg(), "sparks", false).contains("orbit status"));
+}
+
+#[test]
+fn package_path_is_taken_from_the_download_output() {
+    let out = "Resolved package: org/repo@abc / pkg.safetensors\n\
+               Downloading model file: aa.safetensors...\n\
+               Downloading model file: bb.safetensors...\n\
+               /mnt/nas/hub/snapshots/abc/pkg.safetensors\n";
+    assert_eq!(
+        parse_package_path(out).as_deref(),
+        Some("/mnt/nas/hub/snapshots/abc/pkg.safetensors")
+    );
+    assert_eq!(
+        parse_package_path("Downloading model file: x.safetensors..."),
+        None
+    );
+}
+
+#[test]
+fn missing_block_is_an_error() {
+    let mut r = recipe();
+    r.flare = None;
+    assert!(download_cmd(&cfg(), &r).is_err());
+}

+ 6 - 0
src/native/mod.rs

@@ -310,6 +310,12 @@ async fn flare_serve(
     )
     .await?;
     flare::serve(config, recipe, &cluster, opts.dry_run).await?;
+    // The run ledger is the single-launcher serialization point: a FLARE group
+    // is as real as a container, so it must be recorded (otherwise a second
+    // launcher sees "no open run" and starts a second lane on the same nodes).
+    if !opts.dry_run {
+        crate::runlock::mark_start(&recipe.name, &config.recipe_sources, Some(&cluster.name))?;
+    }
     Ok(cluster)
 }
 

+ 21 - 0
src/runlock.rs

@@ -25,6 +25,16 @@ pub enum RunlockCommands {
         /// Recipe name (its directory under the recipe sources)
         recipe: String,
     },
+    /// Record a run that is already serving (adopt it into the ledger) — for
+    /// a lane started before the launcher recorded it (an earlier release, a
+    /// manual `flare orbit run`), so the single-launcher guard sees it
+    Open {
+        /// Recipe name (its directory under the recipe sources)
+        recipe: String,
+        /// Cluster the lane runs on
+        #[arg(long)]
+        cluster: Option<String>,
+    },
     /// List every recipe with an open (running) ledger entry
     List,
 }
@@ -174,6 +184,17 @@ pub(crate) fn run(config: &crate::config::Config, cmd: &RunlockCommands) -> Resu
                 }
             }
         }
+        RunlockCommands::Open { recipe, cluster } => {
+            if open_runs(config).iter().any(|(n, _, _)| n == recipe) {
+                println!("'{recipe}' already has an open run (nothing to do)");
+                return Ok(());
+            }
+            mark_start(recipe, &config.recipe_sources, cluster.as_deref())?;
+            println!(
+                "'{recipe}' recorded as running on {}",
+                cluster.as_deref().unwrap_or("(cluster unknown)")
+            );
+        }
         RunlockCommands::List => {
             let open = open_runs(config);
             if open.is_empty() {

+ 27 - 12
src/web/flare_nodes.rs

@@ -44,19 +44,34 @@ pub fn candidates(config: &Config) -> Vec<&Node> {
     out
 }
 
-/// Probe order for one tick: pinned → last winner → the rest of the candidates.
-pub fn probe_order(config: &Config, remembered: Option<&str>) -> Vec<String> {
-    let mut order: Vec<String> = Vec::new();
-    if let Some(n) = pinned(config) {
-        order.push(n.ssh_ip.clone());
-    }
-    if let Some(ip) = remembered {
-        order.push(ip.to_string());
+/// Rank-0 candidates of ONE cluster (pinned controller first) — the fleet
+/// probe must ask each cluster with its own orbit name, so `prod4` groups are
+/// not looked for in the `sparks` orbit (that silently showed nothing).
+pub fn candidates_for<'a>(
+    config: &'a Config,
+    cluster: &'a crate::config::Cluster,
+) -> Vec<&'a Node> {
+    let pinned_ip = config
+        .flare
+        .as_ref()
+        .and_then(|f| f.clusters.get(&cluster.name))
+        .and_then(|c| c.controller_node.clone())
+        .or_else(|| {
+            config
+                .flare
+                .as_ref()
+                .and_then(|f| f.controller_node.clone())
+        });
+    let mut out: Vec<&Node> = Vec::new();
+    if let Some(ip) = pinned_ip
+        && let Some(n) = cluster.nodes.iter().find(|n| n.name == ip)
+    {
+        out.push(n);
     }
-    for n in candidates(config) {
-        if !order.contains(&n.ssh_ip) {
-            order.push(n.ssh_ip.clone());
+    for n in &cluster.nodes {
+        if n.rank == 0 && !out.iter().any(|x| x.ssh_ip == n.ssh_ip) {
+            out.push(n);
         }
     }
-    order
+    out
 }

+ 12 - 24
src/web/probe.rs

@@ -182,36 +182,24 @@ pub async fn flare_groups(
         return (None, Vec::new());
     };
     let binary = f.binary.clone().unwrap_or_else(|| "flare".to_string());
-    let orbit_cluster = f
-        .orbit_cluster
-        .clone()
-        .unwrap_or_else(|| "sparks".to_string());
-    let cmd = format!("{binary} orbit status --cluster {orbit_cluster} 2>/dev/null");
-    let remembered = LAST_CONTROLLER.lock().await.clone();
-    for ip in crate::web::flare_nodes::probe_order(config, remembered.as_deref()) {
-        let Some(node) = config
-            .clusters
-            .iter()
-            .flat_map(|c| c.nodes.iter())
-            .find(|n| n.ssh_ip == ip)
-        else {
-            continue;
-        };
-        if let Ok(out) = crate::ssh::run_ssh(config, node, cmd.clone(), false).await {
-            let groups = parse_orbit_groups(&out);
-            if !groups.is_empty() {
-                *LAST_CONTROLLER.lock().await = Some(ip.clone());
-                return (Some(ip), groups);
+    // Per cluster, with ITS orbit name: `sparks` groups do not exist in the
+    // `prod4` orbit, and asking the wrong one made a serving prod lane look
+    // empty (health said "no containers", the dashboard showed no rows).
+    for cluster in &config.clusters {
+        let orbit_cluster = crate::native::flare::orbit_cluster_for(config, &cluster.name);
+        let cmd = format!("{binary} orbit status --cluster {orbit_cluster} 2>/dev/null");
+        for node in crate::web::flare_nodes::candidates_for(config, cluster) {
+            if let Ok(out) = crate::ssh::run_ssh(config, node, cmd.clone(), false).await {
+                let groups = parse_orbit_groups(&out);
+                if !groups.is_empty() {
+                    return (Some(node.ssh_ip.clone()), groups);
+                }
             }
         }
     }
     (None, Vec::new())
 }
 
-/// Controller address remembered between probe ticks.
-static LAST_CONTROLLER: std::sync::LazyLock<tokio::sync::Mutex<Option<String>>> =
-    std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
-
 pub struct Fleet {
     pub nodes: HashMap<String, WebNodeState>,
     /// cluster name → a blocking `extra_entity` reports anything but "on".