|
|
@@ -49,6 +49,26 @@ pub fn orbit_cluster_for(config: &Config, cluster: &str) -> String {
|
|
|
.unwrap_or_else(|| "sparks".to_string())
|
|
|
}
|
|
|
|
|
|
+/// Extra `orbit run` flags for a cluster (per-cluster map first, then flat).
|
|
|
+/// Fabric shape is a cluster property: `--roce-ring` emulates a ring on a
|
|
|
+/// switch (our 4nodes-switch) where FLARE cannot find direct links, while the
|
|
|
+/// DAC/switch pair clusters take `--roce-chain`/nothing.
|
|
|
+pub fn extra_flags_for(config: &Config, cluster: &str) -> Vec<String> {
|
|
|
+ let Some(f) = config.flare.as_ref() else {
|
|
|
+ return Vec::new();
|
|
|
+ };
|
|
|
+ let per = f
|
|
|
+ .clusters
|
|
|
+ .get(cluster)
|
|
|
+ .map(|c| c.extra_flags.clone())
|
|
|
+ .unwrap_or_default();
|
|
|
+ if per.is_empty() {
|
|
|
+ f.extra_flags.clone()
|
|
|
+ } else {
|
|
|
+ per
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
/// Local package mirror dir for a cluster (per-cluster map first).
|
|
|
fn local_dir_for(config: &Config, cluster: &str) -> Option<String> {
|
|
|
let f = config.flare.as_ref()?;
|
|
|
@@ -90,9 +110,10 @@ pub fn download_cmd(config: &Config, recipe: &Recipe) -> Result<String> {
|
|
|
if let Some(rev) = &f.revision {
|
|
|
cmd.push_str(&format!(" --revision {}", shq(rev)));
|
|
|
}
|
|
|
- if let Some(key) = config.flare.as_ref().and_then(|c| c.accept_key.clone()) {
|
|
|
- cmd.push_str(&format!(" --dangerously-approve-key {}", shq(&key)));
|
|
|
- }
|
|
|
+ // NOTE: `flare download` does NOT accept the approval flags (it fails with
|
|
|
+ // "unexpected argument") — the signature gate lives on `orbit run`, which
|
|
|
+ // is where `run_cmd` passes them. A download has always been accepted
|
|
|
+ // unattended, so approving a publisher here would break the serve.
|
|
|
Ok(cmd)
|
|
|
}
|
|
|
|
|
|
@@ -100,6 +121,7 @@ pub fn download_cmd(config: &Config, recipe: &Recipe) -> Result<String> {
|
|
|
pub fn run_cmd(
|
|
|
config: &Config,
|
|
|
recipe: &Recipe,
|
|
|
+ our_cluster: &str,
|
|
|
orbit_cluster: &str,
|
|
|
controller: &str,
|
|
|
package: &str,
|
|
|
@@ -130,6 +152,10 @@ pub fn run_cmd(
|
|
|
for flag in &f.serve_flags {
|
|
|
cmd.push_str(&format!(" {}", shq(flag)));
|
|
|
}
|
|
|
+ // fabric shape is a cluster property (a switch-emulated ring, a DAC chain)
|
|
|
+ for flag in extra_flags_for(config, our_cluster) {
|
|
|
+ cmd.push_str(&format!(" {}", shq(&flag)));
|
|
|
+ }
|
|
|
Ok(cmd)
|
|
|
}
|
|
|
|
|
|
@@ -283,7 +309,7 @@ pub async fn serve(
|
|
|
}
|
|
|
}
|
|
|
let orbit_name = orbit_node_name(config, &cluster.name, controller);
|
|
|
- let run = run_cmd(config, recipe, &orbit, &orbit_name, &package)?;
|
|
|
+ let run = run_cmd(config, recipe, &cluster.name, &orbit, &orbit_name, &package)?;
|
|
|
crate::ssh::run_ssh(config, controller, run.clone(), dry_run).await?;
|
|
|
if dry_run {
|
|
|
println!(" [dry-run] {}: {run}", controller.name);
|
|
|
@@ -335,154 +361,5 @@ pub async fn stop(
|
|
|
}
|
|
|
|
|
|
#[cfg(test)]
|
|
|
-mod tests {
|
|
|
- use super::*;
|
|
|
- use crate::config::FlareRecipe;
|
|
|
-
|
|
|
- fn cfg() -> Config {
|
|
|
- let c: Config = toml::from_str(
|
|
|
- "ssh_user = \"u\"\nssh_key_path = \"/k\"\n\n[flare]\naccept_key = \"key-123\"\naccept_publisher = \"local-development\"\norbit_cluster = \"sparks\"\n",
|
|
|
- )
|
|
|
- .expect("test config");
|
|
|
- c
|
|
|
- }
|
|
|
-
|
|
|
- fn recipe() -> Recipe {
|
|
|
- let mut r = crate::engine::tests::base_recipe();
|
|
|
- r.engine = "flare".into();
|
|
|
- r.model = "deepseek-ai/DeepSeek-V4.1-Flash-1017".into();
|
|
|
- r.flare = Some(FlareRecipe {
|
|
|
- package: "local-inference-lab/DeepSeek-V4.1-Flash-1017-FLARE".into(),
|
|
|
- revision: Some("gb10".into()),
|
|
|
- name: Some("deepseek1017".into()),
|
|
|
- http_port: Some(8000),
|
|
|
- serve_flags: vec![
|
|
|
- "--max-concurrency".into(),
|
|
|
- "16".into(),
|
|
|
- "--mtp-adaptive-verification".into(),
|
|
|
- ],
|
|
|
- });
|
|
|
- r
|
|
|
- }
|
|
|
-
|
|
|
- #[test]
|
|
|
- fn download_carries_revision_and_key() {
|
|
|
- let cmd = download_cmd(&cfg(), &recipe()).unwrap();
|
|
|
- assert!(cmd.starts_with("flare download "), "{cmd}");
|
|
|
- // shq() shell-quotes every value, so compare the quoted forms
|
|
|
- assert!(cmd.contains("--revision 'gb10'"), "{cmd}");
|
|
|
- assert!(cmd.contains("--dangerously-approve-key 'key-123'"), "{cmd}");
|
|
|
- }
|
|
|
-
|
|
|
- #[test]
|
|
|
- fn run_passes_flags_verbatim_and_the_controller_package() {
|
|
|
- let cmd = run_cmd(
|
|
|
- &cfg(),
|
|
|
- &recipe(),
|
|
|
- "sparks",
|
|
|
- "dgx5",
|
|
|
- "/pkgs/dsv41.safetensors",
|
|
|
- )
|
|
|
- .unwrap();
|
|
|
- assert!(
|
|
|
- cmd.contains("orbit run 'dgx5:/pkgs/dsv41.safetensors'"),
|
|
|
- "{cmd}"
|
|
|
- );
|
|
|
- assert!(cmd.contains("--cluster 'sparks'"), "{cmd}");
|
|
|
- assert!(cmd.contains("--name 'deepseek1017'"), "{cmd}");
|
|
|
- assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
|
|
|
- assert!(cmd.contains("--detach"), "{cmd}");
|
|
|
- assert!(cmd.contains("'--mtp-adaptive-verification'"), "{cmd}");
|
|
|
- assert!(cmd.contains("--port 8000"), "{cmd}"); // recipe value wins
|
|
|
- assert!(
|
|
|
- cmd.contains("--dangerously-accept-publisher 'local-development'"),
|
|
|
- "{cmd}"
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- #[test]
|
|
|
- fn orbit_node_names_follow_rank_order() {
|
|
|
- let mut c = cfg();
|
|
|
- c.flare.as_mut().unwrap().orbit_nodes = vec!["spark1".into()];
|
|
|
- let node = Node {
|
|
|
- name: "dgx5.lan".into(),
|
|
|
- ssh_ip: "10.0.0.1".into(),
|
|
|
- ib_ip: None,
|
|
|
- rank: 0,
|
|
|
- host_key: None,
|
|
|
- entity: None,
|
|
|
- };
|
|
|
- assert_eq!(orbit_node_name(&c, "dgx58-switch", &node), "spark1");
|
|
|
- let node1 = Node { rank: 1, ..node };
|
|
|
- // no entry for rank 1 → the cluster's own name is used
|
|
|
- assert_eq!(orbit_node_name(&c, "dgx58-switch", &node1), "dgx5.lan");
|
|
|
- }
|
|
|
-
|
|
|
- #[test]
|
|
|
- fn per_cluster_mapping_wins_over_the_flat_block() {
|
|
|
- // The same launcher config must serve FLARE on another cluster: the
|
|
|
- // per-cluster map supplies that cluster's orbit name and node list.
|
|
|
- let mut c = cfg();
|
|
|
- c.flare.as_mut().unwrap().clusters.insert(
|
|
|
- "prod4".into(),
|
|
|
- crate::config::flare_cluster::FlareCluster {
|
|
|
- orbit_cluster: Some("prod4".into()),
|
|
|
- orbit_nodes: vec!["gx10-dgx1".into()],
|
|
|
- controller_node: None,
|
|
|
- local_dir: Some("/data/flare".into()),
|
|
|
- },
|
|
|
- );
|
|
|
- assert_eq!(orbit_cluster_for(&c, "prod4"), "prod4");
|
|
|
- assert_eq!(orbit_cluster_for(&c, "dgx58-switch"), "sparks");
|
|
|
- assert_eq!(local_dir_for(&c, "prod4").as_deref(), Some("/data/flare"));
|
|
|
- let node = Node {
|
|
|
- name: "dgx1.lan".into(),
|
|
|
- ssh_ip: "10.0.0.1".into(),
|
|
|
- ib_ip: None,
|
|
|
- rank: 0,
|
|
|
- host_key: None,
|
|
|
- entity: None,
|
|
|
- };
|
|
|
- assert_eq!(orbit_node_name(&c, "prod4", &node), "gx10-dgx1");
|
|
|
- let cmd = stop_cmd(&c, &recipe(), &orbit_cluster_for(&c, "prod4")).unwrap();
|
|
|
- assert!(cmd.contains("--cluster 'prod4'"), "{cmd}");
|
|
|
- }
|
|
|
-
|
|
|
- #[test]
|
|
|
- fn stop_names_the_group_and_cluster() {
|
|
|
- let cmd = stop_cmd(&cfg(), &recipe(), "sparks").unwrap();
|
|
|
- assert_eq!(cmd, "flare orbit stop 'deepseek1017' --cluster 'sparks'");
|
|
|
- }
|
|
|
-
|
|
|
- #[test]
|
|
|
- fn preflight_checks_agent_and_pairing() {
|
|
|
- let cmd = preflight_cmd(&cfg(), "sparks", true);
|
|
|
- assert!(cmd.contains("is-active flare-agent"), "{cmd}");
|
|
|
- assert!(cmd.contains("orbit status --cluster 'sparks'"), "{cmd}");
|
|
|
- // a peer has no orbit file: binary + agent only
|
|
|
- assert!(!preflight_cmd(&cfg(), "sparks", false).contains("orbit status"));
|
|
|
- }
|
|
|
-
|
|
|
- #[test]
|
|
|
- fn package_path_is_taken_from_the_download_output() {
|
|
|
- let out = "Resolved package: org/repo@abc / pkg.safetensors\n\
|
|
|
- Downloading model file: aa.safetensors...\n\
|
|
|
- Downloading model file: bb.safetensors...\n\
|
|
|
- /mnt/nas/hub/snapshots/abc/pkg.safetensors\n";
|
|
|
- assert_eq!(
|
|
|
- parse_package_path(out).as_deref(),
|
|
|
- Some("/mnt/nas/hub/snapshots/abc/pkg.safetensors")
|
|
|
- );
|
|
|
- assert_eq!(
|
|
|
- parse_package_path("Downloading model file: x.safetensors..."),
|
|
|
- None
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- #[test]
|
|
|
- fn missing_block_is_an_error() {
|
|
|
- let mut r = recipe();
|
|
|
- r.flare = None;
|
|
|
- assert!(download_cmd(&cfg(), &r).is_err());
|
|
|
- }
|
|
|
-}
|
|
|
+#[path = "flare_tests.rs"]
|
|
|
+mod tests;
|