|
|
@@ -49,10 +49,51 @@ pub fn orbit_cluster_for(config: &Config, cluster: &str) -> String {
|
|
|
.unwrap_or_else(|| "sparks".to_string())
|
|
|
}
|
|
|
|
|
|
+/// The orbit cluster a **specific** cluster is paired with, or `None` when the
|
|
|
+/// config does not say (the flat `[flare].orbit_cluster` is then only a
|
|
|
+/// fallback for `serve`, not evidence of a pairing).
|
|
|
+///
|
|
|
+/// This is the identification rule the dashboard and `health` use: a cluster
|
|
|
+/// whose own nodes were never paired with an orbit must not be probed with the
|
|
|
+/// flat default (`sparks`) — `6nodes-switch`, `1node-*`, `ds4` and `qwen38`
|
|
|
+/// share nodes with the paired clusters, so asking one of them `--cluster
|
|
|
+/// sparks` made the sparks lane look like a lane of *that* cluster.
|
|
|
+pub fn orbit_cluster_paired(config: &Config, cluster: &str) -> Option<String> {
|
|
|
+ let f = config.flare.as_ref()?;
|
|
|
+ if let Some(own) = f.clusters.get(cluster) {
|
|
|
+ // A per-cluster block without an orbit name inherits the flat one —
|
|
|
+ // the operator already said "this cluster speaks FLARE".
|
|
|
+ return own
|
|
|
+ .orbit_cluster
|
|
|
+ .clone()
|
|
|
+ .or_else(|| f.orbit_cluster.clone());
|
|
|
+ }
|
|
|
+ // Once the map exists it is the source of truth: the flat block describes
|
|
|
+ // the machine it was written for, and our clusters share nodes (dgx5 is
|
|
|
+ // rank 0 of `dgx58-switch` AND rank 4 of `6nodes-switch`), so a flat
|
|
|
+ // `sparks` would pair `6nodes-switch` too.
|
|
|
+ if !f.clusters.is_empty() {
|
|
|
+ return None;
|
|
|
+ }
|
|
|
+ let flat = f.orbit_cluster.clone()?;
|
|
|
+ match f.controller_node.as_deref() {
|
|
|
+ // Legacy flat block: it speaks for the cluster that holds its
|
|
|
+ // controller node.
|
|
|
+ Some(ctl) => config
|
|
|
+ .clusters
|
|
|
+ .iter()
|
|
|
+ .find(|c| c.name == cluster)
|
|
|
+ .is_some_and(|c| c.nodes.iter().any(|n| n.name == ctl))
|
|
|
+ .then_some(flat),
|
|
|
+ // No controller either: unambiguous only with a single cluster.
|
|
|
+ None => (config.clusters.len() == 1 && config.clusters[0].name == cluster).then_some(flat),
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
/// Extra `orbit run` flags for a cluster (per-cluster map first, then flat).
|
|
|
-/// Fabric shape is a cluster property: `--roce-ring` emulates a ring on a
|
|
|
-/// switch (our 4nodes-switch) where FLARE cannot find direct links, while the
|
|
|
-/// DAC/switch pair clusters take `--roce-chain`/nothing.
|
|
|
+/// Fabric shape is a cluster property. The prod 4-node switch uses no extra
|
|
|
+/// flag: FLARE finds the star links itself once the legacy CX7 ports are down;
|
|
|
+/// `--roce-ring` was a symptom-level fix and is retired.
|
|
|
pub fn extra_flags_for(config: &Config, cluster: &str) -> Vec<String> {
|
|
|
let Some(f) = config.flare.as_ref() else {
|
|
|
return Vec::new();
|
|
|
@@ -70,7 +111,7 @@ pub fn extra_flags_for(config: &Config, cluster: &str) -> Vec<String> {
|
|
|
}
|
|
|
|
|
|
/// Local package mirror dir for a cluster (per-cluster map first).
|
|
|
-fn local_dir_for(config: &Config, cluster: &str) -> Option<String> {
|
|
|
+pub(crate) fn local_dir_for(config: &Config, cluster: &str) -> Option<String> {
|
|
|
let f = config.flare.as_ref()?;
|
|
|
f.clusters
|
|
|
.get(cluster)
|
|
|
@@ -92,7 +133,7 @@ fn orbit_node_name(config: &Config, cluster: &str, node: &Node) -> String {
|
|
|
.unwrap_or_else(|| node.name.clone())
|
|
|
}
|
|
|
|
|
|
-fn binary(config: &Config) -> String {
|
|
|
+pub(crate) fn binary(config: &Config) -> String {
|
|
|
config
|
|
|
.flare
|
|
|
.as_ref()
|
|
|
@@ -240,8 +281,8 @@ pub fn mirror_cmd(local_dir: &str, remote_pkg: &str) -> String {
|
|
|
format!(
|
|
|
"set -e; src=$(dirname {pkg}); dst={dir}; \
|
|
|
if [ \"$src\" = \"$dst\" ]; then echo {pkg}; exit 0; fi; \
|
|
|
- rm -rf \"$dst\"; mkdir -p \"$dst\"; \
|
|
|
- cp -a \"$src\"/. \"$dst\"/; \
|
|
|
+ mkdir -p \"$dst\"; \
|
|
|
+ rsync -a --delete --exclude '.sm121-flare-package.json' \"$src\"/ \"$dst\"/; \
|
|
|
echo \"$dst/$(basename {pkg})\""
|
|
|
)
|
|
|
}
|
|
|
@@ -298,47 +339,82 @@ pub async fn serve(
|
|
|
}
|
|
|
info!("FLARE agents ok on {} node(s)", cluster.nodes.len());
|
|
|
}
|
|
|
- let dl = download_cmd(config, recipe)?;
|
|
|
- let package = crate::ssh::run_ssh_long_running(config, controller, dl, dry_run).await?;
|
|
|
- let mut package = parse_package_path(&package).unwrap_or_default();
|
|
|
- // A cached download prints only "Resolved package: … / model.safetensors"
|
|
|
- // and no final path; resolve the default prepared model from Flare's
|
|
|
- // model cache instead of failing after a successful download.
|
|
|
- if !dry_run && package.is_empty() {
|
|
|
- let cache_cmd = format!(
|
|
|
- "{} models list --json 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); m=d.get(\"models\") or []; vers=(m[0].get(\"versions\") or []) if m else []; v=vers[0] if vers else dict(); print(d.get(\"cache\",\"\") + \"/models/\" + m[0].get(\"model_key\",\"\") + \"/\" + v.get(\"version\",\"\"))'",
|
|
|
- binary(config)
|
|
|
- );
|
|
|
- let dir = crate::ssh::run_ssh(config, controller, cache_cmd, false).await?;
|
|
|
- let dir = dir.trim();
|
|
|
- if !dir.is_empty() {
|
|
|
- let find = format!(
|
|
|
- "find {} -maxdepth 1 -name '*.safetensors' -print -quit",
|
|
|
- shq(dir)
|
|
|
+ let id = crate::native::flare_package::package_id(recipe)?;
|
|
|
+ // Identification: a complete local mirror of this exact package (repo,
|
|
|
+ // revision, TP, and the CURRENT prepared version) means the download
|
|
|
+ // verification and the 500 GB recopy are both unnecessary.
|
|
|
+ let mut package: String;
|
|
|
+ if !dry_run {
|
|
|
+ if let Some(local) =
|
|
|
+ crate::native::flare_package::local_package(config, recipe, &cluster.name, controller)
|
|
|
+ .await?
|
|
|
+ {
|
|
|
+ info!(
|
|
|
+ "FLARE package already local on {}; skipping download and mirror",
|
|
|
+ controller.name
|
|
|
);
|
|
|
- let primary = crate::ssh::run_ssh(config, controller, find, false).await?;
|
|
|
- package = primary.trim().to_string();
|
|
|
- }
|
|
|
- }
|
|
|
- if !dry_run && package.is_empty() {
|
|
|
- bail!(
|
|
|
- "flare download printed no package path on {}",
|
|
|
- controller.name
|
|
|
- );
|
|
|
- }
|
|
|
- // Local mirror: launch from the controller's own disk when configured —
|
|
|
- // engram rows are read at request time, so a shared mount is the wrong
|
|
|
- // place for the package even with a RAM row cache.
|
|
|
- if let Some(dir) = local_dir_for(config, &cluster.name) {
|
|
|
- let dir = dir.replace('~', "$HOME");
|
|
|
- let cmd = mirror_cmd(&dir, &package);
|
|
|
- let out = crate::ssh::run_ssh_long_running(config, controller, cmd, dry_run).await?;
|
|
|
- if !dry_run {
|
|
|
- let local = out.lines().last().unwrap_or_default().trim().to_string();
|
|
|
- if local.is_empty() {
|
|
|
- bail!("flare local mirror produced no path on {}", controller.name);
|
|
|
- }
|
|
|
package = local;
|
|
|
+ } else {
|
|
|
+ let dl = download_cmd(config, recipe)?;
|
|
|
+ let out = crate::ssh::run_ssh_long_running(config, controller, dl, dry_run).await?;
|
|
|
+ package = parse_package_path(&out).unwrap_or_default();
|
|
|
+ // A cached download prints only "Resolved package: … / model.safetensors"
|
|
|
+ // and no final path; resolve the default prepared model from Flare's
|
|
|
+ // model cache instead of failing after a successful download.
|
|
|
+ if package.is_empty() {
|
|
|
+ let cache_cmd = format!(
|
|
|
+ "{} models list --json 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); m=d.get(\"models\") or []; vers=(m[0].get(\"versions\") or []) if m else []; v=vers[0] if vers else dict(); print(d.get(\"cache\",\"\") + \"/models/\" + m[0].get(\"model_key\",\"\") + \"/\" + v.get(\"version\",\"\"))'",
|
|
|
+ binary(config)
|
|
|
+ );
|
|
|
+ let dir = crate::ssh::run_ssh(config, controller, cache_cmd, false).await?;
|
|
|
+ let dir = dir.trim();
|
|
|
+ if !dir.is_empty() {
|
|
|
+ let find = format!(
|
|
|
+ "find {} -maxdepth 1 -name '*.safetensors' -print -quit",
|
|
|
+ shq(dir)
|
|
|
+ );
|
|
|
+ let primary = crate::ssh::run_ssh(config, controller, find, false).await?;
|
|
|
+ package = primary.trim().to_string();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if package.is_empty() {
|
|
|
+ bail!(
|
|
|
+ "flare download printed no package path on {}",
|
|
|
+ controller.name
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ // Local mirror: launch from the controller's own disk when configured —
|
|
|
+ // engram rows are read at request time, so a shared mount is the wrong
|
|
|
+ // place for the package even with a RAM row cache.
|
|
|
+ if let Some(dir) = local_dir_for(config, &cluster.name) {
|
|
|
+ let dir = dir.replace('~', "$HOME");
|
|
|
+ // Version from the NAS package directory (the local mirror root
|
|
|
+ // has no version-named subdir).
|
|
|
+ let version = crate::native::flare_package::version_of(&package);
|
|
|
+ let cmd = mirror_cmd(&dir, &package);
|
|
|
+ let out =
|
|
|
+ crate::ssh::run_ssh_long_running(config, controller, cmd, dry_run).await?;
|
|
|
+ let local = out.lines().last().unwrap_or_default().trim().to_string();
|
|
|
+ if local.is_empty() {
|
|
|
+ bail!("flare local mirror produced no path on {}", controller.name);
|
|
|
+ }
|
|
|
+ package = local;
|
|
|
+ crate::native::flare_package::write_marker(
|
|
|
+ config, controller, &dir, &package, &id, &version,
|
|
|
+ )
|
|
|
+ .await?;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // Dry-run: print the download and mirror commands, no SSH.
|
|
|
+ let dl = download_cmd(config, recipe)?;
|
|
|
+ package = crate::ssh::run_ssh_long_running(config, controller, dl, true).await?;
|
|
|
+ package = parse_package_path(&package).unwrap_or_default();
|
|
|
+ if let Some(dir) = local_dir_for(config, &cluster.name) {
|
|
|
+ let dir = dir.replace('~', "$HOME");
|
|
|
+ let cmd = mirror_cmd(&dir, &package);
|
|
|
+ crate::ssh::run_ssh_long_running(config, controller, cmd, true).await?;
|
|
|
}
|
|
|
}
|
|
|
let orbit_name = orbit_node_name(config, &cluster.name, controller);
|