Quellcode durchsuchen

flare: mirror the package onto the controller and launch from the local copy

[flare].local_dir makes serve copy the package's companion files to the
controller's own disk before launching: the primary is always refreshed, a
companion re-copies only on a size mismatch — a sound check because FLARE names
every companion by its sha256. orbit run then starts from the local path, so a
RAM row-cache miss reads local NVMe instead of the shared mount.

Also: the native (dgpp/flare) path now runs before_serve hooks, so the global
' nvidia-smi -lgc 0,2000' actually applies to FLARE lanes — the night's
two-shot collective timeouts were measured with unlocked clocks.
Alexander Korolev vor 16 Stunden
Ursprung
Commit
cdd141de82
3 geänderte Dateien mit 56 neuen und 0 gelöschten Zeilen
  1. 7 0
      src/config/types.rs
  2. 37 0
      src/native/flare.rs
  3. 12 0
      src/native/mod.rs

+ 7 - 0
src/config/types.rs

@@ -476,6 +476,13 @@ pub struct FlareSettings {
     /// multi-cluster config (the recipes panel then sees no FLARE groups).
     #[serde(default)]
     pub controller_node: Option<String>,
+    /// Local mirror dir for the package on the controller (e.g.
+    /// `~/flare-pkgs`). When set, `serve` copies the package's companion files
+    /// there (they are named by sha256, so a size match means "already there")
+    /// and launches from the local copy — runtime table reads then hit local
+    /// NVMe instead of the shared mount.
+    #[serde(default)]
+    pub local_dir: Option<String>,
     /// `[cluster.<name>]` in `~/.config/flare/orbit.toml` (default `sparks`).
     #[serde(default)]
     pub orbit_cluster: Option<String>,

+ 37 - 0
src/native/flare.rs

@@ -169,6 +169,27 @@ pub fn preflight_cmd(config: &Config, controller: bool) -> String {
     }
 }
 
+/// Mirror a package's files into the controller's local dir and return the
+/// local primary path.
+///
+/// FLARE names every companion by its sha256, so "present and the same size"
+/// is a sound consistency check — a truncated or stale file re-copies, and an
+/// unchanged package costs a `stat` per file. The primary (metadata) is tiny
+/// and always refreshed.
+pub fn mirror_cmd(local_dir: &str, remote_pkg: &str) -> String {
+    let dir = shq(local_dir);
+    let pkg = shq(remote_pkg);
+    format!(
+        "set -e; src=$(dirname {pkg}); dst={dir}; base=$(basename {pkg}); \
+         mkdir -p \"$dst\"; \
+         cp -f {pkg} \"$dst/$base\"; \
+         for f in \"$src\"/*; do b=$(basename \"$f\"); [ \"$b\" = \"$base\" ] && continue; \
+           if [ ! -f \"$dst/$b\" ] || [ \"$(stat -c%s \"$f\")\" != \"$(stat -c%s \"$dst/$b\")\" ]; then \
+             cp -f \"$f\" \"$dst/$b\" || exit 1; fi; done; \
+         echo \"$dst/$base\""
+    )
+}
+
 /// Pick the package path out of `flare download` output: it prints progress
 /// lines ("Downloading model file: …") and finally the resolved package path
 /// ending in `.safetensors`. Taking the last line blindly handed `orbit run`
@@ -227,6 +248,22 @@ pub async fn serve(
             controller.name
         );
     }
+    // Local mirror: launch from the controller's own disk when configured —
+    // engram rows are read at request time, so a shared mount is the wrong
+    // place for the package even with a RAM row cache.
+    let mut package = package;
+    if let Some(dir) = config.flare.as_ref().and_then(|f| f.local_dir.clone()) {
+        let dir = dir.replace('~', "$HOME");
+        let cmd = mirror_cmd(&dir, &package);
+        let out = crate::ssh::run_ssh(config, controller, cmd, dry_run).await?;
+        if !dry_run {
+            let local = out.lines().last().unwrap_or_default().trim().to_string();
+            if local.is_empty() {
+                bail!("flare local mirror produced no path on {}", controller.name);
+            }
+            package = local;
+        }
+    }
     let orbit_name = orbit_node_name(config, controller);
     let run = run_cmd(config, recipe, &orbit_name, &package)?;
     crate::ssh::run_ssh(config, controller, run.clone(), dry_run).await?;

+ 12 - 0
src/native/mod.rs

@@ -297,6 +297,18 @@ async fn flare_serve(
     if recipe.flare.is_none() {
         anyhow::bail!("engine=flare requires a [flare] block");
     }
+    // before_serve hooks (the global GPU clock lock among them): a Spark that
+    // runs unclocked drifts and stalls a collective, so native launches get
+    // the same prepare step docker/tmux lanes do.
+    crate::before_serve::run(
+        config,
+        &cluster,
+        recipe,
+        cluster_name,
+        opts.skip_before_serve,
+        opts.dry_run,
+    )
+    .await?;
     flare::serve(config, recipe, &cluster, opts.dry_run).await?;
     Ok(cluster)
 }