exl3: per-file bind mounts via extra_mounts_manifest — the runtime opens shards with O_NOFOLLOW, so a symlink final component => ELOOP (deterministic, nothing to do with NFS); manifest generated per node by before-serve hook, launcher reads it over ssh after the hooks. EXL3 2x now boots and serves

recipes/dsv41-flash-exl3-2x/recipe.toml

@@ -33,6 +33,7 @@ shm_size = "2g"
 devices = ["/dev/dri/card0"]
 memory_limit = "9g"
 cpus = 6
+extra_mounts_manifest = "/tmp/sm121-exl3-mounts.txt"
 
 # /model + /draft-exl3 are the exact mounts the runtime validates.
 entrypoint = [
@@ -63,8 +64,7 @@ before_serve = [
   # (blobs/ + snapshots/) is mounted at /hub, and the staging dir at /model
   # holds RELATIVE symlinks (../hub/...) into it — the kernel lands on the
   # real correct-depth tree and the inner links resolve.
-  "REPO=/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw && REV=$(cat $REPO/refs/main) && rm -rf /home/kilork/exl3-model && mkdir -p /home/kilork/exl3-model && for f in $REPO/snapshots/$REV/*; do ln -s \"../hub/snapshots/$REV/$(basename $f)\" /home/kilork/exl3-model/$(basename $f); done",
-  "DREPO=/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw && DREV=$(cat $DREPO/refs/main) && rm -rf /home/kilork/exl3-draft && mkdir -p /home/kilork/exl3-draft && for f in $DREPO/snapshots/$DREV/*; do ln -s \"../dh-draft/snapshots/$DREV/$(basename $f)\" /home/kilork/exl3-draft/$(basename $f); done",
+  "python3 - models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw $(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/refs/main) /model models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw $(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/refs/main) /draft-exl3 <<'GENSTAGE'\nimport os, sys\nHUB = '/mnt/nas/ai/huggingface/hub'\nMANIFEST = '/tmp/sm121-exl3-mounts.txt'\ndef gen(repo, rev, out):\n    snap = os.path.join(HUB, repo, 'snapshots', rev)\n    lines = []\n    for name in sorted(os.listdir(snap)):\n        src = os.path.join(snap, name)\n        real = os.path.realpath(src)\n        if os.path.isdir(real):\n            for root, _, files in os.walk(real):\n                for fn in sorted(files):\n                    fp = os.path.join(root, fn)\n                    freal = os.path.realpath(fp)\n                    sub = os.path.relpath(fp, real)\n                    lines.append(freal + '\\t' + os.path.join(out, name, sub))\n        else:\n            lines.append(real + '\\t' + os.path.join(out, name))\n    with open(MANIFEST, 'a') as f:\n        f.write('\\n'.join(lines) + '\\n')\n    print('manifest', MANIFEST, len(lines), 'mounts for', out, flush=True)\nrepo, rev, out, drepo, drev, dout = sys.argv[1:7]\nopen(MANIFEST, 'w').close()  # truncate once before both gens\ngen(repo, rev, out)\ngen(drepo, drev, dout)\nGENSTAGE",
   # The runtime kit (serve.py + conservative.yaml + overlay .so + AOT gemm)
   # is NOT in the image — it is the pinned release/runtime tree, staged once
   # per node and bind-mounted as overlays (serving/ -> /opt/ds41-serving,
@@ -74,28 +74,18 @@ before_serve = [
 ]
 after_stop = [{ script = "restore_display_ram", on_failure = "warn" }]
 
-# Repo dirs (blobs/ + snapshots/) — the real tree the staging symlinks land
-# in; the internal ../../../blobs links resolve here at full depth.
+# Hub dir (both repos + hub-level blobs/): the repo blobs/ is itself a layer
+# of relative symlinks into the sibling hub-level blobs/<xx>/<sha>, so the
+# whole hub tree must be in scope for the chain to resolve.
 [[extra_mounts]]
-host = "/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw"
+host = "/mnt/nas/ai/huggingface/hub"
 container = "/hub"
 read_only = true
 
-[[extra_mounts]]
-host = "/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw"
-container = "/dh-draft"
-read_only = true
-
-# Staging dirs (relative symlinks into /hub and /dh-draft).
-[[extra_mounts]]
-host = "/home/kilork/exl3-model"
-container = "/model"
-read_only = true
-
-[[extra_mounts]]
-host = "/home/kilork/exl3-draft"
-container = "/draft-exl3"
-read_only = true
+# /model and /draft-exl3 are NOT dir-mounted: the runtime opens the shard
+# files with O_NOFOLLOW (symlink final component => ELOOP), so the
+# before-serve hook generates a per-file bind manifest instead
+# (extra_mounts_manifest) — the container sees regular files only.
 
 # Runtime kit overlays (pinned 878e0ee): the image's /opt/ds41-serving is
 # shadowed by the kit's serving tree; the AOT gemm dir is bind-mounted ro.
@@ -171,3 +161,4 @@ NCCL_LL128_BUFFSIZE = "262144"
 NCCL_PROTO = "^LL128"
 NCCL_MAX_NCHANNELS = "8"
 NCCL_IB_ROCE_VERSION_NUM = "2"
+

src/config/recipe.rs

@@ -192,6 +192,14 @@ pub struct Recipe {
     /// is required.
     #[serde(default)]
     pub extra_mounts: Vec<ExtraMount>,
+    /// Path to a per-node mount manifest (host-side file, read over SSH after
+    /// the before-serve hooks): one `host_path<TAB>container_path` line per
+    /// extra bind-mount, all read-only. Lets a recipe's hooks generate
+    /// per-node mount lists that are too dynamic for static `[[extra_mounts]]`
+    /// (e.g. one bind per model shard file, so the container sees regular
+    /// files where the engine opens with O_NOFOLLOW). Docker-only.
+    #[serde(default)]
+    pub extra_mounts_manifest: Option<String>,
 }
 
 impl Recipe {

src/docker/launch_node.rs

@@ -8,7 +8,7 @@
 //! per-cluster.
 
 use crate::config::Node;
-use crate::docker::{command, compose};
+use crate::docker::{command, compose, shq};
 
 /// Per-node inputs for [`launch_node`], grouped so the function stays under
 /// the argument-count lint without scattering fields.
@@ -98,6 +98,62 @@ pub(crate) async fn launch_node(
             Err(e) => return fail(e),
         }
     }
+    // Per-node mount manifest (generated by the recipe's before-serve hooks):
+    // one `host<TAB>container` line per extra read-only bind. Skipped in
+    // dry-run (no SSH); a missing/empty file is a hard error in live runs so
+    // a half-staged node fails loudly instead of starting with half the model.
+    if let Some(manifest_path) = recipe.extra_mounts_manifest.clone() {
+        if dry {
+            tracing::info!("[dry-run] would read mount manifest {} on {}", manifest_path, node.name);
+        } else {
+            let out =
+                match crate::ssh::run_ssh(config, node, format!("cat {}", shq(&manifest_path)), false)
+                    .await
+                {
+                    Ok(o) => o,
+                    Err(e) => {
+                        return fail(anyhow::anyhow!(
+                            "mount manifest {} not readable on {}: {}",
+                            manifest_path,
+                            node.name,
+                            e
+                        ))
+                    }
+                };
+            let mut count = 0;
+            for line in out.lines() {
+                let line = line.trim();
+                if line.is_empty() || line.starts_with('#') {
+                    continue;
+                }
+                let (host, container) = match line.split_once('\t') {
+                    Some(p) => p,
+                    None => {
+                        return fail(anyhow::anyhow!(
+                            "bad mount manifest line on {} (expected host<TAB>container): {}",
+                            node.name,
+                            line
+                        ))
+                    }
+                };
+                count += 1;
+                extra_mounts.push(command::ResolvedMount {
+                    host: host.to_string(),
+                    container: container.to_string(),
+                    read_only: true,
+                    env: None,
+                });
+            }
+            if count == 0 {
+                return fail(anyhow::anyhow!(
+                    "mount manifest {} is empty on {}",
+                    manifest_path,
+                    node.name
+                ));
+            }
+            tracing::info!("mount manifest {} on {}: {} mounts", manifest_path, node.name, count);
+        }
+    }
     let ctx = command::DockerCommandCtx {
         recipe,
         extra_mounts,