pre-serve hooks: serve-local NODE_RANK + TensorFold staging fixes; AGENTS.md monitoring rule

The 2026-10-21 TensorFold bring-up on dgx7-8 exposed three real bugs:

- before_serve handed built-in hooks the CONFIG rank (dgx7 was "rank 2"), while
  the containers are renumbered to the serve-local index on a subset launch.
  A per-rank staging hook (the TensorFold Engram packs) then built rank-2/3
  packs for a 2-rank engine. Hooks now get the launch-local rank too.
- tensorfold-1017-prepare pinned the EXL3 snapshot rev; the NAS cache moved to
  a new snapshot and the hook died on "Cannot open: No such file or directory".
  The rev now resolves from the HF cache's refs/main (pin only as a fallback),
  with an explicit FATAL when the snapshot is absent.
- the Engram packer ran on the host python, which is PEP-668 managed on dgx7/8
  (error: externally-managed-environment). It now prefers the image's python and
  mounts the whole hub (snapshot -> per-model blob -> hub blob symlink chain;
  anything less leaves model.safetensors.index.json dangling).

AGENTS.md gains a "Monitoring discipline" rule (agent-facing): long operations
fail early, so start them in the background and poll in 10-30 s cycles inside
the same tool call, breaking on the first failure marker — never a fixed
sleep-then-check.

AGENTS.md

@@ -23,6 +23,31 @@ and cached (`ssh.rs`), and mimalloc as the backstop. Post-fix RSS is
 created in `main` before anything allocates. Note: axum graceful shutdown
 hangs on live SSE — web closes connections on the signal instead.
 
+## Monitoring discipline (agent rule — long operations fail early)
+
+Do **not** "sleep N minutes, then check". `serve`/`bench`/image builds and the
+staging hooks fail in the first seconds or minutes (a missing mount, a bad rev,
+PEP-668 pip, "not found on any node"), so a 10-15 minute fixed sleep wastes the
+whole window and buries the failure — and a long `sleep` in one tool call
+aborts the call, losing the log position.
+
+- Start the operation in the background (`nohup … > /tmp/<lane>-<phase>.log 2>&1 &`),
+  then **poll in short cycles (10-30 s)** and `break` on the first signal:
+  - failure markers in the log — `Error:`, `error:`, `FATAL`, `Traceback`,
+    `No such file`, `not found on any node`, `died`;
+  - or the phase's success marker (`script:<hook>`, `started on`, `API ready`,
+    `benchmark_complete`).
+- Print the log tail **inside the polling loop** (`tail -1`), so progress is
+  visible while it runs; on failure grep the log for the error and report it —
+  never sleep again after a failure.
+- Prefer the tool's own wait/follow mode when it has one (`sm121 serve` without
+  `--no-wait` waits for readiness and follows liveness; `sm121 logs -f`), and
+  use `--no-wait` only when you intend to poll the readiness URL yourself.
+- Cap the total wait (a readiness wait is minutes, a first-time staging pass is
+  ~40 min — poll for that long, don't `sleep 2400`). On timeout, report the tail
+  instead of sleeping more.
+- Same for `make check`, benches and image pushes: background + short polls.
+
 ## Dev quick-ref
 
 ## Dev quick-ref

host-scripts/tensorfold-1017-prepare.sh

@@ -31,11 +31,22 @@ NAS="${TF1017_NAS:-/mnt/nas/ai/huggingface/hub}"
 PUB="${TF1017_PUB:-/mnt/nas/ai/sm121/dsv41-tf-1017}"
 IMG="${TF1017_IMAGE:-dsv41-tf-1017:v1}"
 RANK="${NODE_RANK:-0}"
-REV="${TF1017_MODEL_REV:-afc7aa0aab98ee735c0f54e0a122019af1ac08f1}"
-EREV="${TF1017_ENGRAM_REV:-1bae0e1088993b9b82de08f59de5be5a34e42170}"
+# Pins are only a fallback: the HF cache's `refs/main` is the truth (a
+# re-cut/re-download moves the snapshot, and a hardcoded rev then points at a
+# directory that no longer exists — exactly what broke the 2026-10-21 serve).
+pin_or_ref() { # <hub dir> <pinned rev>
+    local d="$NAS/$1" ref
+    ref=$(cat "$d/refs/main" 2>/dev/null || true)
+    if [[ -n "$ref" && -d "$d/snapshots/$ref" ]]; then echo "$ref"; else echo "$2"; fi
+}
+REV="${TF1017_MODEL_REV:-$(pin_or_ref models--LibertAIDAI--DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw afc7aa0aab98ee735c0f54e0a122019af1ac08f1)}"
+EREV="${TF1017_ENGRAM_REV:-$(pin_or_ref models--deepseek-ai--DeepSeek-V4.1-Flash-1017 1bae0e1088993b9b82de08f59de5be5a34e42170)}"
 
 WBUNDLE="$NAS/models--LibertAIDAI--DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw/snapshots/$REV"
 EBUNDLE="$NAS/models--deepseek-ai--DeepSeek-V4.1-Flash-1017/snapshots/$EREV"
+[[ -d "$WBUNDLE" ]] || { echo "[tf1017] FATAL: no checkpoint snapshot at $WBUNDLE" >&2; exit 1; }
+[[ -d "$EBUNDLE" ]] || { echo "[tf1017] FATAL: no Engram source snapshot at $EBUNDLE" >&2; exit 1; }
+echo "[tf1017] checkpoint rev $REV, Engram rev $EREV"
 CKPT="$ROOT/models/checkpoint"
 PACKED="$ROOT/engram-packed-1017"
 CACHE="$ROOT/cache"
@@ -77,11 +88,32 @@ fi
 # 3. per-rank Engram packs (host python + numpy; no image needed) ------------
 if [[ "${TF1017_SKIP_PACK:-0}" != "1" ]]; then
     if [[ ! -s "$PACKED/engram-l14-r${RANK}of2.bin" ]]; then
-        python3 -c 'import numpy' 2>/dev/null || python3 -m pip install --user --quiet numpy
         echo "[tf1017] packing Engram tables for rank $RANK (~94 GiB, reads 189 GiB from the NAS)"
-        python3 "$HELP/pack_engram.py" \
-            --src "$EBUNDLE" --config "$CKPT/config.json" \
-            --out "$PACKED" --rank "$RANK"
+        if [[ "${TF1017_PACK_IN_IMAGE:-1}" = "1" ]] && docker image inspect "$IMG" >/dev/null 2>&1; then
+            # Prefer the image: it has numpy + the exact python the packer was
+            # written against. Host python is PEP-668 managed on dgx7/8
+            # (`error: externally-managed-environment`) — no host installs.
+            # Mount the whole repo dir (`snapshots/` + `blobs/`), not just the
+            # snapshot: the snapshot files are symlinks into the hub's blobs,
+            # so a snapshot-only mount makes model.safetensors.index.json
+            # dangle inside the container.
+            # Mount the whole hub: snapshot files are symlinks to the
+            # per-model blobs, which are themselves symlinks to hub-level
+            # blobs (huggingface_hub >= 0.34 layout) — anything less leaves
+            # model.safetensors.index.json dangling in the container.
+            docker run --rm --network none --entrypoint python3 \
+                -v "$NAS":/hub:ro -v "$CKPT":/ckpt:ro -v "$PACKED":/out \
+                -v "$HELP/pack_engram.py":/pack_engram.py:ro \
+                "$IMG" /pack_engram.py --src "/hub/${EBUNDLE#"$NAS"/}" \
+                --config /ckpt/config.json --out /out --rank "$RANK"
+        else
+            python3 -c 'import numpy' 2>/dev/null \
+                || python3 -m pip install --user --quiet --break-system-packages numpy \
+                || { echo "[tf1017] FATAL: numpy missing for the Engram packer and pip refused (no image on this node yet)" >&2; exit 1; }
+            python3 "$HELP/pack_engram.py" \
+                --src "$EBUNDLE" --config "$CKPT/config.json" \
+                --out "$PACKED" --rank "$RANK"
+        fi
     fi
     [[ -s "$PACKED/engram-l1-r${RANK}of2.bin" && -s "$PACKED/engram-l14-r${RANK}of2.bin" ]] \
         || { echo "[tf1017] FATAL: Engram packs missing for rank $RANK" >&2; exit 1; }

src/before_serve.rs

@@ -44,7 +44,7 @@ pub async fn run(
     // Run all commands for one node sequentially (so order matters per
     // node), and the nodes in parallel.
     let mut handles = Vec::with_capacity(cluster.nodes.len());
-    for node in &cluster.nodes {
+    for (local_rank, node) in cluster.nodes.iter().enumerate() {
         let config = config.clone();
         let node = node.clone();
         let cmds = cmds.clone();
@@ -52,7 +52,12 @@ pub async fn run(
         handles.push(crate::locale::spawn_with_locale(async move {
             (
                 node.name.clone(),
-                run_for_node(&config, &node, &cmds, dry).await,
+                // Serve-LOCAL rank, not `node.rank`: a subset launch (2 of a
+                // 4-node cluster) renumbers ranks for the containers, so a
+                // per-rank staging hook must see the same 0..N-1 index —
+                // otherwise dgx7/dgx8 in a 4-node cluster stage rank 2/3
+                // packs for a 2-rank engine (2026-10-21).
+                run_for_node(&config, &node, local_rank as u32, &cmds, dry).await,
             )
         }));
     }
@@ -104,6 +109,7 @@ fn combined_commands(
 async fn run_for_node(
     config: &Config,
     node: &crate::config::Node,
+    local_rank: u32,
     cmds: &[(&'static str, PreServeCommand)],
     dry_run: bool,
 ) -> Result<()> {
@@ -128,7 +134,7 @@ async fn run_for_node(
                 // TensorFold Engram packs) without inventing a second mechanism.
                 (
                     format!("script:{script}"),
-                    format!("sudo NODE_RANK={} bash {remote}", node.rank),
+                    format!("sudo NODE_RANK={local_rank} bash {remote}"),
                 )
             }
             _ => {