glm53 full-model lane: retarget to RedHatAI/GLM-5.3-MXFP4 on the r35 jovian stack; delete the OOM-proven NVFP4-Spark variants

- new recipe glm53-full-mxfp4-4x: full GLM-5.3 (GlmMoeDsaForCausalLM, 78
  layers, hidden 6144, 256 experts top-8, MLA+DSA indexer, MTP head in ckpt,
  403.4 GB MXFP4) on 4x, DCP4 + 512K + 8 GiB KV pin + GMU 0.85, no spec on
  the first boot, instanttensor + B12X, JIT-cache mounts
- base stack = our jovian r35 build (glm53-jovian-r27 dir + glm53-r35
  profile): it registers GlmMoeDsaForCausalLM via the NEW deepseek_v32
  module with the NVIDIA-specific implementation, ships the
  compressed_tensors mxfp4-pack-quantized scheme, and is the same image as
  the Flash 4x prod (reproducible clone-at-pin); lab/aiden images kept as a
  documented fallback
- deleted recipes/glm53-full-nvfp4-spark-lab, -aiden, glm53-spark-nvfp4-4x
  (abandoned local-inference-lab/GLM-5.3-NVFP4-Spark; OOM-thrashed 4 nodes
  2026-09-08, HA power-cycle)
- spec: 2026-09-13 section with the memory arithmetic, why DCP4, the
  double-load hazard, and the boot ladder; prepared only, not started

recipes/glm53-full-mxfp4-4x/recipe.toml

@@ -0,0 +1,240 @@
+# GLM-5.3 (FULL, non-Flash) MXFP4 — RedHatAI official quant — 4× DGX Spark, TP=4.
+#
+# REPLACES the failed full-model variant: `local-inference-lab/GLM-5.3-NVFP4-Spark`
+# (406 GiB → 101.5 GiB weights/node → OOM-proven on 4 nodes 2026-09-08/09: the
+# engine crash-restarted and began a SECOND full load while the first copy was
+# still resident → 2×101.5 > 121.63 GiB → all four nodes OOM-thrashed, sshd down
+# on 3 of 4, HA power-cycle required). That checkpoint is abandoned (recipes
+# glm53-full-nvfp4-spark-lab / -aiden / glm53-spark-nvfp4-4x deleted).
+#
+# THIS checkpoint: RedHatAI/GLM-5.3-MXFP4 — 403.4 GB (375.6 GiB) → 100.85 GB
+# (93.9 GiB) weights/node on 4x. ~7.6 GiB/node more headroom than the NVFP4
+# attempt, which is what makes 4x plausible at all. LLM-Compressor
+# (compressed-tensors) quant: "mxfp4-pack-quantized", weights FP4-e2m1 group-32
+# with E8M0 scales, dynamic per-group FP4 activations; mlp.gate / lm_head /
+# embed_tokens / eh_proj / indexer.{wk,weights_proj} stay bf16.
+#   Model:  https://huggingface.co/RedHatAI/GLM-5.3-MXFP4
+#   Parent: zai-org/GLM-5.3 (FP8). Card: gsm8k 96.36 vs 97.60 (FP8) = 98.7%
+#   recovery; ifeval 89.25 vs 93.53 = 95.4%.
+#   Card's serving recipe (4× B200): --kv-cache-dtype fp8_e4m3 --tp 4
+#   --reasoning-parser glm45 --speculative-config '{"method":"mtp","num_speculative_tokens":5}'
+#
+# GEOMETRY (config.json, verified 2026-09-13): arch GlmMoeDsaForCausalLM
+# (model_type glm_moe_dsa), 78 layers, hidden 6144, 256 routed experts + 1
+# shared, top-8, moe_inter 2048, MLA (kv_lora 512 / q_lora 2048 / qk_nope 192 /
+# qk_rope 64 / v 256), DSA lightning indexer (index_topk 2048), 1M native ctx,
+# first_k_dense_replace 3, num_nextn_predict_layers 1 (MTP head IN the
+# checkpoint), 141 shards, no separate drafter.
+# ~740B total params / ~40B active. The Flash variant we serve in prod is
+# 320B/18B (45 layers, hidden 4096, 288 experts) — this is a different model.
+#
+# ── BASE STACK: our own jovian r35 build (docker-images/glm53-jovian-r27 +
+#    build_profile glm53-r35 → image glm53-jovian-r35:local) ────────────────
+# Selected 2026-09-13. Evidence (all verified by read-only inspection of the
+# images on dgx1):
+#   * registers the full-model arch: registry.py →
+#     "GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", ...) — the NEW
+#     `deepseek_v32` implementation, which is "MLA + a lightning indexer that
+#     selects the top-k tokens for a sparse MLA" and dispatches to an
+#     NVIDIA-specific path (deepseek_v32/nvidia/model.py:
+#     DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM). That is exactly the
+#     GB10/sm_121a target; the lab fork and aiden's image register the same
+#     class from the older `deepseek_v2` module instead.
+#   * ships the MXFP4 scheme: model_executor/layers/quantization/
+#     compressed_tensors/schemes/compressed_tensors_w4a4_mxfp4.py
+#     ("mxfp4-pack-quantized") → the RedHat checkpoint loads natively.
+#   * it is OUR reproducible clone-at-pin build (vLLM de982a50 + b12x 98086604,
+#     r35 lock 2026-09-11 = newest published lab composition), all gates green,
+#     patches ours to write — and it is the SAME image our GLM-5.3-Flash 4x
+#     prod runs, so the A/B vs prod is clean and the JIT caches are warm.
+#   * the b12x MoE/attention backends are the GB10-tuned ones our GLM prod
+#     uses (--moe-backend b12x / --attention-backend B12X).
+# Rejected: `glm53-flash-nvfp4-lab:local` and aiden's
+# `aidendle94/sparkrun-vllm-glm53-gb10:production-1.2` — both older vLLM
+# (0.1.dev20051+g487ecf187, `deepseek_v2` GLM class); the lab fork needed two
+# SM120 mla_attention hotfix mods to boot the full model at all
+# (masked_mha_available AttributeError + forward_mha NotImplementedError, both
+# never re-tested — the lane parked). Keep them as a fallback reference only:
+# if the r35 deepseek_v32 path trips on the DSA/indexer kernels, port those
+# two sed mods (documented in the deleted glm53-full-nvfp4-spark-lab history)
+# onto the lab image and re-point docker_image/docker_image_dir.
+#
+# ── MEMORY PLAN (the binding constraint — read before the first boot) ──────
+#   121.63 GiB unified/node. Idle free: 116-117 GiB.
+#     weights          93.9 GiB (403.4 GB / 4)  ← ~101.5 was fatal
+#     vLLM procs+docker ~12 GiB
+#     KV pin            8 GiB (--kv-cache-memory-bytes)
+#     activations + CUDA graphs ~4 GiB
+#   ⇒ ~118 GiB — tight but inside the node. GMU 0.85 is the ADMISSION
+#   guardrail only (free ≥ 0.85×121.63 = 103.4 GiB; 0.88 = 107.0 GiB fails
+#   once the ~12 GiB of starting processes are counted, the 2026-09-08
+#   #48140 class) — the KV pool is pinned, so GMU does not change capacity.
+#
+#   WHY DCP4: the full model's attention KV is NOT the Flash lane's compact
+#   MLA cache. The parked run measured ~54 KB/token (4 GiB ↔ ~79.6K tokens;
+#   "524288 would need 26.34 GiB") vs the Flash lane's ~2 KB/token at 512K.
+#   Decode-context-parallel-4 shards the KV across the 4 ranks, so the 8 GiB
+#   pin buys 32 GiB aggregate → a single 524288-token request needs ~27 GiB
+#   aggregate (≈6.8 GiB/rank) and just fits. Without DCP a 512K request is
+#   impossible behind these weights. CAVEAT: DCP4 is qualified in r35 for the
+#   Flash model (CP1/CP4 shapes); it is UNVERIFIED for the full model — if the
+#   boot rejects it (the GLM C4 page-stride / interleave-multiple-of-4 gates
+#   that block DFlash2 on 2x), drop --decode-context-parallel-size +
+#   --cp-kv-cache-interleave-size and lower --max-model-len to fit ~10 GiB of
+#   aggregate KV (~180K tokens).
+#
+#   --max-model-len 524288 = the operator's floor (128K/131K profiles are not
+#   usable for us). --max-num-seqs 8 keeps the pool from being shredded by
+#   short requests; the admission scheduler handles the rest.
+#
+# ── BOOT PLAN / HAZARD ─────────────────────────────────────────────────────
+#   The 2026-09-08 disaster was NOT a bad weight footprint per se — it was a
+#   post-load crash + auto-restart that started a SECOND full load on top of
+#   the first. Watch the first boot: if the engine logs a second
+#   "Loading safetensors" pass, kill immediately (`sm121 stop`), do not wait.
+#   Always `sm121 serve --keep` for the first/unknown boot (default is --rm,
+#   which wipes the crash logs).
+#   Ladder if the boot dies: (1) disable DCP4, max-model-len 131072, KV pin
+#   4 GiB (prove stack+checkpoint end-to-end, the parked lane's approach);
+#   (2) then re-add DCP4 + 512K; (3) only then add spec.
+#   Boot markers to record: "Model loading took X GiB", the KV line
+#   ("GPU KV cache size: N tokens, Maximum concurrency for 524288 tokens per
+#   request: Mx"), "Application startup complete". MAX_JOBS=2 below is the
+#   compile-OOM guard (12 parallel nvcc jobs × ~3 GiB behind 94 GiB of weights
+#   = cicc OOM-killed, 2026-09-04 lesson).
+#
+# ── SPECULATIVE DECODING ───────────────────────────────────────────────────
+#   NOT enabled on the first boot: the checkpoint's MTP head is real
+#   (num_nextn_predict_layers=1) and the card suggests mtp k=5, but spec adds
+#   draft KV + graph shapes onto a memory budget that has no room for
+#   surprises, and the full-model MTP path is unproven on this stack. After a
+#   clean boot + smoke + bench, add:
+#     "--speculative-config", """'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'"""
+#   (k=3 = our Flash prod's verified depth; try 5 after k=3 is clean).
+#
+# PREREQUISITE: weights in the shared NAS HF cache on all 4 nodes
+#   (models--RedHatAI--GLM-5.3-MXFP4, 403.4 GB, 141 shards) —
+#   `hf download RedHatAI/GLM-5.3-MXFP4`; HF_HUB_OFFLINE=1 below makes a
+#   missing shard a hard load failure. NAS had 16 TB free at 2026-09-13.
+#
+# A/B TARGET: our GLM-5.3-Flash r35 MTP3 4x prod (dev65/seed 42, hardmode):
+#   163/176 (93), 1,554 pp / 49.6 tg / 1,544 ms TTFT @d0 c1, KV 8.06M @512K.
+#   This lane is a capability/quality lane first (does the full model fit and
+#   hold 512K on 4 nodes), throughput second.
+
+engine = "vllm"
+name = "GLM-5.3 (full) MXFP4 (r35 img, RedHatAI ckpt, 4x DCP4)"
+docker_image = "glm53-jovian-r35:local"
+docker_image_dir = "glm53-jovian-r27"
+# NOTE: build_profile must stay a TOP-LEVEL key — a bare key after
+# [[extra_mounts]] would parse into the last mount table and be silently
+# ignored (2026-09-08).
+build_profile = "glm53-r35"
+entrypoint = ["vllm", "serve"]
+model = "RedHatAI/GLM-5.3-MXFP4"
+parent_model = "zai-org/GLM-5.3"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # R31/r35 delta (their qualified locks): adaptive prefill/decode
+    # compute-time sharing with a fixed 40% prefill floor.
+    "--prefill-compute-share", "0.4",
+    # Stable served name — quant-agnostic (shared across glm53 lanes)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # Admission guardrail only (KV is pinned below): 0.85×121.63 = 103.4 GiB.
+    # 0.88 (107.0) is the Flash lane's value and is too tight here — the full
+    # model's weights leave ~24 GiB before the engine's own ~12 GiB.
+    "--gpu-memory-utilization", "0.85",
+    # 8 GiB/rank. With DCP4 that is 32 GiB aggregate ≈ one 524288-token request
+    # (+headroom). See the memory plan above before changing either number.
+    "--kv-cache-memory-bytes", "8589934592",
+    # DCP4 — required to make 512K fit (KV sharded across the 4 TP ranks).
+    # Unverified for the full model; first fallback is to drop both lines.
+    "--decode-context-parallel-size", "4",
+    "--cp-kv-cache-interleave-size", "4",
+    # No --kv-cache-dtype: let the r35 fork pick its native default for
+    # GlmMoeDsa attention (the Flash lane's explicit fp8/fp8_ds_mla spellings
+    # both failed auto-selection on this build, and those were Flash-MLA
+    # knobs). The card's --kv-cache-dtype fp8_e4m3 is the fallback spelling
+    # if the pool needs to shrink.
+    "--attention-backend", "B12X",
+    # pure I/O loader (kept across every GLM config)
+    "--load-format", "instanttensor",
+    "--block-size", "256",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup (the
+    # autotune scratch at 512K shapes does not fit the headroom, 2026-08-28).
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    "--language-model-only",
+    # NO --speculative-config on the first boot — see the header. MTP head is
+    # in the checkpoint (num_nextn_predict_layers=1); add mtp k=3 after a
+    # clean boot + bench.
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+
+# R31 lock: 16 NCCL channels (their qualified runtime).
+NCCL_NCHANNELS = "16"
+HF_HUB_OFFLINE = "1"
+# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for r27+
+# lanes since the 2026-09-08 A/B (4x DCP4: decode +31..+90%, quality 160 vs
+# 155; specs/tech-architecture/glm53-recipe-state.md). Needs the CX-7 RoCE
+# fabric (all of ours); single-node serves skip it silently.
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
+# Compile-OOM guard: first boot JIT-compiles the mxfp8 gemm module with 12
+# parallel nvcc jobs (~3 GiB each) behind ~94 GiB of resident weights →
+# cicc OOM-killed (2026-09-04). flashinfer's run_ninja reads MAX_JOBS.
+MAX_JOBS = "2"
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+# Upstream launch-script env (memory + compile knobs)
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+VLLM_USE_STANDALONE_COMPILE = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "2"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-jit-121f/b12x"
+container = "/root/.cache/b12x"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"

recipes/glm53-full-nvfp4-spark-aiden/recipe.toml

@@ -1,99 +0,0 @@
-# GLM-5.3-Flash on 4× DGX Spark — aidendle94 sparkrun image (TP=4). DEFERRED.
-#
-# aiden's production 4-node profile, verbatim translation of
-# https://aidenle.com/recipes/glm-5-3-flash-4x-dgx-spark into our 4x launcher
-# shape. Reference numbers (his, single stream, idle cluster, thinking off):
-#   ~84 tok/s decode (FP8, DFlash2 k=7) — NVFP4 +23% (110.7 structured);
-#   prefill ~2,100 cold; TTFT 4 s @ 8K → 40 s @ 94K; KV 2.25M tokens at 16 GiB
-#   pin; 393K context verified. Decode flat with context (34/45 KDA layers +
-#   11 sparse-MLA @ index_topk 2048).
-#
-# DFlash2: acceptance 0.5–0.8; k=7 (drafter block 8 − 1 target token); k=6
-# wins prose, loses structured. Native MTP fallback (production-1.0 image or
-# spec omitted) optimum k=5. ~15 min full relaunch on FP8 (963 s measured).
-# FP8-MLA + DFlash2 combination is STRUCTURALLY broken upstream (three trees
-# fail differently) — hence plain fp8_e4m3 KV.
-#
-# NOT READY TO DEPLOY: needs a 4-node cluster (ds4 + 2nodes-dac freed) and the
-# unsloth/GLM-5.3-Flash-FP8 checkpoint (328 GB — background download running
-# on dgx3, /tmp/dl_fp8.log; ~340 GB free per node required for weights + Triton
-# cache). Fabric: RoCE dual-HCA NCCL profile comes from the launcher (config
-# [nccl.*] / GID auto-detect) — his NCCL_* env block intentionally omitted.
-# NOTE: his launcher unsets VLLM_PREFIX_CACHE_RETENTION_INTERVAL via bash -lc
-# before exec; if our boot rejects it in the KV coordinator, add a mods script.
-
-engine = "vllm"
-name = "GLM-5.3 (full) NVFP4-Spark, aiden image (vLLM, 4x Spark)"
-docker_image = "aidendle94/sparkrun-vllm-glm53-gb10:production-1.2"
-model = "local-inference-lab/GLM-5.3-NVFP4-Spark"
-parent_model = "zai-org/GLM-5.3"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-# image ships ENTRYPOINT ["vllm","serve"]; our launcher passes the recipe
-# entrypoint as CMD words after the image, which would double it. Empty clears.
-entrypoint = []
-
-args = [
-    "--served-model-name", "glm53",
-    "--trust-remote-code",
-    "--reasoning-parser", "glm45",
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--default-chat-template-kwargs", """
-'{"enable_thinking": true, "reasoning_effort": "high"}'""",
-    "--kv-cache-dtype", "fp8_e4m3",
-    "--block-size", "2304",
-    "--moe-backend", "marlin",
-    # full-model weights ~101.5 GiB/node (406G/4): 4 GiB KV pin, ~79K-token
-    # pool (same sizing the lab image computed); 393K context won't fit
-    "--max-model-len", "79616",
-    "--max-num-batched-tokens", "8192",
-    "--max-num-seqs", "16",
-    "--enable-chunked-prefill",
-    "--enforce-eager",
-    "--gpu-memory-utilization", "0.85",
-    "--kv-cache-memory-bytes", "4294967296",
-    # no spec: the DFlash2 drafter is the Flash model's; full-model plain run
-    "--language-model-only",
-    # PARKED 2026-09-04 (user call: stop full-model experiments): the
-    # MIXED_PRECISION full model loads clean on this image (85/85 shards),
-    # but dgx2.lan went down TWICE mid-full-model-boot (once during the first
-    # compile-OOM episode, once during the MAX_JOBS=2 retry's init phase —
-    # whether node instability under this workload or coincidence is open).
-    # Resume when: dgx2 proves stable under this load, and/or the JIT cache
-    # is pre-built (extra_mounts below persist it) / the image ships
-    # prebuilt sm121a mxfp8 gemm cubins.
-    # image requests need the mm template shipped/in-wallpaper on aiden's rigs:
-    # "--chat-template", "<path-to>/chat_template_mm.jinja",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-TRANSFORMERS_OFFLINE = "1"
-TORCH_USE_RTLD_GLOBAL = "1"
-TORCHINDUCTOR_COMPILE_THREADS = "1"
-CUTE_DSL_ARCH = "sm_121a"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-VLLM_NO_USAGE_STATS = "1"
-OMP_NUM_THREADS = "2"
-# Compile-OOM fix (2026-09-04): first boot of the MIXED_PRECISION checkpoint
-# JIT-compiles mxfp8_gemm_cutlass_sm120 (12 nvcc/cicc jobs, ~3 GB RSS each)
-# while 101.5 GiB of weights sit in unified memory -> cicc OOM-killed.
-# flashinfer's run_ninja reads MAX_JOBS and passes -j N: cap at 2.
-MAX_JOBS = "2"
-
-# Persistent JIT caches (containers run with --rm; without these every boot
-# re-runs the ~10 min nvcc compile of the mxfp8 gemm module)
-[[extra_mounts]]
-host = "~/.cache/glm53-full-flashinfer"
-container = "/root/.cache/flashinfer"
-
-[[extra_mounts]]
-host = "~/.cache/glm53-full-triton"
-container = "/root/.triton"

recipes/glm53-full-nvfp4-spark-lab/recipe.toml

@@ -1,148 +0,0 @@
-# GLM-5.3 (FULL, non-Flash) NVFP4-Spark — local-inference-lab quant, 4× DGX Spark, TP=4.
-#
-# Full GLM-5.3 (78 layers / hidden 6144 / 1M ctx, arch GlmMoeDsaForCausalLM)
-# in the lab's GB10-tailored quant: GLM-5.3-NVFP4-Spark (~406 GB →
-# ~101.5 GiB weights/node → 4x only; the plain NVFP4 build is 433 GB).
-# MIXED_PRECISION checkpoint (NVFP4 routed + MXFP8 elsewhere) handled by the
-# lab image's modelopt quant module.
-#
-# Image: glm53-flash-nvfp4-lab:local (same lab vLLM fork — it registers
-# GlmMoeDsaForCausalLM at deepseek_v2.py, which implements SupportsEagle3,
-# unlike the Flash GLM5Next target; that is why the lab's MXFP8 DFlash2
-# drafter is the full-model pairing, not the Flash one).
-#
-# First run: plain (no spec) to prove the stack end-to-end, then add spec.
-# KV pinned 4 GiB (weights leave ~17 GiB/node; pool is small by design —
-# this is a capability/quality lane, not a throughput lane).
-#
-# PREREQUISITES: weights cached in the shared NAS HF cache:
-#   models--local-inference-lab--GLM-5.3-NVFP4-Spark (406G, verified 2026-09-04)
-
-engine = "vllm"
-name = "GLM-5.3 (full) NVFP4-Spark-lab (vLLM, 4x Spark)"
-docker_image = "glm53-flash-nvfp4-lab:local"
-docker_image_dir = "glm53-flash-nvfp4-lab"
-model = "local-inference-lab/GLM-5.3-NVFP4-Spark"
-parent_model = "zai-org/GLM-5.3"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-# required for the launcher's `--entrypoint sh -c` override path (mods below)
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # Stable served name — quant-agnostic (shared across glm53 quants)
-    "--served-model-name", "glm53",
-    "--trust-remote-code",
-    "--distributed-executor-backend", "mp",
-    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
-    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
-    # page cache routinely leave ~108.5 -> flaky
-    # "Free memory ... less than desired" boot deaths (this was the class
-    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
-    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
-    # allocated (weights ~93 + pinned KV 10 + activations).
-    "--gpu-memory-utilization", "0.82",
-    # Pinned 4 GiB KV — full-model weights are ~101.5 GiB/node (406G/4),
-    # far less headroom than the 92.7 GiB/node Flash lane; no MLA-specific
-    # kv-cache-dtype/block-size here (GlmMoeDsa attention picks its own
-    # native KV defaults — the fp8_ds_mla/256 pair was a Flash-MLA knob)
-    "--kv-cache-memory-bytes", "4294967296",
-    # 4 GiB KV pool holds ~79,616 tokens (engine-computed 2026-09-04);
-    # 524288 would need 26.34 GiB — doesn't fit behind 101.5 GiB weights.
-    # 79K context is far beyond anything the bench uses.
-    "--max-model-len", "79616",
-    "--max-num-seqs", "8",
-    "--max-num-batched-tokens", "4096",
-    "--enable-chunked-prefill",
-    "--async-scheduling",
-    "--enable-prefix-caching",
-    "--disable-custom-all-reduce",
-    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
-    "--kernel-config", """
-'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
-    # No spec: the lab DFlash2/MTP drafters are for the Flash model; first
-    # full-model run runs plain to prove the stack, then add spec if clean
-    # instanttensor was removed after the first full-model boot hung mid-load
-    # (dgx2, 2026-09-04), then re-enabled for the parked retry run below
-    # (flag at "--load-format", "instanttensor") with MAX_JOBS=2 + persistent
-    # JIT caches as the anti-OOM guardrails.
-    # NOTE: instanttensor + both mods seds ARE configured below (2026-09-04
-    # retry) — the lane PARKED before that run happened: two sparse-MLA
-    # engine crashes on first requests (masked_mha AttributeError +
-    # forward_mha NotImplementedError), both hotfixed via [mods]; the
-    # force-mqa + instanttensor boot never ran because the full-model work
-    # was parked (dgx2 went down twice under this workload that day).
-    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
-    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
-    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
-    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
-    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
-    "--enforce-eager",
-    # instanttensor loader (lab-fork feature; default loader is ~27 min here,
-    # instanttensor ~2x faster when the NFS cooperates)
-    "--load-format", "instanttensor",
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--reasoning-parser", "glm45",
-]
-
-[mods]
-commands = [
-    # Lab-fork bug: mla_attention.py:810 reads self.impl.masked_mha_available
-    # on the sparse MLA impl, which never sets it -> AttributeError on the
-    # first prefill step (engine crash, 2026-09-04). The attribute is
-    # SM100-only anyway (gate: is_device_capability_family(100)); on GB10
-    # (SM121) the masked-MHA path is off by design, so default to False.
-    # 1-line hotfix until the lab image is rebuilt with the fix baked in.
-    "sed -i \"s/and self\\.impl\\.masked_mha_available/and getattr(self.impl, 'masked_mha_available', False)/\" /usr/local/lib/python3.12/dist-packages/vllm/model_executor/layers/attention/mla_attention.py",
-    # Second wall: the dense-MHA split routes prefill to impl.forward_mha,
-    # which is an unimplemented stub on the SM120 sparse impl (engine crash
-    # #2, 2026-09-04). vllm ships an intended escape hatch for exactly this:
-    # attention_config.sparse_mla_force_mqa = True -> \"always use the MQA
-    # path\" (the model's native sparse attention). No CLI flag exists for it
-    # in this build, so set the documented default via sed.
-    "sed -i \"s/sparse_mla_force_mqa: bool = False/sparse_mla_force_mqa: bool = True/\" /usr/local/lib/python3.12/dist-packages/vllm/config/attention.py",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-# instanttensor loader knobs (lab image bakes the loader; retrying after the
-# 2026-09-04 dgx2 hang — if that node dies mid-load again, drop this)
-INSTANTTENSOR_BACKEND = "BUFFERED"
-INSTANTTENSOR_CONCURRENCY = "1"
-INSTANTTENSOR_IO_DEPTH = "3"
-INSTANTTENSOR_BUFFER_SIZE = "67108864"
-INSTANTTENSOR_CHUNK_SIZE = "8388608"
-# NoPE-MLA rope-pad runtime gate (mod baked into the image)
-VLLM_MLA_NOPE_PAD_ROPE = "1"
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-# Upstream launch-script env (memory + compile knobs)
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-VLLM_USE_AOT_COMPILE = "0"
-TORCH_USE_RTLD_GLOBAL = "1"
-VLLM_NO_USAGE_STATS = "1"
-
-# Architecture (aarch64 / sm_121a DGX Spark)
-TORCH_CUDA_ARCH_LIST = "12.1a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1f"
-CUTE_DSL_ARCH = "sm_121a"
-CMAKE_CUDA_ARCHITECTURES = "121"
-OMP_NUM_THREADS = "4"
-# Compile-OOM fix: first boot JIT-compiles the mxfp8 gemm module (12
-# nvcc/cicc jobs ~3 GB each) behind 101.5 GiB of weights -> cicc OOM-killed
-# (aiden lane, 2026-09-04). flashinfer run_ninja reads MAX_JOBS -> cap at 2.
-MAX_JOBS = "2"
-
-# Persistent JIT caches (containers run with --rm; without these every boot
-# re-runs the ~10 min nvcc compile of the mxfp8 gemm module)
-[[extra_mounts]]
-host = "~/.cache/glm53-full-flashinfer"
-container = "/root/.cache/flashinfer"
-
-[[extra_mounts]]
-host = "~/.cache/glm53-full-triton"
-container = "/root/.triton"
-

recipes/glm53-spark-nvfp4-4x/recipe.toml

@@ -1,251 +0,0 @@
-# r28 4x DCP4 cell (2026-09-08) — clone of the r27 4x-dcp4 recipe (160/176
-# RoCEnante, 46.8 t/s) retargeted at the FULL r28 source lock (cut 2026-09-06
-# ~18:30 UTC across all three forks): vLLM 62f6385c (prefill/decode
-# compute-time sharing + interleaved parallel prefills, chat-template
-# rendering parity, GLM boundary restore, partial prefix-cache hits),
-# b12x 7216f6b9 (21 commits over r27: GLM/DeepSeek loader fixes, W4A4 FC1
-# fusion, M8 route prep, M1 FC2 overlap, GDN/MHC perf), LMCache 934c287a
-# (3 MP-transfer robustness fixes). Same model/flags/KV pin; the image is
-# the r27 build dir with the named version profile profiles/glm53-r28.env
-# (clone-at-pin fetches each commit directly, the tree gates verify them).
-
-# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
-#
-# local-inference-lab mixed quant: NVFP4 routed experts (layers 3–44,
-# group-16) + MXFP8 MTP experts (layer 45) + W4A4 input scales. Best agent
-# quant so far: tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8).
-# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
-#        (~186 GB → 92.72 GiB weights/node; KV pinned at 10 GiB, see below)
-#
-# Image: glm53-flash-nvfp4-lab:local (docker-images/glm53-flash-nvfp4-lab/ —
-# day-0 arm64 image + NoPE-MLA mod + the four FujitsuPolycom lab-checkpoint
-# patches: model naming shim, modelopt MTP namespace fix, persistent_topk
-# CC-12.x guards). Their launch config is mirrored below.
-# PROVENANCE: https://github.com/FujitsuPolycom/glm53-flash-tp2-spark
-#             (Apache-2.0; see patches/UPSTREAM.md) + forum 381350 #41.
-#
-# Deviations from their launch script (intentional):
-#   * no --kda-prefill-backend flashkda (needs their base-image patch — skipped)
-#   * --kv-cache-dtype fp8_ds_mla instead of `fp8` (same packed layout; this
-#     spelling is what the NoPE-MLA mod booted with on the LibertAIDAI stack)
-#   * --language-model-only + no mm-renderer patch/env (text-only)
-#   * no bundled NCCL/LD_PRELOAD, no NCCL_* fabric block (our switched fabric
-#     is configured via [nccl.ds4] + launcher GID auto-detect)
-#   * no --moe-backend marlin: their proven stack uses the default backend for
-#     THIS quant and reports clean evals. Sanity-check output for token loops
-#     anyway (the silent FP4 MoE corruption gotcha) before trusting a new node.
-#   * flashinfer autotune disabled via --kernel-config (their proven boot);
-#     VLLM_ENGINE_READY_TIMEOUT_S stays high regardless.
-#
-# Boot success markers (engine log, in order):
-#   "Model loading took 92.72 GiB" → "[quantprobe] ... algo=MXFP8" (MTP fix
-#   active; algo=None means broken) → "GPU KV cache size: 1,164,369 tokens" →
-#   "Application startup complete". ~700 s sequential weight load.
-#
-# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
-# :"low"} — omitted/other values select MAX reasoning effort (empty content on
-# short max_tokens).
-#
-# Optional: instanttensor loader (~30 s weight load vs ~700 s):
-#   FROM glm53-flash-nvfp4-lab:local + pip install --no-deps --no-build-isolation
-#   instanttensor==0.1.9, then add --load-format instanttensor (--no-deps is
-#   required: resolution would downgrade the image's NCCL).
-#
-# PREREQUISITES: weights cached on both nodes:
-# [A/B CELL 2026-09-03] ship recipe verbatim, checkpoint swapped to the lab
-# non-Spark NVFP4 (isolates checkpoint from image lineage vs the lab-image cell).
-# [BISECT A2 2026-09-03] EXACT 90/100 recipe (236ce5a) + instanttensor + JIT
-# mounts, on the CURRENT eugr-patched image. Isolates image lineage from the
-# execution-env/flag cluster (AOT/V2/b12x/card-env). Comparator: 124/138.
-# GLM-5.3-Flash NVFP4-Spark — local-inference-lab quant with the speculative
-# layers embedded — 2× DGX Spark, TP=2.
-#
-# local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (rev 06972cdee7280cec849aedf05a02062b3e870246):
-# NVFP4 routed experts (layers 3+, group-16) + NVFP4 MTP expert layer in the
-# separate model-mtp-0000x.safetensors shards (mtp_tokens: 5 per their
-# lil.yaml), amax/nonexpert/inputscales side shards, Glm5NextForConditionalGeneration.
-# THE "DSpark speculative embedded" checkpoint — no separate DFlash2 download
-# needed; lil.yaml: default_speculator mtp, dflash2_tokens 7 (r12 doc fr: their
-# standalone-Q: MTP:3 254 tok/s CC1, 333 Sieve, 94 steps/s).
-#
-# Image: glm53-flash-nvfp4-lab:local (same proven day-0 arm64 stack + NoPE-MLA
-# mod + modelopt MTP namespace patch used by the 92/100 lab lane). The MTP
-# layer here is NVFP4 (not MXFP8) — the quantprobe marker may print
-# algo=NVFP4/- different; algo=None = patch missed (fall back to 4p67 image).
-#
-# Deviations from lab (unchanged mechanics): same fabric/env/GMU/KV pinning.
-# lil.yaml launch mirrors: block 256, disable_flashinfer_autotune, mm data
-# mode — our text-only flags identical to glm53-flash-nvfp4-lab.
-#
-# Boot markers: "Model loading took ~XX GiB" → "[quantprobe] ..." →
-# "GPU KV cache size" → "Application startup complete".
-#
-# PREREQUISITES: weights on the NAS hub (175 GiB): hf download local-inference-lab/GLM-5.3-Flash-NVFP4-Spark
-
-engine = "vllm"
-name = "GLM-5.3 NVFP4 Spark 386B (r28 img, 4x DCP4 boot test)"
-docker_image = "glm53-jovian-r28:local"  # full r28 lock (build_profile) over the r27 image tree
-docker_image_dir = "glm53-jovian-r27"
-# NOTE: build_profile must stay a TOP-LEVEL key — a bare key after
-# [[extra_mounts]] would parse into the last mount table and be silently
-# ignored (2026-09-08).
-# FULL r28 source lock (2026-09-06 cut, all three forks): the named version
-# profile docker-images/glm53-jovian-r27/profiles/glm53-r28.env — vLLM
-# 62f6385c + b12x 7216f6b9 (21 commits over r27: GLM/DeepSeek loader fixes,
-# MoE perf) + LMCache 934c287a (3 MP-transfer fixes). Merged over the image
-# dir's versions.env and folded into the build checksum, so this is a
-# distinct image (tag glm53-jovian-r28:local) sharing the r27 build tree.
-# Clone-at-pin fetches each commit directly; the Dockerfile gates HEAD + tree
-# hash per repo. Both r28 branches are strictly ahead of their r27/r26 pins
-# (0 behind) — a clean superset. The r28 vLLM tree has NO requirements/cmake
-# changes vs r27 (verified via the GitHub compare API, 13 commits / 29 files),
-# so the fetch-deps.sh dep gates still hold.
-build_profile = "glm53-r28"
-# Required: the clean r12 image follows the solar contract (ENTRYPOINT []
-# + CMD bash), so the recipe carries the entrypoint (the old day-0-based
-# r12 image had ["vllm","serve"] baked in, which is why this was absent
-# until the clean rebuild).
-entrypoint = ["vllm", "serve"]
-model = "local-inference-lab/GLM-5.3-NVFP4-Spark"
-parent_model = "zai-org/GLM-5.3"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-args = [
-    # Stable served name — quant-agnostic (shared across glm53 quants)
-    "--served-model-name", "glm53",
-    "--trust-remote-code",
-    "--distributed-executor-backend", "mp",
-    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
-    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
-    # page cache routinely leave ~108.5 -> flaky
-    # "Free memory ... less than desired" boot deaths (this was the class
-    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
-    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
-    # allocated (weights ~93 + pinned KV 10 + activations).
-    # 0.88 (not 0.90): the startup admission gate is free-mem >= GMU*total
-    # (107.0 vs 109.5 GiB); ~12 GiB of starting vLLM processes + docker
-    # daemon can push the 0.90 check under the line (2026-09-08, #48140).
-    # KV is explicitly pinned (--kv-cache-memory-bytes) so capacity is unchanged.
-    "--gpu-memory-utilization", "0.88",
-    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
-    # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
-        # 6 GiB (vs 16 for the 184G model): 386B weights leave ~20 GiB/node
-    # of headroom on 4x, so the KV pin must shrink (47.5 → 101.4 GiB/node).
-    "--kv-cache-memory-bytes", "6442450944",  # 16 GiB: 4x NVFP4 has the headroom (47.1 GiB weights/node vs 108 usable); 2x stays 10
-    "--decode-context-parallel-size", "4",
-    "--cp-kv-cache-interleave-size", "4",
-    # No --kv-cache-dtype: let the r12 fork pick its default (the explicit
-    # fp8 / fp8_ds_mla both failed auto-selection on this build).
-    # The GLM5Next model wires the B12X sparse-MLA backend (sm_120/sm_121)
-    # when the global backend is B12X (glm5next/nvidia/attention.py:
-    # _select_sparse_backend); the sm12x auto-priority list never offers it
-    # (FLASHINFER_MLA_SPARSE_SM120 only supports head 576, not GLM's 512).
-    "--attention-backend", "B12X",
-    # pure I/O loader (kept across every config; 171 GB in ~5.5 min vs ~16 NFS)
-    "--load-format", "instanttensor",
-    # [R17 FULL-CLUSTER TEST] lil manifest backends (quality gate: hardmode
-    # must stay ≥88; r12-era this cluster dropped it to 74 — r17 may carry
-    # the fix through — one boot decides).
-    "--block-size", "256",
-    # FINAL BISECT (2026-09-03, hardmode 176pt, r17 img):
-    #   knobs-only 87: d0 c1 24.1, d8192 23.8
-    #   +moe b12x  88: d0 c1 23.5, d8192 26.6  <- SHIP (2 pts d8192)
-    #   +linear b12x 44: W4A4 linear kernel destroys numerics (param-precision 0%)
-    #   +align     88: but d0 c1 18.6 (-5) — perf-negative, dropped
-    # (all three together = 44-74; the lab needs its private packed package)
-    "--max-model-len", "524288",
-    "--max-num-seqs", "16",
-    "--max-num-batched-tokens", "4096",
-    "--enable-chunked-prefill",
-    "--async-scheduling",
-    "--enable-prefix-caching",
-    # NO --disable-custom-all-reduce here: in the r27 fork the RoCEnante
-    # adapter is gated on the custom-all-reduce enable flag
-    # (use_roce_allreduce = use_custom_allreduce && VLLM_ENABLE_ROCE_ALLREDUCE).
-    # On 2x/4x 1-GPU-node clusters vLLM's own CUDA-IPC custom AR is functionally
-    # inert (no cross-node P2P), so dropping the flag only opens the b12x path;
-    # the b12x-PCIe branch stays off (VLLM_ENABLE_PCIE_ALLREDUCE defaults False).
-    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
-    "--kernel-config", """
-'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
-    # Built-in MTP draft, 3 tokens (the r12-qualified "mtp:3" mode — their
-    # x86 numbers: 254 tok/s CC1, 2.37 accept; the checkpoint embeds up to
-    # 5 via the model-mtp-0000x shards but 3 is the verified depth). This is
-    # the checkpoint's own MTP head (Glm5NextMTPModel), NOT dspark — dspark
-    # is the DeepSeek-V4 family's mechanism; GLM's other r12 mode is
-    # dflash2:7 with a separate trained draft checkpoint.
-    # The draft model's vllm_config takes its attention backend from the
-    # SPECULATIVE config, not the global flag (eagle/utils.py:
-    # _make_eagle_draft_vllm_config) — B12X here mirrors the main model's.
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'""",
-    "--language-model-only",
-    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
-    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
-    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
-    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
-    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
-    # graphs moved BEFORE tool-call-parser; enforce-eager removed for the
-    # r17-graphs test (bisect A2 was 88/100 eager; graphs+MTP fix is public
-    # in r17 refs? verify by bench).
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--reasoning-parser", "glm45",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for r27
-# lanes since the 2026-09-08 A/B (4x DCP4: decode +31..+90%, quality 160 vs
-# 155, prefill within noise; specs/tech-architecture/glm53-recipe-state.md,
-# history/night-shifts/2026-09-07.md). Needs the CX-7 RoCE fabric (all of
-# ours); single-node serves skip it silently. Caps default 2MB AR / 16MB AG.
-VLLM_ENABLE_ROCE_ALLREDUCE = "1"
-# FlashInfer JIT (mxfp8_gemm_cutlass_sm120, ~12 kernels) compiles at first
-# boot with 12 parallel nvcc jobs -> OOM-killed (code 137) on GB10's
-# 121 GB unified while the 93 GB weights + 10 GB KV pin are resident.
-# MAX_JOBS caps ninja parallelism so the JIT fits in the remaining ~20 GB.
-MAX_JOBS = "2"
-# NoPE-MLA rope-pad runtime gate (mod baked into the image)
-VLLM_MLA_NOPE_PAD_ROPE = "1"
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-# Upstream launch-script env (memory + compile knobs)
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-VLLM_USE_AOT_COMPILE = "0"
-VLLM_USE_STANDALONE_COMPILE = "1"
-VLLM_USE_FLASHINFER_SAMPLER = "1"
-VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
-TORCH_USE_RTLD_GLOBAL = "1"
-VLLM_NO_USAGE_STATS = "1"
-
-# Architecture (aarch64 / sm_121a DGX Spark)
-TORCH_CUDA_ARCH_LIST = "12.1a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1f"
-CUTE_DSL_ARCH = "sm_121a"
-CMAKE_CUDA_ARCHITECTURES = "121"
-OMP_NUM_THREADS = "2"
-
-[[extra_mounts]]
-host = "~/sm121-cache/glm53-jit-121f/b12x"
-container = "/root/.cache/b12x"
-
-# JIT caches persist per-node across serves (was: cold /root/.cache per
-# container -> ~20 min flashinfer/torch recompile every serve). The host
-# dirs are shared between lanes of the same arch — each cache
-# self-namespaces by its engine version key internally.
-[[extra_mounts]]
-host = "~/sm121-cache/jit-121f/flashinfer"
-container = "/root/.cache/flashinfer"
-
-[[extra_mounts]]
-host = "~/sm121-cache/jit-121f/torchinductor"
-container = "/root/.cache/torchinductor"
-
-[[extra_mounts]]
-host = "~/sm121-cache/jit-121f/vllm"
-container = "/root/.cache/vllm"
-

specs/tech-architecture/glm53-recipe-state.md

@@ -1241,3 +1241,96 @@ working ones — exactly the lab's qualification set. 2x stays MTP3; recipe
 `glm53-flash-r35-2x-dflash2` kept as a canary. Ops note: sm121 `docker run`
 is `--rm` by default (crashed rank-0 logs vanished); `serve --keep` keeps
 containers + logs for post-mortem.
+
+---
+
+## 2026-09-13: FULL GLM-5.3 retargeted to RedHatAI MXFP4 (r35 stack) — old variant deleted
+
+Operator: "download RedHatAI/GLM-5.3-MXFP4, select the base stack to serve it,
+update our recipes for this variant, as our variant did not work, no reason to
+keep it. Do not start it — night shift."
+
+**Deleted** (all three served the abandoned full-model checkpoint
+`local-inference-lab/GLM-5.3-NVFP4-Spark`, 435.6 GB / 406 GiB → 101.5 GiB
+weights/node → OOM-proven on 4x 2026-09-08/09: post-load crash + auto-restart
+began a SECOND full load on top of the first → 2×101.5 > 121.63 GiB → all four
+nodes OOM-thrashed, sshd down on 3 of 4, HA power-cycle. See the 09-08/09-09
+history entries):
+`recipes/glm53-full-nvfp4-spark-lab`, `recipes/glm53-full-nvfp4-spark-aiden`,
+`recipes/glm53-spark-nvfp4-4x`. Kept in git history.
+
+**New**: `recipes/glm53-full-mxfp4-4x` — `RedHatAI/GLM-5.3-MXFP4`
+(403.4 GB / 375.6 GiB, 141 shards, compressed-tensors
+`mxfp4-pack-quantized`, W4A4 group-32 E8M0; card reports 98.7% gsm8k /
+95.4% ifeval recovery vs the FP8 parent).
+
+Geometry (verified from config.json): `GlmMoeDsaForCausalLM` /
+`model_type=glm_moe_dsa`, 78 layers, hidden 6144, 256 routed experts + 1
+shared, top-8, moe_inter 2048, MLA (kv_lora 512), DSA indexer (index_topk
+2048), 1M native ctx, MTP head in-checkpoint (`num_nextn_predict_layers=1`),
+no separate drafter. ~740B total / ~40B active — the config geometry and the
+file size agree at MXFP4's 4.25 bits/param; note the 09-08 history entry's
+"386B" was an estimate that does not match either (a different model class
+from the 320B/18B Flash variant we serve in prod: 45 layers, hidden 4096).
+
+**BASE STACK SELECTED: our own jovian r35 build** —
+`docker_image_dir = "glm53-jovian-r27"` + `build_profile = "glm53-r35"` →
+image `glm53-jovian-r35:local`, the same image as the Flash 4x prod. Evidence
+(read-only image inspection, 2026-09-13):
+- `registry.py` maps `"GlmMoeDsaForCausalLM" → vllm.models.deepseek_v32`, the
+  NEW implementation ("MLA + a lightning indexer selecting top-k for sparse
+  MLA") that dispatches to an **NVIDIA-specific path**
+  (`deepseek_v32/nvidia/model.py: DeepseekV32ForCausalLM as
+  GlmMoeDsaForCausalLM`) — the right target for GB10/sm_121a. The lab fork and
+  aiden's `production-1.2` image register the same class from the older
+  `deepseek_v2` module (vLLM 0.1.dev20051+g487ecf187) and needed two SM120
+  `mla_attention` hotfix mods to boot the full model at all (never re-tested;
+  lane parked).
+- ships `compressed_tensors_w4a4_mxfp4` (mxfp4-pack-quantized) → the RedHat
+  checkpoint loads natively.
+- it is our reproducible clone-at-pin build (r35 lock 2026-09-11: vLLM
+  de982a50 + b12x 98086604), all gates green, and it carries the GB10-tuned
+  b12x backends our GLM prod uses (DCP4, B12X, RoCEnante, MTP3 — all proven
+  on the Flash model). One stack to maintain; clean A/B vs the Flash prod;
+  JIT caches warm.
+Lab fork / aiden image stay as a documented fallback (port the two sed mods
+onto `glm53-flash-nvfp4-lab:local` if the deepseek_v32 DSA/indexer path trips).
+
+**Memory plan** (the binding constraint): 121.63 GiB/node, 116–117 GiB idle
+free. Weights 93.9 GiB (403.4 GB/4) — 7.6 GiB/node lighter than the failed
+attempt, which is what makes 4x plausible. + vLLM/docker ~12 GiB + KV pin
+8 GiB + activations/graphs ~4 GiB ≈ 118 GiB. GMU 0.85 is the admission
+guardrail only (0.88 = 107.0 GiB fails once the ~12 GiB of starting processes
+count — the 2026-09-08 #48140 class); the KV pool is pinned so GMU does not
+change capacity.
+
+**Why DCP4**: the full model's KV is NOT the Flash lane's compact MLA cache —
+the parked run measured ~54 KB/token (4 GiB ↔ ~79.6K tokens; "524288 would
+need 26.34 GiB") vs the Flash's ~2 KB/token at 512K. DCP4 shards KV across
+the 4 TP ranks, so the 8 GiB pin buys 32 GiB aggregate → one 524288-token
+request needs ~27 GiB aggregate (≈6.8 GiB/rank) and just fits. Without DCP,
+512K behind these weights is impossible. CAVEAT: DCP4 is qualified in r35 for
+the Flash model; for the full model it is UNVERIFIED — first fallback is to
+drop `--decode-context-parallel-size`/`--cp-kv-cache-interleave-size` and
+lower `--max-model-len` (~180K at ~10 GiB aggregate KV).
+
+**First-boot plan / hazard**: the 09-08 disaster was a post-load crash +
+auto-restart starting a second full load, not the footprint alone. Watch the
+first boot; a second "Loading safetensors" pass = kill immediately
+(`sm121 stop`), do not wait. Always `--keep` for the first boot (default is
+`--rm`, which wipes the crash logs). Ladder: (1) no-DCP + 131072 + KV 4 GiB
+(prove stack+checkpoint) → (2) DCP4 + 524288 → (3) add MTP k=3 (k=5 after).
+Boot markers to record: "Model loading took X GiB", the KV line, "Application
+startup complete". MAX_JOBS=2 is the compile-OOM guard (~3 GiB × 12 nvcc jobs
+behind 94 GiB of weights, 2026-09-04 lesson).
+
+No speculative decoding on the first boot (memory + full-model MTP
+unproven); `mtp k=3` is documented in the recipe header for step 3.
+
+Prereq: `hf download RedHatAI/GLM-5.3-MXFP4` into the shared NAS HF cache on
+all 4 nodes (was not on the NAS as of 2026-09-13; operator downloading).
+Note: the r35 image now reads as STALE (the r27 build dir gained the r36
+apt-lock/profile edits) — `warn_if_stale` is warn-only and the deployed image
+(label 8bb87348) is the proven Flash-prod build, so the night shift can
+either serve as-is or rebuild with `sm121 build --recipe
+glm53-full-mxfp4-4x --build-node dgx5.lan`.

src/report/mod.rs

@@ -22,7 +22,7 @@ pub(crate) mod parse;
 pub(crate) mod provenance;
 pub(crate) mod quality;
 pub(crate) mod render;
-pub(crate) use render::{html_page, markdown};
+pub(crate) use render::{html_page, html_page_at, markdown};
 pub(crate) mod serve;
 pub(crate) use parse::{PerfCell, bench_complete, parse_bench_json, perf_from};
 pub(crate) use provenance::Provenance;

src/report/render.rs

@@ -102,3 +102,15 @@ pub(crate) fn html_page(rows: &[ReportRow]) -> String {
     let json = serde_json::to_string(rows).unwrap_or_else(|_| "[]".into());
     PAGE_TEMPLATE.replace("{{DATA}}", &json.replace('<', "\\u003c"))
 }
+
+/// Same page mounted under a URL prefix — the dashboard nests it at
+/// `/recipes`, so the page's own absolute paths (its SSE stream and the
+/// `/r/<recipe>/<file>` viewer links) have to move with it.
+pub(crate) fn html_page_at(rows: &[ReportRow], prefix: &str) -> String {
+    let page = html_page(rows);
+    if prefix.is_empty() {
+        return page;
+    }
+    page.replace("/events", &format!("{prefix}/events"))
+        .replace("/r/", &format!("{prefix}/r/"))
+}

src/report/ui_tests.rs

@@ -176,6 +176,28 @@ fn markdown_lists_rows_and_scores() {
     assert!(empty.contains(fluent_i18n::t!("report-title", { "count" => 0 }).as_str()));
 }
 
+/// Mounted variant (the dashboard nests the panel at /recipes): the page's
+/// own absolute paths must move with the prefix or its SSE stream and
+/// viewer links would hit the dashboard's routes.
+#[test]
+fn html_page_at_rewrites_own_paths() {
+    let page = super::html_page_at(&[], "/recipes");
+    assert!(
+        page.contains("new EventSource(\"/recipes/events\")"),
+        "SSE path not rewritten"
+    );
+    assert!(!page.contains("new EventSource(\"/events\")"));
+    assert!(
+        page.contains("href=\"/recipes/r/"),
+        "viewer link not rewritten"
+    );
+    assert!(!page.contains("href=\"/r/"));
+    assert!(
+        !super::html_page(&[]).contains("/recipes"),
+        "unprefixed page must stay clean"
+    );
+}
+
 #[test]
 fn html_page_embeds_data_escaped() {
     let rows = vec![row("r-a", "m/one", "NVFP4", None)];

src/web/mod.rs

@@ -180,10 +180,22 @@ pub fn router(app: AppState) -> Router {
         .route("/api/raw/effective", get(api::raw_effective))
         .route("/events", get(events_stream))
         .layer(middleware::from_fn_with_state(app.clone(), token_guard));
+    // Recipes panel: the very page `sm121 report --serve` serves, nested at
+    // `/recipes` — its own SSE stream and `/r/<recipe>/<file>` viewer move
+    // under the prefix (see `report::html_page_at`), live refresh included.
+    let rows = crate::report::collect(&app.config, false);
+    let recipes = crate::report::serve::router(
+        app.config.clone(),
+        crate::report::html_page_at(&rows, "/recipes"),
+        serde_json::to_string(&rows).unwrap_or_default(),
+        false,
+    )
+    .layer(middleware::from_fn_with_state(app.clone(), token_guard));
     Router::new()
         .route("/", get(|| async { Html(UI_HTML) }))
         .merge(protected)
         .with_state(app)
+        .nest("/recipes", recipes)
 }
 
 /// Run the dashboard until the process exits.

src/web/tests.rs

@@ -98,6 +98,38 @@ async fn api_state_reports_unreachable_node() {
     handle.abort();
 }
 
+/// The recipes panel is mounted under /recipes: the page, its own SSE
+/// stream and the /r/ viewer all live behind the prefix.
+#[tokio::test]
+async fn recipes_panel_is_mounted_under_prefix() {
+    let (addr, handle) = boot().await;
+    let page = reqwest::get(format!("http://{addr}/recipes"))
+        .await
+        .unwrap()
+        .text()
+        .await
+        .unwrap();
+    // its own live stream resolves behind the prefix (headers arrive at once)
+    let sse = reqwest::get(format!("http://{addr}/recipes/events"))
+        .await
+        .unwrap();
+    assert_eq!(sse.status(), 200);
+    assert!(
+        page.contains("new EventSource(\"/recipes/events\")"),
+        "prefix not applied"
+    );
+    assert!(!page.contains("new EventSource(\"/events\")"));
+    // the dashboard's own routes still resolve
+    let ui = reqwest::get(format!("http://{addr}/"))
+        .await
+        .unwrap()
+        .text()
+        .await
+        .unwrap();
+    assert!(ui.contains("recwin") || ui.contains("Recipes"));
+    handle.abort();
+}
+
 #[tokio::test]
 async fn token_gate_enforces_when_set() {
     let app = app_state(Some("s3cret"));

src/web/ui.html

@@ -193,6 +193,12 @@
 
   /* --- raw content windows --------------------------------------------------- */
   .rawwin .win-body { display: flex; flex-direction: column; padding: 0; }
+  .recwin .win-body { padding: 0; }
+  .recwin iframe { width: 100%; height: 100%; border: 0; display: block; background: var(--panel); }
+  /* the embedded report page resolves `color-scheme: light dark` from this
+     element, so it follows the dashboard theme instead of the OS one */
+  :root[data-theme="dark"] .recwin iframe { color-scheme: dark; }
+  :root[data-theme="light"] .recwin iframe { color-scheme: light; }
   .rawbar {
     display: flex; gap: 8px; align-items: center; padding: 8px 10px;
     border-bottom: 1px solid var(--border); flex-shrink: 0;
@@ -520,6 +526,7 @@ function toggleDockMenu() {
   item("topo", () => { (wins.topo.isMinimized() ? wins.topo.restore() : front(wins.topo.win)); });
   item("state", () => { (wins.state.isMinimized() ? wins.state.restore() : front(wins.state.win)); });
   item("jobs", () => { (wins.jobs.isMinimized() ? wins.jobs.restore() : front(wins.jobs.win)); });
+  item("recipes", openRecipes);
   item("logs…", openLogsPicker);
   item("raw", openRaw);
   document.body.appendChild(m);
@@ -1323,6 +1330,23 @@ async function reattachLog(w) {
 // One window, a kind selector: config / effective / recipe:<name>. Content is
 // verbatim from the server (effective is pretty JSON); copy grabs the text.
 let rawWin = null;
+let recipesWin = null;
+// The recipes panel IS the `sm121 report --serve` page, nested by the
+// server at /recipes and embedded as-is: table, grouping, stars, the /r/
+// viewer and live updates stay one implementation (paths prefix-rewritten
+// server-side, theme inherited through the iframe's color-scheme).
+function openRecipes() {
+  if (recipesWin) { front(recipesWin.win); return; }
+  recipesWin = makeWin({
+    id: "win-recipes", title: "Recipes", x: 24, y: 24,
+    w: Math.min(1180, innerWidth - 48), h: Math.min(760, innerHeight - 80),
+    bodyHtml: `<iframe src="/recipes" title="recipes report"></iframe>`,
+    extraCls: "recwin",
+    removeOnClose: true,
+    onClosed: () => { recipesWin = null; },
+  });
+}
+
 async function openRaw() {
   if (rawWin) { front(rawWin); return; }
   const handle = makeWin({

src/web/ui.playwright.mjs

@@ -272,8 +272,8 @@ ok(
 // --- 13. raw window (via the dock ＋ new-window menu) -------------------------------
 await page.click("#dock .plus");
 ok(
-  "dock ＋ menu lists state / jobs / logs / raw",
-  (await page.locator(".dockmenu .dock-chip").count()) === 5, // topo / state / jobs / logs / raw
+  "dock ＋ menu lists topo / state / jobs / recipes / logs / raw",
+  (await page.locator(".dockmenu .dock-chip").count()) === 6,
 );
 await page.click('.dockmenu .dock-chip:has-text("raw")');
 await page.waitForSelector(".rawwin", { timeout: 10000 });