full GLM-5.3 4x lanes (aiden + lab images): PARKED with findings

- both images register GlmMoeDsaForCausalLM; NVFP4-Spark (406G, mixed
  precision) loads clean on both (85/85 shards, ~21 min NFS)
- lab image: two sparse-MLA engine crashes on first requests (masked_mha
  AttributeError, forward_mha NotImplementedError) — hotfixed via [mods]
  (getattr guard + documented sparse_mla_force_mqa=True escape hatch)
- aiden image: first-boot JIT of mxfp8_gemm_cutlass_sm120 OOM-killed cicc
  (12 parallel nvcc jobs behind 101.5 GiB weights) -> MAX_JOBS=2 cap
  (flashinfer run_ninja knob) + persistent flashinfer/triton JIT caches
- parking cause: dgx2.lan went down twice mid-full-model-boot; user call
  to switch back to the proven 4x GLM-5.3-Flash lane

recipes/glm53-full-nvfp4-spark-aiden/recipe.toml

@@ -0,0 +1,98 @@
+# GLM-5.3-Flash on 4× DGX Spark — aidendle94 sparkrun image (TP=4). DEFERRED.
+#
+# aiden's production 4-node profile, verbatim translation of
+# https://aidenle.com/recipes/glm-5-3-flash-4x-dgx-spark into our 4x launcher
+# shape. Reference numbers (his, single stream, idle cluster, thinking off):
+#   ~84 tok/s decode (FP8, DFlash2 k=7) — NVFP4 +23% (110.7 structured);
+#   prefill ~2,100 cold; TTFT 4 s @ 8K → 40 s @ 94K; KV 2.25M tokens at 16 GiB
+#   pin; 393K context verified. Decode flat with context (34/45 KDA layers +
+#   11 sparse-MLA @ index_topk 2048).
+#
+# DFlash2: acceptance 0.5–0.8; k=7 (drafter block 8 − 1 target token); k=6
+# wins prose, loses structured. Native MTP fallback (production-1.0 image or
+# spec omitted) optimum k=5. ~15 min full relaunch on FP8 (963 s measured).
+# FP8-MLA + DFlash2 combination is STRUCTURALLY broken upstream (three trees
+# fail differently) — hence plain fp8_e4m3 KV.
+#
+# NOT READY TO DEPLOY: needs a 4-node cluster (ds4 + 2nodes-dac freed) and the
+# unsloth/GLM-5.3-Flash-FP8 checkpoint (328 GB — background download running
+# on dgx3, /tmp/dl_fp8.log; ~340 GB free per node required for weights + Triton
+# cache). Fabric: RoCE dual-HCA NCCL profile comes from the launcher (config
+# [nccl.*] / GID auto-detect) — his NCCL_* env block intentionally omitted.
+# NOTE: his launcher unsets VLLM_PREFIX_CACHE_RETENTION_INTERVAL via bash -lc
+# before exec; if our boot rejects it in the KV coordinator, add a mods script.
+
+engine = "vllm"
+name = "GLM-5.3 (full) NVFP4-Spark, aiden image (vLLM, 4x Spark)"
+docker_image = "aidendle94/sparkrun-vllm-glm53-gb10:production-1.1"
+model = "local-inference-lab/GLM-5.3-NVFP4-Spark"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+# image ships ENTRYPOINT ["vllm","serve"]; our launcher passes the recipe
+# entrypoint as CMD words after the image, which would double it. Empty clears.
+entrypoint = []
+
+args = [
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--default-chat-template-kwargs", """
+'{"enable_thinking": true, "reasoning_effort": "high"}'""",
+    "--kv-cache-dtype", "fp8_e4m3",
+    "--block-size", "2304",
+    "--moe-backend", "marlin",
+    # full-model weights ~101.5 GiB/node (406G/4): 4 GiB KV pin, ~79K-token
+    # pool (same sizing the lab image computed); 393K context won't fit
+    "--max-model-len", "79616",
+    "--max-num-batched-tokens", "8192",
+    "--max-num-seqs", "16",
+    "--enable-chunked-prefill",
+    "--enforce-eager",
+    "--gpu-memory-utilization", "0.85",
+    "--kv-cache-memory-bytes", "4294967296",
+    # no spec: the DFlash2 drafter is the Flash model's; full-model plain run
+    "--language-model-only",
+    # PARKED 2026-09-04 (user call: stop full-model experiments): the
+    # MIXED_PRECISION full model loads clean on this image (85/85 shards),
+    # but dgx2.lan went down TWICE mid-full-model-boot (once during the first
+    # compile-OOM episode, once during the MAX_JOBS=2 retry's init phase —
+    # whether node instability under this workload or coincidence is open).
+    # Resume when: dgx2 proves stable under this load, and/or the JIT cache
+    # is pre-built (extra_mounts below persist it) / the image ships
+    # prebuilt sm121a mxfp8 gemm cubins.
+    # image requests need the mm template shipped/in-wallpaper on aiden's rigs:
+    # "--chat-template", "<path-to>/chat_template_mm.jinja",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+TORCH_USE_RTLD_GLOBAL = "1"
+TORCHINDUCTOR_COMPILE_THREADS = "1"
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_NO_USAGE_STATS = "1"
+OMP_NUM_THREADS = "2"
+# Compile-OOM fix (2026-09-04): first boot of the MIXED_PRECISION checkpoint
+# JIT-compiles mxfp8_gemm_cutlass_sm120 (12 nvcc/cicc jobs, ~3 GB RSS each)
+# while 101.5 GiB of weights sit in unified memory -> cicc OOM-killed.
+# flashinfer's run_ninja reads MAX_JOBS and passes -j N: cap at 2.
+MAX_JOBS = "2"
+
+# Persistent JIT caches (containers run with --rm; without these every boot
+# re-runs the ~10 min nvcc compile of the mxfp8 gemm module)
+[[extra_mounts]]
+host = "~/.cache/glm53-full-flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/.cache/glm53-full-triton"
+container = "/root/.triton"

recipes/glm53-full-nvfp4-spark-lab/recipe.toml

@@ -0,0 +1,147 @@
+# GLM-5.3 (FULL, non-Flash) NVFP4-Spark — local-inference-lab quant, 4× DGX Spark, TP=4.
+#
+# Full GLM-5.3 (78 layers / hidden 6144 / 1M ctx, arch GlmMoeDsaForCausalLM)
+# in the lab's GB10-tailored quant: GLM-5.3-NVFP4-Spark (~406 GB →
+# ~101.5 GiB weights/node → 4x only; the plain NVFP4 build is 433 GB).
+# MIXED_PRECISION checkpoint (NVFP4 routed + MXFP8 elsewhere) handled by the
+# lab image's modelopt quant module.
+#
+# Image: glm53-flash-nvfp4-lab:local (same lab vLLM fork — it registers
+# GlmMoeDsaForCausalLM at deepseek_v2.py, which implements SupportsEagle3,
+# unlike the Flash GLM5Next target; that is why the lab's MXFP8 DFlash2
+# drafter is the full-model pairing, not the Flash one).
+#
+# First run: plain (no spec) to prove the stack end-to-end, then add spec.
+# KV pinned 4 GiB (weights leave ~17 GiB/node; pool is small by design —
+# this is a capability/quality lane, not a throughput lane).
+#
+# PREREQUISITES: weights cached in the shared NAS HF cache:
+#   models--local-inference-lab--GLM-5.3-NVFP4-Spark (406G, verified 2026-09-04)
+
+engine = "vllm"
+name = "GLM-5.3 (full) NVFP4-Spark-lab (vLLM, 4x Spark)"
+docker_image = "glm53-flash-nvfp4-lab:local"
+docker_image_dir = "glm53-flash-nvfp4-lab"
+model = "local-inference-lab/GLM-5.3-NVFP4-Spark"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+# required for the launcher's `--entrypoint sh -c` override path (mods below)
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
+    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
+    # page cache routinely leave ~108.5 -> flaky
+    # "Free memory ... less than desired" boot deaths (this was the class
+    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
+    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
+    # allocated (weights ~93 + pinned KV 10 + activations).
+    "--gpu-memory-utilization", "0.82",
+    # Pinned 4 GiB KV — full-model weights are ~101.5 GiB/node (406G/4),
+    # far less headroom than the 92.7 GiB/node Flash lane; no MLA-specific
+    # kv-cache-dtype/block-size here (GlmMoeDsa attention picks its own
+    # native KV defaults — the fp8_ds_mla/256 pair was a Flash-MLA knob)
+    "--kv-cache-memory-bytes", "4294967296",
+    # 4 GiB KV pool holds ~79,616 tokens (engine-computed 2026-09-04);
+    # 524288 would need 26.34 GiB — doesn't fit behind 101.5 GiB weights.
+    # 79K context is far beyond anything the bench uses.
+    "--max-model-len", "79616",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    "--disable-custom-all-reduce",
+    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    # No spec: the lab DFlash2/MTP drafters are for the Flash model; first
+    # full-model run runs plain to prove the stack, then add spec if clean
+    # instanttensor was removed after the first full-model boot hung mid-load
+    # (dgx2, 2026-09-04), then re-enabled for the parked retry run below
+    # (flag at "--load-format", "instanttensor") with MAX_JOBS=2 + persistent
+    # JIT caches as the anti-OOM guardrails.
+    # NOTE: instanttensor + both mods seds ARE configured below (2026-09-04
+    # retry) — the lane PARKED before that run happened: two sparse-MLA
+    # engine crashes on first requests (masked_mha AttributeError +
+    # forward_mha NotImplementedError), both hotfixed via [mods]; the
+    # force-mqa + instanttensor boot never ran because the full-model work
+    # was parked (dgx2 went down twice under this workload that day).
+    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
+    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
+    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
+    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
+    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
+    "--enforce-eager",
+    # instanttensor loader (lab-fork feature; default loader is ~27 min here,
+    # instanttensor ~2x faster when the NFS cooperates)
+    "--load-format", "instanttensor",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[mods]
+commands = [
+    # Lab-fork bug: mla_attention.py:810 reads self.impl.masked_mha_available
+    # on the sparse MLA impl, which never sets it -> AttributeError on the
+    # first prefill step (engine crash, 2026-09-04). The attribute is
+    # SM100-only anyway (gate: is_device_capability_family(100)); on GB10
+    # (SM121) the masked-MHA path is off by design, so default to False.
+    # 1-line hotfix until the lab image is rebuilt with the fix baked in.
+    "sed -i \"s/and self\\.impl\\.masked_mha_available/and getattr(self.impl, 'masked_mha_available', False)/\" /usr/local/lib/python3.12/dist-packages/vllm/model_executor/layers/attention/mla_attention.py",
+    # Second wall: the dense-MHA split routes prefill to impl.forward_mha,
+    # which is an unimplemented stub on the SM120 sparse impl (engine crash
+    # #2, 2026-09-04). vllm ships an intended escape hatch for exactly this:
+    # attention_config.sparse_mla_force_mqa = True -> \"always use the MQA
+    # path\" (the model's native sparse attention). No CLI flag exists for it
+    # in this build, so set the documented default via sed.
+    "sed -i \"s/sparse_mla_force_mqa: bool = False/sparse_mla_force_mqa: bool = True/\" /usr/local/lib/python3.12/dist-packages/vllm/config/attention.py",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+# instanttensor loader knobs (lab image bakes the loader; retrying after the
+# 2026-09-04 dgx2 hang — if that node dies mid-load again, drop this)
+INSTANTTENSOR_BACKEND = "BUFFERED"
+INSTANTTENSOR_CONCURRENCY = "1"
+INSTANTTENSOR_IO_DEPTH = "3"
+INSTANTTENSOR_BUFFER_SIZE = "67108864"
+INSTANTTENSOR_CHUNK_SIZE = "8388608"
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+# Upstream launch-script env (memory + compile knobs)
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "4"
+# Compile-OOM fix: first boot JIT-compiles the mxfp8 gemm module (12
+# nvcc/cicc jobs ~3 GB each) behind 101.5 GiB of weights -> cicc OOM-killed
+# (aiden lane, 2026-09-04). flashinfer run_ninja reads MAX_JOBS -> cap at 2.
+MAX_JOBS = "2"
+
+# Persistent JIT caches (containers run with --rm; without these every boot
+# re-runs the ~10 min nvcc compile of the mxfp8 gemm module)
+[[extra_mounts]]
+host = "~/.cache/glm53-full-flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/.cache/glm53-full-triton"
+container = "/root/.triton"
+