glm53 spark recipe: full lil policy alignment (b12x linear/moe backends, k=5 MTP + moe_backend, instanttensor load, mamba-cache-mode align, INSTANTTENSOR env) + mounted pre-baked JIT caches (~/sm121-cache/glm53-jit-121f, dsv4 pattern); boot 35m -> 7m, exact-391 verified

.gitignore

@@ -8,3 +8,4 @@ __pycache__/
 # scratch from docker-images/*/gen-locks.sh
 .gen/
 publish/
+docker-images/*/jit-cache*

recipes/glm53-flash-nvfp4-spark/recipe.toml

@@ -1,70 +1,39 @@
-# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
+# GLM-5.3-Flash NVFP4-Spark — local-inference-lab quant, r12 fork — 2× DGX
+# Spark, TP=2. The lab's own "DSpark" checkpoint (speculative MTP layers
+# embedded, no separate draft download).
 #
-# local-inference-lab mixed quant: NVFP4 routed experts (layers 3–44,
-# group-16) + MXFP8 MTP experts (layer 45) + W4A4 input scales. Best agent
-# quant so far: tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8).
-# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
-#        (~186 GB → 92.72 GiB weights/node; KV pinned at 10 GiB, see below)
+# local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (rev 06972cdee7280cec849aedf05a02062b3e870246):
+# NVFP4 routed experts (layers 3+, group-16) + NVFP4 MTP expert layer in the
+# separate model-mtp-0000x.safetensors shards (mtp_tokens: 5 per their
+# lil.yaml), amax/nonexpert/inputscales side shards, Glm5NextForConditionalGeneration.
+#
+# Image: glm53-jovian-r12:local (docker-images/glm53-jovian-r12/) — clean
+# solar-pattern build of the r12 stack: local-inference-lab/vllm
+# dev/jovian-judgement (base 54f6e98 + PR 550/552/553/554/555/557 by SHA) +
+# b12x 139e040 + PR 260/267/268/269/270, torch 2.13 cu130, flashinfer 0.6.17.
 #
-# Image: glm53-flash-nvfp4-lab:local (docker-images/glm53-flash-nvfp4-lab/ —
-# day-0 arm64 image + NoPE-MLA mod + the four FujitsuPolycom lab-checkpoint
-# patches: model naming shim, modelopt MTP namespace fix, persistent_topk
-# CC-12.x guards). Their launch config is mirrored below.
-# PROVENANCE: https://github.com/FujitsuPolycom/glm53-flash-tp2-spark
-#             (Apache-2.0; see patches/UPSTREAM.md) + forum 381350 #41.
+# Launch flags mirror the lab's lil launcher policy for this family
+# (local-inference-lab/lil, configs/models/_bases.yaml glm base + lil.yaml):
+# attention B12X, linear/moe backend b12x, MTP spec with moe_backend b12x,
+# instanttensor load, block 256, max-num-seqs 8, batched 4096, glm45/glm47
+# parsers, mamba-cache-mode align.
 #
-# Deviations from their launch script (intentional):
-#   * no --kda-prefill-backend flashkda (needs their base-image patch — skipped)
-#   * --kv-cache-dtype fp8_ds_mla instead of `fp8` (same packed layout; this
-#     spelling is what the NoPE-MLA mod booted with on the LibertAIDAI stack)
-#   * --language-model-only + no mm-renderer patch/env (text-only)
-#   * no bundled NCCL/LD_PRELOAD, no NCCL_* fabric block (our switched fabric
-#     is configured via [nccl.ds4] + launcher GID auto-detect)
-#   * no --moe-backend marlin: their proven stack uses the default backend for
-#     THIS quant and reports clean evals. Sanity-check output for token loops
-#     anyway (the silent FP4 MoE corruption gotcha) before trusting a new node.
-#   * flashinfer autotune disabled via --kernel-config (their proven boot);
-#     VLLM_ENGINE_READY_TIMEOUT_S stays high regardless.
+# Boot success markers: "Model loading took ~93 GiB" (instanttensor: ~1 min)
+# → "GPU KV cache size" → "Application startup complete".
 #
-# Boot success markers (engine log, in order):
-#   "Model loading took 92.72 GiB" → "[quantprobe] ... algo=MXFP8" (MTP fix
-#   active; algo=None means broken) → "GPU KV cache size: 1,164,369 tokens" →
-#   "Application startup complete". ~700 s sequential weight load.
+# JIT cache: the flashinfer/b12x JIT caches are MOUNTED from
+# ~/sm121-cache/glm53-jit-121f (see [[extra_mounts]] at the bottom, same
+# pattern as the dsv4 recipe) — pre-baked on both nodes so boot skips the
+# ~15 min first-boot JIT (and the nvcc OOM risk; MAX_JOBS=2 stays as guard).
 #
 # Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
 # :"low"} — omitted/other values select MAX reasoning effort (empty content on
 # short max_tokens).
 #
-# Optional: instanttensor loader (~30 s weight load vs ~700 s):
-#   FROM glm53-flash-nvfp4-lab:local + pip install --no-deps --no-build-isolation
-#   instanttensor==0.1.9, then add --load-format instanttensor (--no-deps is
-#   required: resolution would downgrade the image's NCCL).
-#
 # PREREQUISITES: weights cached on both nodes:
-# GLM-5.3-Flash NVFP4-Spark — local-inference-lab quant with the speculative
-# layers embedded — 2× DGX Spark, TP=2.
-#
-# local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (rev 06972cdee7280cec849aedf05a02062b3e870246):
-# NVFP4 routed experts (layers 3+, group-16) + NVFP4 MTP expert layer in the
-# separate model-mtp-0000x.safetensors shards (mtp_tokens: 5 per their
-# lil.yaml), amax/nonexpert/inputscales side shards, Glm5NextForConditionalGeneration.
-# THE "DSpark speculative embedded" checkpoint — no separate DFlash2 download
-# needed; lil.yaml: default_speculator mtp, dflash2_tokens 7 (r12 doc fr: their
-# standalone-Q: MTP:3 254 tok/s CC1, 333 Sieve, 94 steps/s).
-#
-# Image: glm53-flash-nvfp4-lab:local (same proven day-0 arm64 stack + NoPE-MLA
-# mod + modelopt MTP namespace patch used by the 92/100 lab lane). The MTP
-# layer here is NVFP4 (not MXFP8) — the quantprobe marker may print
-# algo=NVFP4/- different; algo=None = patch missed (fall back to 4p67 image).
-#
-# Deviations from lab (unchanged mechanics): same fabric/env/GMU/KV pinning.
-# lil.yaml launch mirrors: block 256, disable_flashinfer_autotune, mm data
-# mode — our text-only flags identical to glm53-flash-nvfp4-lab.
-#
-# Boot markers: "Model loading took ~XX GiB" → "[quantprobe] ..." →
-# "GPU KV cache size" → "Application startup complete".
-#
-# PREREQUISITES: weights on the NAS hub (175 GiB): hf download local-inference-lab/GLM-5.3-Flash-NVFP4-Spark
+#   ~/dsv4vision-hf is for DeepSeek — for GLM the launcher resolves the repo
+#   from the node's HF cache (NAS hub: 175 GiB) or a local NVMe copy.
+#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4-Spark
 
 engine = "vllm"
 name = "GLM-5.3-Flash NVFP4-Spark (vLLM, 2x Spark, MTP-5)"
@@ -105,6 +74,14 @@ args = [
     # _select_sparse_backend); the sm12x auto-priority list never offers it
     # (FLASHINFER_MLA_SPARSE_SM120 only supports head 576, not GLM's 512).
     "--attention-backend", "B12X",
+    # lil family policy: GDN linear-attention + routed-expert MoE on the
+    # B12X kernels (the GB10 fast path for this quant).
+    "--linear-backend", "b12x",
+    "--moe-backend", "b12x",
+    # lab loader: instanttensor (0.1.9 is already in the r12 image) —
+    # ~1 min weight load vs ~16 min from NFS
+    "--load-format", "instanttensor",
+    "--mamba-cache-mode", "align",
     "--block-size", "256",
     "--max-model-len", "524288",
     "--max-num-seqs", "8",
@@ -116,17 +93,16 @@ args = [
     # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
     "--kernel-config", """
 '{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
-    # Built-in MTP draft, 3 tokens (the r12-qualified "mtp:3" mode — their
-    # x86 numbers: 254 tok/s CC1, 2.37 accept; the checkpoint embeds up to
-    # 5 via the model-mtp-0000x shards but 3 is the verified depth). This is
-    # the checkpoint's own MTP head (Glm5NextMTPModel), NOT dspark — dspark
-    # is the DeepSeek-V4 family's mechanism; GLM's other r12 mode is
-    # dflash2:7 with a separate trained draft checkpoint.
-    # The draft model's vllm_config takes its attention backend from the
+    # Built-in MTP draft, 5 tokens — lil's mtp_tokens for this checkpoint
+    # (all 5 MTP layers are embedded in the model-mtp-0000x shards). The
+    # draft model's vllm_config takes its attention backend from the
     # SPECULATIVE config, not the global flag (eagle/utils.py:
-    # _make_eagle_draft_vllm_config) — B12X here mirrors the main model's.
+    # _make_eagle_draft_vllm_config) — B12X mirrors the main model's;
+    # moe_backend b12x per lil's MTPMoEBackendFromQuantization (nvfp4→b12x).
+    # A/B vs k=3 if accept-rate/quality regresses (k=3 was the r12 x86
+    # reference: 254 tok/s CC1, 2.37 accept).
     "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'""",
+'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"b12x","attention_backend":"B12X"}'""",
     "--language-model-only",
     # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
     # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
@@ -141,10 +117,12 @@ args = [
 
 [env]
 HF_HUB_OFFLINE = "1"
-# FlashInfer JIT (mxfp8_gemm_cutlass_sm120, ~12 kernels) compiles at first
-# boot with 12 parallel nvcc jobs -> OOM-killed (code 137) on GB10's
+# lab lil.yaml INSTANTTENSOR knobs
+INSTANTTENSOR_BUFFER_SIZE = "67108864"
+INSTANTTENSOR_CHUNK_SIZE = "8388608"
+# FlashInfer JIT guard: if a kernel is missing from the mounted JIT cache it
+# compiles at boot — 12-way parallel nvcc got OOM-killed (code 137) on GB10's
 # 121 GB unified while the 93 GB weights + 10 GB KV pin are resident.
-# MAX_JOBS caps ninja parallelism so the JIT fits in the remaining ~20 GB.
 MAX_JOBS = "2"
 # NoPE-MLA rope-pad runtime gate (mod baked into the image)
 VLLM_MLA_NOPE_PAD_ROPE = "1"
@@ -161,3 +139,17 @@ FLASHINFER_CUDA_ARCH_LIST = "12.1f"
 CUTE_DSL_ARCH = "sm_121a"
 CMAKE_CUDA_ARCHITECTURES = "121"
 OMP_NUM_THREADS = "4"
+
+# Pre-baked JIT caches (see header; same pattern as the dsv4 recipe's
+# ~/.cache/dsv4vision-jit mounts). RW on purpose: flashinfer writes lock
+# files into the cache. Missing on a node -> first boot JITs (~15 min at
+# MAX_JOBS=2) and the cache populates for the next boot.
+# NOTE: extra_mounts stay AFTER args (TOML table headers would swallow the
+# args array otherwise).
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-jit-121f/b12x"
+container = "/root/.cache/b12x"