recipes: glm53-full-mxfp4-4x

recipes/glm53-full-mxfp4-4x/recipe.toml

@@ -0,0 +1,300 @@
+# GLM-5.3 (FULL, non-Flash) MXFP4 — RedHatAI official quant — 4× DGX Spark, TP=4.
+#
+# REPLACES the failed full-model variant: `local-inference-lab/GLM-5.3-NVFP4-Spark`
+# (406 GiB → 101.5 GiB weights/node → OOM-proven on 4 nodes 2026-09-08/09: the
+# engine crash-restarted and began a SECOND full load while the first copy was
+# still resident → 2×101.5 > 121.63 GiB → all four nodes OOM-thrashed, sshd down
+# on 3 of 4, HA power-cycle required). That checkpoint is abandoned (recipes
+# glm53-full-nvfp4-spark-lab / -aiden / glm53-spark-nvfp4-4x deleted).
+#
+# THIS checkpoint: RedHatAI/GLM-5.3-MXFP4 — 403.4 GB (375.6 GiB) → 100.85 GB
+# (93.9 GiB) weights/node on 4x. ~7.6 GiB/node more headroom than the NVFP4
+# attempt, which is what makes 4x plausible at all. LLM-Compressor
+# (compressed-tensors) quant: "mxfp4-pack-quantized", weights FP4-e2m1 group-32
+# with E8M0 scales, dynamic per-group FP4 activations; mlp.gate / lm_head /
+# embed_tokens / eh_proj / indexer.{wk,weights_proj} stay bf16.
+#   Model:  https://huggingface.co/RedHatAI/GLM-5.3-MXFP4
+#   Parent: zai-org/GLM-5.3 (FP8). Card: gsm8k 96.36 vs 97.60 (FP8) = 98.7%
+#   recovery; ifeval 89.25 vs 93.53 = 95.4%.
+#   Card's serving recipe (4× B200): --kv-cache-dtype fp8_e4m3 --tp 4
+#   --reasoning-parser glm45 --speculative-config '{"method":"mtp","num_speculative_tokens":5}'
+#
+# GEOMETRY (config.json, verified 2026-09-13): arch GlmMoeDsaForCausalLM
+# (model_type glm_moe_dsa), 78 layers, hidden 6144, 256 routed experts + 1
+# shared, top-8, moe_inter 2048, MLA (kv_lora 512 / q_lora 2048 / qk_nope 192 /
+# qk_rope 64 / v 256), DSA lightning indexer (index_topk 2048), 1M native ctx,
+# first_k_dense_replace 3, num_nextn_predict_layers 1 (MTP head IN the
+# checkpoint), 141 shards, no separate drafter.
+# ~740B total params / ~40B active. The Flash variant we serve in prod is
+# 320B/18B (45 layers, hidden 4096, 288 experts) — this is a different model.
+#
+# ── BASE STACK: our own jovian r35 build (docker-images/jovian-judgement +
+#    build_profile r35 → image jovian-judgement:r35-glm53) ────────────────
+# Selected 2026-09-13. Evidence (all verified by read-only inspection of the
+# images on dgx1):
+#   * registers the full-model arch: registry.py →
+#     "GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", ...) — the NEW
+#     `deepseek_v32` implementation, which is "MLA + a lightning indexer that
+#     selects the top-k tokens for a sparse MLA" and dispatches to an
+#     NVIDIA-specific path (deepseek_v32/nvidia/model.py:
+#     DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM). That is exactly the
+#     GB10/sm_121a target; the lab fork and aiden's image register the same
+#     class from the older `deepseek_v2` module instead.
+#   * ships the MXFP4 scheme: model_executor/layers/quantization/
+#     compressed_tensors/schemes/compressed_tensors_w4a4_mxfp4.py
+#     ("mxfp4-pack-quantized") → the RedHat checkpoint loads natively.
+#   * it is OUR reproducible clone-at-pin build (vLLM de982a50 + b12x 98086604,
+#     r35 lock 2026-09-11 = newest published lab composition), all gates green,
+#     patches ours to write — and it is the SAME image our GLM-5.3-Flash 4x
+#     prod runs, so the A/B vs prod is clean and the JIT caches are warm.
+#   * the b12x MoE/attention backends are the GB10-tuned ones our GLM prod
+#     uses (--moe-backend b12x / --attention-backend B12X).
+# Rejected: `glm53-flash-nvfp4-lab:local` and aiden's
+# `aidendle94/sparkrun-vllm-glm53-gb10:production-1.2` — both older vLLM
+# (0.1.dev20051+g487ecf187, `deepseek_v2` GLM class); the lab fork needed two
+# SM120 mla_attention hotfix mods to boot the full model at all
+# (masked_mha_available AttributeError + forward_mha NotImplementedError, both
+# never re-tested — the lane parked). Keep them as a fallback reference only:
+# if the r35 deepseek_v32 path trips on the DSA/indexer kernels, port those
+# two sed mods (documented in the deleted glm53-full-nvfp4-spark-lab history)
+# onto the lab image and re-point docker_image/docker_image_dir.
+#
+# ── MEMORY PLAN (the binding constraint — read before the first boot) ──────
+#   121.63 GiB unified/node. Idle free: 116-117 GiB.
+#     weights          93.9 GiB (403.4 GB / 4)  ← ~101.5 was fatal
+#     vLLM procs+docker ~12 GiB
+#     KV pin            8 GiB (--kv-cache-memory-bytes)
+#     activations + CUDA graphs ~4 GiB
+#   ⇒ ~118 GiB — tight but inside the node. GMU 0.85 is the ADMISSION
+#   guardrail only (free ≥ 0.85×121.63 = 103.4 GiB; 0.88 = 107.0 GiB fails
+#   once the ~12 GiB of starting processes are counted, the 2026-09-08
+#   #48140 class) — the KV pool is pinned, so GMU does not change capacity.
+#
+#   WHY DCP4: the full model's attention KV is NOT the Flash lane's compact
+#   MLA cache. The parked run measured ~54 KB/token (4 GiB ↔ ~79.6K tokens;
+#   "524288 would need 26.34 GiB") vs the Flash lane's ~2 KB/token at 512K.
+#   Decode-context-parallel-4 shards the KV across the 4 ranks, so the 8 GiB
+#   pin buys 32 GiB aggregate → a single 524288-token request needs ~27 GiB
+#   aggregate (≈6.8 GiB/rank) and just fits. Without DCP a 512K request is
+#   impossible behind these weights. CAVEAT: DCP4 is qualified in r35 for the
+#   Flash model (CP1/CP4 shapes); it is UNVERIFIED for the full model — if the
+#   boot rejects it (the GLM C4 page-stride / interleave-multiple-of-4 gates
+#   that block DFlash2 on 2x), drop --decode-context-parallel-size +
+#   --cp-kv-cache-interleave-size and lower --max-model-len to fit ~10 GiB of
+#   aggregate KV (~180K tokens).
+#
+#   --max-model-len 524288 = the operator's floor (128K/131K profiles are not
+#   usable for us). --max-num-seqs 8 keeps the pool from being shredded by
+#   short requests; the admission scheduler handles the rest.
+#
+# ── BOOT PLAN / HAZARD ─────────────────────────────────────────────────────
+#   The 2026-09-08 disaster was NOT a bad weight footprint per se — it was a
+#   post-load crash + auto-restart that started a SECOND full load on top of
+#   the first. Watch the first boot: if the engine logs a second
+#   "Loading safetensors" pass, kill immediately (`sm121 stop`), do not wait.
+#   Always `sm121 serve --keep` for the first/unknown boot (default is --rm,
+#   which wipes the crash logs).
+#   Ladder if the boot dies: (1) disable DCP4, max-model-len 131072, KV pin
+#   4 GiB (prove stack+checkpoint end-to-end, the parked lane's approach);
+#   (2) then re-add DCP4 + 512K; (3) only then add spec.
+#   Boot markers to record: "Model loading took X GiB", the KV line
+#   ("GPU KV cache size: N tokens, Maximum concurrency for 524288 tokens per
+#   request: Mx"), "Application startup complete". MAX_JOBS=2 below is the
+#   compile-OOM guard (12 parallel nvcc jobs × ~3 GiB behind 94 GiB of weights
+#   = cicc OOM-killed, 2026-09-04 lesson).
+#
+# ── MXFP4 vs NVFP4: what changed, and what is already in our tree ──────────
+# The full-model NVFP4 lane is retired (see above), so this checkpoint is the
+# lab's MXFP4 family: routed-expert weights FP4-e2m1 with power-of-two (E8M0)
+# group-32 scales — the same shape as their own GLM-5.1 MXFP4 repack, which is
+# why their GLM-5.1 runbook (models/glm-5.1-mxfp4.md) is the reference for
+# this lane even though GLM-5.3 moved on: the MXFP4 ingestion path, the
+# FlashInfer Cutlass MXFP4xMXFP8 MoE route, the standard-SwiGLU rule and the
+# "do not set B12X_MOE_FORCE_A16" warning all still apply. That runbook also
+# unsets empty NCCL graph variables before launch
+# (NCCL_GRAPH_FILE / NCCL_GRAPH_DUMP_FILE / VLLM_B12X_MLA_EXTEND_MAX_CHUNKS)
+# — unset them in the shell if a previous lane exported them empty; an empty
+# value is not the same as an unset one to NCCL.
+#
+# ── SPECULATIVE DECODING ───────────────────────────────────────────────────
+#   NOT enabled on the first boot: the checkpoint's MTP head is real
+#   (num_nextn_predict_layers=1) and the card suggests mtp k=5, but spec adds
+#   draft KV + graph shapes onto a memory budget that has no room for
+#   surprises, and the full-model MTP path is unproven on this stack. After a
+#   clean boot + smoke + bench, add:
+#     "--speculative-config", """'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'"""
+#   (k=3 = our Flash prod's verified depth; try 5 after k=3 is clean).
+#
+# PREREQUISITE: weights in the shared NAS HF cache on all 4 nodes
+#   (models--RedHatAI--GLM-5.3-MXFP4, 403.4 GB, 141 shards) —
+#   `hf download RedHatAI/GLM-5.3-MXFP4`; HF_HUB_OFFLINE=1 below makes a
+#   missing shard a hard load failure. NAS had 16 TB free at 2026-09-13.
+#
+# A/B TARGET: our GLM-5.3-Flash r35 MTP3 4x prod (dev65/seed 42, hardmode):
+#   163/176 (93), 1,554 pp / 49.6 tg / 1,544 ms TTFT @d0 c1, KV 8.06M @512K.
+#   This lane is a capability/quality lane first (does the full model fit and
+#   hold 512K on 4 nodes), throughput second.
+
+engine = "vllm"
+name = "GLM-5.3 (full) MXFP4 (r35 img, RedHatAI ckpt, 4x DCP4)"
+# Image: the r37 DS4.1 build (2026-09-14, night shift) - the r35 tree's DCP4
+# sparse-INDEXER path asserts interleave==1 and then crashes in the DCP sparse
+# decode reshape during graph capture ("shape '[16, 16, 512]' is invalid for
+# input of size 524288" - the x4 DCP factor in the attention.py _sparse_
+# decode). r37 is the newer DS4.1 lock whose DCP sparse-MLA path is the
+# qualified one (same tree family as our DCP2/DFlash2 work); the GlmMoeDsa
+# arch + MTP + b12x ship in the r37 image (verified 2026-09-13).
+docker_image = "jovian-judgement:r37-ds41"
+docker_image_dir = "jovian-judgement"
+# NOTE: build_profile must stay a TOP-LEVEL key — a bare key after
+# [[extra_mounts]] would parse into the last mount table and be silently
+# ignored (2026-09-08).
+build_profile = "r37-ds41"
+entrypoint = ["vllm", "serve"]
+model = "RedHatAI/GLM-5.3-MXFP4"
+parent_model = "zai-org/GLM-5.3"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # R31/r35 delta (their qualified locks): adaptive prefill/decode
+    # compute-time sharing with a fixed 40% prefill floor.
+    "--prefill-compute-share", "0.4",
+    # Stable served name — quant-agnostic (shared across glm53 lanes)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # Admission guardrail only (KV is pinned below): 0.85×121.63 = 103.4 GiB.
+    # 0.88 (107.0) is the Flash lane's value and is too tight here — the full
+    # model's weights leave ~24 GiB before the engine's own ~12 GiB.
+    "--gpu-memory-utilization", "0.85",
+    # 8 GiB/rank. With DCP4 that is 32 GiB aggregate ≈ one 524288-token request
+    # (+headroom). See the memory plan above before changing either number.
+    "--kv-cache-memory-bytes", "8589934592",
+    # DCP4 — required to make 512K fit (KV sharded across the 4 TP ranks).
+    # Unverified for the full model; first fallback is to drop both lines.
+    "--decode-context-parallel-size", "4",
+    # interleave MUST be 1 here: the engine (r35) asserts
+    #   DCP sparse indexer currently supports only cp_kv_cache_interleave_size=1
+    # (the DSA top-k sparse-MLA path is not the Flash dense-MLA path where the
+    # x4 interleave / parent-page-stride rule applied; that constraint does not
+    # carry over to this model - learned on the first DCP4 boot, 2026-09-13).
+    "--cp-kv-cache-interleave-size", "1",
+    # No --kv-cache-dtype: let the r35 fork pick its native default for
+    # GlmMoeDsa attention (the Flash lane's explicit fp8/fp8_ds_mla spellings
+    # both failed auto-selection on this build, and those were Flash-MLA
+    # knobs). The card's --kv-cache-dtype fp8_e4m3 is the fallback spelling
+    # if the pool needs to shrink.
+    "--attention-backend", "B12X",
+    # MXFP4 kernel routing (measured on the first boot, 2026-09-13 ~23:10 UTC):
+    # - MoE: the compressed-tensors W4A4-MXFP4 scheme picks MarlinExperts on
+    #   its own (A16 dequant GEMM - correct on GB10). The --moe-backend flag
+    #   does not reach that scheme (verified inert: it still logged "Using
+    #   MarlinExperts for MXFP4 MoE" with the flag set), so we do not set it.
+    # - LINEAR: the default priority [FlashInfer, Marlin, Humming, B12x]
+    #   starts with the FlashInfer kernel, whose is_supported() wrongly
+    #   accepts capability 121 (the numeric has_device_capability(100) check
+    #   passes on GB10) while the cute-dsl mm_fp4 GEMM it calls rejects it at
+    #   runtime with:
+    #     BackendSupportedError: mm_fp4 does not support backend 'cute-dsl'
+    #     with capability 121
+    #   (the first probe boot died there, kv_b_proj dequant in mla_attention.py).
+    # --linear-backend b12x forces the B12xMxFp4LinearKernel - "MXFP4 linear
+    # through the native B12X SM120 dense GEMM" - the right path on GB10,
+    # verified per component: is_device_capability_family(120)=True, b12x
+    # blockscaled.is_supported()=True, and the cute-dsl input quantize
+    # (flashinfer_mxfp4_quantize) works on capability 121; only the cute-dsl
+    # mm_fp4 GEMM is unsupported there. Boot log to look for:
+    #   "Using B12xMxFp4LinearKernel for MXFP4 GEMM"
+    # (The lab's GLM-5.1 runbook describes the same kernel family for the
+    # quant_method=mxfp4 ingestion path - this checkpoint goes through the
+    # compressed-tensors path, which is why --moe-backend was the wrong lever.)
+    # First attempt: b12x (the native SM120 GEMM) - correct selection, but the
+    # unconditional b12x warmup (b12x_warmup.py, no config gate) crashes on
+    # this checkpoint's layer shapes during kernel compile:
+    #   RuntimeError: Expected source and destination tensors to have the same
+    #   rank, but got 3 and 4  (b12x/gemm/blockscaled/_linear.py dense_gemm,
+    #   via _apply_b12x_mxfp4_linear on a synthetic zero source)
+    # so the mxfp4 LINEAR layers run on Marlin (weight-only A16 dequant GEMM -
+    # the same family as the MarlinExperts the MoE scheme picked). Slower than
+    # native FP4 for the attention projections, but the MoE experts (the
+    # dominant cost) are Marlin either way. The b12x warmup rank bug is
+    # filed-worthy for the lab (RedHatAI GLM-5.3-MXFP4 on GB10/sm_121a).
+    "--linear-backend", "marlin",
+    # pure I/O loader (kept across every GLM config)
+    "--load-format", "instanttensor",
+    "--block-size", "256",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup (the
+    # autotune scratch at 512K shapes does not fit the headroom, 2026-08-28).
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    "--language-model-only",
+    # NO --speculative-config on the first boot — see the header. MTP head is
+    # in the checkpoint (num_nextn_predict_layers=1); add mtp k=3 after a
+    # clean boot + bench.
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+
+# R31 lock: 16 NCCL channels (their qualified runtime).
+NCCL_NCHANNELS = "16"
+HF_HUB_OFFLINE = "1"
+# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for r27+
+# lanes since the 2026-09-08 A/B (4x DCP4: decode +31..+90%, quality 160 vs
+# 155; specs/tech-architecture/glm53-recipe-state.md). Needs the CX-7 RoCE
+# fabric (all of ours); single-node serves skip it silently.
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
+# Compile-OOM guard: first boot JIT-compiles the mxfp8 gemm module with 12
+# parallel nvcc jobs (~3 GiB each) behind ~94 GiB of resident weights →
+# cicc OOM-killed (2026-09-04). flashinfer's run_ninja reads MAX_JOBS.
+MAX_JOBS = "2"
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+# Upstream launch-script env (memory + compile knobs)
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+VLLM_USE_STANDALONE_COMPILE = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "2"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-jit-121f/b12x"
+container = "/root/.cache/b12x"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"