launcher/recipes/glm53-full-mxfp4-4x-dspark8/recipe.toml

# GLM-5.3 (FULL, non-Flash) MXFP4 — RedHatAI official quant — 4× DGX Spark, TP=4.
#
# REPLACES the failed full-model variant: `local-inference-lab/GLM-5.3-NVFP4-Spark`
# (406 GiB → 101.5 GiB weights/node → OOM-proven on 4 nodes 2026-09-08/09: the
# engine crash-restarted and began a SECOND full load while the first copy was
# still resident → 2×101.5 > 121.63 GiB → all four nodes OOM-thrashed, sshd down
# on 3 of 4, HA power-cycle required). That checkpoint is abandoned (recipes
# glm53-full-nvfp4-spark-lab / -aiden / glm53-spark-nvfp4-4x deleted).
#
# THIS checkpoint: RedHatAI/GLM-5.3-MXFP4 — 403.4 GB (375.6 GiB) → 100.85 GB
# (93.9 GiB) weights/node on 4x. ~7.6 GiB/node more headroom than the NVFP4
# attempt, which is what makes 4x plausible at all. LLM-Compressor
# (compressed-tensors) quant: "mxfp4-pack-quantized", weights FP4-e2m1 group-32
# with E8M0 scales, dynamic per-group FP4 activations; mlp.gate / lm_head /
# embed_tokens / eh_proj / indexer.{wk,weights_proj} stay bf16.
#   Model:  https://huggingface.co/RedHatAI/GLM-5.3-MXFP4
#   Parent: zai-org/GLM-5.3 (FP8). Card: gsm8k 96.36 vs 97.60 (FP8) = 98.7%
#   recovery; ifeval 89.25 vs 93.53 = 95.4%.
#   Card's serving recipe (4× B200): --kv-cache-dtype fp8_e4m3 --tp 4
#   --reasoning-parser glm45 --speculative-config '{"method":"mtp","num_speculative_tokens":5}'
#
# GEOMETRY (config.json, verified 2026-09-13): arch GlmMoeDsaForCausalLM
# (model_type glm_moe_dsa), 78 layers, hidden 6144, 256 routed experts + 1
# shared, top-8, moe_inter 2048, MLA (kv_lora 512 / q_lora 2048 / qk_nope 192 /
# qk_rope 64 / v 256), DSA lightning indexer (index_topk 2048), 1M native ctx,
# first_k_dense_replace 3, num_nextn_predict_layers 1 (MTP head IN the
# checkpoint), 141 shards, no separate drafter.
# ~740B total params / ~40B active. The Flash variant we serve in prod is
# 320B/18B (45 layers, hidden 4096, 288 experts) — this is a different model.
#
# ── BASE STACK: our own jovian r35 build (docker-images/jovian-judgement +
#    build_profile r35 → image jovian-judgement:r35-glm53) ────────────────
# Selected 2026-09-13. Evidence (all verified by read-only inspection of the
# images on dgx1):
#   * registers the full-model arch: registry.py →
#     "GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", ...) — the NEW
#     `deepseek_v32` implementation, which is "MLA + a lightning indexer that
#     selects the top-k tokens for a sparse MLA" and dispatches to an
#     NVIDIA-specific path (deepseek_v32/nvidia/model.py:
#     DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM). That is exactly the
#     GB10/sm_121a target; the lab fork and aiden's image register the same
#     class from the older `deepseek_v2` module instead.
#   * ships the MXFP4 scheme: model_executor/layers/quantization/
#     compressed_tensors/schemes/compressed_tensors_w4a4_mxfp4.py
#     ("mxfp4-pack-quantized") → the RedHat checkpoint loads natively.
#   * it is OUR reproducible clone-at-pin build (vLLM de982a50 + b12x 98086604,
#     r35 lock 2026-09-11 = newest published lab composition), all gates green,
#     patches ours to write — and it is the SAME image our GLM-5.3-Flash 4x
#     prod runs, so the A/B vs prod is clean and the JIT caches are warm.
#   * the b12x MoE/attention backends are the GB10-tuned ones our GLM prod
#     uses (--moe-backend b12x / --attention-backend B12X).
# Rejected: `glm53-flash-nvfp4-lab:local` and aiden's
# `aidendle94/sparkrun-vllm-glm53-gb10:production-1.2` — both older vLLM
# (0.1.dev20051+g487ecf187, `deepseek_v2` GLM class); the lab fork needed two
# SM120 mla_attention hotfix mods to boot the full model at all
# (masked_mha_available AttributeError + forward_mha NotImplementedError, both
# never re-tested — the lane parked). Keep them as a fallback reference only:
# if the r35 deepseek_v32 path trips on the DSA/indexer kernels, port those
# two sed mods (documented in the deleted glm53-full-nvfp4-spark-lab history)
# onto the lab image and re-point docker_image/docker_image_dir.
#
# ── MEMORY PLAN (the binding constraint — read before the first boot) ──────
#   121.63 GiB unified/node. Idle free: 116-117 GiB.
#     weights          93.9 GiB (403.4 GB / 4)  ← ~101.5 was fatal
#     vLLM procs+docker ~12 GiB
#     KV pin            8 GiB (--kv-cache-memory-bytes)
#     activations + CUDA graphs ~4 GiB
#   ⇒ ~118 GiB — tight but inside the node. GMU 0.85 is the ADMISSION
#   guardrail only (free ≥ 0.85×121.63 = 103.4 GiB; 0.88 = 107.0 GiB fails
#   once the ~12 GiB of starting processes are counted, the 2026-09-08
#   #48140 class) — the KV pool is pinned, so GMU does not change capacity.
#
#   WHY DCP4: the full model's attention KV is NOT the Flash lane's compact
#   MLA cache. The parked run measured ~54 KB/token (4 GiB ↔ ~79.6K tokens;
#   "524288 would need 26.34 GiB") vs the Flash lane's ~2 KB/token at 512K.
#   Decode-context-parallel-4 shards the KV across the 4 ranks, so the 8 GiB
#   pin buys 32 GiB aggregate → a single 524288-token request needs ~27 GiB
#   aggregate (≈6.8 GiB/rank) and just fits. Without DCP a 512K request is
#   impossible behind these weights. CAVEAT: DCP4 is qualified in r35 for the
#   Flash model (CP1/CP4 shapes); it is UNVERIFIED for the full model — if the
#   boot rejects it (the GLM C4 page-stride / interleave-multiple-of-4 gates
#   that block DFlash2 on 2x), drop --decode-context-parallel-size +
#   --cp-kv-cache-interleave-size and lower --max-model-len to fit ~10 GiB of
#   aggregate KV (~180K tokens).
#
#   --max-model-len 524288 = the operator's floor (128K/131K profiles are not
#   usable for us). --max-num-seqs 8 keeps the pool from being shredded by
#   short requests; the admission scheduler handles the rest.
#
# ── BOOT PLAN / HAZARD ─────────────────────────────────────────────────────
#   The 2026-09-08 disaster was NOT a bad weight footprint per se — it was a
#   post-load crash + auto-restart that started a SECOND full load on top of
#   the first. Watch the first boot: if the engine logs a second
#   "Loading safetensors" pass, kill immediately (`sm121 stop`), do not wait.
#   Always `sm121 serve --keep` for the first/unknown boot (default is --rm,
#   which wipes the crash logs).
#   Ladder if the boot dies: (1) disable DCP4, max-model-len 131072, KV pin
#   4 GiB (prove stack+checkpoint end-to-end, the parked lane's approach);
#   (2) then re-add DCP4 + 512K; (3) only then add spec.
#   Boot markers to record: "Model loading took X GiB", the KV line
#   ("GPU KV cache size: N tokens, Maximum concurrency for 524288 tokens per
#   request: Mx"), "Application startup complete". MAX_JOBS=2 below is the
#   compile-OOM guard (12 parallel nvcc jobs × ~3 GiB behind 94 GiB of weights
#   = cicc OOM-killed, 2026-09-04 lesson).
#
# ── MXFP4 vs NVFP4: what changed, and what is already in our tree ──────────
# The full-model NVFP4 lane is retired (see above), so this checkpoint is the
# lab's MXFP4 family: routed-expert weights FP4-e2m1 with power-of-two (E8M0)
# group-32 scales — the same shape as their own GLM-5.1 MXFP4 repack, which is
# why their GLM-5.1 runbook (models/glm-5.1-mxfp4.md) is the reference for
# this lane even though GLM-5.3 moved on: the MXFP4 ingestion path, the
# FlashInfer Cutlass MXFP4xMXFP8 MoE route, the standard-SwiGLU rule and the
# "do not set B12X_MOE_FORCE_A16" warning all still apply. That runbook also
# unsets empty NCCL graph variables before launch
# (NCCL_GRAPH_FILE / NCCL_GRAPH_DUMP_FILE / VLLM_B12X_MLA_EXTEND_MAX_CHUNKS)
# — unset them in the shell if a previous lane exported them empty; an empty
# value is not the same as an unset one to NCCL.
#
# ── SPECULATIVE DECODING ───────────────────────────────────────────────────
#   NOT enabled on the first boot: the checkpoint's MTP head is real
#   (num_nextn_predict_layers=1) and the card suggests mtp k=5, but spec adds
#   draft KV + graph shapes onto a memory budget that has no room for
#   surprises, and the full-model MTP path is unproven on this stack. After a
#   clean boot + smoke + bench, add:
#     "--speculative-config", """'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'"""
#   (k=3 = our Flash prod's verified depth; try 5 after k=3 is clean).
#
# PREREQUISITE: weights in the shared NAS HF cache on all 4 nodes
#   (models--RedHatAI--GLM-5.3-MXFP4, 403.4 GB, 141 shards) —
#   `hf download RedHatAI/GLM-5.3-MXFP4`; HF_HUB_OFFLINE=1 below makes a
#   missing shard a hard load failure. NAS had 16 TB free at 2026-09-13.
#
# A/B TARGET: our GLM-5.3-Flash r35 MTP3 4x prod (dev65/seed 42, hardmode):
#   163/176 (93), 1,554 pp / 49.6 tg / 1,544 ms TTFT @d0 c1, KV 8.06M @512K.
#   This lane is a capability/quality lane first (does the full model fit and
#   hold 512K on 4 nodes), throughput second.

engine = "vllm"
name = "GLM-5.3 (full) MXFP4 (r37 img, 4x, DSpark K8 @131K)"
# Image: the r37 DS4.1 build (2026-09-14, night shift) - the r35 tree's DCP4
# sparse-INDEXER path asserts interleave==1 and then crashes in the DCP sparse
# decode reshape during graph capture ("shape '[16, 16, 512]' is invalid for
# input of size 524288" - the x4 DCP factor in the attention.py _sparse_
# decode). r37 is the newer DS4.1 lock whose DCP sparse-MLA path is the
# qualified one (same tree family as our DCP2/DFlash2 work); the GlmMoeDsa
# arch + MTP + b12x ship in the r37 image (verified 2026-09-13).
docker_image = "jovian-judgement:r37-ds41"
docker_image_dir = "jovian-judgement"
# NOTE: build_profile must stay a TOP-LEVEL key — a bare key after
# [[extra_mounts]] would parse into the last mount table and be silently
# ignored (2026-09-08).
build_profile = "r37-ds41"
entrypoint = ["vllm", "serve"]
model = "RedHatAI/GLM-5.3-MXFP4"
parent_model = "zai-org/GLM-5.3"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # R31/r35 delta (their qualified locks): adaptive prefill/decode
    # compute-time sharing with a fixed 40% prefill floor.
    "--prefill-compute-share", "0.4",
    # Stable served name — quant-agnostic (shared across glm53 lanes)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # Admission guardrail only (KV is pinned below): 0.85×121.63 = 103.4 GiB.
    # 0.88 (107.0) is the Flash lane's value and is too tight here — the full
    # model's weights leave ~24 GiB before the engine's own ~12 GiB.
    "--gpu-memory-utilization", "0.85",
    # 9 GiB/rank pin (DCP1, no context parallel). The no-spec 131K anchor ran
    # at 8 GiB (pool 159,232 tokens); +1 GiB covers the draft model's own
    # KV group: the engine needs 8.08 GiB for one 131072-token request + the
    # drafter, and the 8.0 GiB pin was 80 MB short (boot 5, 2026-09-18).
    "--kv-cache-memory-bytes", "9663676416",
    # NO DCP here: the DCP>1 sparse-decode path (deepseek_v32/attention.py)
    # crashes on both r35 and r37 trees (the 512K-on-4x blocker). 512K on
    # 4x is out; long context is the 8x DCP1 story (P2).
    # No --kv-cache-dtype: let the r35 fork pick its native default for
    # GlmMoeDsa attention (the Flash lane's explicit fp8/fp8_ds_mla spellings
    # both failed auto-selection on this build, and those were Flash-MLA
    # knobs). The card's --kv-cache-dtype fp8_e4m3 is the fallback spelling
    # if the pool needs to shrink.
    # DSpark drafter fix (boot 1, 2026-09-18): the target's B12X sparse-MLA
    # layer normalizes the shared cache config to fp8_ds_mla (mla_attention
    # _canonicalize_sparse_mla_kv_cache_dtype rewrites cache_config.cache
    # dtype in place); the draft model's non-MLA DFlash attention (head 64)
    # is built after that and the r37 selector finds no backend for
    # (head 64, bf16, fp8_ds_mla, non-MLA) → ValueError at init. The
    # drafter's DFlash layers are sliding-window (2048), the target's DSA
    # layers are not — skipping KV quant for the sliding_window pattern
    # gives the draft layers auto (bf16) KV while the target keeps
    # fp8_ds_mla. Explicit --kv-cache-dtype is NOT the lever: the MLA
    # normalization overrode fp8_e4m3 back to fp8_ds_mla (boot 2).
    "--kv-cache-dtype-skip-layers", "sliding_window",
    "--attention-backend", "B12X",
    # MXFP4 kernel routing (measured on the first boot, 2026-09-13 ~23:10 UTC):
    # - MoE: the compressed-tensors W4A4-MXFP4 scheme picks MarlinExperts on
    #   its own (A16 dequant GEMM - correct on GB10). The --moe-backend flag
    #   does not reach that scheme (verified inert: it still logged "Using
    #   MarlinExperts for MXFP4 MoE" with the flag set), so we do not set it.
    # - LINEAR: the default priority [FlashInfer, Marlin, Humming, B12x]
    #   starts with the FlashInfer kernel, whose is_supported() wrongly
    #   accepts capability 121 (the numeric has_device_capability(100) check
    #   passes on GB10) while the cute-dsl mm_fp4 GEMM it calls rejects it at
    #   runtime with:
    #     BackendSupportedError: mm_fp4 does not support backend 'cute-dsl'
    #     with capability 121
    #   (the first probe boot died there, kv_b_proj dequant in mla_attention.py).
    # --linear-backend b12x forces the B12xMxFp4LinearKernel - "MXFP4 linear
    # through the native B12X SM120 dense GEMM" - the right path on GB10,
    # verified per component: is_device_capability_family(120)=True, b12x
    # blockscaled.is_supported()=True, and the cute-dsl input quantize
    # (flashinfer_mxfp4_quantize) works on capability 121; only the cute-dsl
    # mm_fp4 GEMM is unsupported there. Boot log to look for:
    #   "Using B12xMxFp4LinearKernel for MXFP4 GEMM"
    # (The lab's GLM-5.1 runbook describes the same kernel family for the
    # quant_method=mxfp4 ingestion path - this checkpoint goes through the
    # compressed-tensors path, which is why --moe-backend was the wrong lever.)
    # First attempt: b12x (the native SM120 GEMM) - correct selection, but the
    # unconditional b12x warmup (b12x_warmup.py, no config gate) crashes on
    # this checkpoint's layer shapes during kernel compile:
    #   RuntimeError: Expected source and destination tensors to have the same
    #   rank, but got 3 and 4  (b12x/gemm/blockscaled/_linear.py dense_gemm,
    #   via _apply_b12x_mxfp4_linear on a synthetic zero source)
    # so the mxfp4 LINEAR layers run on Marlin (weight-only A16 dequant GEMM -
    # the same family as the MarlinExperts the MoE scheme picked). Slower than
    # native FP4 for the attention projections, but the MoE experts (the
    # dominant cost) are Marlin either way. The b12x warmup rank bug is
    # filed-worthy for the lab (RedHatAI GLM-5.3-MXFP4 on GB10/sm_121a).
    "--linear-backend", "marlin",
    # pure I/O loader (kept across every GLM config)
    "--load-format", "instanttensor",
    "--block-size", "256",
    "--max-model-len", "131072",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup (the
    # autotune scratch at 512K shapes does not fit the headroom, 2026-08-28).
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    "--language-model-only",
    # A/B vs the no-spec 131K anchor (158/176, 934 pp / 19.0 tg / 2,124 ms):
    # external DSpark K8 drafter, RedHatAI/GLM-5.3-speculator.dspark (09-16,
    # v1.0) — a 3-layer Qwen3-style 2.5 GB drafter with Markov + confidence
    # heads, trained on zai-org/GLM-5.3 (BF16) hidden states, layers
    # 2/20/39/58/75 aux. Card contract (num_speculative_tokens 8, method
    # dspark). r37 tree verified to carry the integration (DSparkDraftModel
    # in config/speculative.py + speculators config loader). Drafter rides
    # the TP group (draft_tensor_parallel_size defaults to TP). The repo is
    # in the shared HF cache (snapshot 198b3ced), HF_HUB_OFFLINE=1 below
    # resolves it from there.
    "--speculative-config", """
'{"model":"RedHatAI/GLM-5.3-speculator.dspark","num_speculative_tokens":8,"method":"dspark"}'""",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[mods]
commands = [
    # r37 tree: the draft SlidingWindowSpec inherits the target sparse-MLA
    # shared-page padding (cache_config.skip_page_size_padded) and the KV
    # layout assert fails (page_size_padded < unpadded bf16 draft page).
    # The patcher lives on the shared HF cache (sm121-patches/) and is
    # idempotent per boot; repo copy: patches/patch_dspark_draft_page.py.
    "python3 /root/.cache/huggingface/sm121-patches/patch_dspark_draft_page.py",
]

[env]

# R31 lock: 16 NCCL channels (their qualified runtime).
NCCL_NCHANNELS = "16"
HF_HUB_OFFLINE = "1"
# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for r27+
# lanes since the 2026-09-08 A/B (4x DCP4: decode +31..+90%, quality 160 vs
# 155; specs/tech-architecture/glm53-recipe-state.md). Needs the CX-7 RoCE
# fabric (all of ours); single-node serves skip it silently.
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
# Compile-OOM guard: first boot JIT-compiles the mxfp8 gemm module with 12
# parallel nvcc jobs (~3 GiB each) behind ~94 GiB of resident weights →
# cicc OOM-killed (2026-09-04). flashinfer's run_ninja reads MAX_JOBS.
MAX_JOBS = "2"
# NoPE-MLA rope-pad runtime gate (mod baked into the image)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
# Upstream launch-script env (memory + compile knobs)
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
VLLM_USE_STANDALONE_COMPILE = "1"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "2"

[[extra_mounts]]
host = "~/sm121-cache/glm53-jit-121f/b12x"
container = "/root/.cache/b12x"

# JIT caches persist per-node across serves (was: cold /root/.cache per
# container -> ~20 min flashinfer/torch recompile every serve). The host
# dirs are shared between lanes of the same arch — each cache
# self-namespaces by its engine version key internally.
[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"