recipes: glm53-flash-nvfp4-4p67

recipes/glm53-flash-nvfp4-4p67/recipe.toml

@@ -0,0 +1,116 @@
+# GLM-5.3-Flash NVFP4-4p67 — local-inference-lab HIGHER-FIDELITY quant (vLLM)
+# — 2× DGX Spark, TP=2, MTP-3.
+#
+# "4p67" ≈ 4.67 bits/weight, KLD ~0.04 vs bf16 — the lab's fidelity-grade
+# variant of GLM-5.3-Flash-NVFP4 (our glm53-flash-nvfp4-lab recipe is the
+# workhorse). Quant recipe per the model card:
+#   main model: routed experts NVFP4 (calibrated, ships A4 input scales)
+#               + attention/shared/indexer MXFP8
+#   MTP:        routed experts NVFP4 A16-only + attention/shared MXFP8
+# hf_quant_config declares the routed experts as quant_algo=W4A16_NVFP4 under
+# MIXED_PRECISION — i.e. they execute via the Marlin W4A16 GEMM (bf16
+# activations), not the A4 path.
+# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4-4p67
+#
+# IMAGE: the card says "should be used with local-inference-lab/vllm branch
+# dev/jovian-judgement" — every image they publish is amd64-only (RTX PRO
+# 6000 world, and the fork's model.py hard-imports the b12x backend). Our
+# day-0 lab base already carries the W4A16/MIXED_PRECISION/MXFP8 modelopt
+# methods; the ONE thing 4p67 additionally needs is fused-projection shard
+# resolution: it is the first glm5next checkpoint with quantized attention
+# projections, and without `packed_modules_mapping` on the model classes the
+# fused KDA/MLA/indexer linears load unquantized (KeyError
+# in_proj_qkvbfg_a.weight_scale after ~12 min of weight loading). So:
+# glm53-flash-nvfp4-4p67:local = lab image + patch_4p67.py (5-line class
+# attr on model.py + mtp.py, mirrors the fork's resolution path).
+#
+# Boot markers: weight load ~93 GiB/node (≈700 s) → "GPU KV cache size:
+# 1,164,369 tokens" → "Application startup complete" with the MTP speculator
+# attached. quantprobe lines here show algo=MXFP8 for attention projections
+# and W4A16_NVFP4 for routed experts (lab quant showed algo=None for attn —
+# that difference is the whole point of this image). Watch SpecDecoding
+# acceptance (MTP-3, target len ~3).
+#
+# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
+# :"low"} — omitted/other values select MAX reasoning effort (empty content on
+# short max_tokens).
+#
+# PREREQUISITES: weights cached on both nodes:
+#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4-4p67
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4-4p67 (vLLM, 2x Spark, MTP-3)"
+docker_image = "glm53-flash-nvfp4-4p67:local"
+docker_image_dir = "glm53-flash-nvfp4-4p67"
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4-4p67"
+parent_model = "zai-org/GLM-5.3-Flash"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
+    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
+    # page cache routinely leave ~108.5 -> flaky
+    # "Free memory ... less than desired" boot deaths (this was the class
+    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
+    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
+    # allocated (weights ~93 + pinned KV 10 + activations).
+    "--gpu-memory-utilization", "0.82",
+    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
+    # ~93 GiB weights/node; pinning also keeps warmup off the swap path
+    "--kv-cache-memory-bytes", "10737418240",
+    "--kv-cache-dtype", "fp8_ds_mla",
+    "--block-size", "256",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    "--disable-custom-all-reduce",
+    # Proven lab boot: no flashinfer autotune, no CuTe-DSL warmup
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    # MTP-3 draft (MTP experts are W4A16_NVFP4 in this checkpoint; needs the
+    # modelopt.patch MTP-namespace fix baked into the image)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":3}'""",
+    "--language-model-only",
+    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
+    # (lab quant: graphs+MTP 89 vs eager 92). On the 4p67 quant the bug does
+    # NOT reproduce (measured 2026-08-29: graphs 88 vs eager 87, tg within
+    # ±5%) — but graphs buy nothing either, so eager stays: measurably free,
+    # boots ~2 min faster, immune to the bug class. See
+    # specs/tech-architecture/glm53-recipe-state.md for the A/B table.
+    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
+    # graphs on = 89/100, eager = 92/100). Same build → same guard here;
+    # re-A/B only if the base image is ever refreshed past the fix.
+    "--enforce-eager",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "4"