launcher/recipes/glm53-flash-nvfp4-4p67/recipe.toml

# GLM-5.3-Flash NVFP4-4p67 — local-inference-lab HIGHER-FIDELITY quant (vLLM)
# — 2× DGX Spark, TP=2, MTP-3.
#
# "4p67" ≈ 4.67 bits/weight, KLD ~0.04 vs bf16 — the lab's fidelity-grade
# variant of GLM-5.3-Flash-NVFP4 (our glm53-flash-nvfp4-lab recipe is the
# workhorse). Quant recipe per the model card:
#   main model: routed experts NVFP4 (calibrated, ships A4 input scales)
#               + attention/shared/indexer MXFP8
#   MTP:        routed experts NVFP4 A16-only + attention/shared MXFP8
# hf_quant_config declares the routed experts as quant_algo=W4A16_NVFP4 under
# MIXED_PRECISION — i.e. they execute via the Marlin W4A16 GEMM (bf16
# activations), not the A4 path.
# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4-4p67
#
# IMAGE: the card says "should be used with local-inference-lab/vllm branch
# dev/jovian-judgement" — every image they publish is amd64-only (RTX PRO
# 6000 world, and the fork's model.py hard-imports the b12x backend). Our
# day-0 lab base already carries the W4A16/MIXED_PRECISION/MXFP8 modelopt
# methods; the ONE thing 4p67 additionally needs is fused-projection shard
# resolution: it is the first glm5next checkpoint with quantized attention
# projections, and without `packed_modules_mapping` on the model classes the
# fused KDA/MLA/indexer linears load unquantized (KeyError
# in_proj_qkvbfg_a.weight_scale after ~12 min of weight loading). So:
# glm53-flash-nvfp4-4p67:local = lab image + patch_4p67.py (5-line class
# attr on model.py + mtp.py, mirrors the fork's resolution path).
#
# Boot markers: weight load ~93 GiB/node (≈700 s) → "GPU KV cache size:
# 1,164,369 tokens" → "Application startup complete" with the MTP speculator
# attached. quantprobe lines here show algo=MXFP8 for attention projections
# and W4A16_NVFP4 for routed experts (lab quant showed algo=None for attn —
# that difference is the whole point of this image). Watch SpecDecoding
# acceptance (MTP-3, target len ~3).
#
# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
# :"low"} — omitted/other values select MAX reasoning effort (empty content on
# short max_tokens).
#
# PREREQUISITES: weights cached on both nodes:
#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4-4p67

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-4p67 (vLLM, 2x Spark, MTP-3)"
docker_image = "glm53-flash-nvfp4-4p67:local"
docker_image_dir = "glm53-flash-nvfp4-4p67"
model = "local-inference-lab/GLM-5.3-Flash-NVFP4-4p67"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
    # page cache routinely leave ~108.5 -> flaky
    # "Free memory ... less than desired" boot deaths (this was the class
    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
    # allocated (weights ~93 + pinned KV 10 + activations).
    "--gpu-memory-utilization", "0.82",
    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
    # ~93 GiB weights/node; pinning also keeps warmup off the swap path
    "--kv-cache-memory-bytes", "10737418240",
    "--kv-cache-dtype", "fp8_ds_mla",
    "--block-size", "256",
    "--max-model-len", "524288",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    "--disable-custom-all-reduce",
    # Proven lab boot: no flashinfer autotune, no CuTe-DSL warmup
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    # MTP-3 draft (MTP experts are W4A16_NVFP4 in this checkpoint; needs the
    # modelopt.patch MTP-namespace fix baked into the image)
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":3}'""",
    "--language-model-only",
    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
    # graphs on = 89/100, eager = 92/100). Same build → same guard here;
    # re-A/B only if the base image is ever refreshed past the fix.
    "--enforce-eager",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
# NoPE-MLA rope-pad runtime gate (mod baked into the image)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "4"