launcher/recipes/glm53-flash-nvfp4-4p67/recipe.toml

# GLM-5.3-Flash NVFP4-4p67 — local-inference-lab HIGHER-FIDELITY quant (vLLM)
# — 2× DGX Spark, TP=2, MTP-3.
#
# "4p67" ≈ 4.67 bits/weight, KLD ~0.04 vs bf16 — the lab's fidelity-grade
# variant of GLM-5.3-Flash-NVFP4 (our glm53-flash-nvfp4-lab recipe is the
# workhorse). Quant recipe per the model card:
#   main model: routed experts NVFP4 (calibrated, ships A4 input scales)
#               + attention/shared/indexer MXFP8
#   MTP:        routed experts NVFP4 A16-only + attention/shared MXFP8
# hf_quant_config declares the routed experts as quant_algo=W4A16_NVFP4 under
# MIXED_PRECISION — i.e. they execute via the Marlin W4A16 GEMM (bf16
# activations), not the A4 path.
# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4-4p67
#
# IMAGE: the card says "should be used with local-inference-lab/vllm branch
# dev/jovian-judgement" — every image they publish is amd64-only (RTX PRO
# 6000 world). Our glm53-flash-nvfp4-lab:local base is the same day-0 family
# and verifiably carries the whole required surface (W4A16_NVFP4 linear+MoE
# methods, MIXED_PRECISION config, ModelOptMxFp8LinearMethod all present;
# the FujitsuPolycom patches ride on top), so this quant runs on the SAME
# image as the lab quant — zero extra build. Verified-by-inspection
# 2026-08-29; first boot must pass the token-loop sanity check.
#
# Boot markers (engine log): weight load ~93 GiB/node (≈700 s) →
# "GPU KV cache size: 1,164,369 tokens" → "Application startup complete"
# with the MTP speculator attached. The lab-quant "[quantprobe] ...
# algo=MXFP8" line is NOT expected: this checkpoint's MTP experts are
# W4A16_NVFP4. Watch SpecDecoding acceptance lines (target: len ~3, same
# MTP-3 draft path as the lab quant).
#
# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
# :"low"} — omitted/other values select MAX reasoning effort (empty content on
# short max_tokens).
#
# PREREQUISITES: weights cached on both nodes:
#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4-4p67

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-4p67 (vLLM, 2x Spark, MTP-3)"
docker_image = "glm53-flash-nvfp4-lab:local"
docker_image_dir = "glm53-flash-nvfp4-lab"
model = "local-inference-lab/GLM-5.3-Flash-NVFP4-4p67"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GMU is superseded by the pinned KV pool; 0.90 as upstream boots with it
    "--gpu-memory-utilization", "0.90",
    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
    # ~93 GiB weights/node; pinning also keeps warmup off the swap path
    "--kv-cache-memory-bytes", "10737418240",
    "--kv-cache-dtype", "fp8_ds_mla",
    "--block-size", "256",
    "--max-model-len", "524288",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    "--disable-custom-all-reduce",
    # Proven lab boot: no flashinfer autotune, no CuTe-DSL warmup
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    # MTP-3 draft (MTP experts are W4A16_NVFP4 in this checkpoint; needs the
    # modelopt.patch MTP-namespace fix baked into the image)
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":3}'""",
    "--language-model-only",
    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
    # graphs on = 89/100, eager = 92/100). Same build → same guard here;
    # re-A/B only if the base image is ever refreshed past the fix.
    "--enforce-eager",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
# NoPE-MLA rope-pad runtime gate (mod baked into the image)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "4"