launcher/recipes/glm53-flash-nvfp4-spark/recipe.toml

# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
#
# local-inference-lab mixed quant: NVFP4 routed experts (layers 3–44,
# group-16) + MXFP8 MTP experts (layer 45) + W4A4 input scales. Best agent
# quant so far: tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8).
# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
#        (~186 GB → 92.72 GiB weights/node; KV pinned at 10 GiB, see below)
#
# Image: glm53-flash-nvfp4-lab:local (docker-images/glm53-flash-nvfp4-lab/ —
# day-0 arm64 image + NoPE-MLA mod + the four FujitsuPolycom lab-checkpoint
# patches: model naming shim, modelopt MTP namespace fix, persistent_topk
# CC-12.x guards). Their launch config is mirrored below.
# PROVENANCE: https://github.com/FujitsuPolycom/glm53-flash-tp2-spark
#             (Apache-2.0; see patches/UPSTREAM.md) + forum 381350 #41.
#
# Deviations from their launch script (intentional):
#   * no --kda-prefill-backend flashkda (needs their base-image patch — skipped)
#   * --kv-cache-dtype fp8_ds_mla instead of `fp8` (same packed layout; this
#     spelling is what the NoPE-MLA mod booted with on the LibertAIDAI stack)
#   * --language-model-only + no mm-renderer patch/env (text-only)
#   * no bundled NCCL/LD_PRELOAD, no NCCL_* fabric block (our switched fabric
#     is configured via [nccl.ds4] + launcher GID auto-detect)
#   * no --moe-backend marlin: their proven stack uses the default backend for
#     THIS quant and reports clean evals. Sanity-check output for token loops
#     anyway (the silent FP4 MoE corruption gotcha) before trusting a new node.
#   * flashinfer autotune disabled via --kernel-config (their proven boot);
#     VLLM_ENGINE_READY_TIMEOUT_S stays high regardless.
#
# Boot success markers (engine log, in order):
#   "Model loading took 92.72 GiB" → "[quantprobe] ... algo=MXFP8" (MTP fix
#   active; algo=None means broken) → "GPU KV cache size: 1,164,369 tokens" →
#   "Application startup complete". ~700 s sequential weight load.
#
# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
# :"low"} — omitted/other values select MAX reasoning effort (empty content on
# short max_tokens).
#
# Optional: instanttensor loader (~30 s weight load vs ~700 s):
#   FROM glm53-flash-nvfp4-lab:local + pip install --no-deps --no-build-isolation
#   instanttensor==0.1.9, then add --load-format instanttensor (--no-deps is
#   required: resolution would downgrade the image's NCCL).
#
# PREREQUISITES: weights cached on both nodes:
# GLM-5.3-Flash NVFP4-Spark — local-inference-lab quant with the speculative
# layers embedded — 2× DGX Spark, TP=2.
#
# local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (rev 06972cdee7280cec849aedf05a02062b3e870246):
# NVFP4 routed experts (layers 3+, group-16) + NVFP4 MTP expert layer in the
# separate model-mtp-0000x.safetensors shards (mtp_tokens: 5 per their
# lil.yaml), amax/nonexpert/inputscales side shards, Glm5NextForConditionalGeneration.
# THE "DSpark speculative embedded" checkpoint — no separate DFlash2 download
# needed; lil.yaml: default_speculator mtp, dflash2_tokens 7 (r12 doc fr: their
# standalone-Q: MTP:3 254 tok/s CC1, 333 Sieve, 94 steps/s).
#
# Image: glm53-flash-nvfp4-lab:local (same proven day-0 arm64 stack + NoPE-MLA
# mod + modelopt MTP namespace patch used by the 92/100 lab lane). The MTP
# layer here is NVFP4 (not MXFP8) — the quantprobe marker may print
# algo=NVFP4/- different; algo=None = patch missed (fall back to 4p67 image).
#
# Deviations from lab (unchanged mechanics): same fabric/env/GMU/KV pinning.
# lil.yaml launch mirrors: block 256, disable_flashinfer_autotune, mm data
# mode — our text-only flags identical to glm53-flash-nvfp4-lab.
#
# Boot markers: "Model loading took ~XX GiB" → "[quantprobe] ..." →
# "GPU KV cache size" → "Application startup complete".
#
# PREREQUISITES: weights on the NAS hub (175 GiB): hf download local-inference-lab/GLM-5.3-Flash-NVFP4-Spark

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-Spark (vLLM, 2x Spark, MTP-5)"
docker_image = "glm53-jovian-r12:local"  # r12 fork build (see docker-images/glm53-jovian-r12)
docker_image_dir = "glm53-jovian-r12"
# Required: the clean r12 image follows the solar contract (ENTRYPOINT []
# + CMD bash), so the recipe carries the entrypoint (the old day-0-based
# r12 image had ["vllm","serve"] baked in, which is why this was absent
# until the clean rebuild).
entrypoint = ["vllm", "serve"]
model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
    # page cache routinely leave ~108.5 -> flaky
    # "Free memory ... less than desired" boot deaths (this was the class
    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
    # allocated (weights ~93 + pinned KV 10 + activations).
    "--gpu-memory-utilization", "0.82",
    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
    # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
    "--kv-cache-memory-bytes", "10737418240",
    # No --kv-cache-dtype: let the r12 fork pick its default (the explicit
    # fp8 / fp8_ds_mla both failed auto-selection on this build).
    # The GLM5Next model wires the B12X sparse-MLA backend (sm_120/sm_121)
    # when the global backend is B12X (glm5next/nvidia/attention.py:
    # _select_sparse_backend); the sm12x auto-priority list never offers it
    # (FLASHINFER_MLA_SPARSE_SM120 only supports head 576, not GLM's 512).
    "--attention-backend", "B12X",
    "--block-size", "256",
    "--max-model-len", "524288",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    "--disable-custom-all-reduce",
    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    # Built-in MTP draft, 3 tokens (the r12-qualified "mtp:3" mode — their
    # x86 numbers: 254 tok/s CC1, 2.37 accept; the checkpoint embeds up to
    # 5 via the model-mtp-0000x shards but 3 is the verified depth). This is
    # the checkpoint's own MTP head (Glm5NextMTPModel), NOT dspark — dspark
    # is the DeepSeek-V4 family's mechanism; GLM's other r12 mode is
    # dflash2:7 with a separate trained draft checkpoint.
    # The draft model's vllm_config takes its attention backend from the
    # SPECULATIVE config, not the global flag (eagle/utils.py:
    # _make_eagle_draft_vllm_config) — B12X here mirrors the main model's.
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'""",
    "--language-model-only",
    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
    "--enforce-eager",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
# FlashInfer JIT (mxfp8_gemm_cutlass_sm120, ~12 kernels) compiles at first
# boot with 12 parallel nvcc jobs -> OOM-killed (code 137) on GB10's
# 121 GB unified while the 93 GB weights + 10 GB KV pin are resident.
# MAX_JOBS caps ninja parallelism so the JIT fits in the remaining ~20 GB.
MAX_JOBS = "2"
# NoPE-MLA rope-pad runtime gate (mod baked into the image)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
# Upstream launch-script env (memory + compile knobs)
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "4"