launcher/recipes/glm53-full-nvfp4-spark-lab/recipe.toml

# GLM-5.3 (FULL, non-Flash) NVFP4-Spark — local-inference-lab quant, 4× DGX Spark, TP=4.
#
# Full GLM-5.3 (78 layers / hidden 6144 / 1M ctx, arch GlmMoeDsaForCausalLM)
# in the lab's GB10-tailored quant: GLM-5.3-NVFP4-Spark (~406 GB →
# ~101.5 GiB weights/node → 4x only; the plain NVFP4 build is 433 GB).
# MIXED_PRECISION checkpoint (NVFP4 routed + MXFP8 elsewhere) handled by the
# lab image's modelopt quant module.
#
# Image: glm53-flash-nvfp4-lab:local (same lab vLLM fork — it registers
# GlmMoeDsaForCausalLM at deepseek_v2.py, which implements SupportsEagle3,
# unlike the Flash GLM5Next target; that is why the lab's MXFP8 DFlash2
# drafter is the full-model pairing, not the Flash one).
#
# First run: plain (no spec) to prove the stack end-to-end, then add spec.
# KV pinned 4 GiB (weights leave ~17 GiB/node; pool is small by design —
# this is a capability/quality lane, not a throughput lane).
#
# PREREQUISITES: weights cached in the shared NAS HF cache:
#   models--local-inference-lab--GLM-5.3-NVFP4-Spark (406G, verified 2026-09-04)

engine = "vllm"
name = "GLM-5.3 (full) NVFP4-Spark-lab (vLLM, 4x Spark)"
docker_image = "glm53-flash-nvfp4-lab:local"
docker_image_dir = "glm53-flash-nvfp4-lab"
model = "local-inference-lab/GLM-5.3-NVFP4-Spark"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# required for the launcher's `--entrypoint sh -c` override path (mods below)
entrypoint = ["vllm", "serve"]

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
    # page cache routinely leave ~108.5 -> flaky
    # "Free memory ... less than desired" boot deaths (this was the class
    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
    # allocated (weights ~93 + pinned KV 10 + activations).
    "--gpu-memory-utilization", "0.82",
    # Pinned 4 GiB KV — full-model weights are ~101.5 GiB/node (406G/4),
    # far less headroom than the 92.7 GiB/node Flash lane; no MLA-specific
    # kv-cache-dtype/block-size here (GlmMoeDsa attention picks its own
    # native KV defaults — the fp8_ds_mla/256 pair was a Flash-MLA knob)
    "--kv-cache-memory-bytes", "4294967296",
    # 4 GiB KV pool holds ~79,616 tokens (engine-computed 2026-09-04);
    # 524288 would need 26.34 GiB — doesn't fit behind 101.5 GiB weights.
    # 79K context is far beyond anything the bench uses.
    "--max-model-len", "79616",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    "--disable-custom-all-reduce",
    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    # No spec: the lab DFlash2/MTP drafters are for the Flash model; first
    # full-model run runs plain to prove the stack, then add spec if clean
    # instanttensor was removed after the first full-model boot hung mid-load
    # (dgx2, 2026-09-04), then re-enabled for the parked retry run below
    # (flag at "--load-format", "instanttensor") with MAX_JOBS=2 + persistent
    # JIT caches as the anti-OOM guardrails.
    # NOTE: instanttensor + both mods seds ARE configured below (2026-09-04
    # retry) — the lane PARKED before that run happened: two sparse-MLA
    # engine crashes on first requests (masked_mha AttributeError +
    # forward_mha NotImplementedError), both hotfixed via [mods]; the
    # force-mqa + instanttensor boot never ran because the full-model work
    # was parked (dgx2 went down twice under this workload that day).
    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
    "--enforce-eager",
    # instanttensor loader (lab-fork feature; default loader is ~27 min here,
    # instanttensor ~2x faster when the NFS cooperates)
    "--load-format", "instanttensor",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[mods]
commands = [
    # Lab-fork bug: mla_attention.py:810 reads self.impl.masked_mha_available
    # on the sparse MLA impl, which never sets it -> AttributeError on the
    # first prefill step (engine crash, 2026-09-04). The attribute is
    # SM100-only anyway (gate: is_device_capability_family(100)); on GB10
    # (SM121) the masked-MHA path is off by design, so default to False.
    # 1-line hotfix until the lab image is rebuilt with the fix baked in.
    "sed -i \"s/and self\\.impl\\.masked_mha_available/and getattr(self.impl, 'masked_mha_available', False)/\" /usr/local/lib/python3.12/dist-packages/vllm/model_executor/layers/attention/mla_attention.py",
    # Second wall: the dense-MHA split routes prefill to impl.forward_mha,
    # which is an unimplemented stub on the SM120 sparse impl (engine crash
    # #2, 2026-09-04). vllm ships an intended escape hatch for exactly this:
    # attention_config.sparse_mla_force_mqa = True -> \"always use the MQA
    # path\" (the model's native sparse attention). No CLI flag exists for it
    # in this build, so set the documented default via sed.
    "sed -i \"s/sparse_mla_force_mqa: bool = False/sparse_mla_force_mqa: bool = True/\" /usr/local/lib/python3.12/dist-packages/vllm/config/attention.py",
]

[env]
HF_HUB_OFFLINE = "1"
# instanttensor loader knobs (lab image bakes the loader; retrying after the
# 2026-09-04 dgx2 hang — if that node dies mid-load again, drop this)
INSTANTTENSOR_BACKEND = "BUFFERED"
INSTANTTENSOR_CONCURRENCY = "1"
INSTANTTENSOR_IO_DEPTH = "3"
INSTANTTENSOR_BUFFER_SIZE = "67108864"
INSTANTTENSOR_CHUNK_SIZE = "8388608"
# NoPE-MLA rope-pad runtime gate (mod baked into the image)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
# Upstream launch-script env (memory + compile knobs)
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "4"
# Compile-OOM fix: first boot JIT-compiles the mxfp8 gemm module (12
# nvcc/cicc jobs ~3 GB each) behind 101.5 GiB of weights -> cicc OOM-killed
# (aiden lane, 2026-09-04). flashinfer run_ninja reads MAX_JOBS -> cap at 2.
MAX_JOBS = "2"

# Persistent JIT caches (containers run with --rm; without these every boot
# re-runs the ~10 min nvcc compile of the mxfp8 gemm module)
[[extra_mounts]]
host = "~/.cache/glm53-full-flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/.cache/glm53-full-triton"
container = "/root/.triton"