launcher/recipes/glm53-flash-r35-4x-dflash2/recipe.toml

# GLM-5.3-Flash r35 + DFlash2 K7 draft — A/B vs the r35 MTP3 prod recipe.
#
# Clone of recipes/glm53-flash-r35-4x-dcp4 with ONLY the speculative-config
# swapped: MTP3 -> DFlash2 (lab's block-diffusion ~1B draft, offline MXFP8,
# on the shared NAS from the 2026-08-28 port work). Same image
# (jovian-judgement:r35-glm53, checksum 8bb87348 — DFlash2 machinery is in-tree),
# same 4x DCP4 shape, same KV/scheduler pins. Night-shift A/B: stop the MTP3
# prod, serve this, bench (dev65, seed 42, hardmode, c1/c2) vs the 163/176
# r35 anchor, decide, restore.
#
# R35 is lab-qualified for DFlash2 at TP4/DCP1 AND TP4/DCP4 (RTX quartet;
# C1/C8, cold 32K prefill). GB10 is our own claim: the 08-29 2x blocker
# (draft-KV page unification on the day-0 base) is void here — the r35 tree
# has the dedicated dflash_attn backend + b12x built for GB10.

engine = "vllm"
name = "GLM-5.3-Flash NVFP4 (r35 img, lab ckpt, 4x DCP4, DFlash2 K7)"
docker_image = "jovian-judgement:r35-glm53"  # full r35 lock (build_profile) over the jovian-judgement image tree
docker_image_dir = "jovian-judgement"
# NOTE: build_profile must stay a TOP-LEVEL key — a bare key after
# [[extra_mounts]] would parse into the last mount table and be silently
# ignored (2026-09-08).
# FULL r28 source lock (2026-09-06 cut, all three forks): the named version
# profile docker-images/jovian-judgement/profiles/r28.env — vLLM
# 62f6385c + b12x 7216f6b9 (21 commits over r27: GLM/DeepSeek loader fixes,
# MoE perf) + LMCache 934c287a (3 MP-transfer fixes). Merged over the image
# dir's versions.env and folded into the build checksum, so this is a
# distinct image (tag jovian-judgement:r28-glm53) sharing the r27 build tree.
# Clone-at-pin fetches each commit directly; the Dockerfile gates HEAD + tree
# hash per repo. Both r28 branches are strictly ahead of their r27/r26 pins
# (0 behind) — a clean superset. The r28 vLLM tree has NO requirements/cmake
# changes vs r27 (verified via the GitHub compare API, 13 commits / 29 files),
# so the fetch-deps.sh dep gates still hold.
build_profile = "r35"
# Required: the clean r12 image follows the solar contract (ENTRYPOINT []
# + CMD bash), so the recipe carries the entrypoint (the old day-0-based
# r12 image had ["vllm","serve"] baked in, which is why this was absent
# until the clean rebuild).
entrypoint = ["vllm", "serve"]
model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
parent_model = "zai-org/GLM-5.3-Flash"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # R31 delta (their r28.1-r31 locks): adaptive prefill/decode
    # compute-time sharing with a fixed 40% prefill floor.
    "--prefill-compute-share", "0.4",
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
    # page cache routinely leave ~108.5 -> flaky
    # "Free memory ... less than desired" boot deaths (this was the class
    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
    # allocated (weights ~93 + pinned KV 10 + activations).
    # 0.88 (not 0.90): the startup admission gate is free-mem >= GMU*total
    # (107.0 vs 109.5 GiB); ~12 GiB of starting vLLM processes + docker
    # daemon can push the 0.90 check under the line (2026-09-08, #48140).
    # KV is explicitly pinned (--kv-cache-memory-bytes) so capacity is unchanged.
    "--gpu-memory-utilization", "0.88",
    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
    # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
    "--kv-cache-memory-bytes", "17179869184",  # 16 GiB: 4x NVFP4 has the headroom (47.1 GiB weights/node vs 108 usable); 2x stays 10
    "--decode-context-parallel-size", "4",
    "--cp-kv-cache-interleave-size", "4",
    # No --kv-cache-dtype: let the r12 fork pick its default (the explicit
    # fp8 / fp8_ds_mla both failed auto-selection on this build).
    # The GLM5Next model wires the B12X sparse-MLA backend (sm_120/sm_121)
    # when the global backend is B12X (glm5next/nvidia/attention.py:
    # _select_sparse_backend); the sm12x auto-priority list never offers it
    # (FLASHINFER_MLA_SPARSE_SM120 only supports head 576, not GLM's 512).
    "--attention-backend", "B12X",
    # pure I/O loader (kept across every config; 171 GB in ~5.5 min vs ~16 NFS)
    "--load-format", "instanttensor",
    # [R17 FULL-CLUSTER TEST] lil manifest backends (quality gate: hardmode
    # must stay ≥88; r12-era this cluster dropped it to 74 — r17 may carry
    # the fix through — one boot decides).
    "--block-size", "256",
    # FINAL BISECT (2026-09-03, hardmode 176pt, r17 img):
    #   knobs-only 87: d0 c1 24.1, d8192 23.8
    #   +moe b12x  88: d0 c1 23.5, d8192 26.6  <- SHIP (2 pts d8192)
    #   +linear b12x 44: W4A4 linear kernel destroys numerics (param-precision 0%)
    #   +align     88: but d0 c1 18.6 (-5) — perf-negative, dropped
    # (all three together = 44-74; the lab needs its private packed package)
    "--max-model-len", "524288",
    "--max-num-seqs", "16",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    # NO --disable-custom-all-reduce here: in the r27 fork the RoCEnante
    # adapter is gated on the custom-all-reduce enable flag
    # (use_roce_allreduce = use_custom_allreduce && VLLM_ENABLE_ROCE_ALLREDUCE).
    # On 2x/4x 1-GPU-node clusters vLLM's own CUDA-IPC custom AR is functionally
    # inert (no cross-node P2P), so dropping the flag only opens the b12x path;
    # the b12x-PCIe branch stays off (VLLM_ENABLE_PCIE_ALLREDUCE defaults False).
    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    # DFlash2 A/B: lab-qualified DFlash2 K7 draft (block-diffusion, ~1B,
    # offline MXFP8 — local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8,
    # already on the shared NAS). Everything else is byte-identical to the
    # r35 MTP3 prod recipe, so the bench delta isolates the speculator.
    # R35 tree ships the DFlash2 machinery natively (dflash proposer,
    # DFlash2DraftModel, dedicated dflash_attn graph-safe split-KV
    # backend); K7 is lab-qualified at TP4/DCP1 and TP4/DCP4 (RTX quartet).
    # Draft attention: left to the tree default (the r35 wrapper maps
    # DFLASH_ATTENTION_BACKEND, defaulting to the dflash path); fallbacks
    # if boot fails: "FLASH_ATTN" (card's explicit choice) or "B12X".
    # kv_cache_dtype auto = BF16 draft pages (the card's DFLASH_KV_CACHE_DTYPE).
    "--speculative-config", """
'{"method":"dflash","model":"local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8","num_speculative_tokens":7,"kv_cache_dtype":"auto"}'""",
    "--language-model-only",
    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
    # graphs moved BEFORE tool-call-parser; enforce-eager removed for the
    # r17-graphs test (bisect A2 was 88/100 eager; graphs+MTP fix is public
    # in r17 refs? verify by bench).
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]

# R31 lock: 16 NCCL channels (their qualified runtime).
NCCL_NCHANNELS = "16"
HF_HUB_OFFLINE = "1"
# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for r27
# lanes since the 2026-09-08 A/B (4x DCP4: decode +31..+90%, quality 160 vs
# 155, prefill within noise; specs/tech-architecture/glm53-recipe-state.md,
# history/night-shifts/2026-09-07.md). Needs the CX-7 RoCE fabric (all of
# ours); single-node serves skip it silently. Caps default 2MB AR / 16MB AG.
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
# FlashInfer JIT (mxfp8_gemm_cutlass_sm120, ~12 kernels) compiles at first
# boot with 12 parallel nvcc jobs -> OOM-killed (code 137) on GB10's
# 121 GB unified while the 93 GB weights + 10 GB KV pin are resident.
# MAX_JOBS caps ninja parallelism so the JIT fits in the remaining ~20 GB.
MAX_JOBS = "2"
# NoPE-MLA rope-pad runtime gate (mod baked into the image)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
# Upstream launch-script env (memory + compile knobs)
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
VLLM_USE_STANDALONE_COMPILE = "1"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "2"

[[extra_mounts]]
host = "~/sm121-cache/glm53-jit-121f/b12x"
container = "/root/.cache/b12x"

# JIT caches persist per-node across serves (was: cold /root/.cache per
# container -> ~20 min flashinfer/torch recompile every serve). The host
# dirs are shared between lanes of the same arch — each cache
# self-namespaces by its engine version key internally.
[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"