launcher/recipes/glm53-full-mxfp4-8x/recipe.toml

# GLM-5.3 (full) MXFP4 — 8x DGX Spark @512K, no DCP.
#
# P2.1 of the 2026-09-18 night: the full model's first 8x trial. The 4x
# 512K state is blocked by the DCP>1 sparse-decode crash
# (deepseek_v32/attention.py:596, r35+r37); at 8x no DCP is needed:
#
#   weights  403.4 GB / 8 = 50.4 GB/node
#   budget   GMU 0.85 x 121.63 = 103.4 GiB
#   KV pool  103.4 - 50.4 - 12 (process) = ~41 GiB/rank
#   512K req 26.3 GiB  -> 1.5x concurrency (768K would be ~39.5, tight)
#
# Same image/contract as the qualified 4x 131K state (marlin linear, B12X
# attention, instanttensor, block 256, RoCE allreduce) minus DCP and the KV
# pin (the pool is GMU-driven at 8x). The 4x 131K anchor (dev67 no-spec):
# 158/176, 934 pp / 19.0 tg / 2,124 ms.
engine = "vllm"
name = "GLM-5.3 (full) MXFP4 (r37 img, RedHatAI ckpt, 8x DCP1 @512K)"
# Image: the r37 DS4.1 build (2026-09-14, night shift) - the r35 tree's DCP4
# sparse-INDEXER path asserts interleave==1 and then crashes in the DCP sparse
# decode reshape during graph capture ("shape '[16, 16, 512]' is invalid for
# input of size 524288" - the x4 DCP factor in the attention.py _sparse_
# decode). r37 is the newer DS4.1 lock whose DCP sparse-MLA path is the
# qualified one (same tree family as our DCP2/DFlash2 work); the GlmMoeDsa
# arch + MTP + b12x ship in the r37 image (verified 2026-09-13).
docker_image = "jovian-judgement:r37-ds41"
docker_image_dir = "jovian-judgement"
# NOTE: build_profile must stay a TOP-LEVEL key — a bare key after
# [[extra_mounts]] would parse into the last mount table and be silently
# ignored (2026-09-08).
build_profile = "r37-ds41"
entrypoint = ["vllm", "serve"]
model = "RedHatAI/GLM-5.3-MXFP4"
parent_model = "zai-org/GLM-5.3"
required_nodes = 8
tensor_parallel_size = 8
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # R31/r35 delta (their qualified locks): adaptive prefill/decode
    # compute-time sharing with a fixed 40% prefill floor.
    "--prefill-compute-share", "0.4",
    # Stable served name — quant-agnostic (shared across glm53 lanes)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # Admission guardrail only (KV is pinned below): 0.85×121.63 = 103.4 GiB.
    # 0.88 (107.0) is the Flash lane's value and is too tight here — the full
    # model's weights leave ~24 GiB before the engine's own ~12 GiB.
    "--gpu-memory-utilization", "0.85",
    # No KV pin: the pool is GMU-driven at 8x (~41 GiB/rank, see header).
    # No --kv-cache-dtype: let the r35 fork pick its native default for
    # GlmMoeDsa attention (the Flash lane's explicit fp8/fp8_ds_mla spellings
    # both failed auto-selection on this build, and those were Flash-MLA
    # knobs). The card's --kv-cache-dtype fp8_e4m3 is the fallback spelling
    # if the pool needs to shrink.
    "--attention-backend", "B12X",
    # MXFP4 kernel routing (measured on the first boot, 2026-09-13 ~23:10 UTC):
    # - MoE: the compressed-tensors W4A4-MXFP4 scheme picks MarlinExperts on
    #   its own (A16 dequant GEMM - correct on GB10). The --moe-backend flag
    #   does not reach that scheme (verified inert: it still logged "Using
    #   MarlinExperts for MXFP4 MoE" with the flag set), so we do not set it.
    # - LINEAR: the default priority [FlashInfer, Marlin, Humming, B12x]
    #   starts with the FlashInfer kernel, whose is_supported() wrongly
    #   accepts capability 121 (the numeric has_device_capability(100) check
    #   passes on GB10) while the cute-dsl mm_fp4 GEMM it calls rejects it at
    #   runtime with:
    #     BackendSupportedError: mm_fp4 does not support backend 'cute-dsl'
    #     with capability 121
    #   (the first probe boot died there, kv_b_proj dequant in mla_attention.py).
    # --linear-backend b12x forces the B12xMxFp4LinearKernel - "MXFP4 linear
    # through the native B12X SM120 dense GEMM" - the right path on GB10,
    # verified per component: is_device_capability_family(120)=True, b12x
    # blockscaled.is_supported()=True, and the cute-dsl input quantize
    # (flashinfer_mxfp4_quantize) works on capability 121; only the cute-dsl
    # mm_fp4 GEMM is unsupported there. Boot log to look for:
    #   "Using B12xMxFp4LinearKernel for MXFP4 GEMM"
    # (The lab's GLM-5.1 runbook describes the same kernel family for the
    # quant_method=mxfp4 ingestion path - this checkpoint goes through the
    # compressed-tensors path, which is why --moe-backend was the wrong lever.)
    # First attempt: b12x (the native SM120 GEMM) - correct selection, but the
    # unconditional b12x warmup (b12x_warmup.py, no config gate) crashes on
    # this checkpoint's layer shapes during kernel compile:
    #   RuntimeError: Expected source and destination tensors to have the same
    #   rank, but got 3 and 4  (b12x/gemm/blockscaled/_linear.py dense_gemm,
    #   via _apply_b12x_mxfp4_linear on a synthetic zero source)
    # so the mxfp4 LINEAR layers run on Marlin (weight-only A16 dequant GEMM -
    # the same family as the MarlinExperts the MoE scheme picked). Slower than
    # native FP4 for the attention projections, but the MoE experts (the
    # dominant cost) are Marlin either way. The b12x warmup rank bug is
    # filed-worthy for the lab (RedHatAI GLM-5.3-MXFP4 on GB10/sm_121a).
    "--linear-backend", "marlin",
    # pure I/O loader (kept across every GLM config)
    "--load-format", "instanttensor",
    "--block-size", "256",
    "--max-model-len", "524288",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup (the
    # autotune scratch at 512K shapes does not fit the headroom, 2026-08-28).
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    "--language-model-only",
    # NO --speculative-config on the first boot — see the header. MTP head is
    # in the checkpoint (num_nextn_predict_layers=1); add mtp k=3 after a
    # clean boot + bench.
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]

# R31 lock: 16 NCCL channels (their qualified runtime).
NCCL_NCHANNELS = "16"
HF_HUB_OFFLINE = "1"
# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for r27+
# lanes since the 2026-09-08 A/B (4x DCP4: decode +31..+90%, quality 160 vs
# 155; specs/tech-architecture/glm53-recipe-state.md). Needs the CX-7 RoCE
# fabric (all of ours); single-node serves skip it silently.
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
# Compile-OOM guard: first boot JIT-compiles the mxfp8 gemm module with 12
# parallel nvcc jobs (~3 GiB each) behind ~94 GiB of resident weights →
# cicc OOM-killed (2026-09-04). flashinfer's run_ninja reads MAX_JOBS.
MAX_JOBS = "2"
# NoPE-MLA rope-pad runtime gate (mod baked into the image)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
# Upstream launch-script env (memory + compile knobs)
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
VLLM_USE_STANDALONE_COMPILE = "1"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "2"

[[extra_mounts]]
host = "~/sm121-cache/glm53-jit-121f/b12x"
container = "/root/.cache/b12x"

# JIT caches persist per-node across serves (was: cold /root/.cache per
# container -> ~20 min flashinfer/torch recompile every serve). The host
# dirs are shared between lanes of the same arch — each cache
# self-namespaces by its engine version key internally.
[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"