launcher/recipes/glm53-flash-nvfp4-spark/recipe.toml

# GLM-5.3-Flash NVFP4-Spark — local-inference-lab quant, r12 fork — 2× DGX
# Spark, TP=2. The lab's own "DSpark" checkpoint (speculative MTP layers
# embedded, no separate draft download).
#
# local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (rev 06972cdee7280cec849aedf05a02062b3e870246):
# NVFP4 routed experts (layers 3+, group-16) + NVFP4 MTP expert layer in the
# separate model-mtp-0000x.safetensors shards (mtp_tokens: 5 per their
# lil.yaml), amax/nonexpert/inputscales side shards, Glm5NextForConditionalGeneration.
#
# Image: glm53-jovian-r12:local (docker-images/glm53-jovian-r12/) — clean
# solar-pattern build of the r12 stack: local-inference-lab/vllm
# dev/jovian-judgement (base 54f6e98 + PR 550/552/553/554/555/557 by SHA) +
# b12x 139e040 + PR 260/267/268/269/270, torch 2.13 cu130, flashinfer 0.6.17.
#
# Launch flags mirror the lab's lil launcher policy for this family
# (local-inference-lab/lil, configs/models/_bases.yaml glm base + lil.yaml):
# attention B12X, linear/moe backend b12x, MTP spec with moe_backend b12x,
# instanttensor load, block 256, max-num-seqs 8, batched 4096, glm45/glm47
# parsers, mamba-cache-mode align.
#
# Boot success markers: "Model loading took ~93 GiB" (instanttensor: ~1 min)
# → "GPU KV cache size" → "Application startup complete".
#
# JIT cache: the flashinfer/b12x JIT caches are MOUNTED from
# ~/sm121-cache/glm53-jit-121f (see [[extra_mounts]] at the bottom, same
# pattern as the dsv4 recipe) — pre-baked on both nodes so boot skips the
# ~15 min first-boot JIT (and the nvcc OOM risk; MAX_JOBS=2 stays as guard).
#
# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
# :"low"} — omitted/other values select MAX reasoning effort (empty content on
# short max_tokens).
#
# PREREQUISITES: weights cached on both nodes:
#   ~/dsv4vision-hf is for DeepSeek — for GLM the launcher resolves the repo
#   from the node's HF cache (NAS hub: 175 GiB) or a local NVMe copy.
#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4-Spark

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-Spark (vLLM, 2x Spark, MTP-5)"
docker_image = "glm53-jovian-r12:local"  # r12 fork build (see docker-images/glm53-jovian-r12)
docker_image_dir = "glm53-jovian-r12"
# Required: the clean r12 image follows the solar contract (ENTRYPOINT []
# + CMD bash), so the recipe carries the entrypoint (the old day-0-based
# r12 image had ["vllm","serve"] baked in, which is why this was absent
# until the clean rebuild).
entrypoint = ["vllm", "serve"]
model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
    # page cache routinely leave ~108.5 -> flaky
    # "Free memory ... less than desired" boot deaths (this was the class
    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
    # allocated (weights ~93 + pinned KV 10 + activations).
    "--gpu-memory-utilization", "0.82",
    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
    # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
    "--kv-cache-memory-bytes", "10737418240",
    # No --kv-cache-dtype: let the r12 fork pick its default (the explicit
    # fp8 / fp8_ds_mla both failed auto-selection on this build).
    # The GLM5Next model wires the B12X sparse-MLA backend (sm_120/sm_121)
    # when the global backend is B12X (glm5next/nvidia/attention.py:
    # _select_sparse_backend); the sm12x auto-priority list never offers it
    # (FLASHINFER_MLA_SPARSE_SM120 only supports head 576, not GLM's 512).
    "--attention-backend", "B12X",
    # lab loader: instanttensor (0.1.9 is already in the r12 image) —
    # ~5 min weight load vs ~16 min from NFS; stays in every bisection run
    "--load-format", "instanttensor",
    # PERF RUN (2026-09-02): full lab execution engine per lil render
    # (vllm-node-eugr reference): b12x linear+MoE kernels, mamba state
    # cache align, no enforce-eager (lab runs CUDA graphs; the enforce-eager
    # quality bug was day-0-build-specific, r12 is the lab-qualified build).
    "--linear-backend", "b12x",
    "--moe-backend", "b12x",
    "--mamba-cache-mode", "align",
    "--block-size", "256",
    "--max-model-len", "524288",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    "--disable-custom-all-reduce",
    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    # lab spec config verbatim (lil render): k=5, moe_backend b12x
    # (nvfp4->b12x), attention B12X (draft's backend comes from the spec
    # config, not the global flag — eagle/utils.py _make_eagle_draft_vllm_config)
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"b12x","attention_backend":"B12X"}'""",
    "--language-model-only",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
# lab INSTANTTENSOR loader knobs (lil render: BACKEND/CONCURRENCY/IO_DEPTH
# + the lil.yaml BUFFER_SIZE/CHUNK)
INSTANTTENSOR_BACKEND = "BUFFERED"
INSTANTTENSOR_CONCURRENCY = "1"
INSTANTTENSOR_IO_DEPTH = "3"
INSTANTTENSOR_BUFFER_SIZE = "67108864"
INSTANTTENSOR_CHUNK_SIZE = "8388608"
# FlashInfer JIT guard: if a kernel is missing from the mounted JIT cache it
# compiles at boot — 12-way parallel nvcc got OOM-killed (code 137) on GB10's
# 121 GB unified while the 93 GB weights + 10 GB KV pin are resident.
MAX_JOBS = "2"
# NoPE-MLA rope-pad runtime gate (mod baked into the image)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
# PERF RUN: lab execution engine (lil render reference)
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_STANDALONE_COMPILE = "1"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
# NOTE: no VLLM_USE_MEGA_AOT_ARTIFACT — the prebuilt mega-AOT artifact ships
# in the lab image, not in ours; the flag would only serialize.
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "4"

# Pre-baked JIT caches (see header; same pattern as the dsv4 recipe's
# ~/.cache/dsv4vision-jit mounts). RW on purpose: flashinfer writes lock
# files into the cache. Missing on a node -> first boot JITs (~15 min at
# MAX_JOBS=2) and the cache populates for the next boot.
# NOTE: extra_mounts stay AFTER args (TOML table headers would swallow the
# args array otherwise).
[[extra_mounts]]
host = "~/sm121-cache/glm53-jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/glm53-jit-121f/b12x"
container = "/root/.cache/b12x"