launcher/recipes/glm53-flash-nvfp4-lab/reports/2026-09-20_090824_512k-kv10g.recipe.toml

# GLM-5.3-Flash NVFP4-lab — the lab's NVFP4 quant (2026-09-16 RE-CUT,
# local-inference-lab/GLM-5.3-Flash-NVFP4), 2× DGX Spark, TP=2.
#
# The 09-16 re-cut replaced the old ~186 GB upload in place: 199.4 GB
# (36 expert shards + 4 nonexpert + 4 MTP shards, modelopt NVFP4).
# Operator direction (2026-09-20): "they improved it a lot" — this lane
# re-tests the re-cut against the old 89/100 anchor on the proven r35
# engine (same interim mode as the Spark lanes).
#
# Structure (verified from model.safetensors.index.json, identical to the
# Spark checkpoint): `model.language_model.layers.*` with SPLIT GDN naming
# (q/k/g_a/g_b/f_a/f_b/b_proj + hc_* hybrid-conv), a `model.visual.*`
# tower (skipped via --language-model-only), and the MTP head embedded at
# layer 45 (eh_proj/enorm/hnorm) — no separate drafter download. Same
# loader path as the Spark lane: the JJ fork lineage (r35) maps it natively
# (no KeyError); r38 is out — b12x preparation-pool deadlock on 2x GB10.
#
# Image: glm53-jovian-r35:local + VLLM_DISABLED_KERNELS (the small GDN
# mxfp8 GEMMs reject flashinfer cutlass' min n/k=128 — routed to the b12x
# Mxfp8 backend; proven on the Spark lanes 162/176 + 161/176).
#
# Memory: 199.4 GB / 2 ranks ≈ 93 GiB weights/node at GMU 0.88 (107.0 GiB)
# — no KV pin (auto-sizing takes the remaining ~10 GiB; a 10 GiB pin as in
# the old 186 GB cut would overshoot the tighter budget).
#
# PREREQUISITES: weights on each node's LOCAL hub (the NAS disk died
# 2026-09-19): /home/kilork/hf-local, ~/.cache/huggingface symlink
# re-pointed per node. Re-cut re-download in progress on all 4 nodes
# (dgx1 is the tightest: ~40 GiB left after the model).

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-lab re-cut (vLLM, 2x Spark, MTP-3, r35)"
docker_image = "glm53-jovian-r35:local"  # r35 loads the lab quants (r38 pool-deadlock on 2x, see bug report)
docker_image_dir = "glm53-jovian-r27"
build_profile = "glm53-r35"
entrypoint = ["vllm", "serve"]
model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
parent_model = "zai-org/GLM-5.3-Flash"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GMU is ONLY a startup guardrail: 0.88 = 107.0 GiB, below any
    # realistic free-memory dip on GB10's 121.63 GiB (#48140).
    "--gpu-memory-utilization", "0.88",
    # no KV pin: ~93 GiB weights/rank leaves only ~2.4 GiB KV at GMU 0.88
    # (the re-cut is heavier than the old 186 GB cut). 512K TEST (2026-09-20,
    # local-only, uncommitted until the boot proves the pool fits): max-num-seqs
    # 16->4 and prefill budget 4096->2048 to free the ~1.3 GiB/rank that one
    # 524288-token request needs (profile-run peak + CUDA-graph captures)
    "--block-size", "256",
    "--max-model-len", "524288",
    "--max-num-seqs", "4",
    "--max-num-batched-tokens", "2048",
    # explicit KV pin (same as the Spark variant): auto-sizing left headroom
    # on the table (logs showed more free than the ~2.4 GiB auto-pick); 10 GiB
    # per rank -> ~1.4M-token pool, 2.7 full 512K contexts
    "--kv-cache-memory-bytes", "10737418240",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    # B12X sparse-MLA (required for head-512 on sm_12x)
    "--attention-backend", "B12X",
    # pure I/O loader (baked into the r27-lineage images; ~1 min off local NVMe)
    "--load-format", "instanttensor",
    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    # Built-in MTP draft, 3 tokens (layer 45, in-checkpoint). The draft
    # model's vllm_config takes its attention backend from the SPECULATIVE
    # config, not the global flag — B12X mirrors the main model.
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'""",
    "--language-model-only",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for the
# r27 lanes since the 2026-09-08 A/B (4x: decode +31..+90%, quality 160 vs
# 155). Needs the RoCE fabric; single-node serves skip it silently.
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
# Route mxfp8 linears away from flashinfer cutlass (rejects small GDN GEMMs: min n/k=128)
VLLM_DISABLED_KERNELS = "FlashInferCutlassMxfp8LinearKernel"
# FlashInfer JIT compiles at first boot with N parallel nvcc jobs ->
# OOM-killed on GB10 while weights+KV are resident; cap the ninja jobs.
MAX_JOBS = "2"
# NoPE-MLA rope-pad runtime gate (mod baked into the image lineage)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
# Upstream launch-script env (memory + compile knobs)
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
VLLM_USE_STANDALONE_COMPILE = "1"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "2"

# JIT caches persist per-node across serves (cold /root/.cache per container
# = ~20 min flashinfer/torch recompile every serve)
[[extra_mounts]]
host = "~/sm121-cache/glm53-jit-121f/b12x"
container = "/root/.cache/b12x"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"