launcher/recipes/glm53-flash-nvfp4/recipe.toml

# GLM-5.3-Flash NVFP4 (vLLM) — 2× DGX Spark / GB10 (sm_121), TP=2.
#
# 320B/18B-active, hybrid 34 KDA linear-attention + 11 sparse-MLA layers,
# glm5_next arch (NoPE-MLA, qk_rope_head_dim=0), in-checkpoint MTP head.
# Text-only here (--language-model-only — the multimodal processor balloons
# the API front-end to ~15.7 GB and the GB10 kernel OOM-kills it).
#
# WHY NVFP4 on 2 nodes (not the official FP8 on 4): LibertAIDAI's weight-only
# NVFP4-A16 quant halves the footprint (~155 GB) so it fits 2 Sparks; the
# sparse+linear attention keeps KV nearly free (KV pool ~738K tok at GMU 0.85).
# Model: https://huggingface.co/LibertAIDAI/GLM-5.3-Flash-NVFP4
#
# Engine: vLLM day-0 per-model image (glm5_next support baked in). The arch
# fails every sm_121 attention backend out of the box; a 4-part build-time
# mod (docker-images/glm53-flash-nvfp4/patches/patch_mla.py) dresses GLM's
# NoPE-MLA in DeepSeek's fp8_ds_mla 512+64 layout and compacts the sm120
# sparse-MLA topk table 2176->2048. Applied at image build; this recipe
# activates it at runtime via VLLM_MLA_NOPE_PAD_ROPE=1.
#
# TWO SILENT GB10 GOTCHAS (from the reference — both already handled here):
#   1. SILENT FP4 MoE CORRUPTION: the auto-selected FLASHINFER_CUTLASS NVFP4
#      MoE backend emits degenerate token loops with ZERO errors (model loads,
#      /health green, every completion garbage). --moe-backend marlin fixes it.
#      If a GB10 NVFP4 serve "works" but loops, check which MoE backend engaged
#      before blaming the quant.
#   2. A 60+ MIN "No available shared memory broadcast block found" loop at
#      boot is NOT a hang — the worker is FlashInfer-autotuning a kernel flavor
#      (tune persists to ~/.cache/vllm, pays once). VLLM_ENGINE_READY_TIMEOUT_S
#      is raised to 3600 so the launcher doesn't give up first.
#
# Measured (median of 3, temp 0, single stream, 2x Spark, MTP-5 acceptance ~2.55):
#   24.74 code / 30.30 structured / 19.58 prose tok/s; no-spec baseline 14.6 flat.
#   256K needle-exact (30K/100K/200K deep); ~1150 tok/s prefill.
#
# Image: glm53-flash-nvfp4:local (built from vllm/vllm-openai:glm53-flash-arm64-cu130
#   + the patch). `sm121 serve` builds it on the build node if the checksum label
#   is stale (docker_image_dir below).
#
# PREREQUISITES:
#   Weights cached on both nodes (NAS ~/.cache/huggingface):
#       huggingface-cli download LibertAIDAI/GLM-5.3-Flash-NVFP4
#
# REFERENCE (provenance):
#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark
#   https://forums.developer.nvidia.com/t/381433

engine = "vllm"
name = "GLM-5.3-Flash NVFP4 (vLLM, 2x Spark)"
docker_image = "glm53-flash-nvfp4:local"
docker_image_dir = "glm53-flash-nvfp4"
model = "LibertAIDAI/GLM-5.3-Flash-NVFP4"
parent_model = "zai-org/GLM-5.3-Flash"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    # GLM tokenizer
    "--trust-remote-code",
    # Executor (mp, no ray)
    "--distributed-executor-backend", "mp",
    # Memory — weights ~77.5 GiB of 119.6 GiB/node; sparse+linear KV nearly free
    "--gpu-memory-utilization", "0.85",
    "--max-model-len", "262144",
    "--max-num-seqs", "1",
    # DeepSeek-MLA KV layout the NoPE-pad mod dresses the attention in
    "--kv-cache-dtype", "fp8_ds_mla",
    # Text-only (multimodal processor OOM-kills the GB10 front-end)
    "--language-model-only",
    # GOTCHA 1: marlin avoids the silent FP4 MoE corruption
    "--moe-backend", "marlin",
    # MTP=5 (in-checkpoint drafter; acceptance length ~2.55)
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":5}'""",
    # Tokenizer / reasoning / tool calling (GLM family)
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
# Offline — weights must be cached on each node
HF_HUB_OFFLINE = "1"
# Activates the build-time NoPE-MLA rope-pad mod (mathematically a no-op:
# zeros are rotation-free and add exactly 0 to every logit)
VLLM_MLA_NOPE_PAD_ROPE = "1"
# GOTCHA 2: survive the 60+ min FlashInfer-autotune boot window
VLLM_ENGINE_READY_TIMEOUT_S = "3600"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"