recipes: glm53-flash-nvfp4

recipes/glm53-flash-nvfp4/recipe.toml

@@ -0,0 +1,97 @@
+# GLM-5.3-Flash NVFP4 (vLLM) — 2× DGX Spark / GB10 (sm_121), TP=2.
+#
+# 320B/18B-active, hybrid 34 KDA linear-attention + 11 sparse-MLA layers,
+# glm5_next arch (NoPE-MLA, qk_rope_head_dim=0), in-checkpoint MTP head.
+# Text-only here (--language-model-only — the multimodal processor balloons
+# the API front-end to ~15.7 GB and the GB10 kernel OOM-kills it).
+#
+# WHY NVFP4 on 2 nodes (not the official FP8 on 4): LibertAIDAI's weight-only
+# NVFP4-A16 quant halves the footprint (~155 GB) so it fits 2 Sparks; the
+# sparse+linear attention keeps KV nearly free (KV pool ~738K tok at GMU 0.85).
+# Model: https://huggingface.co/LibertAIDAI/GLM-5.3-Flash-NVFP4
+#
+# Engine: vLLM day-0 per-model image (glm5_next support baked in). The arch
+# fails every sm_121 attention backend out of the box; a 4-part build-time
+# mod (docker-images/glm53-flash-nvfp4/patches/patch_mla.py) dresses GLM's
+# NoPE-MLA in DeepSeek's fp8_ds_mla 512+64 layout and compacts the sm120
+# sparse-MLA topk table 2176->2048. Applied at image build; this recipe
+# activates it at runtime via VLLM_MLA_NOPE_PAD_ROPE=1.
+#
+# TWO SILENT GB10 GOTCHAS (from the reference — both already handled here):
+#   1. SILENT FP4 MoE CORRUPTION: the auto-selected FLASHINFER_CUTLASS NVFP4
+#      MoE backend emits degenerate token loops with ZERO errors (model loads,
+#      /health green, every completion garbage). --moe-backend marlin fixes it.
+#      If a GB10 NVFP4 serve "works" but loops, check which MoE backend engaged
+#      before blaming the quant.
+#   2. A 60+ MIN "No available shared memory broadcast block found" loop at
+#      boot is NOT a hang — the worker is FlashInfer-autotuning a kernel flavor
+#      (tune persists to ~/.cache/vllm, pays once). VLLM_ENGINE_READY_TIMEOUT_S
+#      is raised to 3600 so the launcher doesn't give up first.
+#
+# Measured (median of 3, temp 0, single stream, 2x Spark, MTP-5 acceptance ~2.55):
+#   24.74 code / 30.30 structured / 19.58 prose tok/s; no-spec baseline 14.6 flat.
+#   256K needle-exact (30K/100K/200K deep); ~1150 tok/s prefill.
+#
+# Image: glm53-flash-nvfp4:local (built from vllm/vllm-openai:glm53-flash-arm64-cu130
+#   + the patch). `sm121 serve` builds it on the build node if the checksum label
+#   is stale (docker_image_dir below).
+#
+# PREREQUISITES:
+#   Weights cached on both nodes (NAS ~/.cache/huggingface):
+#       huggingface-cli download LibertAIDAI/GLM-5.3-Flash-NVFP4
+#
+# REFERENCE (provenance):
+#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark
+#   https://forums.developer.nvidia.com/t/381433
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4 (vLLM, 2x Spark)"
+docker_image = "glm53-flash-nvfp4:local"
+docker_image_dir = "glm53-flash-nvfp4"
+model = "LibertAIDAI/GLM-5.3-Flash-NVFP4"
+parent_model = "zai-org/GLM-5.3-Flash"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    # GLM tokenizer
+    "--trust-remote-code",
+    # Executor (mp, no ray)
+    "--distributed-executor-backend", "mp",
+    # Memory — weights ~77.5 GiB of 119.6 GiB/node; sparse+linear KV nearly free
+    "--gpu-memory-utilization", "0.85",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "1",
+    # DeepSeek-MLA KV layout the NoPE-pad mod dresses the attention in
+    "--kv-cache-dtype", "fp8_ds_mla",
+    # Text-only (multimodal processor OOM-kills the GB10 front-end)
+    "--language-model-only",
+    # GOTCHA 1: marlin avoids the silent FP4 MoE corruption
+    "--moe-backend", "marlin",
+    # MTP=5 (in-checkpoint drafter; acceptance length ~2.55)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":5}'""",
+    # Tokenizer / reasoning / tool calling (GLM family)
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+# Offline — weights must be cached on each node
+HF_HUB_OFFLINE = "1"
+# Activates the build-time NoPE-MLA rope-pad mod (mathematically a no-op:
+# zeros are rotation-free and add exactly 0 to every logit)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+# GOTCHA 2: survive the 60+ min FlashInfer-autotune boot window
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"