glm53-flash-fp4local-2x-dflash: DFlash2 A/B lane — incoai/GLM-5.3-Flash-DFlash2 drafter (2.2G, DFlash2DraftModel 5L/4096H, registered in the eugr build) vs the card's embedded MTP n5; rejection sampling is lossless, acceptance is the open question (DFlash2 distilled on the original zai distribution vs the QAD+nemotron-post-trained target)

recipes/glm53-flash-fp4local-2x-dflash/recipe.toml

@@ -0,0 +1,130 @@
+# NVIDIA GLM-5.3-Flash-FP4-Local — official NVFP4 (QAD) GLM-5.3-Flash, 2x DGX Spark.
+# Source: model card https://huggingface.co/nvidia/GLM-5.3-Flash-FP4-Local
+# ("2 x NVIDIA DGX Spark" section, transcribed). Quantized with nvidia-modelopt
+# v0.47.0 via quantization-aware distillation (QAD). NVFP4 routed experts
+# (E2M1 + E4M3 block scales + FP32 global scale). 2M tokens FP8 KV on 2x Spark.
+#
+# CRITICAL: the NVFP4 scale factors use a NEW lossless-compression format that
+# upstream vLLM does NOT yet support — MUST run on eugr/spark-vllm-b12x (b12x
+# backend + --load-format b12x). Digest-pinned (arm64, built 2026-10-15).
+# This is the official NVFP4 counterpart to our kindling 2x lane (our own r35
+# build) — a direct A/B vs the 95/100 kindling anchor.
+#
+# The launcher supplies the NCCL/socket/IB env from the cluster config
+# (NCCL_IB_HCA, NCCL_SOCKET_IFNAME, MN_IF_NAME, UCX_NET_DEVICES, GLOO/TP_SOCKET,
+# NCCL_IB_DISABLE, master addr/port). Only the b12x + RoCE-allreduce + AOT env
+# from the card's "both nodes" block is added here.
+
+engine = "vllm"
+name = "GLM-5.3-Flash-FP4-Local (2x Spark, NVFP4 QAD, DFlash2 k7 A/B)"
+docker_image = "eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595"
+entrypoint = ["vllm", "serve"]
+model = "nvidia/GLM-5.3-Flash-FP4-Local"
+parent_model = "zai-org/GLM-5.3-Flash"
+port = 30000
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "16g"
+# distributed_backend is MANDATORY for multi-node: without it the launcher
+# skips ALL dist flags (--tensor-parallel-size/--nnodes/--node-rank/--master-addr)
+# and each node boots an INDEPENDENT single-node engine that tries to load the
+# full 166 GB checkpoint into a 121 GiB box — the root cause of the four
+# "stalls" (b12x routing OOM) before this was caught.
+# GB10 adaptation of the card's DGX-Spark-x2 block: the card ships
+# The card's DGX-Spark-x2 block: --max-model-len 500000 + GMU 0.87 +
+# --max-num-seqs 4. The earlier "GB10 adaptation" (GMU 0.75 / 32768 / 2 seqs)
+# was based on a misdiagnosis: the four stalls were the missing
+# distributed_backend (each node loading the FULL model), not memory pressure
+# from the card's values. With correct TP2 (~87 GiB/rank) the card config
+# boots fine (proven by the manual card-flow run: 42/42 routed, 92/100).
+# Recovery if a boot ever wedges: HA plug cycle of switch.dgx3/dgx4 ONLY
+# (never the shared fabric switch).
+args = [
+    "--served-model-name", "glm53-fp4-local",
+    "--dtype", "bfloat16",
+    "--mamba-cache-mode", "align",
+    "--enable-prefix-caching",
+    "--enable-chunked-prefill",
+    "--kv-cache-dtype", "fp8",
+    "--quantization", "modelopt_mixed",
+    "--attention-backend", "B12X",
+    "--block-size", "256",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    "--no-enable-flashinfer-autotune",
+    "--load-format", "b12x",
+    "--max-model-len", "500000",
+    "--max-num-seqs", "4",
+    "--max-num-batched-tokens", "4096",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--gpu-memory-utilization", "0.87",
+    "--speculative-config", """
+'{"method":"dflash","model":"{{dflash2}}","num_speculative_tokens":7,"rejection_sample_method":"standard","attention_backend":"FLASHINFER","draft_load_config":{"load_format":"auto"}}'""",
+]
+
+# DFlash2 drafter (A/B vs the card's embedded MTP head): distilled against the
+# ORIGINAL zai GLM-5.3-Flash; the FP4-Local target is QAD-distilled toward the
+# same teacher + nemotron post-training, so acceptance should hold (rejection
+# sampling is lossless — worst case is less speedup, never worse quality).
+[[extra_mounts]]
+repo = "incoai/GLM-5.3-Flash-DFlash2"
+container = "dflash2"
+
+# `serve --cold` overlays these over `args` (by flag name) for an extra-safe
+# first boot on a memory-tight node — smaller KV than the card's default.
+# The default `args` are the card verbatim; `--cold` is the cautious first
+# launch, then drop the flag once the lane is proven to boot.
+cold_args = [
+    "--max-model-len", "16384",
+    "--max-num-seqs", "1",
+    "--max-num-batched-tokens", "2048",
+    # GMU 0.75: with the CORRECT TP2 the per-rank footprint is ~87 GiB
+    # (83 GiB weights + overhead), so 0.72 left only 0.47 GiB KV vs the
+    # 1.07 GiB needed at 16384 ctx — the boot-safe floor is the default GMU;
+    # the cold tier is the ctx/seq/batched reduction. (The earlier stalls were
+    # the missing distributed_backend — each node loaded the full model.)
+    "--gpu-memory-utilization", "0.75",
+]
+
+[env]
+# b12x MoE/FP4 path (card "both nodes" block)
+CUTE_DSL_ARCH = "sm_121a"
+SAFETENSORS_FAST_GPU = "1"
+B12X_POLICY_MODE = "auto"
+VLLM_B12X_MOE_FP4_FORCE_A16 = "1"
+B12X_W4A16_A4_PREFILL = "1"
+# RoCE allreduce + NCCL tuning (card)
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
+VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2MB"
+NCCL_BUFFSIZE = "1048576"
+NCCL_MAX_CTAS = "8"
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+# vLLM runner / compile flags (card)
+VLLM_WORKER_MULTIPROC_METHOD = "spawn"
+VLLM_SSM_CONV_STATE_LAYOUT = "DS"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+
+# JIT / AOT / b12x-autotune caches the card mounts on every node (speeds boot +
+# re-serve; docker creates the host dir if missing). Card: mkdir -p + mount.
+[[extra_mounts]]
+host = "~/.cache/vllm"
+container = "/root/.cache/vllm"
+
+[[extra_mounts]]
+host = "~/.cache/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/.triton"
+container = "/root/.triton"
+
+[[extra_mounts]]
+host = "~/.cache/b12x"
+container = "/root/.cache/b12x"