recipes: glm53-flash-dflash2-mxfp8

recipes/glm53-flash-dflash2-mxfp8/recipe.toml

@@ -0,0 +1,139 @@
+# GLM-5.3-Flash + DFlash2 draft (MXFP8) — speculative decoding — 2× Spark TP=2
+#
+# DFlash2-MXFP8 (local-inference-lab) is a ~1B BLOCK-DIFFUSION DRAFT model,
+# NOT a standalone LLM. It is loaded beside the TARGET
+# `local-inference-lab/GLM-5.3-Flash-NVFP4` (the lab quant) and every drafted
+# token is verified against it. Draft: 5 sliding-attention layers, block_size
+# 8, proposal length 7, reads target hidden states at layers 5/14/24/33/42.
+# Model card: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
+#
+# DRAFT LICENSE: source incoai/GLM-5.3-Flash-DFlash2 is CC BY-NC-ND 4.0
+# (non-commercial, no derivatives) — this image is a quantization of it; the
+# MXFP8 conversion itself is the "qualified" upstream artifact. Keep to
+# personal/non-commercial use.
+#
+# WHY THIS IMAGE: the card's qualified image (voipmonitor/vllm, TP=4 B12X on
+# 4× RTX PRO 6000) is amd64-only and its B12X attention/moe/linear/allreduce
+# backends do not exist on GB10. Neither DFlash2DraftModel nor the DFlash2
+# serialized-MXFP8 projection path was in our day-0 base. So we PORTED the
+# three DFlash2 source files from local-inference-lab/vllm
+# dev/jovian-judgement @ da4d7be6 (Apache-2.0, see docker-images/.../UPSTREAM.md)
+# onto our proven lab image → glm53-flash-dflash2:local. The day-0 base
+# already shipped W4A16_NVFP4 / MIXED_PRECISION / ModelOptMxFp8LinearMethod /
+# the dflash proposer, so the whole port is those 3 files + 1 registry line.
+#
+# DEVIATIONS from the card's qualified config (forced by hardware):
+#   * TP=4 → TP=2; no B12X attention/moe/linear/allreduce (fall back to the
+#     lab image's proven SM121 backends).
+#   * --enforce-eager ON: CUDA-graph+spec-decode quality regression is
+#     build-specific to this day-0 base (our 2026-08-28 A/B: graphs+MTP
+#     89 vs eager 92). Card runs CUDAGRAPH_MODE=FULL on a newer build.
+#   * TP=2 keeps draft KV in BF16 (auto), same as the card's config.
+# Adopted verbatim from the card: --speculative-config {method:dflash,
+# model:...DFlash2-MXFP8, num_speculative_tokens:7, kv_cache_dtype:auto}.
+#
+# =====================================================================
+# STATUS 2026-08-29: BLOCKED on GB10 — boots to weight-load + draft load
+# (both fine, MXFP8 draft kernel binds), then dies in KV-cache setup:
+#
+#   NotImplementedError: layers.3.self_attn.indexer.k_cache: page size is
+#   not divisible by the maximum page size and cannot be padded
+#   (indexes_kv_by_block_stride is False).
+#
+# MATH (block size cancels, so no --block-size / draft kv dtype fixes it):
+# per-token pages on this base are indexer 132 B, MLA fp8_ds_mla 660 B,
+# DFlash2 draft (8 kv heads x 128) bf16 4096 B / fp8 2048 B. vLLM unifies
+# target+draft pages by integer block-size scaling, which needs divisio-
+# nility: 4096%660 != 0, 2048%660 != 0. Tried and eliminated: draft
+# kv_cache_dtype fp8, target kv-cache-dtype variants, --disable-hybrid-
+# kv-cache-manager (then: "failed to convert the KV cache specs to one
+# unified type" — the kpool indexer spec is not coercible either).
+#
+# WHY THE FORK DOES NOT HIT THIS: its qualified run (voipmonitor image,
+# sm_120) uses B12X attention/indexer backends which opt into
+# indexes_kv_by_block_stride, so unify PADS the strided pages instead.
+# The b12x kernels are not part of the arm64 GB10 base and are compiled
+# for sm_120 consumer Blackwell.
+#
+# PATHS FORWARD (pick before spending boot time on this again):
+#   a) build local-inference-lab/vllm dev/jovian-judgement for arm64/cu130
+#      incl. b12x kernels (hours; sm_120->sm_121 compat unknown);
+#   b) wait for a GB10 indexer/MLA backend that sets
+#      indexes_kv_by_block_stride (upstream);
+#   c) give up — acceptance-rate comparison falls back to MTP (already
+#      characterized in specs/tech-architecture/glm53-recipe-state.md).
+#
+# What IS done and committed regardless: glm53-flash-dflash2:local image
+# (DFlash2DraftModel port + registry), patch_eagle3.py (target-side
+# EAGLE3/aux-hidden-state support for Glm5Next, fork parity), verified
+# config plumbing (spec JSON parsed, aux layers 5/14/24/33/42 extracted
+# from the draft config, FlashInferCutlassMxfp8LinearKernel bound).
+#
+# PREREQUISITES: BOTH models cached on both nodes:
+#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
+#   hf download local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
+
+engine = "vllm"
+name = "GLM-5.3-Flash + DFlash2 draft (vLLM, 2x Spark, spec-k7)"
+docker_image = "glm53-flash-dflash2:local"
+docker_image_dir = "glm53-flash-dflash2"
+# TARGET model — the DFlash2 draft model is referenced only via
+# --speculative-config below, never as the top-level model.
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
+parent_model = "zai-org/GLM-5.3-Flash"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail (KV pinned below): 0.90 demands
+    # 109.46 GiB free on GB10's 121.63 and driver reserve + page cache
+    # routinely leave ~108.5 -> flaky boot deaths. 0.82 always fits.
+    "--gpu-memory-utilization", "0.82",
+    # Pinned 10 GiB KV; the draft adds 5 small sliding-window layers on top
+    "--kv-cache-memory-bytes", "10737418240",
+    "--kv-cache-dtype", "fp8_ds_mla",
+    "--block-size", "256",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    "--disable-custom-all-reduce",
+    # DFlash2 draft from local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
+    # (block-diffusion drafter, proposal length 7 — verbatim from the card)
+    # Card: kv_cache_dtype auto (bf16 draft pages) — kept faithful.
+    "--speculative-config", """
+'{"method":"dflash","model":"local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8","num_speculative_tokens":7,"kv_cache_dtype":"auto"}'""",
+    "--language-model-only",
+    "--load-format", "instanttensor",
+    # See header: eager is the safe control on the day-0 base (graphs+spec
+    # quality regression). The card's newer build runs FULL graphs.
+    "--enforce-eager",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "4"