launcher/recipes/glm53-flash-dflash2-mxfp8/recipe.toml

# GLM-5.3-Flash + DFlash2 draft (MXFP8) — speculative decoding — 2× Spark TP=2
#
# DFlash2-MXFP8 (local-inference-lab) is a ~1B BLOCK-DIFFUSION DRAFT model,
# NOT a standalone LLM. It is loaded beside the TARGET
# `local-inference-lab/GLM-5.3-Flash-NVFP4` (the lab quant) and every drafted
# token is verified against it. Draft: 5 sliding-attention layers, block_size
# 8, proposal length 7, reads target hidden states at layers 5/14/24/33/42.
# Model card: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
#
# DRAFT LICENSE: source incoai/GLM-5.3-Flash-DFlash2 is CC BY-NC-ND 4.0
# (non-commercial, no derivatives) — this image is a quantization of it; the
# MXFP8 conversion itself is the "qualified" upstream artifact. Keep to
# personal/non-commercial use.
#
# WHY THIS IMAGE: the card's qualified image (voipmonitor/vllm, TP=4 B12X on
# 4× RTX PRO 6000) is amd64-only and its B12X attention/moe/linear/allreduce
# backends do not exist on GB10. Neither DFlash2DraftModel nor the DFlash2
# serialized-MXFP8 projection path was in our day-0 base. So we PORTED the
# three DFlash2 source files from local-inference-lab/vllm
# dev/jovian-judgement @ da4d7be6 (Apache-2.0, see docker-images/.../UPSTREAM.md)
# onto our proven lab image → glm53-flash-dflash2:local. The day-0 base
# already shipped W4A16_NVFP4 / MIXED_PRECISION / ModelOptMxFp8LinearMethod /
# the dflash proposer, so the whole port is those 3 files + 1 registry line.
#
# DEVIATIONS from the card's qualified config (forced by hardware):
#   * TP=4 → TP=2; no B12X attention/moe/linear/allreduce (fall back to the
#     lab image's proven SM121 backends).
#   * --enforce-eager ON: CUDA-graph+spec-decode quality regression is
#     build-specific to this day-0 base (our 2026-08-28 A/B: graphs+MTP
#     89 vs eager 92). Card runs CUDAGRAPH_MODE=FULL on a newer build.
#   * TP=2 keeps draft KV in BF16 (auto), same as the card's config.
# Adopted verbatim from the card: --speculative-config {method:dflash,
# model:...DFlash2-MXFP8, num_speculative_tokens:7, kv_cache_dtype:auto}.
#
# =====================================================================
# STATUS 2026-08-29: BLOCKED on GB10 — boots to weight-load + draft load
# (both fine, MXFP8 draft kernel binds), then dies in KV-cache setup:
#
#   NotImplementedError: layers.3.self_attn.indexer.k_cache: page size is
#   not divisible by the maximum page size and cannot be padded
#   (indexes_kv_by_block_stride is False).
#
# MATH (block size cancels, so no --block-size / draft kv dtype fixes it):
# per-token pages on this base are indexer 132 B, MLA fp8_ds_mla 660 B,
# DFlash2 draft (8 kv heads x 128) bf16 4096 B / fp8 2048 B. vLLM unifies
# target+draft pages by integer block-size scaling, which needs divisio-
# nility: 4096%660 != 0, 2048%660 != 0. Tried and eliminated: draft
# kv_cache_dtype fp8, target kv-cache-dtype variants, --disable-hybrid-
# kv-cache-manager (then: "failed to convert the KV cache specs to one
# unified type" — the kpool indexer spec is not coercible either).
#
# WHY THE FORK DOES NOT HIT THIS: its qualified run (voipmonitor image,
# sm_120) uses B12X attention/indexer backends which opt into
# indexes_kv_by_block_stride, so unify PADS the strided pages instead.
# The b12x kernels are not part of the arm64 GB10 base and are compiled
# for sm_120 consumer Blackwell.
#
# PATHS FORWARD (pick before spending boot time on this again):
#   a) build local-inference-lab/vllm dev/jovian-judgement for arm64/cu130
#      incl. b12x kernels (hours; sm_120->sm_121 compat unknown);
#   b) wait for a GB10 indexer/MLA backend that sets
#      indexes_kv_by_block_stride (upstream);
#   c) give up — acceptance-rate comparison falls back to MTP (already
#      characterized in specs/tech-architecture/glm53-recipe-state.md).
#
# What IS done and committed regardless: glm53-flash-dflash2:local image
# (DFlash2DraftModel port + registry), patch_eagle3.py (target-side
# EAGLE3/aux-hidden-state support for Glm5Next, fork parity), verified
# config plumbing (spec JSON parsed, aux layers 5/14/24/33/42 extracted
# from the draft config, FlashInferCutlassMxfp8LinearKernel bound).
#
# PREREQUISITES: BOTH models cached on both nodes:
#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
#   hf download local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8

engine = "vllm"
name = "GLM-5.3-Flash + DFlash2 draft (vLLM, 2x Spark, spec-k7)"
docker_image = "glm53-flash-dflash2:local"
docker_image_dir = "glm53-flash-dflash2"
# TARGET model — the DFlash2 draft model is referenced only via
# --speculative-config below, never as the top-level model.
model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GMU is ONLY a startup guardrail (KV pinned below): 0.90 demands
    # 109.46 GiB free on GB10's 121.63 and driver reserve + page cache
    # routinely leave ~108.5 -> flaky boot deaths. 0.82 always fits.
    "--gpu-memory-utilization", "0.82",
    # Pinned 10 GiB KV; the draft adds 5 small sliding-window layers on top
    "--kv-cache-memory-bytes", "10737418240",
    "--kv-cache-dtype", "fp8_ds_mla",
    "--block-size", "256",
    "--max-model-len", "524288",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    "--disable-custom-all-reduce",
    # DFlash2 draft from local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
    # (block-diffusion drafter, proposal length 7 — verbatim from the card)
    # Card: kv_cache_dtype auto (bf16 draft pages) — kept faithful.
    "--speculative-config", """
'{"method":"dflash","model":"local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8","num_speculative_tokens":7,"kv_cache_dtype":"auto"}'""",
    "--language-model-only",
    "--load-format", "instanttensor",
    # See header: eager is the safe control on the day-0 base (graphs+spec
    # quality regression). The card's newer build runs FULL graphs.
    "--enforce-eager",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "4"