launcher/recipes/glm53-flash-dflash2-mxfp8/recipe.toml

# GLM-5.3-Flash + DFlash2 draft (MXFP8) — speculative decoding — 2× Spark TP=2
#
# DFlash2-MXFP8 (local-inference-lab) is a ~1B BLOCK-DIFFUSION DRAFT model,
# NOT a standalone LLM. It is loaded beside the TARGET
# `local-inference-lab/GLM-5.3-Flash-NVFP4` (the lab quant) and every drafted
# token is verified against it. Draft: 5 sliding-attention layers, block_size
# 8, proposal length 7, reads target hidden states at layers 5/14/24/33/42.
# Model card: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
#
# DRAFT LICENSE: source incoai/GLM-5.3-Flash-DFlash2 is CC BY-NC-ND 4.0
# (non-commercial, no derivatives) — this image is a quantization of it; the
# MXFP8 conversion itself is the "qualified" upstream artifact. Keep to
# personal/non-commercial use.
#
# WHY THIS IMAGE: the card's qualified image (voipmonitor/vllm, TP=4 B12X on
# 4× RTX PRO 6000) is amd64-only and its B12X attention/moe/linear/allreduce
# backends do not exist on GB10. Neither DFlash2DraftModel nor the DFlash2
# serialized-MXFP8 projection path was in our day-0 base. So we PORTED the
# three DFlash2 source files from local-inference-lab/vllm
# dev/jovian-judgement @ da4d7be6 (Apache-2.0, see docker-images/.../UPSTREAM.md)
# onto our proven lab image → glm53-flash-dflash2:local. The day-0 base
# already shipped W4A16_NVFP4 / MIXED_PRECISION / ModelOptMxFp8LinearMethod /
# the dflash proposer, so the whole port is those 3 files + 1 registry line.
#
# DEVIATIONS from the card's qualified config (forced by hardware):
#   * TP=4 → TP=2; no B12X attention/moe/linear/allreduce (fall back to the
#     lab image's proven SM121 backends).
#   * --enforce-eager ON: CUDA-graph+spec-decode quality regression is
#     build-specific to this day-0 base (our 2026-08-28 A/B: graphs+MTP
#     89 vs eager 92). Card runs CUDAGRAPH_MODE=FULL on a newer build.
#   * TP=2 keeps draft KV in BF16 (auto), same as the card's config.
# Adopted verbatim from the card: --speculative-config {method:dflash,
# model:...DFlash2-MXFP8, num_speculative_tokens:7, kv_cache_dtype:auto}.
#
# Boot markers: target loads as in glm53-flash-nvfp4-lab (92.72 GiB/node) →
# a "DFlash2" speculator/drafter line in the spec-decode init → "Application
# startup complete". A "DFlash2DraftModel not in registry" error means the
# image overlay didn't apply (rebuild glm53-flash-dflash2:local). Verify the
# DFlash proposal acceptance rate is > the lab MTP-3 baseline (that's the
# whole point of DFlash2); if accuracy looks off, this recipe is the eager
# control.
#
# PREREQUISITES: BOTH models cached on both nodes:
#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
#   hf download local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8

engine = "vllm"
name = "GLM-5.3-Flash + DFlash2 draft (vLLM, 2x Spark, spec-k7)"
docker_image = "glm53-flash-dflash2:local"
docker_image_dir = "glm53-flash-dflash2"
# TARGET model — the DFlash2 draft model is referenced only via
# --speculative-config below, never as the top-level model.
model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    "--gpu-memory-utilization", "0.90",
    # Pinned 10 GiB KV; the draft adds 5 small sliding-window layers on top
    "--kv-cache-memory-bytes", "10737418240",
    "--kv-cache-dtype", "fp8_ds_mla",
    "--block-size", "256",
    "--max-model-len", "262144",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    "--disable-custom-all-reduce",
    # DFlash2 draft from local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
    # (block-diffusion drafter, proposal length 7 — verbatim from the card)
    "--speculative-config", """
'{"method":"dflash","model":"local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8","num_speculative_tokens":7,"kv_cache_dtype":"auto"}'""",
    "--language-model-only",
    # See header: eager is the safe control on the day-0 base (graphs+spec
    # quality regression). The card's newer build runs FULL graphs.
    "--enforce-eager",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "4"