launcher/recipes/glm53-flash-nvfp4-redhat/recipe.toml

# GLM-5.3-Flash NVFP4 — RedHatAI quant (vLLM) — 2× DGX Spark, TP=2.
#
# RedHatAI/LLM-Compressor quant: MoE experts W4A4 NVFP4 (group-16, local
# per-group FP8 scales, compressed-tensors nvfp4-pack-quantized). Vision /
# embeddings / lm_head kept original precision. Quantizes via regex targets,
# so no name-prefix matching bugs.
# Model: https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4
#        (~177 GB → ~88 GB/node on 2 Sparks; GMU 0.85 = ~101 GiB budget)
#
# ⚠ NO MTP DRAFTER IN THIS CHECKPOINT — the index has 146,745 tensors and
#   zero layer-45/MTP weights. `--speculative-config mtp` WILL fail to load.
#   This recipe is baseline decode only (~15 tok/s class on this arch —
#   spec decode buys 1.3–2×; use glm53-flash-nvfp4[-lab] when MTP matters).
#
# Image: shared glm53-flash-nvfp4:local overlay (day-0 vLLM image + the
# NoPE-MLA / sm120 sparse-MLA mod). The mod is arch-level, quant-independent.
#
# GB10 gotchas handled (see glm53-flash-nvfp4 header for detail):
#   --moe-backend marlin → if vLLM rejects marlin for W4A4 at startup (marlin
#   is mainly W4A16), remove the flag — but then verify output is not
#   token-loop garbage (the silent FLASHINFER_CUTLASS corruption) before
#   trusting it; `--moe-backend triton` is the other candidate.
#
# Vendor note: RedHatAI's own README suggests --no-enable-flashinfer-autotune
# for a fast boot — deliberately NOT set here: the GB10 sparse-MLA kernels
# rely on the autotune result (first boot is slow by design, tune persists to
# ~/.cache/vllm). Add it if you want to skip re-tuning after the first run.
#
# PREREQUISITES: weights cached on both nodes:
#   huggingface-cli download RedHatAI/GLM-5.3-Flash-NVFP4
#
# REFERENCE:
#   https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4 (vendor vLLM cmd)
#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark (mod)

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-redhat (vLLM, 2x Spark, no MTP)"
docker_image = "glm53-flash-nvfp4:local"
docker_image_dir = "glm53-flash-nvfp4"
model = "RedHatAI/GLM-5.3-Flash-NVFP4"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    "--gpu-memory-utilization", "0.85",
    "--max-model-len", "262144",
    "--max-num-seqs", "1",
    "--kv-cache-dtype", "fp8_ds_mla",
    "--language-model-only",
    "--moe-backend", "marlin",
    # NO --speculative-config: checkpoint ships no MTP weights (verified index)
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"

TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"