launcher/recipes/glm53-flash-nvfp4-redhat/recipe.toml

# GLM-5.3-Flash NVFP4 — RedHatAI quant (vLLM) — 2× DGX Spark, TP=2.
#
# RedHatAI/LLM-Compressor quant: MoE experts W4A4 NVFP4 (group-16, local
# per-group FP8 scales, compressed-tensors nvfp4-pack-quantized). Vision /
# embeddings / lm_head kept original precision. Quantizes via regex targets,
# so no name-prefix matching bugs.
# Model: https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4
#        (~177 GB → ~88 GB/node on 2 Sparks)
# KV: pinned --kv-cache-memory-bytes 10 GiB (same as -lab, identical MLA
#   geometry) → ~1.16 M-token pool @ fp8_ds_mla = 2.22× concurrency at the
#   524288 ctx. GMU 0.90 matches the proven lab config; the pin overrides
#   GMU-based KV sizing.
#
# MTP (added 2026-09-05): the checkpoint now ships the MTP drafter as
#   model_mtp.safetensors (7.6 GB, added to the repo after our 2026-08-28
#   index scan that found zero MTP weights) — vendor card section "Enable
#   Speculative Decoding" recommends k=5. Cost: +~7.6 GB weights/node on top
#   of the 88 GB base; if the boot dies on free-memory or KV shrinks, drop
#   the KV pin 10 → 8 GiB. k=3 is the fallback if k=5 regresses quality.
#
# Image: the -lab overlay (day-0 vLLM image + NoPE-MLA mod + FujitsuPolycom
# arch patches). The sparse_attn_indexer CC-12.0 guards are REQUIRED at the
# 512K ctx: at 262K the indexer topk fit 48 SMs, at 512K it wants 62 CTAs and
# the fallback needs 128 KB smem (GB10 has 99 KB) → hard boot abort
# (launch_persistent_topk oversubscribe, 2026-08-28). The patches are
# arch-level, not quant-level: the modelopt quantprobe is a no-op for this
# compressed-tensors checkpoint, the model.py naming shim anchors on
# arch-internal names.
#
# GB10 gotchas handled (see glm53-flash-nvfp4 header for detail):
#   --moe-backend marlin → if vLLM rejects marlin for W4A4 at startup (marlin
#   is mainly W4A16), remove the flag — but then verify output is not
#   token-loop garbage (the silent FLASHINFER_CUTLASS corruption) before
#   trusting it; `--moe-backend triton` is the other candidate.
#
# GB10 gotcha (MEASURED): at 524288 ctx WITH flashinfer autotune on, rank 0
# dies mid-boot on a driver OOM (NVRM NV_ERR_NO_MEMORY _memdescAllocInternal,
# 2026-08-28 05:01) — autotune/graph scratch at the 512K verify shapes doesn't
# fit the unified-memory headroom. The 262144 ctx + autotune combo booted
# fine. Fix = the lab-proven boot: autotune + cutedsl warmup off (a perf
# micro-opt only; the sparse-MLA kernels have a non-autotuned fallback) and
# expandable_segments to cut fragmentation.
#
# PREREQUISITES: weights cached on both nodes:
#   huggingface-cli download RedHatAI/GLM-5.3-Flash-NVFP4
# (re-run the download on nodes cached before 2026-09-04: model_mtp.safetensors
#  7.6 GB was added to the repo after the initial pulls; HF_HUB_OFFLINE=1 means
#  a missing file is a hard load failure)
#
# REFERENCE:
#   https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4 (vendor vLLM cmd)
#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark (mod)

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-redhat (vLLM, 2x Spark, MTP k=5)"
docker_image = "glm53-flash-nvfp4-lab:local"
docker_image_dir = "glm53-flash-nvfp4-lab"
model = "RedHatAI/GLM-5.3-Flash-NVFP4"
parent_model = "zai-org/GLM-5.3-Flash"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
    # page cache routinely leave ~108.5 (and ~106.6 after a crashed boot)
    # -> flaky "Free memory ... less than desired" boot deaths. 0.82 is
    # below any realistic dip and does NOT change what gets allocated.
    "--gpu-memory-utilization", "0.82",
    "--kv-cache-memory-bytes", "10737418240",
    "--max-model-len", "524288",
    # Scan/graph scratch scales with the prefill chunk: 8192 OOMs the driver
    # at 512K ctx (05:38 NVRM OOM), 4096 is the lab-proven ceiling here.
    "--max-num-batched-tokens", "4096",
    "--block-size", "256",
    "--max-num-seqs", "1",
    "--kv-cache-dtype", "fp8_ds_mla",
    "--language-model-only",
    "--moe-backend", "marlin",
    # MTP drafter: model_mtp.safetensors in the main repo (vendor card
    # "Enable Speculative Decoding": k=5; no separate drafter repo needed)
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":5}'""",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
]

[env]
HF_HUB_OFFLINE = "1"
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"

TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"

# JIT caches persist per-node across serves (was: cold /root/.cache per
# container -> ~20 min flashinfer/torch recompile every serve). The host
# dirs are shared between lanes of the same arch — each cache
# self-namespaces by its engine version key internally.
[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"