recipes: glm53-flash-nvfp4-redhat

recipes/glm53-flash-nvfp4-redhat/recipe.toml

@@ -0,0 +1,122 @@
+# GLM-5.3-Flash NVFP4 — RedHatAI quant (vLLM) — 2× DGX Spark, TP=2.
+#
+# RedHatAI/LLM-Compressor quant: MoE experts W4A4 NVFP4 (group-16, local
+# per-group FP8 scales, compressed-tensors nvfp4-pack-quantized). Vision /
+# embeddings / lm_head kept original precision. Quantizes via regex targets,
+# so no name-prefix matching bugs.
+# Model: https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4
+#        (~177 GB → ~88 GB/node on 2 Sparks)
+# KV: pinned --kv-cache-memory-bytes 10 GiB (same as -lab, identical MLA
+#   geometry) → ~1.16 M-token pool @ fp8_ds_mla = 2.22× concurrency at the
+#   524288 ctx. GMU 0.90 matches the proven lab config; the pin overrides
+#   GMU-based KV sizing.
+#
+# MTP (added 2026-09-05): the checkpoint now ships the MTP drafter as
+#   model_mtp.safetensors (7.6 GB, added to the repo after our 2026-08-28
+#   index scan that found zero MTP weights) — vendor card section "Enable
+#   Speculative Decoding" recommends k=5. Cost: +~7.6 GB weights/node on top
+#   of the 88 GB base; if the boot dies on free-memory or KV shrinks, drop
+#   the KV pin 10 → 8 GiB. k=3 is the fallback if k=5 regresses quality.
+#
+# Image: the -lab overlay (day-0 vLLM image + NoPE-MLA mod + FujitsuPolycom
+# arch patches). The sparse_attn_indexer CC-12.0 guards are REQUIRED at the
+# 512K ctx: at 262K the indexer topk fit 48 SMs, at 512K it wants 62 CTAs and
+# the fallback needs 128 KB smem (GB10 has 99 KB) → hard boot abort
+# (launch_persistent_topk oversubscribe, 2026-08-28). The patches are
+# arch-level, not quant-level: the modelopt quantprobe is a no-op for this
+# compressed-tensors checkpoint, the model.py naming shim anchors on
+# arch-internal names.
+#
+# GB10 gotchas handled (see glm53-flash-nvfp4 header for detail):
+#   --moe-backend marlin → if vLLM rejects marlin for W4A4 at startup (marlin
+#   is mainly W4A16), remove the flag — but then verify output is not
+#   token-loop garbage (the silent FLASHINFER_CUTLASS corruption) before
+#   trusting it; `--moe-backend triton` is the other candidate.
+#
+# GB10 gotcha (MEASURED): at 524288 ctx WITH flashinfer autotune on, rank 0
+# dies mid-boot on a driver OOM (NVRM NV_ERR_NO_MEMORY _memdescAllocInternal,
+# 2026-08-28 05:01) — autotune/graph scratch at the 512K verify shapes doesn't
+# fit the unified-memory headroom. The 262144 ctx + autotune combo booted
+# fine. Fix = the lab-proven boot: autotune + cutedsl warmup off (a perf
+# micro-opt only; the sparse-MLA kernels have a non-autotuned fallback) and
+# expandable_segments to cut fragmentation.
+#
+# PREREQUISITES: weights cached on both nodes:
+#   huggingface-cli download RedHatAI/GLM-5.3-Flash-NVFP4
+# (re-run the download on nodes cached before 2026-09-04: model_mtp.safetensors
+#  7.6 GB was added to the repo after the initial pulls; HF_HUB_OFFLINE=1 means
+#  a missing file is a hard load failure)
+#
+# REFERENCE:
+#   https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4 (vendor vLLM cmd)
+#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark (mod)
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4-redhat (vLLM, 2x Spark, MTP k=5)"
+docker_image = "glm53-flash-nvfp4-lab:local"
+docker_image_dir = "glm53-flash-nvfp4-lab"
+model = "RedHatAI/GLM-5.3-Flash-NVFP4"
+parent_model = "zai-org/GLM-5.3-Flash"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
+    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
+    # page cache routinely leave ~108.5 (and ~106.6 after a crashed boot)
+    # -> flaky "Free memory ... less than desired" boot deaths. 0.82 is
+    # below any realistic dip and does NOT change what gets allocated.
+    "--gpu-memory-utilization", "0.82",
+    "--kv-cache-memory-bytes", "10737418240",
+    "--max-model-len", "524288",
+    # Scan/graph scratch scales with the prefill chunk: 8192 OOMs the driver
+    # at 512K ctx (05:38 NVRM OOM), 4096 is the lab-proven ceiling here.
+    "--max-num-batched-tokens", "4096",
+    "--block-size", "256",
+    "--max-num-seqs", "1",
+    "--kv-cache-dtype", "fp8_ds_mla",
+    "--language-model-only",
+    "--moe-backend", "marlin",
+    # MTP drafter: model_mtp.safetensors in the main repo (vendor card
+    # "Enable Speculative Decoding": k=5; no separate drafter repo needed)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":5}'""",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"