|
|
@@ -0,0 +1,236 @@
|
|
|
+# r28 4x DCP4 cell (2026-09-08) — clone of the r27 4x-dcp4 recipe (160/176
|
|
|
+# RoCEnante, 46.8 t/s) retargeted at the r28 engine: 13 commits over the r27
|
|
|
+# pin (prefill/decode compute-time sharing + interleaved parallel prefills,
|
|
|
+# chat-template rendering parity, GLM boundary restore across DCP/DFlash
|
|
|
+# groups, partial prefix-cache hits). Same model/flags/KV pin; the image is
|
|
|
+# the r27 build dir with a [build.versions] override of the vLLM pin (clone-
|
|
|
+# at-pin fetches the commit directly, the tree gate verifies 6361a741).
|
|
|
+
|
|
|
+# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
|
|
|
+#
|
|
|
+# local-inference-lab mixed quant: NVFP4 routed experts (layers 3–44,
|
|
|
+# group-16) + MXFP8 MTP experts (layer 45) + W4A4 input scales. Best agent
|
|
|
+# quant so far: tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8).
|
|
|
+# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
|
|
|
+# (~186 GB → 92.72 GiB weights/node; KV pinned at 10 GiB, see below)
|
|
|
+#
|
|
|
+# Image: glm53-flash-nvfp4-lab:local (docker-images/glm53-flash-nvfp4-lab/ —
|
|
|
+# day-0 arm64 image + NoPE-MLA mod + the four FujitsuPolycom lab-checkpoint
|
|
|
+# patches: model naming shim, modelopt MTP namespace fix, persistent_topk
|
|
|
+# CC-12.x guards). Their launch config is mirrored below.
|
|
|
+# PROVENANCE: https://github.com/FujitsuPolycom/glm53-flash-tp2-spark
|
|
|
+# (Apache-2.0; see patches/UPSTREAM.md) + forum 381350 #41.
|
|
|
+#
|
|
|
+# Deviations from their launch script (intentional):
|
|
|
+# * no --kda-prefill-backend flashkda (needs their base-image patch — skipped)
|
|
|
+# * --kv-cache-dtype fp8_ds_mla instead of `fp8` (same packed layout; this
|
|
|
+# spelling is what the NoPE-MLA mod booted with on the LibertAIDAI stack)
|
|
|
+# * --language-model-only + no mm-renderer patch/env (text-only)
|
|
|
+# * no bundled NCCL/LD_PRELOAD, no NCCL_* fabric block (our switched fabric
|
|
|
+# is configured via [nccl.ds4] + launcher GID auto-detect)
|
|
|
+# * no --moe-backend marlin: their proven stack uses the default backend for
|
|
|
+# THIS quant and reports clean evals. Sanity-check output for token loops
|
|
|
+# anyway (the silent FP4 MoE corruption gotcha) before trusting a new node.
|
|
|
+# * flashinfer autotune disabled via --kernel-config (their proven boot);
|
|
|
+# VLLM_ENGINE_READY_TIMEOUT_S stays high regardless.
|
|
|
+#
|
|
|
+# Boot success markers (engine log, in order):
|
|
|
+# "Model loading took 92.72 GiB" → "[quantprobe] ... algo=MXFP8" (MTP fix
|
|
|
+# active; algo=None means broken) → "GPU KV cache size: 1,164,369 tokens" →
|
|
|
+# "Application startup complete". ~700 s sequential weight load.
|
|
|
+#
|
|
|
+# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
|
|
|
+# :"low"} — omitted/other values select MAX reasoning effort (empty content on
|
|
|
+# short max_tokens).
|
|
|
+#
|
|
|
+# Optional: instanttensor loader (~30 s weight load vs ~700 s):
|
|
|
+# FROM glm53-flash-nvfp4-lab:local + pip install --no-deps --no-build-isolation
|
|
|
+# instanttensor==0.1.9, then add --load-format instanttensor (--no-deps is
|
|
|
+# required: resolution would downgrade the image's NCCL).
|
|
|
+#
|
|
|
+# PREREQUISITES: weights cached on both nodes:
|
|
|
+# [A/B CELL 2026-09-03] ship recipe verbatim, checkpoint swapped to the lab
|
|
|
+# non-Spark NVFP4 (isolates checkpoint from image lineage vs the lab-image cell).
|
|
|
+# [BISECT A2 2026-09-03] EXACT 90/100 recipe (236ce5a) + instanttensor + JIT
|
|
|
+# mounts, on the CURRENT eugr-patched image. Isolates image lineage from the
|
|
|
+# execution-env/flag cluster (AOT/V2/b12x/card-env). Comparator: 124/138.
|
|
|
+# GLM-5.3-Flash NVFP4-Spark — local-inference-lab quant with the speculative
|
|
|
+# layers embedded — 2× DGX Spark, TP=2.
|
|
|
+#
|
|
|
+# local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (rev 06972cdee7280cec849aedf05a02062b3e870246):
|
|
|
+# NVFP4 routed experts (layers 3+, group-16) + NVFP4 MTP expert layer in the
|
|
|
+# separate model-mtp-0000x.safetensors shards (mtp_tokens: 5 per their
|
|
|
+# lil.yaml), amax/nonexpert/inputscales side shards, Glm5NextForConditionalGeneration.
|
|
|
+# THE "DSpark speculative embedded" checkpoint — no separate DFlash2 download
|
|
|
+# needed; lil.yaml: default_speculator mtp, dflash2_tokens 7 (r12 doc fr: their
|
|
|
+# standalone-Q: MTP:3 254 tok/s CC1, 333 Sieve, 94 steps/s).
|
|
|
+#
|
|
|
+# Image: glm53-flash-nvfp4-lab:local (same proven day-0 arm64 stack + NoPE-MLA
|
|
|
+# mod + modelopt MTP namespace patch used by the 92/100 lab lane). The MTP
|
|
|
+# layer here is NVFP4 (not MXFP8) — the quantprobe marker may print
|
|
|
+# algo=NVFP4/- different; algo=None = patch missed (fall back to 4p67 image).
|
|
|
+#
|
|
|
+# Deviations from lab (unchanged mechanics): same fabric/env/GMU/KV pinning.
|
|
|
+# lil.yaml launch mirrors: block 256, disable_flashinfer_autotune, mm data
|
|
|
+# mode — our text-only flags identical to glm53-flash-nvfp4-lab.
|
|
|
+#
|
|
|
+# Boot markers: "Model loading took ~XX GiB" → "[quantprobe] ..." →
|
|
|
+# "GPU KV cache size" → "Application startup complete".
|
|
|
+#
|
|
|
+# PREREQUISITES: weights on the NAS hub (175 GiB): hf download local-inference-lab/GLM-5.3-Flash-NVFP4-Spark
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "GLM-5.3-Flash NVFP4 (r28 img, lab ckpt, 4x DCP4)"
|
|
|
+docker_image = "glm53-jovian-r28:local" # r28 vLLM pin over the r27 image tree
|
|
|
+docker_image_dir = "glm53-jovian-r27"
|
|
|
+# Required: the clean r12 image follows the solar contract (ENTRYPOINT []
|
|
|
+# + CMD bash), so the recipe carries the entrypoint (the old day-0-based
|
|
|
+# r12 image had ["vllm","serve"] baked in, which is why this was absent
|
|
|
+# until the clean rebuild).
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
|
|
|
+required_nodes = 4
|
|
|
+tensor_parallel_size = 4
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+
|
|
|
+args = [
|
|
|
+ # Stable served name — quant-agnostic (shared across glm53 quants)
|
|
|
+ "--served-model-name", "glm53",
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--distributed-executor-backend", "mp",
|
|
|
+ # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
|
|
|
+ # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
|
|
|
+ # page cache routinely leave ~108.5 -> flaky
|
|
|
+ # "Free memory ... less than desired" boot deaths (this was the class
|
|
|
+ # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
|
|
|
+ # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
|
|
|
+ # allocated (weights ~93 + pinned KV 10 + activations).
|
|
|
+ "--gpu-memory-utilization", "0.90",
|
|
|
+ # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
|
|
|
+ # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
|
|
|
+ "--kv-cache-memory-bytes", "17179869184", # 16 GiB: 4x NVFP4 has the headroom (47.1 GiB weights/node vs 108 usable); 2x stays 10
|
|
|
+ "--decode-context-parallel-size", "4",
|
|
|
+ "--cp-kv-cache-interleave-size", "4",
|
|
|
+ # No --kv-cache-dtype: let the r12 fork pick its default (the explicit
|
|
|
+ # fp8 / fp8_ds_mla both failed auto-selection on this build).
|
|
|
+ # The GLM5Next model wires the B12X sparse-MLA backend (sm_120/sm_121)
|
|
|
+ # when the global backend is B12X (glm5next/nvidia/attention.py:
|
|
|
+ # _select_sparse_backend); the sm12x auto-priority list never offers it
|
|
|
+ # (FLASHINFER_MLA_SPARSE_SM120 only supports head 576, not GLM's 512).
|
|
|
+ "--attention-backend", "B12X",
|
|
|
+ # pure I/O loader (kept across every config; 171 GB in ~5.5 min vs ~16 NFS)
|
|
|
+ "--load-format", "instanttensor",
|
|
|
+ # [R17 FULL-CLUSTER TEST] lil manifest backends (quality gate: hardmode
|
|
|
+ # must stay ≥88; r12-era this cluster dropped it to 74 — r17 may carry
|
|
|
+ # the fix through — one boot decides).
|
|
|
+ "--block-size", "256",
|
|
|
+ # FINAL BISECT (2026-09-03, hardmode 176pt, r17 img):
|
|
|
+ # knobs-only 87: d0 c1 24.1, d8192 23.8
|
|
|
+ # +moe b12x 88: d0 c1 23.5, d8192 26.6 <- SHIP (2 pts d8192)
|
|
|
+ # +linear b12x 44: W4A4 linear kernel destroys numerics (param-precision 0%)
|
|
|
+ # +align 88: but d0 c1 18.6 (-5) — perf-negative, dropped
|
|
|
+ # (all three together = 44-74; the lab needs its private packed package)
|
|
|
+ "--max-model-len", "524288",
|
|
|
+ "--max-num-seqs", "16",
|
|
|
+ "--max-num-batched-tokens", "4096",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--async-scheduling",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ # NO --disable-custom-all-reduce here: in the r27 fork the RoCEnante
|
|
|
+ # adapter is gated on the custom-all-reduce enable flag
|
|
|
+ # (use_roce_allreduce = use_custom_allreduce && VLLM_ENABLE_ROCE_ALLREDUCE).
|
|
|
+ # On 2x/4x 1-GPU-node clusters vLLM's own CUDA-IPC custom AR is functionally
|
|
|
+ # inert (no cross-node P2P), so dropping the flag only opens the b12x path;
|
|
|
+ # the b12x-PCIe branch stays off (VLLM_ENABLE_PCIE_ALLREDUCE defaults False).
|
|
|
+ # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
|
|
|
+ "--kernel-config", """
|
|
|
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
|
|
|
+ # Built-in MTP draft, 3 tokens (the r12-qualified "mtp:3" mode — their
|
|
|
+ # x86 numbers: 254 tok/s CC1, 2.37 accept; the checkpoint embeds up to
|
|
|
+ # 5 via the model-mtp-0000x shards but 3 is the verified depth). This is
|
|
|
+ # the checkpoint's own MTP head (Glm5NextMTPModel), NOT dspark — dspark
|
|
|
+ # is the DeepSeek-V4 family's mechanism; GLM's other r12 mode is
|
|
|
+ # dflash2:7 with a separate trained draft checkpoint.
|
|
|
+ # The draft model's vllm_config takes its attention backend from the
|
|
|
+ # SPECULATIVE config, not the global flag (eagle/utils.py:
|
|
|
+ # _make_eagle_draft_vllm_config) — B12X here mirrors the main model's.
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'""",
|
|
|
+ "--language-model-only",
|
|
|
+ # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
|
|
|
+ # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
|
|
|
+ # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
|
|
|
+ # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
|
|
|
+ # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
|
|
|
+ # graphs moved BEFORE tool-call-parser; enforce-eager removed for the
|
|
|
+ # r17-graphs test (bisect A2 was 88/100 eager; graphs+MTP fix is public
|
|
|
+ # in r17 refs? verify by bench).
|
|
|
+ "--tool-call-parser", "glm47",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--reasoning-parser", "glm45",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for r27
|
|
|
+# lanes since the 2026-09-08 A/B (4x DCP4: decode +31..+90%, quality 160 vs
|
|
|
+# 155, prefill within noise; specs/tech-architecture/glm53-recipe-state.md,
|
|
|
+# history/night-shifts/2026-09-07.md). Needs the CX-7 RoCE fabric (all of
|
|
|
+# ours); single-node serves skip it silently. Caps default 2MB AR / 16MB AG.
|
|
|
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
|
|
|
+# FlashInfer JIT (mxfp8_gemm_cutlass_sm120, ~12 kernels) compiles at first
|
|
|
+# boot with 12 parallel nvcc jobs -> OOM-killed (code 137) on GB10's
|
|
|
+# 121 GB unified while the 93 GB weights + 10 GB KV pin are resident.
|
|
|
+# MAX_JOBS caps ninja parallelism so the JIT fits in the remaining ~20 GB.
|
|
|
+MAX_JOBS = "2"
|
|
|
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
|
|
|
+VLLM_MLA_NOPE_PAD_ROPE = "1"
|
|
|
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
|
+# Upstream launch-script env (memory + compile knobs)
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+VLLM_USE_AOT_COMPILE = "0"
|
|
|
+VLLM_USE_STANDALONE_COMPILE = "1"
|
|
|
+VLLM_USE_FLASHINFER_SAMPLER = "1"
|
|
|
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
|
|
|
+TORCH_USE_RTLD_GLOBAL = "1"
|
|
|
+VLLM_NO_USAGE_STATS = "1"
|
|
|
+
|
|
|
+# Architecture (aarch64 / sm_121a DGX Spark)
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+CMAKE_CUDA_ARCHITECTURES = "121"
|
|
|
+OMP_NUM_THREADS = "2"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/glm53-jit-121f/b12x"
|
|
|
+container = "/root/.cache/b12x"
|
|
|
+
|
|
|
+# JIT caches persist per-node across serves (was: cold /root/.cache per
|
|
|
+# container -> ~20 min flashinfer/torch recompile every serve). The host
|
|
|
+# dirs are shared between lanes of the same arch — each cache
|
|
|
+# self-namespaces by its engine version key internally.
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/flashinfer"
|
|
|
+container = "/root/.cache/flashinfer"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/torchinductor"
|
|
|
+container = "/root/.cache/torchinductor"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/vllm"
|
|
|
+container = "/root/.cache/vllm"
|
|
|
+
|
|
|
+# r28 engine pin (the image dir pins r27 63a82f8d): recipe pins win over the
|
|
|
+# image's versions.env, the merged file folds into the build checksum, so this
|
|
|
+# is a distinct image (tag glm53-jovian-r28:local) sharing the r27 build tree.
|
|
|
+# The r28 tree has NO requirements/cmake changes vs r27 (verified via the
|
|
|
+# GitHub compare API, 13 commits / 29 files), so the tree gates in
|
|
|
+# fetch-deps.sh still hold with the same dep pins.
|
|
|
+[build.versions]
|
|
|
+VLLM_PIN = "62f6385c2ab633041347925b174e463345bf39b1"
|
|
|
+VLLM_TREE = "6361a741337f60bed8a0431d655c78b90f658210"
|
|
|
+VLLM_PRETEND_VERSION = "0.1.dev0+glm53.r28.62f6385c"
|
