|
@@ -1,52 +1,39 @@
|
|
|
-# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
|
|
|
|
|
|
|
+# GLM-5.3-Flash NVFP4-lab — the lab's NVFP4 quant (2026-09-16 RE-CUT,
|
|
|
|
|
+# local-inference-lab/GLM-5.3-Flash-NVFP4), 2× DGX Spark, TP=2.
|
|
|
#
|
|
#
|
|
|
-# local-inference-lab mixed quant: NVFP4 routed experts (layers 3–44,
|
|
|
|
|
-# group-16) + MXFP8 MTP experts (layer 45) + W4A4 input scales. Best agent
|
|
|
|
|
-# quant so far: tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8).
|
|
|
|
|
-# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
|
|
|
|
|
-# (~186 GB → 92.72 GiB weights/node; KV pinned at 10 GiB, see below)
|
|
|
|
|
|
|
+# The 09-16 re-cut replaced the old ~186 GB upload in place: 199.4 GB
|
|
|
|
|
+# (36 expert shards + 4 nonexpert + 4 MTP shards, modelopt NVFP4).
|
|
|
|
|
+# Operator direction (2026-09-20): "they improved it a lot" — this lane
|
|
|
|
|
+# re-tests the re-cut against the old 89/100 anchor on the proven r35
|
|
|
|
|
+# engine (same interim mode as the Spark lanes).
|
|
|
#
|
|
#
|
|
|
-# Image: glm53-flash-nvfp4-lab:local (docker-images/glm53-flash-nvfp4-lab/ —
|
|
|
|
|
-# day-0 arm64 image + NoPE-MLA mod + the four FujitsuPolycom lab-checkpoint
|
|
|
|
|
-# patches: model naming shim, modelopt MTP namespace fix, persistent_topk
|
|
|
|
|
-# CC-12.x guards). Their launch config is mirrored below.
|
|
|
|
|
-# PROVENANCE: https://github.com/FujitsuPolycom/glm53-flash-tp2-spark
|
|
|
|
|
-# (Apache-2.0; see patches/UPSTREAM.md) + forum 381350 #41.
|
|
|
|
|
|
|
+# Structure (verified from model.safetensors.index.json, identical to the
|
|
|
|
|
+# Spark checkpoint): `model.language_model.layers.*` with SPLIT GDN naming
|
|
|
|
|
+# (q/k/g_a/g_b/f_a/f_b/b_proj + hc_* hybrid-conv), a `model.visual.*`
|
|
|
|
|
+# tower (skipped via --language-model-only), and the MTP head embedded at
|
|
|
|
|
+# layer 45 (eh_proj/enorm/hnorm) — no separate drafter download. Same
|
|
|
|
|
+# loader path as the Spark lane: the JJ fork lineage (r35) maps it natively
|
|
|
|
|
+# (no KeyError); r38 is out — b12x preparation-pool deadlock on 2x GB10.
|
|
|
#
|
|
#
|
|
|
-# Deviations from their launch script (intentional):
|
|
|
|
|
-# * no --kda-prefill-backend flashkda (needs their base-image patch — skipped)
|
|
|
|
|
-# * --kv-cache-dtype fp8_ds_mla instead of `fp8` (same packed layout; this
|
|
|
|
|
-# spelling is what the NoPE-MLA mod booted with on the LibertAIDAI stack)
|
|
|
|
|
-# * --language-model-only + no mm-renderer patch/env (text-only)
|
|
|
|
|
-# * no bundled NCCL/LD_PRELOAD, no NCCL_* fabric block (our switched fabric
|
|
|
|
|
-# is configured via [nccl.ds4] + launcher GID auto-detect)
|
|
|
|
|
-# * no --moe-backend marlin: their proven stack uses the default backend for
|
|
|
|
|
-# THIS quant and reports clean evals. Sanity-check output for token loops
|
|
|
|
|
-# anyway (the silent FP4 MoE corruption gotcha) before trusting a new node.
|
|
|
|
|
-# * flashinfer autotune disabled via --kernel-config (their proven boot);
|
|
|
|
|
-# VLLM_ENGINE_READY_TIMEOUT_S stays high regardless.
|
|
|
|
|
|
|
+# Image: glm53-jovian-r35:local + VLLM_DISABLED_KERNELS (the small GDN
|
|
|
|
|
+# mxfp8 GEMMs reject flashinfer cutlass' min n/k=128 — routed to the b12x
|
|
|
|
|
+# Mxfp8 backend; proven on the Spark lanes 162/176 + 161/176).
|
|
|
#
|
|
#
|
|
|
-# Boot success markers (engine log, in order):
|
|
|
|
|
-# "Model loading took 92.72 GiB" → "[quantprobe] ... algo=MXFP8" (MTP fix
|
|
|
|
|
-# active; algo=None means broken) → "GPU KV cache size: 1,164,369 tokens" →
|
|
|
|
|
-# "Application startup complete". ~700 s sequential weight load.
|
|
|
|
|
|
|
+# Memory: 199.4 GB / 2 ranks ≈ 93 GiB weights/node at GMU 0.88 (107.0 GiB)
|
|
|
|
|
+# — no KV pin (auto-sizing takes the remaining ~10 GiB; a 10 GiB pin as in
|
|
|
|
|
+# the old 186 GB cut would overshoot the tighter budget).
|
|
|
#
|
|
#
|
|
|
-# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
|
|
|
|
|
-# :"low"} — omitted/other values select MAX reasoning effort (empty content on
|
|
|
|
|
-# short max_tokens).
|
|
|
|
|
-#
|
|
|
|
|
-# instanttensor loader — baked into the image (Dockerfile step 3) and enabled
|
|
|
|
|
-# via args + env knobs below. MEASURED 2026-09-03: 631 s total load vs ~700 s
|
|
|
|
|
-# sequential (lab model resolves TWO passes: target + MTP/draft shards; the
|
|
|
|
|
-# r12 spark stack loads once in ~5.5 min). Modest gain, worth keeping.
|
|
|
|
|
-#
|
|
|
|
|
-# PREREQUISITES: weights cached on both nodes:
|
|
|
|
|
-# hf download local-inference-lab/GLM-5.3-Flash-NVFP4
|
|
|
|
|
|
|
+# PREREQUISITES: weights on each node's LOCAL hub (the NAS disk died
|
|
|
|
|
+# 2026-09-19): /home/kilork/hf-local, ~/.cache/huggingface symlink
|
|
|
|
|
+# re-pointed per node. Re-cut re-download in progress on all 4 nodes
|
|
|
|
|
+# (dgx1 is the tightest: ~40 GiB left after the model).
|
|
|
|
|
|
|
|
engine = "vllm"
|
|
engine = "vllm"
|
|
|
-name = "GLM-5.3-Flash NVFP4-lab (vLLM, 2x Spark, MTP-3)"
|
|
|
|
|
-docker_image = "glm53-flash-nvfp4-lab:local"
|
|
|
|
|
-docker_image_dir = "glm53-flash-nvfp4-lab"
|
|
|
|
|
|
|
+name = "GLM-5.3-Flash NVFP4-lab re-cut (vLLM, 2x Spark, MTP-3, r35)"
|
|
|
|
|
+docker_image = "glm53-jovian-r35:local" # r35 loads the lab quants (r38 pool-deadlock on 2x, see bug report)
|
|
|
|
|
+docker_image_dir = "glm53-jovian-r27"
|
|
|
|
|
+build_profile = "glm53-r35"
|
|
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
|
|
model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
|
|
|
parent_model = "zai-org/GLM-5.3-Flash"
|
|
parent_model = "zai-org/GLM-5.3-Flash"
|
|
|
required_nodes = 2
|
|
required_nodes = 2
|
|
@@ -60,41 +47,31 @@ args = [
|
|
|
"--served-model-name", "glm53",
|
|
"--served-model-name", "glm53",
|
|
|
"--trust-remote-code",
|
|
"--trust-remote-code",
|
|
|
"--distributed-executor-backend", "mp",
|
|
"--distributed-executor-backend", "mp",
|
|
|
- # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
|
|
|
|
|
- # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
|
|
|
|
|
- # page cache routinely leave ~108.5 -> flaky
|
|
|
|
|
- # "Free memory ... less than desired" boot deaths (this was the class
|
|
|
|
|
- # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
|
|
|
|
|
- # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
|
|
|
|
|
- # allocated (weights ~93 + pinned KV 10 + activations).
|
|
|
|
|
- "--gpu-memory-utilization", "0.82",
|
|
|
|
|
- # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
|
|
|
|
|
- # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
|
|
|
|
|
- "--kv-cache-memory-bytes", "10737418240",
|
|
|
|
|
- "--kv-cache-dtype", "fp8_ds_mla",
|
|
|
|
|
|
|
+ # GMU is ONLY a startup guardrail: 0.88 = 107.0 GiB, below any
|
|
|
|
|
+ # realistic free-memory dip on GB10's 121.63 GiB (#48140).
|
|
|
|
|
+ "--gpu-memory-utilization", "0.88",
|
|
|
|
|
+ # no KV pin: ~93 GiB weights/rank leaves ~10 GiB — let the engine
|
|
|
|
|
+ # auto-size the pool (a fixed pin overshoots the tighter budget)
|
|
|
"--block-size", "256",
|
|
"--block-size", "256",
|
|
|
"--max-model-len", "524288",
|
|
"--max-model-len", "524288",
|
|
|
- "--max-num-seqs", "8",
|
|
|
|
|
|
|
+ "--max-num-seqs", "16",
|
|
|
"--max-num-batched-tokens", "4096",
|
|
"--max-num-batched-tokens", "4096",
|
|
|
"--enable-chunked-prefill",
|
|
"--enable-chunked-prefill",
|
|
|
"--async-scheduling",
|
|
"--async-scheduling",
|
|
|
"--enable-prefix-caching",
|
|
"--enable-prefix-caching",
|
|
|
- "--disable-custom-all-reduce",
|
|
|
|
|
|
|
+ # B12X sparse-MLA (required for head-512 on sm_12x)
|
|
|
|
|
+ "--attention-backend", "B12X",
|
|
|
|
|
+ # pure I/O loader (baked into the r27-lineage images; ~1 min off local NVMe)
|
|
|
|
|
+ "--load-format", "instanttensor",
|
|
|
# Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
|
|
# Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
|
|
|
"--kernel-config", """
|
|
"--kernel-config", """
|
|
|
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
|
|
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
|
|
|
- # MTP-3 draft from the MXFP8 layer-45 shard (needs modelopt.patch in image)
|
|
|
|
|
|
|
+ # Built-in MTP draft, 3 tokens (layer 45, in-checkpoint). The draft
|
|
|
|
|
+ # model's vllm_config takes its attention backend from the SPECULATIVE
|
|
|
|
|
+ # config, not the global flag — B12X mirrors the main model.
|
|
|
"--speculative-config", """
|
|
"--speculative-config", """
|
|
|
-'{"method":"mtp","num_speculative_tokens":3}'""",
|
|
|
|
|
|
|
+'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'""",
|
|
|
"--language-model-only",
|
|
"--language-model-only",
|
|
|
- # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
|
|
|
|
|
- # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
|
|
|
|
|
- # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
|
|
|
|
|
- # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
|
|
|
|
|
- # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
|
|
|
|
|
- "--enforce-eager",
|
|
|
|
|
- # instanttensor loader (0.1.9 baked in image; ~30 s load vs ~700 s NFS)
|
|
|
|
|
- "--load-format", "instanttensor",
|
|
|
|
|
"--tool-call-parser", "glm47",
|
|
"--tool-call-parser", "glm47",
|
|
|
"--enable-auto-tool-choice",
|
|
"--enable-auto-tool-choice",
|
|
|
"--reasoning-parser", "glm45",
|
|
"--reasoning-parser", "glm45",
|
|
@@ -102,19 +79,24 @@ args = [
|
|
|
|
|
|
|
|
[env]
|
|
[env]
|
|
|
HF_HUB_OFFLINE = "1"
|
|
HF_HUB_OFFLINE = "1"
|
|
|
-# lab INSTANTTENSOR loader knobs (same values the spark recipe validated:
|
|
|
|
|
-# lil render BACKEND/CONCURRENCY/IO_DEPTH + lil.yaml BUFFER_SIZE/CHUNK)
|
|
|
|
|
-INSTANTTENSOR_BACKEND = "BUFFERED"
|
|
|
|
|
-INSTANTTENSOR_CONCURRENCY = "1"
|
|
|
|
|
-INSTANTTENSOR_IO_DEPTH = "3"
|
|
|
|
|
-INSTANTTENSOR_BUFFER_SIZE = "67108864"
|
|
|
|
|
-INSTANTTENSOR_CHUNK_SIZE = "8388608"
|
|
|
|
|
-# NoPE-MLA rope-pad runtime gate (mod baked into the image)
|
|
|
|
|
|
|
+# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for the
|
|
|
|
|
+# r27 lanes since the 2026-09-08 A/B (4x: decode +31..+90%, quality 160 vs
|
|
|
|
|
+# 155). Needs the RoCE fabric; single-node serves skip it silently.
|
|
|
|
|
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
|
|
|
|
|
+# Route mxfp8 linears away from flashinfer cutlass (rejects small GDN GEMMs: min n/k=128)
|
|
|
|
|
+VLLM_DISABLED_KERNELS = "FlashInferCutlassMxfp8LinearKernel"
|
|
|
|
|
+# FlashInfer JIT compiles at first boot with N parallel nvcc jobs ->
|
|
|
|
|
+# OOM-killed on GB10 while weights+KV are resident; cap the ninja jobs.
|
|
|
|
|
+MAX_JOBS = "2"
|
|
|
|
|
+# NoPE-MLA rope-pad runtime gate (mod baked into the image lineage)
|
|
|
VLLM_MLA_NOPE_PAD_ROPE = "1"
|
|
VLLM_MLA_NOPE_PAD_ROPE = "1"
|
|
|
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
|
# Upstream launch-script env (memory + compile knobs)
|
|
# Upstream launch-script env (memory + compile knobs)
|
|
|
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
VLLM_USE_AOT_COMPILE = "0"
|
|
VLLM_USE_AOT_COMPILE = "0"
|
|
|
|
|
+VLLM_USE_STANDALONE_COMPILE = "1"
|
|
|
|
|
+VLLM_USE_FLASHINFER_SAMPLER = "1"
|
|
|
|
|
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
|
|
|
TORCH_USE_RTLD_GLOBAL = "1"
|
|
TORCH_USE_RTLD_GLOBAL = "1"
|
|
|
VLLM_NO_USAGE_STATS = "1"
|
|
VLLM_NO_USAGE_STATS = "1"
|
|
|
|
|
|
|
@@ -123,4 +105,22 @@ TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
|
|
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
|
|
|
CUTE_DSL_ARCH = "sm_121a"
|
|
CUTE_DSL_ARCH = "sm_121a"
|
|
|
CMAKE_CUDA_ARCHITECTURES = "121"
|
|
CMAKE_CUDA_ARCHITECTURES = "121"
|
|
|
-OMP_NUM_THREADS = "4"
|
|
|
|
|
|
|
+OMP_NUM_THREADS = "2"
|
|
|
|
|
+
|
|
|
|
|
+# JIT caches persist per-node across serves (cold /root/.cache per container
|
|
|
|
|
+# = ~20 min flashinfer/torch recompile every serve)
|
|
|
|
|
+[[extra_mounts]]
|
|
|
|
|
+host = "~/sm121-cache/glm53-jit-121f/b12x"
|
|
|
|
|
+container = "/root/.cache/b12x"
|
|
|
|
|
+
|
|
|
|
|
+[[extra_mounts]]
|
|
|
|
|
+host = "~/sm121-cache/jit-121f/flashinfer"
|
|
|
|
|
+container = "/root/.cache/flashinfer"
|
|
|
|
|
+
|
|
|
|
|
+[[extra_mounts]]
|
|
|
|
|
+host = "~/sm121-cache/jit-121f/torchinductor"
|
|
|
|
|
+container = "/root/.cache/torchinductor"
|
|
|
|
|
+
|
|
|
|
|
+[[extra_mounts]]
|
|
|
|
|
+host = "~/sm121-cache/jit-121f/vllm"
|
|
|
|
|
+container = "/root/.cache/vllm"
|
