|
|
@@ -0,0 +1,167 @@
|
|
|
+# GLM-5.3 (full) MXFP4 — 8x DGX Spark @512K, no DCP.
|
|
|
+#
|
|
|
+# P2.1 of the 2026-09-18 night: the full model's first 8x trial. The 4x
|
|
|
+# 512K state is blocked by the DCP>1 sparse-decode crash
|
|
|
+# (deepseek_v32/attention.py:596, r35+r37); at 8x no DCP is needed:
|
|
|
+#
|
|
|
+# weights 403.4 GB / 8 = 50.4 GB/node
|
|
|
+# budget GMU 0.85 x 121.63 = 103.4 GiB
|
|
|
+# KV pool 103.4 - 50.4 - 12 (process) = ~41 GiB/rank
|
|
|
+# 512K req 26.3 GiB -> 1.5x concurrency (768K would be ~39.5, tight)
|
|
|
+#
|
|
|
+# Same image/contract as the qualified 4x 131K state (marlin linear, B12X
|
|
|
+# attention, instanttensor, block 256, RoCE allreduce) minus DCP and the KV
|
|
|
+# pin (the pool is GMU-driven at 8x). The 4x 131K anchor (dev67 no-spec):
|
|
|
+# 158/176, 934 pp / 19.0 tg / 2,124 ms.
|
|
|
+engine = "vllm"
|
|
|
+name = "GLM-5.3 (full) MXFP4 (r37 img, RedHatAI ckpt, 8x DCP1 @512K)"
|
|
|
+# Image: the r37 DS4.1 build (2026-09-14, night shift) - the r35 tree's DCP4
|
|
|
+# sparse-INDEXER path asserts interleave==1 and then crashes in the DCP sparse
|
|
|
+# decode reshape during graph capture ("shape '[16, 16, 512]' is invalid for
|
|
|
+# input of size 524288" - the x4 DCP factor in the attention.py _sparse_
|
|
|
+# decode). r37 is the newer DS4.1 lock whose DCP sparse-MLA path is the
|
|
|
+# qualified one (same tree family as our DCP2/DFlash2 work); the GlmMoeDsa
|
|
|
+# arch + MTP + b12x ship in the r37 image (verified 2026-09-13).
|
|
|
+docker_image = "jovian-r37-ds41:local"
|
|
|
+docker_image_dir = "glm53-jovian-r27"
|
|
|
+# NOTE: build_profile must stay a TOP-LEVEL key — a bare key after
|
|
|
+# [[extra_mounts]] would parse into the last mount table and be silently
|
|
|
+# ignored (2026-09-08).
|
|
|
+build_profile = "jovian-r37-ds41"
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+model = "RedHatAI/GLM-5.3-MXFP4"
|
|
|
+parent_model = "zai-org/GLM-5.3"
|
|
|
+required_nodes = 8
|
|
|
+tensor_parallel_size = 8
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+
|
|
|
+args = [
|
|
|
+ # R31/r35 delta (their qualified locks): adaptive prefill/decode
|
|
|
+ # compute-time sharing with a fixed 40% prefill floor.
|
|
|
+ "--prefill-compute-share", "0.4",
|
|
|
+ # Stable served name — quant-agnostic (shared across glm53 lanes)
|
|
|
+ "--served-model-name", "glm53",
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--distributed-executor-backend", "mp",
|
|
|
+ # Admission guardrail only (KV is pinned below): 0.85×121.63 = 103.4 GiB.
|
|
|
+ # 0.88 (107.0) is the Flash lane's value and is too tight here — the full
|
|
|
+ # model's weights leave ~24 GiB before the engine's own ~12 GiB.
|
|
|
+ "--gpu-memory-utilization", "0.85",
|
|
|
+ # No KV pin: the pool is GMU-driven at 8x (~41 GiB/rank, see header).
|
|
|
+ # No --kv-cache-dtype: let the r35 fork pick its native default for
|
|
|
+ # GlmMoeDsa attention (the Flash lane's explicit fp8/fp8_ds_mla spellings
|
|
|
+ # both failed auto-selection on this build, and those were Flash-MLA
|
|
|
+ # knobs). The card's --kv-cache-dtype fp8_e4m3 is the fallback spelling
|
|
|
+ # if the pool needs to shrink.
|
|
|
+ "--attention-backend", "B12X",
|
|
|
+ # MXFP4 kernel routing (measured on the first boot, 2026-09-13 ~23:10 UTC):
|
|
|
+ # - MoE: the compressed-tensors W4A4-MXFP4 scheme picks MarlinExperts on
|
|
|
+ # its own (A16 dequant GEMM - correct on GB10). The --moe-backend flag
|
|
|
+ # does not reach that scheme (verified inert: it still logged "Using
|
|
|
+ # MarlinExperts for MXFP4 MoE" with the flag set), so we do not set it.
|
|
|
+ # - LINEAR: the default priority [FlashInfer, Marlin, Humming, B12x]
|
|
|
+ # starts with the FlashInfer kernel, whose is_supported() wrongly
|
|
|
+ # accepts capability 121 (the numeric has_device_capability(100) check
|
|
|
+ # passes on GB10) while the cute-dsl mm_fp4 GEMM it calls rejects it at
|
|
|
+ # runtime with:
|
|
|
+ # BackendSupportedError: mm_fp4 does not support backend 'cute-dsl'
|
|
|
+ # with capability 121
|
|
|
+ # (the first probe boot died there, kv_b_proj dequant in mla_attention.py).
|
|
|
+ # --linear-backend b12x forces the B12xMxFp4LinearKernel - "MXFP4 linear
|
|
|
+ # through the native B12X SM120 dense GEMM" - the right path on GB10,
|
|
|
+ # verified per component: is_device_capability_family(120)=True, b12x
|
|
|
+ # blockscaled.is_supported()=True, and the cute-dsl input quantize
|
|
|
+ # (flashinfer_mxfp4_quantize) works on capability 121; only the cute-dsl
|
|
|
+ # mm_fp4 GEMM is unsupported there. Boot log to look for:
|
|
|
+ # "Using B12xMxFp4LinearKernel for MXFP4 GEMM"
|
|
|
+ # (The lab's GLM-5.1 runbook describes the same kernel family for the
|
|
|
+ # quant_method=mxfp4 ingestion path - this checkpoint goes through the
|
|
|
+ # compressed-tensors path, which is why --moe-backend was the wrong lever.)
|
|
|
+ # First attempt: b12x (the native SM120 GEMM) - correct selection, but the
|
|
|
+ # unconditional b12x warmup (b12x_warmup.py, no config gate) crashes on
|
|
|
+ # this checkpoint's layer shapes during kernel compile:
|
|
|
+ # RuntimeError: Expected source and destination tensors to have the same
|
|
|
+ # rank, but got 3 and 4 (b12x/gemm/blockscaled/_linear.py dense_gemm,
|
|
|
+ # via _apply_b12x_mxfp4_linear on a synthetic zero source)
|
|
|
+ # so the mxfp4 LINEAR layers run on Marlin (weight-only A16 dequant GEMM -
|
|
|
+ # the same family as the MarlinExperts the MoE scheme picked). Slower than
|
|
|
+ # native FP4 for the attention projections, but the MoE experts (the
|
|
|
+ # dominant cost) are Marlin either way. The b12x warmup rank bug is
|
|
|
+ # filed-worthy for the lab (RedHatAI GLM-5.3-MXFP4 on GB10/sm_121a).
|
|
|
+ "--linear-backend", "marlin",
|
|
|
+ # pure I/O loader (kept across every GLM config)
|
|
|
+ "--load-format", "instanttensor",
|
|
|
+ "--block-size", "256",
|
|
|
+ "--max-model-len", "524288",
|
|
|
+ "--max-num-seqs", "8",
|
|
|
+ "--max-num-batched-tokens", "4096",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--async-scheduling",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup (the
|
|
|
+ # autotune scratch at 512K shapes does not fit the headroom, 2026-08-28).
|
|
|
+ "--kernel-config", """
|
|
|
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
|
|
|
+ "--language-model-only",
|
|
|
+ # NO --speculative-config on the first boot — see the header. MTP head is
|
|
|
+ # in the checkpoint (num_nextn_predict_layers=1); add mtp k=3 after a
|
|
|
+ # clean boot + bench.
|
|
|
+ "--tool-call-parser", "glm47",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--reasoning-parser", "glm45",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+
|
|
|
+# R31 lock: 16 NCCL channels (their qualified runtime).
|
|
|
+NCCL_NCHANNELS = "16"
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for r27+
|
|
|
+# lanes since the 2026-09-08 A/B (4x DCP4: decode +31..+90%, quality 160 vs
|
|
|
+# 155; specs/tech-architecture/glm53-recipe-state.md). Needs the CX-7 RoCE
|
|
|
+# fabric (all of ours); single-node serves skip it silently.
|
|
|
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
|
|
|
+# Compile-OOM guard: first boot JIT-compiles the mxfp8 gemm module with 12
|
|
|
+# parallel nvcc jobs (~3 GiB each) behind ~94 GiB of resident weights →
|
|
|
+# cicc OOM-killed (2026-09-04). flashinfer's run_ninja reads MAX_JOBS.
|
|
|
+MAX_JOBS = "2"
|
|
|
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
|
|
|
+VLLM_MLA_NOPE_PAD_ROPE = "1"
|
|
|
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
|
+# Upstream launch-script env (memory + compile knobs)
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+VLLM_USE_AOT_COMPILE = "0"
|
|
|
+VLLM_USE_STANDALONE_COMPILE = "1"
|
|
|
+VLLM_USE_FLASHINFER_SAMPLER = "1"
|
|
|
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
|
|
|
+TORCH_USE_RTLD_GLOBAL = "1"
|
|
|
+VLLM_NO_USAGE_STATS = "1"
|
|
|
+
|
|
|
+# Architecture (aarch64 / sm_121a DGX Spark)
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+CMAKE_CUDA_ARCHITECTURES = "121"
|
|
|
+OMP_NUM_THREADS = "2"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/glm53-jit-121f/b12x"
|
|
|
+container = "/root/.cache/b12x"
|
|
|
+
|
|
|
+# JIT caches persist per-node across serves (was: cold /root/.cache per
|
|
|
+# container -> ~20 min flashinfer/torch recompile every serve). The host
|
|
|
+# dirs are shared between lanes of the same arch — each cache
|
|
|
+# self-namespaces by its engine version key internally.
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/flashinfer"
|
|
|
+container = "/root/.cache/flashinfer"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/torchinductor"
|
|
|
+container = "/root/.cache/torchinductor"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/vllm"
|
|
|
+container = "/root/.cache/vllm"
|
