|
|
@@ -0,0 +1,119 @@
|
|
|
+# GLM-5.3-Flash NVFP4-lab RE-CUT on r39 (jovian-judgement tip 2026-09-21),
|
|
|
+# 4× DGX Spark, TP=4.
|
|
|
+#
|
|
|
+# 4-node twin of glm53-flash-nvfp4-lab-r39 (2x, benched 164/176 on v2.7.0):
|
|
|
+# same model, same 512K config as the proven r35 4x lane (159/176) — only
|
|
|
+# the engine image moves (r35 -> r39, dir jovian-judgement-r39, profile
|
|
|
+# r39), so the bench is a clean engine A/B on the 4-rank fabric. The r39
|
|
|
+# tip is also the first r38+ engine that boots on 4x GB10 at all (r38
|
|
|
+# deadlocks at the b12x preparation pool, BUG-2026-09-19, reproduced on
|
|
|
+# both 2x and 4x).
|
|
|
+#
|
|
|
+# Note: r39 moved the external-dep contract (all cmake pins + DeepSelect),
|
|
|
+# so the image builds from docker-images/jovian-judgement-r39 (its own
|
|
|
+# fetch-deps.sh + deps.lock).
|
|
|
+#
|
|
|
+# Structure (verified from model.safetensors.index.json, identical to the
|
|
|
+# Spark checkpoint): `model.language_model.layers.*` with SPLIT GDN naming
|
|
|
+# (q/k/g_a/g_b/f_a/f_b/b_proj + hc_* hybrid-conv), a `model.visual.*`
|
|
|
+# tower (skipped via --language-model-only), and the MTP head embedded at
|
|
|
+# layer 45 (eh_proj/enorm/hnorm) — no separate drafter download.
|
|
|
+#
|
|
|
+# Memory: 199.4 GB / 4 ranks ≈ 46.5 GiB weights/node at GMU 0.88 —
|
|
|
+# no KV pin, auto-sized pool takes the remaining ~55 GiB/node.
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "GLM-5.3-Flash NVFP4-lab re-cut (vLLM, 4x Spark, MTP-3, r39 tip)"
|
|
|
+docker_image = "jovian-judgement:r39-glm53"
|
|
|
+docker_image_dir = "jovian-judgement-r39"
|
|
|
+build_profile = "r39"
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
|
|
|
+parent_model = "zai-org/GLM-5.3-Flash"
|
|
|
+required_nodes = 4
|
|
|
+tensor_parallel_size = 4
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+
|
|
|
+args = [
|
|
|
+ # Stable served name — quant-agnostic (shared across glm53 quants)
|
|
|
+ "--served-model-name", "glm53",
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--distributed-executor-backend", "mp",
|
|
|
+ # GMU is ONLY a startup guardrail: 0.88 = 107.0 GiB, below any
|
|
|
+ # realistic free-memory dip on GB10's 121.63 GiB (#48140).
|
|
|
+ "--gpu-memory-utilization", "0.88",
|
|
|
+ # no KV pin: ~46.5 GiB weights/rank leaves ~55 GiB — auto-sized pool
|
|
|
+ "--block-size", "256",
|
|
|
+ "--max-model-len", "524288",
|
|
|
+ "--max-num-seqs", "16",
|
|
|
+ "--max-num-batched-tokens", "4096",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--async-scheduling",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ # B12X sparse-MLA (required for head-512 on sm_12x)
|
|
|
+ "--attention-backend", "B12X",
|
|
|
+ # pure I/O loader (baked into the r27-lineage images; ~1 min off local NVMe)
|
|
|
+ "--load-format", "instanttensor",
|
|
|
+ # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
|
|
|
+ "--kernel-config", """
|
|
|
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
|
|
|
+ # Built-in MTP draft, 3 tokens (layer 45, in-checkpoint). The draft
|
|
|
+ # model's vllm_config takes its attention backend from the SPECULATIVE
|
|
|
+ # config, not the global flag — B12X mirrors the main model.
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'""",
|
|
|
+ "--language-model-only",
|
|
|
+ "--tool-call-parser", "glm47",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--reasoning-parser", "glm45",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for the
|
|
|
+# r27 lanes since the 2026-09-08 A/B (4x: decode +31..+90%, quality 160 vs
|
|
|
+# 155). Needs the RoCE fabric; single-node serves skip it silently.
|
|
|
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
|
|
|
+# Route mxfp8 linears away from flashinfer cutlass (rejects small GDN GEMMs: min n/k=128)
|
|
|
+VLLM_DISABLED_KERNELS = "FlashInferCutlassMxfp8LinearKernel"
|
|
|
+# FlashInfer JIT compiles at first boot with N parallel nvcc jobs ->
|
|
|
+# OOM-killed on GB10 while weights+KV are resident; cap the ninja jobs.
|
|
|
+MAX_JOBS = "2"
|
|
|
+# NoPE-MLA rope-pad runtime gate (mod baked into the image lineage)
|
|
|
+VLLM_MLA_NOPE_PAD_ROPE = "1"
|
|
|
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
|
+# Upstream launch-script env (memory + compile knobs)
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+VLLM_USE_AOT_COMPILE = "0"
|
|
|
+VLLM_USE_STANDALONE_COMPILE = "1"
|
|
|
+VLLM_USE_FLASHINFER_SAMPLER = "1"
|
|
|
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
|
|
|
+TORCH_USE_RTLD_GLOBAL = "1"
|
|
|
+VLLM_NO_USAGE_STATS = "1"
|
|
|
+
|
|
|
+# Architecture (aarch64 / sm_121a DGX Spark)
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+CMAKE_CUDA_ARCHITECTURES = "121"
|
|
|
+OMP_NUM_THREADS = "2"
|
|
|
+
|
|
|
+# JIT caches persist per-node across serves (cold /root/.cache per container
|
|
|
+# = ~20 min flashinfer/torch recompile every serve)
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/glm53-jit-121f/b12x"
|
|
|
+container = "/root/.cache/b12x"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/flashinfer"
|
|
|
+container = "/root/.cache/flashinfer"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/torchinductor"
|
|
|
+container = "/root/.cache/torchinductor"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/vllm"
|
|
|
+container = "/root/.cache/vllm"
|
