|
|
@@ -0,0 +1,116 @@
|
|
|
+# GLM-5.3-Flash NVFP4 — eugr B12X full-cluster variant (vLLM) — 2× DGX Spark, TP=2.
|
|
|
+#
|
|
|
+# Port of eugr's dual-Spark recipe (eugr/spark-vllm-docker @ 94b3c30,
|
|
|
+# 2026-09-04, recipes/glm-5.3-flash.yaml) onto our launcher, on their
|
|
|
+# prebuilt image. A/B cell against our r18 fork build: same lab branch line
|
|
|
+# (dev/jovian-judgement) — their image = 09-04 tip + lukealonso/b12x master
|
|
|
+# + their patch set; our r18 pin = 09-05 tip + lab b12x + eugr-derived
|
|
|
+# patches. Compare quality (hardmode) + decode + TTFT before trusting either.
|
|
|
+#
|
|
|
+# Image: eugr/spark-vllm-b12x:nightly-20260904 (digest c3b44d3f…, = :latest).
|
|
|
+# Built from local-inference-lab/vllm dev/jovian-judgement (their e91bc76,
|
|
|
+# 2026-09-03, switched the ref from dev/infernal-invocation) +
|
|
|
+# lukealonso/b12x @ master; torch 2.13.0, CUDA 13.0.2,
|
|
|
+# nvidia-cutlass-dsl 4.7.0; their docker/patch_vllm_*.py set +
|
|
|
+# instanttensor + fastsafetensors; system-NCCL symlink fix. Exact b12x
|
|
|
+# commit baked in-image: /workspace/b12x-source-commit (+ build-metadata.yaml).
|
|
|
+# NOTE: the image ships NO ENTRYPOINT — this recipe must carry
|
|
|
+# entrypoint = ["vllm","serve"] (unlike the sparkrun/aiden images).
|
|
|
+#
|
|
|
+# Model: local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (their recipe's model —
|
|
|
+# NOT the GLM-5.3-NVFP4-Spark repo the aiden lane uses). ~180 GB class:
|
|
|
+# huggingface-cli download on BOTH nodes before the first serve
|
|
|
+# (HF_HUB_OFFLINE=1 below).
|
|
|
+#
|
|
|
+# What this lane tests (vs our r17/r18 findings):
|
|
|
+# - FULL b12x cluster: --attention-backend B12X + --moe-backend b12x +
|
|
|
+# --linear-backend b12x. On our r17 image, --linear-backend b12x was THE
|
|
|
+# quality TOXIN (44/100 hardmode). Their 09-04 b12x predates the SM121
|
|
|
+# A16-precision commits, so the answer is open — measure, don't trust.
|
|
|
+# - RoCEnante one-shot RoCE collectives: VLLM_ENABLE_ROCE_ALLREDUCE=1
|
|
|
+# (their vllm #597 / b12x #295 pair; −5–19% decode per-step,
|
|
|
+# +15–23% c16 measured on 4× Spark TP4).
|
|
|
+# - MTP k=5 with drafter moe_backend=humming + B12X attention (vendor-card
|
|
|
+# parity), AOT=1 + MEGA_AOT=1 + V2 runner, mamba-cache-mode align,
|
|
|
+# instanttensor buffered loader, prefix caching ON (we run prefix-cache
|
|
|
+# off elsewhere — perf-relevant, not quality-relevant).
|
|
|
+#
|
|
|
+# Deliberate parity deviations from our house lanes:
|
|
|
+# - NO --language-model-only (their recipe loads the vision tower; add the
|
|
|
+# flag only if memory is tight)
|
|
|
+# - --kv-cache-dtype fp8 (plain), not fp8_ds_mla
|
|
|
+# - no --decode-context-parallel-size (their DCP=1 is the default; no-op)
|
|
|
+# - NCCL/cluster env comes from the launcher (their NCCL_* block omitted)
|
|
|
+#
|
|
|
+# REFERENCE:
|
|
|
+# https://github.com/eugr/spark-vllm-docker (recipes/glm-5.3-flash.yaml)
|
|
|
+# https://hub.docker.com/r/eugr/spark-vllm-b12x/tags
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "GLM-5.3-Flash NVFP4 eugr B12X (vLLM, 2x Spark, MTP k=5, RoCE)"
|
|
|
+docker_image = "eugr/spark-vllm-b12x:nightly-20260904"
|
|
|
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark"
|
|
|
+required_nodes = 2
|
|
|
+tensor_parallel_size = 2
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+# image ships no ENTRYPOINT — the launcher supplies the vllm command
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+
|
|
|
+args = [
|
|
|
+ # Stable served name — quant-agnostic (shared across glm53 quants)
|
|
|
+ "--served-model-name", "glm53",
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--distributed-executor-backend", "mp",
|
|
|
+ # eugr parity: FULL b12x cluster (linear-b12x = the r17 quality toxin —
|
|
|
+ # measure before trusting)
|
|
|
+ "--attention-backend", "B12X",
|
|
|
+ "--moe-backend", "b12x",
|
|
|
+ "--linear-backend", "b12x",
|
|
|
+ "--mamba-cache-mode", "align",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--dtype", "bfloat16",
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
+ "--quantization", "modelopt_mixed",
|
|
|
+ "--no-enable-flashinfer-autotune",
|
|
|
+ "--load-format", "instanttensor",
|
|
|
+ "--model-loader-extra-config", """
|
|
|
+'{"instanttensor_copy":false}'""",
|
|
|
+ "--block-size", "256",
|
|
|
+ "--max-model-len", "1048576",
|
|
|
+ "--max-num-seqs", "4",
|
|
|
+ "--max-num-batched-tokens", "4096",
|
|
|
+ # MTP k=5 (eugr recipe / vendor card); drafter on humming MoE + B12X attn
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}'""",
|
|
|
+ "--reasoning-parser", "glm45",
|
|
|
+ "--tool-call-parser", "glm47",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--kv-cache-memory-bytes", "10737418240",
|
|
|
+ "--gpu-memory-utilization", "0.87",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+SAFETENSORS_FAST_GPU = "1"
|
|
|
+# RoCEnante one-shot RoCE collectives (ON in eugr's recipe; off-by-default flag)
|
|
|
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
|
|
|
+VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2MB"
|
|
|
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
|
|
|
+VLLM_WORKER_MULTIPROC_METHOD = "spawn"
|
|
|
+VLLM_SSM_CONV_STATE_LAYOUT = "DS"
|
|
|
+VLLM_USE_AOT_COMPILE = "1"
|
|
|
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
|
|
|
+VLLM_USE_V2_MODEL_RUNNER = "1"
|
|
|
+B12X_POLICY_MODE = "auto"
|
|
|
+# instanttensor buffered loader (eugr's defaults)
|
|
|
+INSTANTTENSOR_BACKEND = "BUFFERED"
|
|
|
+INSTANTTENSOR_BUFFER_SIZE = "67108864"
|
|
|
+INSTANTTENSOR_CHUNK_SIZE = "8388608"
|
|
|
+INSTANTTENSOR_CONCURRENCY = "1"
|
|
|
+INSTANTTENSOR_IO_DEPTH = "3"
|
