recipes: glm53-flash-nvfp4-eugr

recipes/glm53-flash-nvfp4-eugr/recipe.toml

@@ -0,0 +1,142 @@
+# GLM-5.3-Flash NVFP4 — eugr B12X full-cluster variant (vLLM) — 2× DGX Spark, TP=2.
+#
+# PRODUCTION DEFAULT for the eugr stack: run this 2x lane until the 4x test
+# variant (recipes/glm53-flash-nvfp4-eugr-4x) delivers a dramatic
+# improvement (night shift compares them).
+#
+# Port of eugr's dual-Spark recipe (eugr/spark-vllm-docker @ 94b3c30,
+# 2026-09-04, recipes/glm-5.3-flash.yaml) onto our launcher, on their
+# prebuilt image. A/B cell against our r18 fork build: same lab branch line
+# (dev/jovian-judgement) — their image = 09-04 tip + lukealonso/b12x master
+# + their patch set; our r18 pin = 09-05 tip + lab b12x + eugr-derived
+# patches. Compare quality (hardmode) + decode + TTFT before trusting either.
+#
+# Image: eugr/spark-vllm-b12x:nightly-20260904 (digest c3b44d3f…, = :latest).
+#   Built from local-inference-lab/vllm dev/jovian-judgement (their e91bc76,
+#   2026-09-03, switched the ref from dev/infernal-invocation) +
+#   lukealonso/b12x @ master; torch 2.13.0, CUDA 13.0.2,
+#   nvidia-cutlass-dsl 4.7.0; their docker/patch_vllm_*.py set +
+#   instanttensor + fastsafetensors; system-NCCL symlink fix. Exact b12x
+#   commit baked in-image: /workspace/b12x-source-commit (+ build-metadata.yaml).
+#   NOTE: the image ships NO ENTRYPOINT — this recipe must carry
+#   entrypoint = ["vllm","serve"] (unlike the sparkrun/aiden images).
+#
+# Model: local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (their recipe's model —
+#   NOT the GLM-5.3-NVFP4-Spark repo the aiden lane uses). ~180 GB class:
+#   huggingface-cli download on BOTH nodes before the first serve
+#   (HF_HUB_OFFLINE=1 below).
+#
+# What this lane tests (vs our r17/r18 findings):
+#   - FULL b12x cluster: --attention-backend B12X + --moe-backend b12x +
+#     --linear-backend b12x. On our r17 image, --linear-backend b12x was THE
+#     quality TOXIN (44/100 hardmode). Their 09-04 b12x predates the SM121
+#     A16-precision commits, so the answer is open — measure, don't trust.
+#   - RoCEnante one-shot RoCE collectives: VLLM_ENABLE_ROCE_ALLREDUCE=1
+#     (their vllm #597 / b12x #295 pair; −5–19% decode per-step,
+#     +15–23% c16 measured on 4× Spark TP4).
+#   - MTP k=5 with drafter moe_backend=humming + B12X attention (vendor-card
+#     parity), AOT=1 + MEGA_AOT=1 + V2 runner, mamba-cache-mode align,
+#     instanttensor buffered loader, prefix caching ON (we run prefix-cache
+#     off elsewhere — perf-relevant, not quality-relevant).
+#
+# Deliberate parity deviations from our house lanes:
+#   - NO --language-model-only (their recipe loads the vision tower; add the
+#     flag only if memory is tight)
+#   - --kv-cache-dtype fp8 (plain), not fp8_ds_mla
+#   - no --decode-context-parallel-size (their DCP=1 is the default; no-op)
+#   - NCCL/cluster env comes from the launcher (their NCCL_* block omitted)
+#
+# REFERENCE:
+#   https://github.com/eugr/spark-vllm-docker  (recipes/glm-5.3-flash.yaml)
+#   https://hub.docker.com/r/eugr/spark-vllm-b12x/tags
+#   https://forums.developer.nvidia.com/t/381350/364 — public confirmation of
+#   this flag set (2026-09-05, tea24864 on 4x/TP4, perf-only: ~60 tok/s c1,
+#   ~73 c4 decode, prefill 2.3-2.8k tok/s) + the DFlash2 alternative spec
+#   ("method":"dflash" + "incoai/GLM-5.3-Flash-DFlash2" drafter, k=7, see
+#   night queue 3b)
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4 eugr B12X (vLLM, 2x Spark, MTP k=5, RoCE)"
+docker_image = "eugr/spark-vllm-b12x:nightly-20260904"
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark"
+parent_model = "zai-org/GLM-5.3-Flash"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+# image ships no ENTRYPOINT — the launcher supplies the vllm command
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # eugr parity: FULL b12x cluster (linear-b12x = the r17 quality toxin —
+    # measure before trusting)
+    "--attention-backend", "B12X",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    "--mamba-cache-mode", "align",
+    "--enable-prefix-caching",
+    "--enable-chunked-prefill",
+    "--dtype", "bfloat16",
+    "--kv-cache-dtype", "fp8",
+    "--quantization", "modelopt_mixed",
+    "--no-enable-flashinfer-autotune",
+    "--load-format", "instanttensor",
+    "--model-loader-extra-config", """
+'{"instanttensor_copy":false}'""",
+    "--block-size", "256",
+    "--max-model-len", "1048576",
+    "--max-num-seqs", "4",
+    "--max-num-batched-tokens", "4096",
+    # MTP k=5 (eugr recipe / vendor card); drafter on humming MoE + B12X attn
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}'""",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--kv-cache-memory-bytes", "10737418240",
+    "--gpu-memory-utilization", "0.87",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+CUTE_DSL_ARCH = "sm_121a"
+SAFETENSORS_FAST_GPU = "1"
+# RoCEnante one-shot RoCE collectives (ON in eugr's recipe; off-by-default flag)
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
+VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2MB"
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+VLLM_WORKER_MULTIPROC_METHOD = "spawn"
+VLLM_SSM_CONV_STATE_LAYOUT = "DS"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+B12X_POLICY_MODE = "auto"
+# instanttensor buffered loader (eugr's defaults)
+INSTANTTENSOR_BACKEND = "BUFFERED"
+INSTANTTENSOR_BUFFER_SIZE = "67108864"
+INSTANTTENSOR_CHUNK_SIZE = "8388608"
+INSTANTTENSOR_CONCURRENCY = "1"
+INSTANTTENSOR_IO_DEPTH = "3"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"