launcher/recipes/glm53-flash-nvfp4-eugr/recipe.toml

# GLM-5.3-Flash NVFP4 — eugr B12X full-cluster variant (vLLM) — 2× DGX Spark, TP=2.
#
# PRODUCTION DEFAULT for the eugr stack: run this 2x lane until the 4x test
# variant (recipes/glm53-flash-nvfp4-eugr-4x) delivers a dramatic
# improvement (night shift compares them).
#
# Port of eugr's dual-Spark recipe (eugr/spark-vllm-docker @ 94b3c30,
# 2026-09-04, recipes/glm-5.3-flash.yaml) onto our launcher, on their
# prebuilt image. A/B cell against our r18 fork build: same lab branch line
# (dev/jovian-judgement) — their image = 09-04 tip + lukealonso/b12x master
# + their patch set; our r18 pin = 09-05 tip + lab b12x + eugr-derived
# patches. Compare quality (hardmode) + decode + TTFT before trusting either.
#
# Image: eugr/spark-vllm-b12x:nightly-20260904 (digest c3b44d3f…, = :latest).
#   Built from local-inference-lab/vllm dev/jovian-judgement (their e91bc76,
#   2026-09-03, switched the ref from dev/infernal-invocation) +
#   lukealonso/b12x @ master; torch 2.13.0, CUDA 13.0.2,
#   nvidia-cutlass-dsl 4.7.0; their docker/patch_vllm_*.py set +
#   instanttensor + fastsafetensors; system-NCCL symlink fix. Exact b12x
#   commit baked in-image: /workspace/b12x-source-commit (+ build-metadata.yaml).
#   NOTE: the image ships NO ENTRYPOINT — this recipe must carry
#   entrypoint = ["vllm","serve"] (unlike the sparkrun/aiden images).
#
# Model: local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (their recipe's model —
#   NOT the GLM-5.3-NVFP4-Spark repo the aiden lane uses). ~180 GB class:
#   huggingface-cli download on BOTH nodes before the first serve
#   (HF_HUB_OFFLINE=1 below).
#
# What this lane tests (vs our r17/r18 findings):
#   - FULL b12x cluster: --attention-backend B12X + --moe-backend b12x +
#     --linear-backend b12x. On our r17 image, --linear-backend b12x was THE
#     quality TOXIN (44/100 hardmode). Their 09-04 b12x predates the SM121
#     A16-precision commits, so the answer is open — measure, don't trust.
#   - RoCEnante one-shot RoCE collectives: VLLM_ENABLE_ROCE_ALLREDUCE=1
#     (their vllm #597 / b12x #295 pair; −5–19% decode per-step,
#     +15–23% c16 measured on 4× Spark TP4).
#   - MTP k=5 with drafter moe_backend=humming + B12X attention (vendor-card
#     parity), AOT=1 + MEGA_AOT=1 + V2 runner, mamba-cache-mode align,
#     instanttensor buffered loader, prefix caching ON (we run prefix-cache
#     off elsewhere — perf-relevant, not quality-relevant).
#
# Deliberate parity deviations from our house lanes:
#   - NO --language-model-only (their recipe loads the vision tower; add the
#     flag only if memory is tight)
#   - --kv-cache-dtype fp8 (plain), not fp8_ds_mla
#   - no --decode-context-parallel-size (their DCP=1 is the default; no-op)
#   - NCCL/cluster env comes from the launcher (their NCCL_* block omitted)
#
# REFERENCE:
#   https://github.com/eugr/spark-vllm-docker  (recipes/glm-5.3-flash.yaml)
#   https://hub.docker.com/r/eugr/spark-vllm-b12x/tags
#   https://forums.developer.nvidia.com/t/381350/364 — public confirmation of
#   this flag set (2026-09-05, tea24864 on 4x/TP4, perf-only: ~60 tok/s c1,
#   ~73 c4 decode, prefill 2.3-2.8k tok/s) + the DFlash2 alternative spec
#   ("method":"dflash" + "incoai/GLM-5.3-Flash-DFlash2" drafter, k=7, see
#   night queue 3b)

engine = "vllm"
name = "GLM-5.3-Flash NVFP4 eugr B12X (vLLM, 2x Spark, MTP k=5, RoCE)"
docker_image = "eugr/spark-vllm-b12x:nightly-20260904"
model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# image ships no ENTRYPOINT — the launcher supplies the vllm command
entrypoint = ["vllm", "serve"]

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # eugr parity: FULL b12x cluster (linear-b12x = the r17 quality toxin —
    # measure before trusting)
    "--attention-backend", "B12X",
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--mamba-cache-mode", "align",
    "--enable-prefix-caching",
    "--enable-chunked-prefill",
    "--dtype", "bfloat16",
    "--kv-cache-dtype", "fp8",
    "--quantization", "modelopt_mixed",
    "--no-enable-flashinfer-autotune",
    "--load-format", "instanttensor",
    "--model-loader-extra-config", """
'{"instanttensor_copy":false}'""",
    "--block-size", "256",
    "--max-model-len", "1048576",
    "--max-num-seqs", "4",
    "--max-num-batched-tokens", "4096",
    # MTP k=5 (eugr recipe / vendor card); drafter on humming MoE + B12X attn
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}'""",
    "--reasoning-parser", "glm45",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--kv-cache-memory-bytes", "10737418240",
    "--gpu-memory-utilization", "0.87",
]

[env]
HF_HUB_OFFLINE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
CUTE_DSL_ARCH = "sm_121a"
SAFETENSORS_FAST_GPU = "1"
# RoCEnante one-shot RoCE collectives (ON in eugr's recipe; off-by-default flag)
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2MB"
VLLM_ENABLE_PCIE_ALLREDUCE = "0"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_SSM_CONV_STATE_LAYOUT = "DS"
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_MEGA_AOT_ARTIFACT = "1"
VLLM_USE_V2_MODEL_RUNNER = "1"
B12X_POLICY_MODE = "auto"
# instanttensor buffered loader (eugr's defaults)
INSTANTTENSOR_BACKEND = "BUFFERED"
INSTANTTENSOR_BUFFER_SIZE = "67108864"
INSTANTTENSOR_CHUNK_SIZE = "8388608"
INSTANTTENSOR_CONCURRENCY = "1"
INSTANTTENSOR_IO_DEPTH = "3"

# JIT caches persist per-node across serves (was: cold /root/.cache per
# container -> ~20 min flashinfer/torch recompile every serve). The host
# dirs are shared between lanes of the same arch — each cache
# self-namespaces by its engine version key internally.
[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"