launcher/recipes/glm53-flash-eugr-2x/recipe.toml

# eugr b12x lane (2026-09-10) — verbatim mirror of eugr/spark-vllm-docker
# recipes/glm-5.3-flash.yaml (their cluster_only dual-Spark B12X recipe),
# run on the eugr/prebuilt image instead of their build-and-copy.sh flow.
#
# Image: eugr/spark-vllm-b12x:nightly-20260909 (Docker Hub, ~11 GB). Built
# by their --exp-b12x path from the FLOATING refs: vLLM
# local-inference-lab/vllm @ dev/jovian-judgement (41ea64ae at 09-10, the
# mainline line — NOT the integration/glm53-r31 release cut) + b12x master.
# Provenance: https://github.com/eugr/spark-vllm-docker (MIT).
#
# This is a third-party independent b12x stack, A/B'd against our r28/r31
# lanes on the SAME model (GLM-5.3-Flash-NVFP4-Spark) and fabric. Their
# notable deviations vs our stack:
#   * --load-format b12x (their experimental b12x loader, 09-06: "faster and
#     more memory efficient than Instanttensor") — NOT instanttensor
#   * --moe-backend b12x + --linear-backend b12x (W4A4 linear kernels — our
#     r17 bisect found these destructive on the r17 stack; their new loader
#     may have fixed numerics, the bench decides)
#   * MTP k=5 with moe_backend "humming" (vs our k=3)
#   * 1M ctx, max-num-seqs 4 (vs our 512K/16), KV pinned 8 GiB, GMU 0.87
#   * V2 model runner + AOT/MEGA-AOT on, spawn workers, PCIE AR off
#
# Baseline for the A/B: glm53-flash-r31-2x / glm53-flash-r28-spark on
# 2nodes-dac (r28: 158/176, 1475 pp / 27.2 tg / 1362 ms d0 c1).

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-Spark (eugr b12x img, 2x)"
docker_image = "eugr/spark-vllm-b12x:nightly-20260909"  # prebuilt (no build: no docker_image_dir)
model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
entrypoint = ["vllm", "serve"]

args = [
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    "--mamba-cache-mode", "align",
    "--enable-prefix-caching",
    "--enable-chunked-prefill",
    "--dtype", "bfloat16",
    "--kv-cache-dtype", "fp8",
    "--quantization", "modelopt_mixed",
    "--attention-backend", "B12X",
    "--block-size", "256",
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--no-enable-flashinfer-autotune",
    "--load-format", "b12x",
    "--max-model-len", "1048576",
    "--max-num-seqs", "4",
    "--max-num-batched-tokens", "4096",
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}'""",
    "--reasoning-parser", "glm45",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--kv-cache-memory-bytes", "8589934592",
    "--gpu-memory-utilization", "0.87",
    # Their launcher bakes these envs into the image; kept explicit here so
    # the recipe stays self-documenting (the image's own env is the default,
    # ours overrides with identical values).
]

[env]
HF_HUB_OFFLINE = "1"
CUTE_DSL_ARCH = "sm_121a"
SAFETENSORS_FAST_GPU = "1"
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2MB"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_SSM_CONV_STATE_LAYOUT = "DS"
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_MEGA_AOT_ARTIFACT = "1"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_ENABLE_PCIE_ALLREDUCE = "0"
B12X_POLICY_MODE = "auto"
INSTANTTENSOR_BACKEND = "BUFFERED"
INSTANTTENSOR_BUFFER_SIZE = "67108864"
INSTANTTENSOR_CHUNK_SIZE = "8388608"
INSTANTTENSOR_CONCURRENCY = "1"
INSTANTTENSOR_IO_DEPTH = "3"
# JIT caches persist per-node (self-namespace by engine version key)
OMP_NUM_THREADS = "2"
MAX_JOBS = "2"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"

[[extra_mounts]]
host = "~/sm121-cache/glm53-jit-121f/b12x"
container = "/root/.cache/b12x"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"