recipes/recipes/glm53-flash-fp4-local-2x/recipe.toml

# NVIDIA GLM-5.3-Flash-FP4-Local — official NVFP4 (QAD) GLM-5.3-Flash, 2x DGX Spark.
# Source: model card https://huggingface.co/nvidia/GLM-5.3-Flash-FP4-Local
# ("2 x NVIDIA DGX Spark" section, transcribed). Quantized with nvidia-modelopt
# v0.47.0 via quantization-aware distillation (QAD). NVFP4 routed experts
# (E2M1 + E4M3 block scales + FP32 global scale). 2M tokens FP8 KV on 2x Spark.
#
# CRITICAL: the NVFP4 scale factors use a NEW lossless-compression format that
# upstream vLLM does NOT yet support — MUST run on eugr/spark-vllm-b12x (b12x
# backend + --load-format b12x). Digest-pinned (arm64, built 2026-10-15).
# This is the official NVFP4 counterpart to our kindling 2x lane (our own r35
# build) — a direct A/B vs the 95/100 kindling anchor.
#
# The launcher supplies the NCCL/socket/IB env from the cluster config
# (NCCL_IB_HCA, NCCL_SOCKET_IFNAME, MN_IF_NAME, UCX_NET_DEVICES, GLOO/TP_SOCKET,
# NCCL_IB_DISABLE, master addr/port). Only the b12x + RoCE-allreduce + AOT env
# from the card's "both nodes" block is added here.

engine = "vllm"
name = "GLM-5.3-Flash-FP4-Local (2x Spark, NVFP4 QAD, MTP n5)"
docker_image = "eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595"
entrypoint = ["vllm", "serve"]
model = "nvidia/GLM-5.3-Flash-FP4-Local"
parent_model = "zai-org/GLM-5.3-Flash"
port = 30000
required_nodes = 2
tensor_parallel_size = 2
args = [
    "--served-model-name", "glm53-fp4-local",
    "--dtype", "bfloat16",
    "--mamba-cache-mode", "align",
    "--enable-prefix-caching",
    "--enable-chunked-prefill",
    "--kv-cache-dtype", "fp8",
    "--quantization", "modelopt_mixed",
    "--attention-backend", "B12X",
    "--block-size", "256",
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--no-enable-flashinfer-autotune",
    "--load-format", "b12x",
    "--max-model-len", "500000",
    "--max-num-seqs", "4",
    "--max-num-batched-tokens", "4096",
    "--reasoning-parser", "glm45",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--gpu-memory-utilization", "0.87",
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}'""",
]

[env]
# b12x MoE/FP4 path (card "both nodes" block)
CUTE_DSL_ARCH = "sm_121a"
SAFETENSORS_FAST_GPU = "1"
B12X_POLICY_MODE = "auto"
VLLM_B12X_MOE_FP4_FORCE_A16 = "1"
B12X_W4A16_A4_PREFILL = "1"
# RoCE allreduce + NCCL tuning (card)
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2MB"
NCCL_BUFFSIZE = "1048576"
NCCL_MAX_CTAS = "8"
VLLM_ENABLE_PCIE_ALLREDUCE = "0"
# vLLM runner / compile flags (card)
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_SSM_CONV_STATE_LAYOUT = "DS"
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_MEGA_AOT_ARTIFACT = "1"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"

# JIT / AOT / b12x-autotune caches the card mounts on every node (speeds boot +
# re-serve; docker creates the host dir if missing). Card: mkdir -p + mount.
[[extra_mounts]]
host = "~/.cache/vllm"
container = "/root/.cache/vllm"

[[extra_mounts]]
host = "~/.cache/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/.triton"
container = "/root/.triton"

[[extra_mounts]]
host = "~/.cache/b12x"
container = "/root/.cache/b12x"