launcher/recipes/glm53-flash-intel-w4a16/recipe.toml

# GLM-5.3-Flash NVFP4 (lab quant) on the COMMUNITY sparkrun image — 2× DGX
# Spark, TP=2.
#
# Image: aidendle94/sparkrun-vllm-glm53-gb10 (Spark Arena / sparkrun
# orchestrator's GLM-5.3 vLLM build). Verified 2026-08-29: same day-0 vLLM
# dev base as our lab image reports (v0.1.dev20051+g487ecf187, torch
# 2.13.0+cu130), arm64-only (no manifest list — sm_121 target), generic
# `vllm serve` entrypoint, no baked weights. Digest pin (single-arch
# manifest, checked with `docker manifest inspect`):
#   sha256:9b0592293a0b0161a040568578aca330227a03d140fa1c5f8965f64292431ef0
#
# WHY THIS RECIPE: cross-checks our home-built lab image against the Spark
# Arena community stack — same checkpoint, same proven flag set, different
# runtime. If both boot and score alike, the flag set (not our patches) is
# the load-bearing part; any delta localizes to the image overlay.
# Flags are mirrored verbatim from recipes/glm53-flash-nvfp4-lab (their
# launch config + our fabric/env deviations — see that header for the full
# rationale of each knob).
#
# Weights live on the NAS (mounted at /mnt/nas on the nodes) and are
# exposed to the container via extra_mounts + HF_HOME — no per-node
# `hf download` needed (config.toml [mount_aliases] hf_cache repoints this
# site-wide).
#
# Boot markers: "Model loading took ~92.7 GiB" → "[quantprobe] …
# algo=MXFP8" (MTP/modelopt fix active in this image; algo=None means the
# image lacks the mod — report and fall back to our lab image) →
# "GPU KV cache size" → "Application startup complete".
#
# Chat needs "chat_template_kwargs":{"reasoning_effort":"low"} — omitted or
# other values select MAX reasoning effort (empty content on short
# max_tokens).

engine = "vllm"
name = "GLM-5.3-Flash W4A16-AutoRound Intel (vLLM, 2x Spark, r12 fork)"
# glm53-jovian-r12:local — our in-docker build of local-inference-lab/vllm
# (jovian-judgement composition + PR heads 550-557) with the native
# Glm5NextForConditionalGeneration arch and B12X sparse-MLA attention.
# The timothystewart6 image was abandoned: upstream vLLM 0.28 there has NO
# native glm5_next arch (its Transformers-backend fallback lacks the
# per-stream q_conv1d weight mapping — boot dies in the weight loader).
docker_image = "glm53-jovian-r12:local"
docker_image_dir = "glm53-jovian-r12"
model = "Intel/GLM-5.3-Flash-W4A16-AutoRound"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    # AutoRound int4 pack → GPTQ-style safetensors; explicit quant route (the
    # timothystewart6 upstream-stack build knows quant_method auto-round).
    "--quantization", "gptq_marlin",
    # Required for the head-512 MLA layers on sm_12x: the auto-selection
    # list (TRITON_MLA / FLASHINFER_MLA_SPARSE_SM120) has no backend that
    # fits GLM's 512 (SM120 flashinfer supports head 576 only).
    "--attention-backend", "B12X",
    "--distributed-executor-backend", "mp",
    # GMU is ONLY a startup guardrail (KV pool is pinned below): 0.90
    # flaked boots on GB10 ("Free memory ... less than desired") because
    # driver reserve + page cache leave ~106-109 GiB free of 121.63.
    # 0.82 = 99.7 GiB, below any realistic dip; changes nothing allocated.
    "--gpu-memory-utilization", "0.82",
    "--kv-cache-memory-bytes", "10737418240",
    # DEVIATION from lab: plain fp8 KV (this image's mla cache kernel asserts
    # "pe_dim must be 64 for fp8_ds_mla" — its NoPE handling differs from the
    # lab image's rope-pad mod; plain fp8 is what the vendor GLM-5.3 recipe
    # uses and doesn't assert pe layout).
    "--kv-cache-dtype", "fp8",
    "--block-size", "256",
    "--max-model-len", "524288",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    "--disable-custom-all-reduce",
    # Pin the MoE backend: the lab image's `auto` resolves to MARLIN for
    # this quant; this image's `auto` resolves to a TileLang/deepgemm JIT
    # path that compiles 97 CUDA kernels AT BOOT (all cores, evicted the
    # kernel's own worker via OOM — dmesg 08:30, cicc/cudafe++ storms).
    # MARLIN = proven lab path, no JIT at boot.
    "--moe-backend", "marlin",
    # Proven day-0 boot: no flashinfer autotune, no CuTe-DSL warmup
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    # DEVIATION from lab recipe: no MTP here. This image lacks the lab
    # image's FuijitsuPolycom modelopt-MTP namespace fix (boot dies at
    # layers.45.mtp_block.mlp.experts.routed_experts.w2_weight_scale —
    # verified 2026-08-30 on ds4). Spec layers are skipped entirely when
    # no spec config is set, so the base model loads clean.
    "--language-model-only",
    # Day-0 guard: on the lab image, graphs+MTP regressed tool quality. Keep
    # while unverified on this build; A/B if you want graphs.
    "--enforce-eager",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
# NoPE-MLA rope-pad runtime gate (baked into our lab image; no-op if this
# image gates it differently — boot marker will tell)
VLLM_MLA_NOPE_PAD_ROPE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_NO_USAGE_STATS = "1"
# Upstream launch-script env (memory + compile knobs)
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
TORCH_USE_RTLD_GLOBAL = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "4"

# NAS-backed HF cache (read-only use; HF writes nothing in offline mode).
# Direct host path — NAS is mounted at /mnt/nas on every node; use
# [mount_aliases] in config.toml when paths differ per site.
[[extra_mounts]]
host = "/mnt/nas/ai/huggingface"
container = "/mnt/nas/ai/huggingface"
env = "HF_HOME"

# The image ships a CUDA 13 runtime WITHOUT dev headers (no nvrtc.h —
# 105 headers total) and flashinfer's deepgemm JIT compiles at warmup with
# nvcc -> "nvrtc.h: No such file or directory" (verified 2026-08-30 on
# ds4). Bind the host's nvrtc.h through; it is an API-decl header, ABI-
# independent of the 13.x minor in this image.
[[extra_mounts]]
host = "/usr/local/cuda/include/nvrtc.h"
container = "/usr/local/cuda/include/nvrtc.h"
read_only = true