recipes: glm53-flash-intel-w4a16

recipes/glm53-flash-intel-w4a16/recipe.toml

@@ -0,0 +1,143 @@
+# GLM-5.3-Flash NVFP4 (lab quant) on the COMMUNITY sparkrun image — 2× DGX
+# Spark, TP=2.
+#
+# Image: aidendle94/sparkrun-vllm-glm53-gb10 (Spark Arena / sparkrun
+# orchestrator's GLM-5.3 vLLM build). Verified 2026-08-29: same day-0 vLLM
+# dev base as our lab image reports (v0.1.dev20051+g487ecf187, torch
+# 2.13.0+cu130), arm64-only (no manifest list — sm_121 target), generic
+# `vllm serve` entrypoint, no baked weights. Digest pin (single-arch
+# manifest, checked with `docker manifest inspect`):
+#   sha256:9b0592293a0b0161a040568578aca330227a03d140fa1c5f8965f64292431ef0
+#
+# WHY THIS RECIPE: cross-checks our home-built lab image against the Spark
+# Arena community stack — same checkpoint, same proven flag set, different
+# runtime. If both boot and score alike, the flag set (not our patches) is
+# the load-bearing part; any delta localizes to the image overlay.
+# Flags are mirrored verbatim from recipes/glm53-flash-nvfp4-lab (their
+# launch config + our fabric/env deviations — see that header for the full
+# rationale of each knob).
+#
+# Weights live on the NAS (mounted at /mnt/nas on the nodes) and are
+# exposed to the container via extra_mounts + HF_HOME — no per-node
+# `hf download` needed (config.toml [mount_aliases] hf_cache repoints this
+# site-wide).
+#
+# Boot markers: "Model loading took ~92.7 GiB" → "[quantprobe] …
+# algo=MXFP8" (MTP/modelopt fix active in this image; algo=None means the
+# image lacks the mod — report and fall back to our lab image) →
+# "GPU KV cache size" → "Application startup complete".
+#
+# Chat needs "chat_template_kwargs":{"reasoning_effort":"low"} — omitted or
+# other values select MAX reasoning effort (empty content on short
+# max_tokens).
+
+engine = "vllm"
+name = "GLM-5.3-Flash W4A16-AutoRound Intel (vLLM, 2x Spark, r12 fork)"
+# glm53-jovian-r12:local — our in-docker build of local-inference-lab/vllm
+# (jovian-judgement composition + PR heads 550-557) with the native
+# Glm5NextForConditionalGeneration arch and B12X sparse-MLA attention.
+# The timothystewart6 image was abandoned: upstream vLLM 0.28 there has NO
+# native glm5_next arch (its Transformers-backend fallback lacks the
+# per-stream q_conv1d weight mapping — boot dies in the weight loader).
+docker_image = "glm53-jovian-r12:local"
+docker_image_dir = "glm53-jovian-r12"
+model = "Intel/GLM-5.3-Flash-W4A16-AutoRound"
+parent_model = "zai-org/GLM-5.3-Flash"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    # AutoRound int4 pack → GPTQ-style safetensors; explicit quant route (the
+    # timothystewart6 upstream-stack build knows quant_method auto-round).
+    "--quantization", "gptq_marlin",
+    # Required for the head-512 MLA layers on sm_12x: the auto-selection
+    # list (TRITON_MLA / FLASHINFER_MLA_SPARSE_SM120) has no backend that
+    # fits GLM's 512 (SM120 flashinfer supports head 576 only).
+    "--attention-backend", "B12X",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail (KV pool is pinned below): 0.90
+    # flaked boots on GB10 ("Free memory ... less than desired") because
+    # driver reserve + page cache leave ~106-109 GiB free of 121.63.
+    # 0.82 = 99.7 GiB, below any realistic dip; changes nothing allocated.
+    "--gpu-memory-utilization", "0.82",
+    "--kv-cache-memory-bytes", "10737418240",
+    # DEVIATION from lab: plain fp8 KV (this image's mla cache kernel asserts
+    # "pe_dim must be 64 for fp8_ds_mla" — its NoPE handling differs from the
+    # lab image's rope-pad mod; plain fp8 is what the vendor GLM-5.3 recipe
+    # uses and doesn't assert pe layout).
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    "--disable-custom-all-reduce",
+    # Pin the MoE backend: the lab image's `auto` resolves to MARLIN for
+    # this quant; this image's `auto` resolves to a TileLang/deepgemm JIT
+    # path that compiles 97 CUDA kernels AT BOOT (all cores, evicted the
+    # kernel's own worker via OOM — dmesg 08:30, cicc/cudafe++ storms).
+    # MARLIN = proven lab path, no JIT at boot.
+    "--moe-backend", "marlin",
+    # Proven day-0 boot: no flashinfer autotune, no CuTe-DSL warmup
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    # DEVIATION from lab recipe: no MTP here. This image lacks the lab
+    # image's FuijitsuPolycom modelopt-MTP namespace fix (boot dies at
+    # layers.45.mtp_block.mlp.experts.routed_experts.w2_weight_scale —
+    # verified 2026-08-30 on ds4). Spec layers are skipped entirely when
+    # no spec config is set, so the base model loads clean.
+    "--language-model-only",
+    # Day-0 guard: on the lab image, graphs+MTP regressed tool quality. Keep
+    # while unverified on this build; A/B if you want graphs.
+    "--enforce-eager",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+archived = true
+
+[env]
+HF_HUB_OFFLINE = "1"
+# NoPE-MLA rope-pad runtime gate (baked into our lab image; no-op if this
+# image gates it differently — boot marker will tell)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_NO_USAGE_STATS = "1"
+# Upstream launch-script env (memory + compile knobs)
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "4"
+
+# NAS-backed HF cache (read-only use; HF writes nothing in offline mode).
+# Direct host path — NAS is mounted at /mnt/nas on every node; use
+# [mount_aliases] in config.toml when paths differ per site.
+[[extra_mounts]]
+host = "/mnt/nas/ai/huggingface"
+container = "/mnt/nas/ai/huggingface"
+env = "HF_HOME"
+
+# The image ships a CUDA 13 runtime WITHOUT dev headers (no nvrtc.h —
+# 105 headers total) and flashinfer's deepgemm JIT compiles at warmup with
+# nvcc -> "nvrtc.h: No such file or directory" (verified 2026-08-30 on
+# ds4). Bind the host's nvrtc.h through; it is an API-decl header, ABI-
+# independent of the 13.x minor in this image.
+[[extra_mounts]]
+host = "/usr/local/cuda/include/nvrtc.h"
+container = "/usr/local/cuda/include/nvrtc.h"
+read_only = true
+