new lanes: aiden 4x + lab DFlash2 drafter (incoai swap); lab image + lab DFlash2 (MTP swap) — competitive A/Bs on the lab's speculative model

recipes/glm53-flash-aiden-4x-labdrafter/recipe.toml

@@ -0,0 +1,78 @@
+# GLM-5.3-Flash on 4× DGX Spark — aiden image with the LAB's own DFlash2
+# drafter (local-inference-lab/GLM-5.3-Flash-DFlash2, 1.2 GB, on the shared
+# NAS HF) instead of incoai's — the competitive A/B the user asked for:
+# same image, same NVFP4 checkpoint, only the speculative model differs.
+# (This recipe = glm53-flash-aiden-4x with the drafter swapped.)
+#
+# aiden's production 4-node profile, verbatim translation of
+# https://aidenle.com/recipes/glm-5-3-flash-4x-dgx-spark into our 4x launcher
+# shape. Reference numbers (his, single stream, idle cluster, thinking off):
+#   ~84 tok/s decode (FP8, DFlash2 k=7) — NVFP4 +23% (110.7 structured);
+#   prefill ~2,100 cold; TTFT 4 s @ 8K → 40 s @ 94K; KV 2.25M tokens at 16 GiB
+#   pin; 393K context verified. Decode flat with context (34/45 KDA layers +
+#   11 sparse-MLA @ index_topk 2048).
+#
+# DFlash2: acceptance 0.5–0.8; k=7 (drafter block 8 − 1 target token); k=6
+# wins prose, loses structured. Native MTP fallback (production-1.0 image or
+# spec omitted) optimum k=5. ~15 min full relaunch on FP8 (963 s measured).
+# FP8-MLA + DFlash2 combination is STRUCTURALLY broken upstream (three trees
+# fail differently) — hence plain fp8_e4m3 KV.
+#
+# NOT READY TO DEPLOY: needs a 4-node cluster (ds4 + 2nodes-dac freed) and the
+# unsloth/GLM-5.3-Flash-FP8 checkpoint (328 GB — background download running
+# on dgx3, /tmp/dl_fp8.log; ~340 GB free per node required for weights + Triton
+# cache). Fabric: RoCE dual-HCA NCCL profile comes from the launcher (config
+# [nccl.*] / GID auto-detect) — his NCCL_* env block intentionally omitted.
+# NOTE: his launcher unsets VLLM_PREFIX_CACHE_RETENTION_INTERVAL via bash -lc
+# before exec; if our boot rejects it in the KV coordinator, add a mods script.
+
+engine = "vllm"
+name = "GLM-5.3-Flash aiden image (vLLM, 4x Spark, LAB DFlash2-7, NVFP4)"
+docker_image = "aidendle94/sparkrun-vllm-glm53-gb10:production-1.1"
+model = "LibertAIDAI/GLM-5.3-Flash-NVFP4"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+# image ships ENTRYPOINT ["vllm","serve"]; our launcher passes the recipe
+# entrypoint as CMD words after the image, which would double it. Empty clears.
+entrypoint = []
+
+args = [
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--default-chat-template-kwargs", """
+'{"enable_thinking": true, "reasoning_effort": "high"}'""",
+    "--kv-cache-dtype", "fp8_e4m3",
+    "--block-size", "2304",
+    "--moe-backend", "marlin",
+    "--max-model-len", "393216",
+    "--max-num-batched-tokens", "8192",
+    "--max-num-seqs", "16",
+    "--enable-chunked-prefill",
+    "--enforce-eager",
+    "--gpu-memory-utilization", "0.85",
+    "--kv-cache-memory-bytes", "17179869184",
+    "--speculative-config", """
+'{"method":"dflash","model":"local-inference-lab/GLM-5.3-Flash-DFlash2","num_speculative_tokens":7}'""",
+    "--language-model-only",
+    # image requests need the mm template shipped/in-wallpaper on aiden's rigs:
+    # "--chat-template", "<path-to>/chat_template_mm.jinja",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+TORCH_USE_RTLD_GLOBAL = "1"
+TORCHINDUCTOR_COMPILE_THREADS = "1"
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_NO_USAGE_STATS = "1"
+OMP_NUM_THREADS = "2"

recipes/glm53-flash-nvfp4-lab-dflash2/recipe.toml

@@ -0,0 +1,125 @@
+# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
+#
+# local-inference-lab mixed quant: NVFP4 routed experts (layers 3–44,
+# group-16) + MXFP8 MTP experts (layer 45) + W4A4 input scales. Best agent
+# quant so far: tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8).
+# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
+#        (~186 GB → 92.72 GiB weights/node; KV pinned at 10 GiB, see below)
+#
+# Image: glm53-flash-nvfp4-lab:local (docker-images/glm53-flash-nvfp4-lab/ —
+# day-0 arm64 image + NoPE-MLA mod + the four FujitsuPolycom lab-checkpoint
+# patches: model naming shim, modelopt MTP namespace fix, persistent_topk
+# CC-12.x guards). Their launch config is mirrored below.
+# PROVENANCE: https://github.com/FujitsuPolycom/glm53-flash-tp2-spark
+#             (Apache-2.0; see patches/UPSTREAM.md) + forum 381350 #41.
+#
+# Deviations from their launch script (intentional):
+#   * no --kda-prefill-backend flashkda (needs their base-image patch — skipped)
+#   * --kv-cache-dtype fp8_ds_mla instead of `fp8` (same packed layout; this
+#     spelling is what the NoPE-MLA mod booted with on the LibertAIDAI stack)
+#   * --language-model-only + no mm-renderer patch/env (text-only)
+#   * no bundled NCCL/LD_PRELOAD, no NCCL_* fabric block (our switched fabric
+#     is configured via [nccl.ds4] + launcher GID auto-detect)
+#   * no --moe-backend marlin: their proven stack uses the default backend for
+#     THIS quant and reports clean evals. Sanity-check output for token loops
+#     anyway (the silent FP4 MoE corruption gotcha) before trusting a new node.
+#   * flashinfer autotune disabled via --kernel-config (their proven boot);
+#     VLLM_ENGINE_READY_TIMEOUT_S stays high regardless.
+#
+# Boot success markers (engine log, in order):
+#   "Model loading took 92.72 GiB" → "[quantprobe] ... algo=MXFP8" (MTP fix
+#   active; algo=None means broken) → "GPU KV cache size: 1,164,369 tokens" →
+#   "Application startup complete". ~700 s sequential weight load.
+#
+# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
+# :"low"} — omitted/other values select MAX reasoning effort (empty content on
+# short max_tokens).
+#
+# instanttensor loader — baked into the image (Dockerfile step 3) and enabled
+# via args + env knobs below. MEASURED 2026-09-03: 631 s total load vs ~700 s
+# sequential (lab model resolves TWO passes: target + MTP/draft shards; the
+# r12 spark stack loads once in ~5.5 min). Modest gain, worth keeping.
+#
+# PREREQUISITES: weights cached on both nodes:
+#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4-lab (vLLM, 2x Spark, MTP-3)"
+docker_image = "glm53-flash-nvfp4-lab:local"
+docker_image_dir = "glm53-flash-nvfp4-lab"
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
+    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
+    # page cache routinely leave ~108.5 -> flaky
+    # "Free memory ... less than desired" boot deaths (this was the class
+    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
+    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
+    # allocated (weights ~93 + pinned KV 10 + activations).
+    "--gpu-memory-utilization", "0.82",
+    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
+    # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
+    "--kv-cache-memory-bytes", "10737418240",
+    "--kv-cache-dtype", "fp8_ds_mla",
+    "--block-size", "256",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    "--disable-custom-all-reduce",
+    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    # MTP-3 draft from the MXFP8 layer-45 shard (needs modelopt.patch in image)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":3}'""",
+    "--language-model-only",
+    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
+    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
+    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
+    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
+    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
+    "--enforce-eager",
+    # instanttensor loader (0.1.9 baked in image; ~30 s load vs ~700 s NFS)
+    "--load-format", "instanttensor",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+# lab INSTANTTENSOR loader knobs (same values the spark recipe validated:
+# lil render BACKEND/CONCURRENCY/IO_DEPTH + lil.yaml BUFFER_SIZE/CHUNK)
+INSTANTTENSOR_BACKEND = "BUFFERED"
+INSTANTTENSOR_CONCURRENCY = "1"
+INSTANTTENSOR_IO_DEPTH = "3"
+INSTANTTENSOR_BUFFER_SIZE = "67108864"
+INSTANTTENSOR_CHUNK_SIZE = "8388608"
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+# Upstream launch-script env (memory + compile knobs)
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "4"