restore recipes/ in-tree — recipe + docker-images must share one revision

Reverts the recipe-data relocation from d0d210e (the earlier message only
asked to populate git.kilork.org/sm121/recipes, not to split the source).
A recipe.toml is meaningless without the exact Dockerfile/patches/pins in
docker-images/<dir>/ that build its image, and vice versa — they move in
one commit or the image/recipe pair can silently drift.

sm121/recipes remains the registry data repo: it stores `sm121 publish`
bundles, which ARE self-contained (overlay flattened in), so publishing
stays revision-consistent by construction. Launcher-side test fixtures
and the config-path fixes from d0d210e stay (they were orthogonal).

Also re-applies redhat GMU 0.90 -> 0.82 (pinned-KV guardrail, same flake
class as lab/4p67/dflash2).

AGENTS.md

@@ -89,8 +89,10 @@ per node instead of running it.
 
 ## Recipe format (`recipes/<name>/recipe.toml`)
 
-Recipe data lives in <https://git.kilork.org/sm121/recipes>; this repo
-keeps only the image build contexts (`docker-images/`).
+Recipes and their image build contexts (`docker-images/`) both live here
+and must move together — the served image must match the recipe revision.
+<https://git.kilork.org/sm121/recipes> holds only `sm121 publish` output
+(self-contained bundles) as the registry's storage.
 
 ```toml
 engine = "vllm"                     # or "sglang"

CLAUDE.md

@@ -51,7 +51,7 @@ See `specs/tech-architecture/tech-stack.md` for the full architectural map.
 
 | Glob | Source of truth |
 |------|-----------------|
-| `docker-images/**`, `host-scripts/**` | `AGENTS.md` (recipe data itself: git.kilork.org/sm121/recipes) |
+| `recipes/**`, `docker-images/**`, `host-scripts/**` | `AGENTS.md` |
 | `src/**`, `locales/**` | `CONVENTIONS.md` + `specs/` |
 | Cluster/recipe/config format questions | `AGENTS.md` (cluster ops SoT) |
 | Dev workflow / planning / lifecycle | `specs/` (bigpowers SoT) |

README.md

@@ -60,13 +60,15 @@ Prefix must be unambiguous — error if multiple clusters match.
 
 ## Recipes
 
-Recipes are TOML files in `recipes/<name>/recipe.toml` — resolved relative
-to the current directory (then `~/.config/sm121-launcher/recipes/`). The
-canonical collection lives in its own repo:
-<https://git.kilork.org/sm121/recipes> (one flat bundle per recipe; also
-the storage repo of the recipe registry). Clone it, drop your
-`config.toml` beside `recipes/`, and run `sm121` from there — or fetch
-single recipes with `sm121 registry list|install`.
+Recipes are TOML files in `recipes/<name>/recipe.toml`, resolved relative
+to the current directory (then `~/.config/sm121-launcher/recipes/`).
+This repo is the source of truth: a recipe and its image build context
+(`docker-images/<dir>/` — Dockerfile, patches, version pins) are
+versioned together, since the served image must match the recipe's
+revision exactly. `sm121 publish` flattens both into a self-contained
+bundle; the registry data repo <https://git.kilork.org/sm121/recipes>
+stores those published bundles (discovery/sharing artifact, never the
+source).
 
 ### Creating a recipe
 

recipes/deepseek-v4-flash-dspark38/recipe.toml

@@ -0,0 +1,131 @@
+# DeepSeek-V4-Flash-0731 DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
+#
+# Variant targeting aidendle94's production-3.8 image ("dsv4/apostolic" b12x
+# harness). This is the BUG-2026-08-03T204037 bring-up recipe — the two images
+# are *different b12x forks*, not a version bump from production-3.75, so the
+# launch contract differs:
+#
+#   - production-3.8 is a clean-overlay base (null ENTRYPOINT, CMD sleep
+#     infinity) whose intended launcher is /usr/local/bin/dsv4-vllm-entrypoint.
+#     We set that as the recipe entrypoint so the JIT toolchain env
+#     (DG_JIT_NVCC_COMPILER, NVCC_PREPEND_FLAGS + CCCL include path, CUDAHOSTCXX)
+#     and the /cache JIT dir layout are active. The 3.75 recipe ran vLLM directly
+#     because that image was self-contained with everything baked in.
+#   - VLLM_USE_V2_MODEL_RUNNER is dropped (experimental in this fork; the
+#     working glm52 recipe does not set it).
+#   - tiny-decode (VLLM_B12X_TINY_DECODE) is enabled at GMU 0.78 + CHUNK=8
+#     (the load-time repack needs headroom; it OOM'd at GMU 0.9).
+#   - /cache is mounted so the wrapper's JIT caches persist across restarts.
+
+engine = "vllm"
+name = "DeepSeek V4 Flash DSpark (production-3.8)"
+# Local overlay of the production-3.8 base built from
+# docker-images/deepseek-v4-flash-dspark38 (root-cause patch for the b12x
+# cutlass from_dlpack grad bug that blocked boot; see the Dockerfile).
+docker_image = "sparkrun-vllm-ds4-gb10:dspark38"
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+# production-3.8 launch contract: the clean-base image must be started via the
+# dsv4-vllm-entrypoint wrapper, which sets the JIT toolchain + /cache dirs and
+# `exec vllm "$@"`. `serve <recipe-args/auto flags>` land as "$@".
+entrypoint = ["/usr/local/bin/dsv4-vllm-entrypoint", "serve"]
+
+args = [
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--load-format", "auto",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "8192",
+    "--max-cudagraph-capture-size", "64",
+    # 0.78 is the safe GMU ceiling on GB10 (weights ~77.7 GiB of ~119.6 GiB);
+    # higher boots but dies under traffic. Also leaves headroom for the tiny
+    # decode kernel's GPU load-time repack.
+    "--gpu-memory-utilization", "0.78",
+    "--served-model-name", "deepseek-v4-flash",
+    # Engine backends — b12x across attention / MoE / linear (this fork's path)
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    # Prefix caching
+    "--enable-prefix-caching",
+    # Compilation
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    # DSpark speculative decoding — k=5 (DeepSeek-V4-Flash native
+    # dspark_block_size). draft_sample_method greedy.
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"greedy"}'""",
+    # Tokenizer / reasoning
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--reasoning-parser", "deepseek_v4",
+    "--enable-auto-tool-choice",
+    "--generation-config", "vllm",
+    "--default-chat-template-kwargs.thinking=true",
+    "--default-chat-template-kwargs.reasoning_effort=high",
+    "--disable-custom-all-reduce",
+    "--trust-remote-code",
+]
+
+# Persistent JIT/compile caches. The wrapper `mkdir -p`s under /cache
+# (DG_JIT_CACHE_DIR=/cache/deep_gemm_jit, VLLM_CACHE_ROOT=/cache/vllm,
+# TRITON_CACHE_DIR=/cache/triton, etc.), so /cache must be a mounted volume to
+# survive restarts. The HF cache is already mounted at /root/.cache/huggingface
+# by the launcher.
+[[extra_mounts]]
+host = "~/.cache/ds4-38-vllm"
+container = "/cache"
+
+[env]
+# --- b12x fork env family (shared with the proven eugr b12x recipe) ---
+VLLM_ATTENTION_BACKEND = "B12X_MLA_SPARSE"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_MEGA_AOT_ARTIFACT = "0"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_DENSE_SPLITK_TURBO = "1"
+B12X_W4A16_TC_DECODE = "1"
+# Tiny A8 decode kernel: DISABLED — its GPU load-time MoE-weight repack, on
+# top of the tight host RAM (~32 GiB free) + dual-node 155 GiB load, OOM'd both
+# hosts (had to power-cycle via HA). Revisit separately if host RAM headroom is
+# available. Keep the CHUNK pin for reference.
+# VLLM_B12X_TINY_DECODE = "1"
+# VLLM_B12X_TINY_CHUNK = "8"
+
+# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+OMP_NUM_THREADS = "4"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"
+
+# --- offline ---
+HF_HUB_OFFLINE = "1"
+
+# --- NCCL overrides (image-agnostic; launcher injects IB/HCA/socket from
+#     cluster config, these are recipe-level extras) ---
+NCCL_P2P_LEVEL = "SYS"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IB_TIMEOUT = "24"
+NCCL_IB_QPS_PER_CONNECTION = "8"
+NCCL_BUFFERSIZE = "16777216"
+NCCL_SOCKET_FAMILY = "inet"
+
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"

recipes/deepseek-v4-flash-eugr-agent/recipe.toml

@@ -0,0 +1,167 @@
+# DeepSeek-V4-Flash-0731 DSpark — AGENT-SERVING profile (2× GB10, TP=2).
+#
+# Fork of deepseek-v4-flash-eugr tuned for interactive agent traffic, applying
+# the validated "agent-2048" profile from the SvangenStudios DGX-Spark
+# agent-serving benchmarks (v0.1.1, measured 2026-08-02 on this exact stack:
+# DeepSeek-V4-Flash-0731, DSpark k=5, 2× DGX Spark, TP=2, RoCE):
+#   https://github.com/SvangenStudios/dgx-spark-agent-serving-benchmarks
+#   configs/agent-2048.env  /  results/RESULTS.md §4
+#
+# The benchmark's headline finding: --max-num-batched-tokens is a FAIRNESS
+# lever for agent workloads, not just a jitter lever. Against an ongoing
+# decode stream, a 262K-token prefill at 2048 delivers 2.9× the decode share
+# (5.0% vs 1.7%), 3.1× the decode tokens, 73% lower p95 output-chunk jitter,
+# and +67% KV-pool capacity — for 7.7% prefill throughput. The launcher's
+# whole purpose is agent serving (a model may be serving this very agent), so
+# 2048 is the more correct default here.
+#
+# Mental model (benchmark §3): an LLM server is three nearly-independent
+# engines — prefill (compute-bound, serializes), decode (bandwidth-bound,
+# scales to ~N=4), admission (scheduler-bound, BLOCKED during long prefill).
+# Chunk size governs how often decode gets a scheduling slot between prefill
+# chunks; smaller = more slots. It does NOT fix new-request admission (a
+# short request submitted during a large prefill waits until prefill
+# completes — 221× normal TTFT in both profiles).
+#
+# Knob changes vs deepseek-v4-flash-eugr (everything else identical — same
+# eugr image, same backends, same env):
+#   --max-num-batched-tokens  8192 -> 2048   (the fairness lever)
+#   --max-num-seqs            8    -> 6      (decode saturates ~N=4; 6 gives
+#                                             headroom without inflating
+#                                             worst-case latency)
+#   --max-cudagraph-capture-size 64 -> 36    (= max_num_seqs * (k+1) = 6*6;
+#                                             the k+1 multiple rule — the
+#                                             benchmark notes k=5 rejects
+#                                             MAX_NUM_SEQS=1 at boot)
+#   --gpu-memory-utilization  0.85 -> 0.78   (benchmark: "0.80 boots but dies
+#                                             under traffic"; GMU is a KV
+#                                             lever, not a safety lever, when
+#                                             weights dominate memory)
+#   --max-model-len           auto -> 1048576  (benchmark's 1M working capacity
+#                                             — verified 12/12 needles at
+#                                             32K/128K/512K/900K)
+#
+# Trade-off: this profile costs ~7.7% prefill throughput vs the 8192 batch
+# profile. Use deepseek-v4-flash-eugr (8192) for exclusive long-context batch
+# jobs (512K–1M) with no concurrent agent traffic. See the batch-8192 profile
+# in the benchmark repo for the opposite extreme.
+#
+# 3× Sparks (TP=3): bump tensor_parallel_size + required_nodes to 3 (the eugr
+# image supports it per forum comment #28). k=5 unchanged (model-native).
+
+engine = "vllm"
+name = "DeepSeek V4 Flash DSpark (eugr b12x, agent 2048)"
+docker_image = "eugr/spark-vllm-b12x:sm121"
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+docker_image_dir = "deepseek-v4-flash-eugr"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--load-format", "instanttensor",
+    # 1M working capacity (benchmark §2: 12/12 needles at 32K–900K). A 900K
+    # prefill takes ~15 min and starves everything — batch capacity, not an
+    # interactive mode.
+    "--max-model-len", "1048576",
+    # Decode saturates around N≈4 (benchmark §3); 6 gives headroom without
+    # inflating worst-case latency. cudagraph capture = max_num_seqs*(k+1)=36.
+    "--max-num-seqs", "6",
+    # FAIRNESS LEVER (benchmark §4): 2048 gives 2.9× the decode share under a
+    # concurrent prefill vs 8192, +67% KV pool, 73% lower jitter, for 7.7%
+    # prefill throughput. Use 8192 only for exclusive long-context batch jobs.
+    "--max-num-batched-tokens", "2048",
+    # = max_num_seqs * (k+1) = 6 * 6 (the k+1 multiple rule).
+    "--max-cudagraph-capture-size", "36",
+    # Benchmark finding: "0.80 boots but dies under traffic" — 0.78 is the
+    # safe ceiling. GMU is a KV lever, not a safety lever, when weights
+    # dominate memory (~77.7 GiB of ~119.6 GiB node).
+    "--gpu-memory-utilization", "0.78",
+    "--served-model-name", "deepseek-v4-flash",
+    # Engine backends — b12x across attention / MoE / linear
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    # Prefix caching — agent frameworks build prompts append-only, so reuse is
+    # 95.7–98.4% per turn (benchmark §9: divergence at the last token of the
+    # previous prompt; only ~270–660 tokens re-prefilled per agent step).
+    "--enable-prefix-caching",
+    # Compilation
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    # DSpark speculative decoding — k=5 (DeepSeek-V4-Flash native
+    # dspark_block_size). draft_sample_method per eugr recipe; draft attention
+    # backend B12X_MLA_SPARSE. The baked mod makes the embedded draft load via
+    # lazy safetensors while the target stays on InstantTensor.
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"B12X_MLA_SPARSE"}'""",
+    # Tokenizer / reasoning — reasoning-config pins the parser + empty
+    # reasoning markers (deepseek_v4 tokenizer mode emits them itself).
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--reasoning-parser", "deepseek_v4",
+    "--reasoning-config", """
+'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
+    "--enable-auto-tool-choice",
+    "--generation-config", "vllm",
+    "--default-chat-template-kwargs.thinking=true",
+    "--default-chat-template-kwargs.reasoning_effort=high",
+    "--disable-custom-all-reduce",
+    "--trust-remote-code",
+]
+
+# Persistent vLLM compile cache (speeds up restarts). Mounted at vLLM's
+# default cache root for root (/root/.cache/vllm); the HF cache is already
+# mounted at /root/.cache/huggingface by the launcher. Shares the eugr
+# recipe's cache dir (same image/backends — compile artifacts are reusable).
+[[extra_mounts]]
+host = "~/.cache/vllm-ds4-eugr"
+container = "/root/.cache/vllm"
+
+[env]
+# --- b12x + compile (from eugr's recipe env, verbatim) ---
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_USE_MEGA_AOT_ARTIFACT = "-1"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_MOE_FORCE_A8 = "1"
+
+# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+OMP_NUM_THREADS = "4"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"
+
+# --- offline ---
+HF_HUB_OFFLINE = "1"
+
+# --- NCCL overrides (GB10 single-NIC tuning; image-agnostic — mirrors
+#     recipes/deepseek-v4-flash. The launcher still injects IB/HCA/socket
+#     vars from the cluster config; these are recipe-level extras.) ---
+NCCL_P2P_LEVEL = "SYS"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IB_TIMEOUT = "24"
+NCCL_IB_QPS_PER_CONNECTION = "8"
+NCCL_BUFFERSIZE = "16777216"
+NCCL_SOCKET_FAMILY = "inet"
+
+# PCIe allreduce not needed on GB10 single-NIC
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"

recipes/deepseek-v4-flash-eugr/recipe.toml

@@ -0,0 +1,149 @@
+# DeepSeek-V4-Flash-0731 DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
+#
+# This variant uses eugr's prebuilt b12x image (eugr/spark-vllm-b12x) and
+# mirrors eugr's recipes/deepseek-v4-flash-0731.yaml on the b12x branch — the
+# stack published in NVIDIA forum comment #28:
+#   https://forums.developer.nvidia.com/t/instructions-for-running-deepseek-v4-flash-with-dspark-using-eugrs-repo/376220/28
+# ("While I'm working on integrating b12x into the build/test pipeline and
+# merging into main, I pushed prebuilt container as eugr/spark-vllm-b12x …
+# this version supports 3x Sparks with TP=3! To use, just add --tp 3.")
+#
+# Image: eugr/spark-vllm-b12x:sm121 — the prebuilt base with the
+# instanttensor-hybrid-draft-loader mod baked in at build time (see
+# docker-images/deepseek-v4-flash-eugr). eugr's recipe applies that mod via
+# run-recipe.sh --apply-mod; sm121 has no runtime mod-upload step for Docker
+# recipes, so the overlay patches vLLM's model_loader get_model() at image
+# build time instead. The recipe therefore carries no [mods] block.
+#
+# The mod keeps InstantTensor for the primary model while letting the embedded
+# dspark/MTP draft use lazy safetensors (avoids a second InstantTensor
+# GPU-streaming pass over the checkpoint). It activates only when
+# --load-format instanttensor is set (which this recipe does).
+#
+# Model: deepseek-ai/DeepSeek-V4-Flash-0731 — official release; native
+# dspark_block_size = 5, so num_speculative_tokens = 5.
+#
+# Differences vs recipes/deepseek-v4-flash (aidendle94 production-3.76.1 image):
+#   - attention backend B12X_MLA_SPARSE (eugr) vs FLASHINFER_MLA_SPARSE_DSV4
+#   - --linear-backend b12x + --load-format instanttensor (the mod's trigger)
+#   - --reasoning-config pinned to deepseek_v4 parser w/ empty start/end strs
+#   - draft_sample_method probabilistic + draft attention_backend B12X_MLA_SPARSE
+#   - max_num_seqs=8 / batched=8192 / cudagraph=64 (eugr defaults)
+#   - VLLM_USE_MEGA_AOT_ARTIFACT="-1" (eugr) vs "0"
+#
+# 3× Sparks (TP=3): bump tensor_parallel_size + required_nodes to 3 (the eugr
+# image supports it per comment #28). k=5 is unchanged (model-native).
+
+engine = "vllm"
+name = "DeepSeek V4 Flash DSpark (eugr b12x)"
+docker_image = "eugr/spark-vllm-b12x:sm121"
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+docker_image_dir = "deepseek-v4-flash-eugr"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--kv-cache-dtype", "fp8",
+    "--kv-cache-memory-bytes", "21474836480",
+    "--block-size", "256",
+    "--load-format", "instanttensor",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--long-prefill-token-threshold", "1024",
+    "--max-cudagraph-capture-size", "64",
+    # 0.78, not higher: the SvangenStudios agent-serving benchmark (v0.1.1) on
+    # this exact stack found "0.80 boots but dies under traffic". GMU is a KV
+    # lever, not a safety lever, when weights dominate memory (~77.7 GiB of
+    # ~119.6 GiB node). For interactive agent traffic prefer the
+    # deepseek-v4-flash-eugr-agent profile (2048 chunk). See
+    # specs/tech-architecture/dgx-spark-agent-serving-findings.md.
+    "--gpu-memory-utilization", "0.78",
+    "--served-model-name", "deepseek-v4-flash",
+    # Engine backends — b12x across attention / MoE / linear
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    # Prefix caching
+    "--enable-prefix-caching",
+    # Compilation
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    # DSpark speculative decoding — k=5 (DeepSeek-V4-Flash native
+    # dspark_block_size). draft_sample_method per eugr recipe; draft attention
+    # backend B12X_MLA_SPARSE. The baked mod makes the embedded draft load via
+    # lazy safetensors while the target stays on InstantTensor.
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"B12X_MLA_SPARSE"}'""",
+    # Tokenizer / reasoning — reasoning-config pins the parser + empty
+    # reasoning markers (deepseek_v4 tokenizer mode emits them itself).
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--reasoning-parser", "deepseek_v4",
+    "--reasoning-config", """
+'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
+    "--enable-auto-tool-choice",
+    "--generation-config", "vllm",
+    "--default-chat-template-kwargs.thinking=true",
+    "--default-chat-template-kwargs.reasoning_effort=max",
+    "--disable-custom-all-reduce",
+    "--trust-remote-code",
+]
+
+# Persistent vLLM compile cache (speeds up restarts). Mounted at vLLM's
+# default cache root for root (/root/.cache/vllm); the HF cache is already
+# mounted at /root/.cache/huggingface by the launcher.
+[[extra_mounts]]
+host = "~/.cache/vllm-ds4-eugr"
+container = "/root/.cache/vllm"
+
+[env]
+# --- b12x + compile (from eugr's recipe env, verbatim) ---
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_USE_MEGA_AOT_ARTIFACT = "-1"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_MOE_FORCE_A8 = "1"
+
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
+
+# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+OMP_NUM_THREADS = "4"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"
+
+# --- offline ---
+HF_HUB_OFFLINE = "1"
+
+# --- NCCL overrides (GB10 single-NIC tuning; image-agnostic — mirrors
+#     recipes/deepseek-v4-flash. The launcher still injects IB/HCA/socket
+#     vars from the cluster config; these are recipe-level extras.) ---
+NCCL_P2P_LEVEL = "SYS"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IB_TIMEOUT = "24"
+NCCL_IB_QPS_PER_CONNECTION = "8"
+NCCL_BUFFERSIZE = "16777216"
+NCCL_SOCKET_FAMILY = "inet"
+
+# PCIe allreduce not needed on GB10 single-NIC
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"
+
+VLLM_DSV4_REASONING_CONTENT = "1"

recipes/deepseek-v4-flash-src/recipe.toml

@@ -0,0 +1,97 @@
+# EXPERIMENTAL — DeepSeek-V4-Flash-0731 DSpark, built FROM SOURCE from the
+# newest local-inference-lab b12x fork ("eldritch"), instead of the prebuilt
+# aidendle94 image.
+#
+#   fork branch: feat/gg-ds4-0731-launcher (e63a190ee) — torch 2.12.0,
+#                flashinfer 0.6.15, cutlass-dsl 4.6.0, sm_121a
+#   image:       vllm-ds4-dspark-src:probe (built on demand via docker_image_dir)
+#
+# Serving flags/env mirror the fork's authoritative serve-ds4-flash-spark.sh at
+# that commit (DSpark k=5, draft attention backend B12X_MLA_SPARSE, b12x MoE/MLA,
+# cudagraph capture size = max_num_seqs * (k+1)). Adapt cluster-side: TP=2,
+# 1 node per rank, no DCP.
+#
+# ⚠️ NOT production. The image is a multi-hour from-source build. Boot may need
+# tuning; verify acceptance/throughput before any real traffic.
+
+engine = "vllm"
+name = "DeepSeek V4 Flash DSpark (EXPERIMENTAL, from source)"
+docker_image = "vllm-ds4-dspark-src:probe"
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+docker_image_dir = "deepseek-v4-flash-dspark-src"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--load-format", "auto",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "4",
+    "--max-num-batched-tokens", "8192",
+    # fork rule: max_num_seqs * (k+1) = 4 * 6
+    "--max-cudagraph-capture-size", "24",
+    # 0.78, not 0.80: the SvangenStudios agent-serving benchmark (v0.1.1) on
+    # this exact stack found "0.80 boots but dies under traffic". GMU is a KV
+    # lever, not a safety lever, when weights dominate memory. See
+    # specs/tech-architecture/dgx-spark-agent-serving-findings.md.
+    "--gpu-memory-utilization", "0.78",
+    "--served-model-name", "deepseek-v4-flash", "default",
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    "--async-scheduling",
+    "--no-scheduler-reserve-full-isl",
+    # DSpark k=5 (dspark_block_size == 5); draft attention backend per fork
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":5,"attention_backend":"B12X_MLA_SPARSE"}'""",
+    "--generation-config", "vllm",
+    "--default-chat-template-kwargs.thinking=true",
+    "--default-chat-template-kwargs.reasoning_effort=high",
+    "--default-chat-template-kwargs.temperature=1",
+    "--default-chat-template-kwargs.top_p=1",
+    "--disable-custom-all-reduce",
+    "--trust-remote-code",
+]
+
+[env]
+# --- from the fork's serve-ds4-flash-spark.sh (e63a190ee) ---
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
+VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_PCIE_ALLREDUCE_BACKEND = "b12x"
+VLLM_ENABLE_PCIE_ALLREDUCE = "1"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_DENSE_SPLITK_TURBO = "1"
+B12X_W4A16_TC_DECODE = "1"
+B12X_MOE_FORCE_A8 = "1"
+
+# --- build/arch + memory ---
+CUTE_DSL_ARCH = "sm_121a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+DG_JIT_USE_NVRTC = "0"
+USE_CUDNN = "1"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+OMP_NUM_THREADS = "4"
+
+# --- offline ---
+HF_HUB_OFFLINE = "1"

recipes/deepseek-v4-flash/recipe.toml

@@ -0,0 +1,153 @@
+# DeepSeek-V4-Flash-DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
+# DSpark speculative decoding on the official DeepSeek-V4-Flash-0731 release
+# via the local :fixed overlay on aidendle94's production-3.76.1 image (see
+# docker-images/deepseek-v4-flash-dspark).
+#
+# Image: sparkrun-vllm-ds4-gb10:fixed
+#   production-3.76.1 base (DSpark, b12x MoE, patched NCCL 2.30.4, multi-node
+#   fixes) + instanttensor for faster model loading.
+# Model: deepseek-ai/DeepSeek-V4-Flash-0731 — official release.
+#   0731 (and the -DSpark preview) both have dspark_block_size = 5. This image's
+#   speculator hard-rejects num_speculative_tokens > block_size
+#   (v1/worker/gpu/spec_decode/dspark/speculator.py, "must be <= dspark_block_size"),
+#   so k=7 (as suggested by the 0731 model card) FAILS to boot. k=5 is the
+#   model's native draft block and the correct value here.
+#
+# Note: this image lineage (vLLM 0.2x with the upstream DSpark speculator) does
+# NOT have the shared-expert draft-loader bug reported for the older fork
+# (tonyd2wild Patch 4). Its loader already maps mtp.*.shared_experts.w1/w3 into
+# gate_up_proj (dspark.py load_weights), so no patch is needed.
+#
+# Reference: Nvidia Developer Forum — DeepSeek-V4-Flash-0731 recipe
+#   https://forums.developer.nvidia.com/t/deepseek-v4-flash-0731-dspark-1m-nvfp4-kv-2x-dgx-spark/378824
+# Encoder reasoning-effort fix source: 0rand's serving stack (thread 372268/708)
+#   https://forums.developer.nvidia.com/t/deepseek-v4-flash-aiden-recipe-from-reddit-1m-token-session-operational-cuda-12-1-tailored-for-dgx-spark-gb10/372268/708
+#   (three-level low/high/max ladder; bundled into the :fixed overlay here)
+
+engine = "vllm"
+name = "DeepSeek V4 Flash DSpark (vLLM)"
+docker_image = "sparkrun-vllm-ds4-gb10:fixed"
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+docker_image_dir = "deepseek-v4-flash-dspark"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--load-format", "auto",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "16",
+    "--max-num-batched-tokens", "4096",
+    "--max-cudagraph-capture-size", "128",
+    "--gpu-memory-utilization", "0.78",
+    "--served-model-name", "deepseek-v4-flash",
+    # Engine backends
+    "--attention-backend", "FLASHINFER_MLA_SPARSE_DSV4",
+    "--moe-backend", "b12x",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # Compilation
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    # Async scheduling
+    "--async-scheduling",
+    "--no-scheduler-reserve-full-isl",
+    # DSpark speculative decoding — k=5 is DeepSeek-V4-Flash's native
+    # dspark_block_size (0731 == preview). Do NOT raise to k=7 (0731 model-card
+    # rec): this image hard-rejects k > block_size at boot. draft_sample_method
+    # matches the image CMD (probabilistic).
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic"}'""",
+    # Tokenizer / reasoning
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--reasoning-parser", "deepseek_v4",
+    "--enable-auto-tool-choice",
+    # Proven generation defaults (thread 372268/708 serving stack; with
+    # --generation-config vllm only the override params are applied, so these
+    # set the served defaults). repetition_penalty 1.0 = identity: the field is
+    # accepted by the spec but never deviates from no-op during req handling.
+    "--override-generation-config", """
+'{"temperature":0.8,"top_p":0.25,"repetition_penalty":1.0}'""",
+    "--generation-config", "vllm",
+    "--default-chat-template-kwargs.thinking=true",
+    "--default-chat-template-kwargs.reasoning_effort=max",
+    # Disable custom all-reduce (not needed on GB10 single-NIC)
+    "--disable-custom-all-reduce",
+    "--trust-remote-code",
+]
+
+# Persistent compile caches (speed up restarts — see guide §2/§7). The HF
+# cache is already mounted at /root/.cache/huggingface by the launcher.
+[[extra_mounts]]
+host = "~/.cache/vllm-ds4-dspark"   # compiled vLLM/attention kernels
+container = "/cache"
+
+[[extra_mounts]]
+host = "~/.cache/tilelang-ds4"      # DSpark speculative-decode kernels
+container = "/root/.tilelang"
+
+[env]
+# DSpark speculative decoding
+VLLM_DSPARK_REPLICATE_MARKOV_W1 = "1"
+# b12x MoE (this image's primary path)
+VLLM_USE_B12X_MOE = "1"
+# VLLM_USE_B12X_MOE = "0"              # b12x MoE OFF -> cutlass W4A8 (b12x MoE prefill stalls on GB10)
+
+# b12x suite = REQUIRED by this image
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_MEGA_AOT_ARTIFACT = "0"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_DENSE_SPLITK_TURBO = "1"
+B12X_W4A16_TC_DECODE = "1"
+B12X_MHC_MAX_TOKENS = "16384"
+
+# GPU memory management
+CUTE_DSL_ARCH = "sm_121a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+OMP_NUM_THREADS = "4"
+
+# Offline mode
+HF_HUB_OFFLINE = "1"
+
+# Compile-time arch hints
+TORCH_CUDA_ARCH_LIST = "12.1a"
+
+# NCCL overrides (cluster config ib_hca/socket_ifname used for most; recipe wins last)
+# NCCL_CROSS_NIC / NCCL_PROTO come from the cluster [nccl] section (2nodes-dac:
+# cross_nic=0, proto=LL,LL128,Simple) — matches the reference compose except
+# cross_nic (reference runs 1; our cluster config deliberately uses 0 for this
+# single-RoCE topology). NCCL_NVLS_ENABLE mirrors the reference compose.
+NCCL_P2P_LEVEL = "SYS"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_NVLS_ENABLE = "0"
+NCCL_IB_TIMEOUT = "24"
+NCCL_IB_QPS_PER_CONNECTION = "8"
+NCCL_BUFFERSIZE = "16777216"
+NCCL_SOCKET_FAMILY = "inet"
+
+# PCIe allreduce not needed on GB10 single-NIC
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"
+
+# GPU device isolation (single GPU per node)
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/gemma4-12b/recipe.toml

@@ -0,0 +1,38 @@
+engine = "vllm"
+name = "Gemma 4 12B IT (MTP)"
+docker_image = "vllm/vllm-openai:gemma4-unified"
+model = "google/gemma-4-12B-it"
+# docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
+# model = "RedHatAI/gemma-4-12B-it-NVFP4"
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+
+# Single-node recipe: Gemma 4 12B with BF16, FP8 KV cache,
+# MTP speculative decoding (5 tokens), 196K context.
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--dtype", "bfloat16",
+    "--kv-cache-dtype", "fp8",
+    "--gpu-memory-utilization", "0.75",
+    "--max-model-len", "196608",
+    "--max-num-seqs", "2",
+    "--max-num-batched-tokens", "8192",
+    "--enable-prefix-caching",
+    "--enable-chunked-prefill",
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "gemma4",
+    "--reasoning-parser", "gemma4",
+    "--generation-config", "vllm",
+    "--safetensors-load-strategy", "prefetch",
+    "--speculative-config", """
+'{"method":"mtp","model":"google/gemma-4-12B-it-assistant","num_speculative_tokens":5}'""",
+    "--limit-mm-per-prompt", """
+'{"image":4,"audio":1,"video":1}'""",
+    "--trust-remote-code",
+    "--served-model-name", "gemma4-12b",
+]
+
+[env]
+TRITON_CACHE_DIR = "/root/.cache/triton"

recipes/glm51/recipe.toml

@@ -0,0 +1,34 @@
+engine = "vllm"
+name = "GLM-5.1 NVFP8"
+docker_image = "vllm/vllm-openai:glm51-cu130"
+model = "lukealonso/GLM-5.1-NVFP4"
+
+required_nodes = 4
+distributed_backend = "mp"
+pipeline_parallel_size = 0
+
+args = [
+    "--speculative-config.method", "mtp",
+    "--speculative-config.num_speculative_tokens", "3",
+    "--tool-call-parser", "glm47",
+    "--reasoning-parser", "glm45",
+    "--enable-auto-tool-choice",
+    "--chat-template-content-format", "string",
+    "--attention-backend", "TRITON_MLA",
+    "--disable-flashinfer",
+    "--disable-custom-all-reduce",
+    "--enable-prefix-caching",
+    "--enable-chunked-prefill",
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "32768",
+    "--max-num-batched-tokens", "6144",
+    "--max-num-seqs", "48",
+    "--gpu-memory-utilization", "0.88",
+    "--trust-remote-code",
+    "--served-model-name", "glm51", "default",
+]
+
+[env]
+NCCL_DEBUG = "WARN"
+NCCL_IB_MERGE_NICS = "1"
+NCCL_MAX_NCHANNELS = "12"

recipes/glm52-hybrid-fp8-mxfp4/recipe.toml

@@ -0,0 +1,151 @@
+# GLM-5.2 Hybrid FP8+NVFP4+MXFP4 — 4× DGX Spark (GB10, sm_121a), TP=4, DCP=4.
+#
+# Model: aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
+#   → https://huggingface.co/aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
+#
+# Mixed-precision GLM-5.2 checkpoint:
+#   - Attention (MLA projections, indexer):  block-FP8
+#   - Dense MLPs & shared experts:           NVFP4 (group-16)
+#   - Routed experts, layers 3–77:           MXFP4 (e2m1 + ue8m0 per-32)
+#   - MTP draft experts (layer 78):          MXFP4
+#   - Norms, gates, lm_head, MTP dense:      BF16
+#
+# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.2
+#   vLLM + b12x GB10 kernels with hybrid_mxfp4_ct quant config,
+#   index-authoritative loader, NVFP4 KV cache with per-token scaling,
+#   sparse-MLA CKV gather, boot-time kernel prewarm.
+#
+# PREREQUISITES:
+#   - Models cached on every node via HF_HOME:
+#       huggingface-cli download aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
+#       (382 GiB, requires free disk on each node)
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/glm-5-2-hybrid-fp8-nvfp4-mxfp4-optimal-runtime-recipe/377598
+#   https://huggingface.co/aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
+
+engine = "vllm"
+name = "GLM-5.2 Hybrid FP8-MXFP4 (vLLM)"
+docker_image = "sparkrun-vllm-ds4-gb10:glm52"
+model = "aidendle94/GLM-5.2-Hybrid-FP8-MXFP4"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "10gb"
+docker_image_dir = "glm52"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Model name
+    "--served-model-name", "glm52",
+    # General
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--enable-prefix-caching",
+    # DCP (Decode Context Parallelism)
+    "--decode-context-parallel-size", "4",
+    "--dcp-comm-backend", "ag_rs",
+    # Engine backends
+    "--attention-backend", "B12X_MLA_SPARSE",
+    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
+    "--hf-overrides", """
+'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
+    # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
+    "--kv-cache-dtype", "nvfp4_ds_mla",
+    "--kv-cache-memory-bytes", "5500000000",
+    # Context & batch config
+    "--max-model-len", "262144",
+    "--max-num-batched-tokens", "2048",
+    "--max-num-seqs", "3",
+    "--gpu-memory-utilization", "0.89",
+    # Speculative decode — MTP k=4 with adaptive depths
+    # {{mtp_draft}} is substituted per-node with the resolved HF cache path
+    # (refs/main → latest snapshot) so symlinks resolve correctly.
+    "--speculative-config", """
+'{"model":"{{mtp_draft}}","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
+    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
+    # Timeout safety net (cold boot: ranks finish minutes apart)
+    "--distributed-timeout-seconds", "3600",
+    "--cpu-distributed-timeout-seconds", "3600",
+]
+
+# Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot).
+# `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent,
+# falls back to host "~/.cache/jit" (expanded to /home/<ssh_user>/.cache/jit).
+# `env` injects -e XDG_CACHE_HOME=/root/.cache/jit so vLLM/torch write there.
+[[extra_mounts]]
+alias = "jit_cache"
+host = "~/.cache/jit"
+container = "/root/.cache/jit"
+env = "XDG_CACHE_HOME"
+
+# MTP draft model — a subfolder of the main GLM-5.2 repo. Resolved at serve
+# time, per node, from the HF cache (refs/main → latest snapshot), so the
+# path survives `huggingface-cli download` updates without a fragile symlink.
+# The recipe references {{mtp_draft}} in --speculative-config; the launcher
+# substitutes the resolved full HF cache path per-node (no bind-mount — the
+# HF cache is already mounted at /root/.cache/huggingface, so symlinks
+# resolve correctly).
+[[extra_mounts]]
+repo = "aidendle94/GLM-5.2-Hybrid-FP8-MXFP4"
+subfolder = "mtp-draft"
+container = "mtp_draft"
+
+
+[env]
+# ── b12x suite ────────────────────────────────────────────────────────────
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_DCP_SHARD_DRAFT = "1"
+VLLM_DCP_GLOBAL_TOPK = "1"
+VLLM_DSV4_INDEXER_SP = "1"
+
+# ── MLA CKV gather (sparse gather for decode) ─────────────────────────────
+VLLM_B12X_MLA_CKV_GATHER = "1"
+VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
+VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
+VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
+
+# ── Speculative decode ────────────────────────────────────────────────────
+VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
+
+# ── NVFP4 KV cache ────────────────────────────────────────────────────────
+VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
+
+# ── Timeout safety net ────────────────────────────────────────────────────
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+
+# ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ─────────────────
+# The image bakes a value that GLM-5.2's KV coordinator rejects.
+# Empty string = unset at runtime.
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = ""
+
+# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+
+# ── JIT cache ──────────────────────────────────────────────────────────────
+# Persisted via the [[extra_mounts]] jit_cache entry above: XDG_CACHE_HOME is
+# injected from that mount, pointing at /root/.cache/jit (no hardcoded dupe).
+
+# ── NCCL overrides ─────────────────────────────────────────────────────────
+# Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME
+# Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN
+NCCL_IB_DISABLE = "0"             # enable IB (override launcher default)
+NCCL_MAX_NCHANNELS = "4"
+NCCL_MIN_NCHANNELS = "4"
+NCCL_CUMEM_ENABLE = "0"
+
+# ── Compute ────────────────────────────────────────────────────────────────
+OMP_NUM_THREADS = "4"
+
+# Volumes not needed — model is a Hugging Face Hub ID, downloaded via HF_HOME

recipes/glm52-int4-int8/recipe.toml

@@ -0,0 +1,109 @@
+# GLM-5.2 QuantTrio Int4-Int8Mix — 4× DGX Spark / GB10 (sm_121), TP=4.
+#
+# EXPERIMENT: Using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6
+# instead of locally-built sparkrun-vllm-ds4-gb10:glm52.
+# Pre-built on Docker Hub with CUDA 13.2.1, custom NCCL, b12x suite.
+#
+# Model: QuantTrio/GLM-5.2-Int4-Int8Mix (mixed Int4/Int8, 10% expert-pruned)
+#   → https://huggingface.co/QuantTrio/GLM-5.2-Int4-Int8Mix
+#   Includes in-checkpoint MTP draft model.
+#
+# PREREQUISITES:
+#   - Weights cached on every node:
+#       huggingface-cli download QuantTrio/GLM-5.2-Int4-Int8Mix
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/glm-5-2-int4-int8-on-8x-gb10-1200-t-s-prefill-33-54-t-s-avg-decode-generic-coding-structured/376831
+#   https://github.com/ciprianveg/gb10-glm-5.2
+
+engine = "vllm"
+name = "GLM-5.2 Int4-Int8Mix (vLLM, DS4 image)"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6"
+model = "QuantTrio/GLM-5.2-Int4-Int8Mix"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "64gb"
+# docker_image_dir = "glm52"             # removed for experiment
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "150000",
+    "--max-num-seqs", "1",
+    "--max-num-batched-tokens", "4096",
+    "--gpu-memory-utilization", "0.91",
+    # Engine backends
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    # Compilation
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # MTP speculative decoding (in-checkpoint draft model)
+#    "--speculative-config", """
+#'{"method":"mtp","num_speculative_tokens":4,"moe_backend":"b12x","draft_sample_method":"probabilistic"}'""",
+    # Tokenizer / reasoning (GLM-5.2 family)
+    "--tool-call-parser", "glm47",
+    "--reasoning-parser", "glm45",
+    "--enable-auto-tool-choice",
+    "--trust-remote-code",
+    "--served-model-name", "glm52",
+]
+
+[env]
+# b12x suite (image requires this)
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_MEGA_AOT_ARTIFACT = "0"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_DENSE_SPLITK_TURBO = "1"
+B12X_W4A16_TC_DECODE = "1"
+B12X_MHC_MAX_TOKENS = "16384"
+
+# GLM-5.2 support flags
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "256"
+
+# Timeout safety net for cold boot (cudagraph + JIT)
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
+
+# Offline mode — weights must be cached on each node
+HF_HUB_OFFLINE = "1"
+
+# Architecture
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+
+# GPU memory management
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+
+# NCCL overrides
+NCCL_CROSS_NIC = "1"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IGNORE_CPU_AFFINITY = "1"
+NCCL_DEBUG = "WARN"
+
+# PCIe allreduce not needed on GB10 single-NIC
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"
+
+# GPU device isolation
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/glm52-mxfp4-experts-gptq/recipe.toml

@@ -0,0 +1,148 @@
+# GLM-5.2 MXFP4-Experts-GPTQ — 4× DGX Spark (GB10, sm_121a), TP=4.
+#
+# Model: aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
+#   → https://huggingface.co/aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
+#
+# GPTQ-quantized MXFP4 experts for GLM-5.2 (744B MoE, 256 experts/layer,
+# 8 activated per token, 78 layers, sparse-MLA). Experts quantized to
+# MXFP4 via GPTQ; non-expert layers (attention, indexer, router, lm_head)
+# kept in BF16.
+#
+# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.2
+#   vLLM + b12x suite with hybrid_mxfp4_ct quant config, index-authoritative
+#   loader, NVFP4 KV cache with per-token scaling, sparse-MLA CKV gather.
+#
+# PREREQUISITES:
+#   - Weights cached on every node via HF_HOME:
+#       huggingface-cli download aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
+#     (requires free disk on each node)
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/glm-5-2-hybrid-fp8-nvfp4-mxfp4-optimal-runtime-recipe/377598
+#   https://huggingface.co/aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
+
+engine = "vllm"
+name = "GLM-5.2 MXFP4-Experts-GPTQ (vLLM)"
+docker_image = "sparkrun-vllm-ds4-gb10:glm52"
+model = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "10gb"
+docker_image_dir = "glm52"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Model name
+    "--served-model-name", "glm52",
+    # General
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--enable-prefix-caching",
+    # DCP (Decode Context Parallelism)
+    "--decode-context-parallel-size", "4",
+    "--dcp-comm-backend", "ag_rs",
+    # Engine backends
+    "--attention-backend", "B12X_MLA_SPARSE",
+    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
+    "--hf-overrides", """
+'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
+    # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
+    "--kv-cache-dtype", "nvfp4_ds_mla",
+    "--kv-cache-memory-bytes", "5500000000",
+    # Context & batch config
+    "--max-model-len", "262144",
+    "--max-num-batched-tokens", "2048",
+    "--max-num-seqs", "3",
+    "--gpu-memory-utilization", "0.89",
+    # Speculative decode — MTP k=4 with adaptive depths
+    # {{mtp_draft}} is substituted per-node with the resolved HF cache path
+    # (refs/main → latest snapshot) so symlinks resolve correctly.
+    "--speculative-config", """
+'{"model":"{{mtp_draft}}","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
+    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
+    # Timeout safety net (cold boot: ranks finish minutes apart)
+    "--distributed-timeout-seconds", "3600",
+    "--cpu-distributed-timeout-seconds", "3600",
+]
+
+# Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot).
+# `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent,
+# falls back to host "~/.cache/jit" (expanded to /home/<ssh_user>/.cache/jit).
+# `env` injects -e XDG_CACHE_HOME=/root/.cache/jit so vLLM/torch write there.
+[[extra_mounts]]
+alias = "jit_cache"
+host = "~/.cache/jit"
+container = "/root/.cache/jit"
+env = "XDG_CACHE_HOME"
+
+# MTP draft model — a subfolder of the main GLM-5.2 repo. Resolved at serve
+# time, per node, from the HF cache (refs/main → latest snapshot), so the
+# path survives `huggingface-cli download` updates without a fragile symlink.
+# The recipe references {{mtp_draft}} in --speculative-config; the launcher
+# substitutes the resolved full HF cache path per-node (no bind-mount — the
+# HF cache is already mounted at /root/.cache/huggingface, so symlinks
+# resolve correctly).
+[[extra_mounts]]
+repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
+subfolder = "mtp-draft"
+container = "mtp_draft"
+
+
+[env]
+# ── b12x suite ────────────────────────────────────────────────────────────
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_DCP_SHARD_DRAFT = "1"
+VLLM_DCP_GLOBAL_TOPK = "1"
+VLLM_DSV4_INDEXER_SP = "1"
+
+# ── MLA CKV gather (sparse gather for decode) ─────────────────────────────
+VLLM_B12X_MLA_CKV_GATHER = "1"
+VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
+VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
+VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
+
+# ── Speculative decode ────────────────────────────────────────────────────
+VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
+
+# ── NVFP4 KV cache ────────────────────────────────────────────────────────
+VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
+
+# ── Timeout safety net ────────────────────────────────────────────────────
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+
+# ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ─────────────────
+# The image bakes a value that GLM-5.2's KV coordinator rejects.
+# Empty string = unset at runtime.
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = ""
+
+# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+
+# ── JIT cache ──────────────────────────────────────────────────────────────
+# Persisted via the [[extra_mounts]] jit_cache entry above: XDG_CACHE_HOME is
+# injected from that mount, pointing at /root/.cache/jit (no hardcoded dupe).
+
+# ── NCCL overrides ─────────────────────────────────────────────────────────
+# Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME
+# Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN
+NCCL_IB_DISABLE = "0"             # enable IB (override launcher default)
+NCCL_MAX_NCHANNELS = "4"
+NCCL_MIN_NCHANNELS = "4"
+NCCL_CUMEM_ENABLE = "0"
+
+# ── Compute ────────────────────────────────────────────────────────────────
+OMP_NUM_THREADS = "4"
+
+# Volumes not needed — model is a Hugging Face Hub ID, downloaded via HF_HOME

recipes/glm52-nf3-hybrid-context/recipe.toml

@@ -0,0 +1,140 @@
+# GLM-5.2 NF3-Hybrid — 🧠 CONTEXT LANE (DCP4, 800K max-len)
+# 4× DGX Spark (GB10, aarch64/sm_121a), TP=4, DCP=4.
+#
+# Model: madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
+#   → https://huggingface.co/madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
+#
+# Unpruned GLM-5.2 (753B/40B MoE, all 256 experts) in 3-format hybrid quant:
+#   327GB — top-64 experts/layer in NVFP4, remaining 192 in NF3 (3-bit),
+#   non-expert layers in MXFP8.
+#
+# EXPERIMENT: Using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6
+# instead of locally-built vllm-nf3-hybrid:probe. Pre-built on Docker Hub.
+# NOTE: production-hybrid-1.6 lacks NF3 hybrid overlay files
+# (hybrid_loader.py, nf3_kernel.py, nvfp4_kernel.py, mxfp8_tier.json).
+# If it fails, revert to vllm-nf3-hybrid:probe.
+#
+# DCP4 lane: KV sharded across all 4 nodes → 877K-token pool.
+#   Every decode step pays ag_rs allreduce on all 78 layers.
+#   MTP k=3 (needs fuse_allreduce_rms) — near-perfect acceptance ~3.0
+#   beat k=4/5 by ~8% on long network-taxed steps.
+#
+# Performance (measured 2026-07-07, 512-tok gens, temp 0, shallow ctx):
+#   c1: 21.4–21.6 tok/s  |  c2: 32.9  |  c3: 38.9
+#   c4: 45.4  |  c5: 37.4  |  c6: 40.1 aggregate
+#   KV pool: 877,056 tokens  |  max-model-len: 800,000
+#   Boot ~20 min, MTP k=3 acceptance ~3.0
+#
+# TUNING NOTES:
+#   - gpu-memory-utilization 0.88 (GB10 unified memory; author's 0.96 is
+#     dedicated-VRAM territory)
+#   - max-num-batched-tokens ≥2048 or NF3 graph capture breaks
+#   - max-cudagraph-capture-size must cover seqs×(1+k) = 4×4 = 16
+#   - VLLM_DCP_SHARD_DRAFT=1 mandatory for MTP draft KV under DCP
+#   - k is lane-specific: CONTEXT lane runs k=3 + fuse_allreduce_rms
+#
+# PREREQUISITES:
+#   - Weights cached on every node:
+#       huggingface-cli download madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
+#     (327GB, 334GB on disk, 96 safetensors shards)
+
+engine = "vllm"
+name = "GLM-5.2 NF3-Hybrid CONTEXT LANE (vLLM)"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6"
+model = "madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+# docker_image_dir = "glm52-nf3-hybrid"             # removed for experiment
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Quantization & KV cache
+    "--quantization", "modelopt_fp4",
+    "--kv-cache-dtype", "fp8",
+#    "--load-format", "auto",
+    "--load-format", "fastsafetensors",
+    # Context length (800K max-len, 877K KV pool)
+    "--max-model-len", "200000",
+    # Batch & memory
+    "--max-num-seqs", "4",
+    "--max-num-batched-tokens", "4096",
+    "--max-cudagraph-capture-size", "32",
+    "--gpu-memory-utilization", "0.84",
+    # Engine backends
+    #"--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    # Scheduling
+    "--async-scheduling",
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    # Compilation (CONTEXT: fuse_allreduce_rms for DCP network tax)
+    "--compilation-config", """
+'{"pass_config":{"fuse_allreduce_rms":true}}'""",
+    # Speculative decode — CONTEXT: MTP k=3 (near-perfect acceptance ~3.0)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":3,"moe_backend":"b12x","draft_sample_method":"probabilistic"}'""",
+    # Tool calling & reasoning (verified working from first boot)
+    "--tool-call-parser", "glm47",
+    "--reasoning-parser", "glm45",
+    "--enable-auto-tool-choice",
+    # General
+    "--trust-remote-code",
+    "--enable-flashinfer-autotune",
+    "--served-model-name", "glm-5.2",
+]
+
+[env]
+# Memory stability
+#PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+#VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+
+# ── DCP (Decode Context Parallelism) — CONTEXT: DCP4 ──────────────────────
+# KV sharded across all 4 nodes — 4× pool, decode pays ag_rs gather
+DECODE_CONTEXT_PARALLEL_SIZE = "4"
+DCP_COMM_BACKEND = "ag_rs"
+DCP_KV_CACHE_INTERLEAVE_SIZE = "1"
+
+# ── b12x suite (required) ─────────────────────────────────────────────────
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_DCP_GLOBAL_TOPK = "1"
+VLLM_DCP_SHARD_DRAFT = "1"
+B12X_W4A16_TC_DECODE = "1"
+B12X_MOE_FORCE_A16 = "1"
+B12X_DENSE_SPLITK_TURBO = "1"
+
+# ── Hybrid loader knobs (v2) ──────────────────────────────────────────────
+HYBRID_TIER = "both"
+HYBRID_KEPT = "b12x_nf3"
+HYBRID_NF3 = "b12x_nf3"
+HYBRID_B12X_MAX_TOKENS = "4096"
+HYBRID_MXFP8_NATIVE = "1"
+# Real path to tier json inside the image (loader hardcodes /opt/venv layout)
+HYBRID_MXFP8_TIER_JSON = "/usr/local/lib/python3.12/dist-packages/mxfp8_tier.json"
+
+# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+
+# ── Offline mode (weights must be cached on each node) ────────────────────
+HF_HOME = "/root/.cache/huggingface"
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+
+# ── Timeout safety net (cold boot: cudagraph + JIT) ───────────────────────
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "3600"
+
+# ── NCCL / RoCE (cluster config ib_hca/socket_ifname used for most) ──────
+NCCL_CROSS_NIC = "1"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IGNORE_CPU_AFFINITY = "1"
+NCCL_DEBUG = "WARN"
+
+# ── Compute ────────────────────────────────────────────────────────────────
+OMP_NUM_THREADS = "4"

recipes/glm52-nf3-hybrid-fast/recipe.toml

@@ -0,0 +1,133 @@
+# GLM-5.2 NF3-Hybrid — 🏎️ FAST LANE (DCP1, 200K max-len)
+# 4× DGX Spark (GB10, aarch64/sm_121a), TP=4, DCP=1.
+#
+# Model: madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
+#   → https://huggingface.co/madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
+#
+# Unpruned GLM-5.2 (753B/40B MoE, all 256 experts) in 3-format hybrid quant:
+#   327GB — top-64 experts/layer in NVFP4, remaining 192 in NF3 (3-bit),
+#   non-expert layers in MXFP8.
+#
+# EXPERIMENT: Using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6
+# instead of locally-built vllm-nf3-hybrid:probe. Pre-built on Docker Hub.
+# NOTE: production-hybrid-1.6 lacks NF3 hybrid overlay files
+# (hybrid_loader.py, nf3_kernel.py, nvfp4_kernel.py, mxfp8_tier.json).
+# If it fails, revert to vllm-nf3-hybrid:probe.
+#
+# DCP1 lane: each sequence's KV cache lives entirely on one node → zero
+#   cross-node attention traffic per decode step. ~1/4 the KV pool of DCP4.
+#   Best for interactive chat, agents, throughput-sensitive workloads.
+#
+# Performance (measured 2026-07-07, 512-tok gens, temp 0, MTP acceptance 3.3–4.0):
+#   c1: 24–29 tok/s (content-dependent, peaks to 29.1 @ accept 4.02)
+#   c2: 36.6  |  c3: 45.7  |  c4: 54.2  |  c5: 59.2  |  c6: 67.4
+#   KV pool: 219,264 tokens  |  max-model-len: 200,000
+#
+# TUNING NOTES:
+#   - gpu-memory-utilization 0.88 (GB10 unified memory; author's 0.96 is
+#     dedicated-VRAM territory; 0.88 = safe for 200K + 12e9 KV allocation)
+#   - kv-cache-memory-bytes 12e9 per node caps KV pool at 219K tokens
+#   - max-num-batched-tokens ≥2048 or NF3 graph capture breaks
+#   - max-cudagraph-capture-size must cover seqs×(1+k) = 6×5 = 30
+#   - FAST lane uses MTP k=4, NO fuse_allreduce_rms (no benefit without
+#     DCP network tax)
+#   - k is lane-specific: FAST runs k=4, no fuse
+#
+# PREREQUISITES:
+#   - Weights cached on every node:
+#       huggingface-cli download madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
+#     (327GB, 334GB on disk, 96 safetensors shards)
+
+engine = "vllm"
+name = "GLM-5.2 NF3-Hybrid FAST LANE (vLLM)"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6"
+model = "madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "64gb"
+# docker_image_dir = "glm52-nf3-hybrid"             # removed for experiment
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Quantization & KV cache
+    "--quantization", "modelopt_fp4",
+    "--kv-cache-dtype", "fp8",
+    "--load-format", "fastsafetensors",
+    # Context length (200K max-len, 219K KV pool)
+    "--max-model-len", "200000",
+    # Batch & memory — FAST lane: 6 concurrent, 12e9 KV bytes
+    "--max-num-seqs", "6",
+    "--max-num-batched-tokens", "4096",
+    "--max-cudagraph-capture-size", "32",
+    "--gpu-memory-utilization", "0.88",
+    "--kv-cache-memory-bytes", "12000000000",
+    # Engine backends
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    # Scheduling
+    "--async-scheduling",
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    # Speculative decode — FAST: MTP k=4 (accept 3.3–4.0, best aggregate)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":4,"moe_backend":"b12x","draft_sample_method":"probabilistic"}'""",
+    # Tool calling & reasoning (verified working from first boot)
+    "--tool-call-parser", "glm47",
+    "--reasoning-parser", "glm45",
+    "--enable-auto-tool-choice",
+    # General
+    "--trust-remote-code",
+    "--enable-flashinfer-autotune",
+    "--served-model-name", "glm-5.2",
+]
+
+[env]
+# ── DCP (Decode Context Parallelism) — FAST: DCP1 ────────────────────────
+# KV lives locally — zero cross-node attention traffic, ~1/4 pool of DCP4
+DECODE_CONTEXT_PARALLEL_SIZE = "1"
+DCP_COMM_BACKEND = "ag_rs"
+DCP_KV_CACHE_INTERLEAVE_SIZE = "1"
+
+# ── b12x suite (required) ─────────────────────────────────────────────────
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_DCP_GLOBAL_TOPK = "1"
+VLLM_DCP_SHARD_DRAFT = "1"
+B12X_W4A16_TC_DECODE = "1"
+B12X_MOE_FORCE_A16 = "1"
+B12X_DENSE_SPLITK_TURBO = "1"
+
+# ── Hybrid loader knobs (v2) ──────────────────────────────────────────────
+HYBRID_TIER = "both"
+HYBRID_KEPT = "b12x_nf3"
+HYBRID_NF3 = "b12x_nf3"
+HYBRID_B12X_MAX_TOKENS = "4096"
+HYBRID_MXFP8_NATIVE = "1"
+HYBRID_MXFP8_TIER_JSON = "/usr/local/lib/python3.12/dist-packages/mxfp8_tier.json"
+
+# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+
+# ── Offline mode (weights must be cached on each node) ────────────────────
+HF_HOME = "/root/.cache/huggingface"
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+
+# ── Timeout safety net (cold boot: cudagraph + JIT) ───────────────────────
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "3600"
+
+# ── NCCL / RoCE (cluster config ib_hca/socket_ifname used for most) ──────
+NCCL_CROSS_NIC = "1"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IGNORE_CPU_AFFINITY = "1"
+NCCL_DEBUG = "WARN"
+
+# ── Compute ────────────────────────────────────────────────────────────────
+OMP_NUM_THREADS = "4"

recipes/glm52-nf4-lloyd-experts-gptq/recipe.toml

@@ -0,0 +1,205 @@
+# GLM-5.2 NF4 Lloyd-Experts-GPTQ — 4× DGX Spark (GB10, sm_121a), TP=4, DCP=4.
+#
+# Model: aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
+#   → https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
+#
+# Lloyd-codebook NF4 routed-expert overlay for GLM-5.2 (744B MoE, 256
+# experts/layer, 8 activated/token, 78 layers, sparse-MLA). Expert weights
+# quantized to 4-bit with a fitted Lloyd–Max codebook (−16.8% expert
+# reconstruction error vs E2M1/MXFP4 baseline on the same GPTQ pipeline).
+# Codes are indices into per-layer grids/L{n}.json — NOT the E2M1 table.
+#
+# LOADING via [composed_model]: the Lloyd repo is experts-only (no config,
+# no tokenizer, no non-expert weights). The launcher builds a /tmp dir per
+# node with symlinks into the HF cache and bind-mounts it at /root/model:
+#   1. aidendle94/GLM-5.2-MXFP4-Experts-GPTQ (base checkpoint — config.json
+#      with quant_method="hybrid_mxfp4_ct", hybrid-ct-*.safetensors non-expert
+#      weights, tokenizer, mtp-draft) → all files symlinked.
+#   2. aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ (Lloyd NF4 experts) →
+#      L*.safetensors symlinked, OVERWRITING the MXFP4 E2M1 experts
+#      (later sources win). Same filenames, same tensor names — the Lloyd
+#      codes replace the E2M1 codes.
+#   3. nf4_layers.json (generated) — tells the kernel which layers use the
+#      nf4_e8m0_k32 decode path vs the default fp4_e8m0_k32 E2M1 bit-twiddle.
+#
+# The symlinks point to container-side paths (/root/.cache/huggingface/...)
+# so they resolve inside the container where the HF cache is mounted. The
+# _NF4_CODEBOOK is GLOBAL and hardcoded in the b12x binary (fitted as the
+# mean of 71 per-layer Lloyd–Max tables) — the per-layer grids/L{n}.json
+# files are reference artifacts, NOT loaded at runtime.
+#
+# Image: sparkrun-vllm-ds4-gb10:glm52
+#   Built from docker-images/glm52/Dockerfile, which FROMs
+#   aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.4 (bakes the
+#   nf4_e8m0_k32 / nf4_packed kernel — inert without VLLM_NF4_LAYERS).
+#
+# PREREQUISITES:
+#   - Both repos cached on every node via HF_HOME:
+#       huggingface-cli download aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
+#       huggingface-cli download aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
+#     (MXFP4 base ~390 GB, Lloyd experts ~385 GB)
+#   No manual overlay or JSON creation — [composed_model] handles everything.
+
+engine = "vllm"
+name = "GLM-5.2 NF4 Lloyd-Experts-GPTQ (vLLM)"
+docker_image = "sparkrun-vllm-ds4-gb10:glm52"
+model = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "10gb"
+docker_image_dir = "glm52"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Model name
+    "--served-model-name", "glm52",
+    # General
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--enable-prefix-caching",
+    # DCP (Decode Context Parallelism)
+    "--decode-context-parallel-size", "4",
+    "--dcp-comm-backend", "ag_rs",
+    # Engine backends
+    "--attention-backend", "B12X_MLA_SPARSE",
+    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
+    "--hf-overrides", """
+'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
+    # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
+    "--kv-cache-dtype", "nvfp4_ds_mla",
+    "--kv-cache-memory-bytes", "5500000000",
+    # Context & batch config
+    "--max-model-len", "262144",
+    "--max-num-batched-tokens", "2048",
+    "--max-num-seqs", "3",
+    "--gpu-memory-utilization", "0.89",
+    # Speculative decode — MTP k=4 with adaptive depths
+    # /root/model/mtp-draft is symlinked from the MXFP4 base checkpoint
+    # (the composed dir includes it via source #1).
+    #
+    # NOTE: MTP + reasoning parser (glm45) + response_format (guided decoding)
+    # requires the vLLM structured-output reasoning-boundary fix (upstream
+    # PRs #44297 + #44993, forward-ported in docker-images/glm52/). Without
+    # it, the grammar FSM fails to advance across the reasoning-end marker
+    # (|◊|) under MTP, so the content channel is unconstrained and the model
+    # emits prose/markdown-fence prefixes before JSON. The fix is baked into
+    # the sparkrun-vllm-ds4-gb10:glm52 image.
+    "--speculative-config", """
+'{"model":"/root/model/mtp-draft","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
+    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
+    # Timeout safety net (cold boot: ranks finish minutes apart)
+    "--distributed-timeout-seconds", "3600",
+    "--cpu-distributed-timeout-seconds", "3600",
+]
+
+[composed_model]
+mount = "/root/model"
+
+[[composed_model.sources]]
+repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
+
+[[composed_model.sources]]
+repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ"
+glob = "L*.safetensors"
+# Exclude the 5 layers with mis-packed experts (per model card known-defect
+# workaround). These layers' Lloyd repo L{n}.safetensors contain 255 Lloyd-
+# coded experts + 1 E2M1 expert — decoding as E2M1 (which nf4_layers.json
+# does) would garbage-decode the 255 Lloyd experts. Instead, skip the Lloyd
+# overlay for these files so the MXFP4 base's all-E2M1 L{n}.safetensors
+# survives, and the excluded-from-NF4 E2M1 decode path matches their bytes.
+exclude = [
+    "L5.safetensors",
+    "L8.safetensors",
+    "L9.safetensors",
+    "L32.safetensors",
+    "L46.safetensors",
+]
+
+# NF4 layer list — all 75 MoE layers (3–77) EXCEPT the 5 layers with a
+# known packing defect (per model card 2026-07-27). Layers 5, 8, 9, 32, 46
+# each contain 1 expert whose bytes were packed with E2M1/MXFP4 nibbles
+# (quantizer arm won per-expert, but decoding is per-layer). These layers
+# are ALSO excluded from the Lloyd source above (exclude = [...]) so the
+# MXFP4 base's all-E2M1 L{n}.safetensors survives — decoding as E2M1
+# (which this list does) matches their bytes. Layer 10 had the same defect
+# but was already caught + fixed upstream by the packing guard, so it
+# stays in the NF4 set.
+# Ref: https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
+# ("Known defect: 5 experts are packed on the wrong grid")
+[[composed_model.files]]
+name = "nf4_layers.json"
+content = '{"layers":[3,4,6,7,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,33,34,35,36,37,38,39,40,41,42,43,44,45,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77]}'
+
+# Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot).
+# `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent,
+# falls back to host "~/.cache/jit" (expanded to /home/<ssh_user>/.cache/jit).
+# `env` injects -e XDG_CACHE_HOME=/root/.cache/jit so vLLM/torch write there.
+[[extra_mounts]]
+alias = "jit_cache"
+host = "~/.cache/jit"
+container = "/root/.cache/jit"
+env = "XDG_CACHE_HOME"
+
+
+[env]
+# ── NF4 Lloyd expert overlay ──────────────────────────────────────────────
+# Path to a JSON file listing which layers use the nf4_e8m0_k32 decode path
+# (vs the default fp4_e8m0_k32 E2M1 bit-twiddle). Generated by [composed_model]
+# at /root/model/nf4_layers.json — no placeholder needed (known container path).
+VLLM_NF4_LAYERS = "/root/model/nf4_layers.json"
+
+# ── b12x suite ────────────────────────────────────────────────────────────
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_DCP_SHARD_DRAFT = "1"
+VLLM_DCP_GLOBAL_TOPK = "1"
+VLLM_DSV4_INDEXER_SP = "1"
+
+# ── MLA CKV gather (sparse gather for decode) ─────────────────────────────
+VLLM_B12X_MLA_CKV_GATHER = "1"
+VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
+VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
+VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
+
+# ── Speculative decode ────────────────────────────────────────────────────
+VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
+
+# ── NVFP4 KV cache ────────────────────────────────────────────────────────
+VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
+
+# ── Timeout safety net ────────────────────────────────────────────────────
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+
+# ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ─────────────────
+# The image bakes a value that GLM-5.2's KV coordinator rejects.
+# Empty string = unset at runtime.
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = ""
+
+# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+
+# ── JIT cache ──────────────────────────────────────────────────────────────
+# Persisted via the [[extra_mounts]] jit_cache entry above: XDG_CACHE_HOME is
+# injected from that mount, pointing at /root/.cache/jit (no hardcoded dupe).
+
+# ── NCCL overrides ─────────────────────────────────────────────────────────
+# Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME
+# Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN
+NCCL_IB_DISABLE = "0"             # enable IB (override launcher default)
+NCCL_MAX_NCHANNELS = "4"
+NCCL_MIN_NCHANNELS = "4"
+NCCL_CUMEM_ENABLE = "0"
+
+# ── Compute ────────────────────────────────────────────────────────────────
+OMP_NUM_THREADS = "4"

recipes/glm52/recipe.toml

@@ -0,0 +1,35 @@
+engine = "vllm"
+name = "GLM-5.2 NVFP4 (vLLM)"
+docker_image = "vllm/vllm-openai:glm52"
+model = "Mapika/GLM-5.2-NVFP4"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+
+# 410B NVFP4 model on 4× DGX Spark (GB10 Blackwell), 128 GB shared each.
+# TP=4 → ~103 GB/GPU for weights, ~25 GB for KV cache at 0.90 util.
+#entrypoint = ["vllm", "serve"]
+
+args = [
+#    "--kv-cache-dtype", "fp8",
+    "--gpu-memory-utilization", "0.85",
+    "--load-format", "safetensors",
+    "--enable-prefix-caching",
+#    "--speculative-config", """
+#'{"method":"mtp","num_speculative_tokens":5}'""",
+    "--tool-call-parser", "glm47",
+    "--reasoning-parser", "glm45",
+    "--enable-auto-tool-choice",
+#    "--enable-expert-parallel",
+    "--trust-remote-code",
+    "--served-model-name", "glm52",
+]
+
+[env]
+# Fast safetensors loading (GPU‑side decompress)
+SAFETENSORS_FAST_GPU = "1"
+# Allow long context lengths
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+# Blackwell MoE kernels
+#VLLM_USE_B12X_MOE = "1"

recipes/glm53-flash-dflash2-mxfp8/recipe.toml

@@ -0,0 +1,137 @@
+# GLM-5.3-Flash + DFlash2 draft (MXFP8) — speculative decoding — 2× Spark TP=2
+#
+# DFlash2-MXFP8 (local-inference-lab) is a ~1B BLOCK-DIFFUSION DRAFT model,
+# NOT a standalone LLM. It is loaded beside the TARGET
+# `local-inference-lab/GLM-5.3-Flash-NVFP4` (the lab quant) and every drafted
+# token is verified against it. Draft: 5 sliding-attention layers, block_size
+# 8, proposal length 7, reads target hidden states at layers 5/14/24/33/42.
+# Model card: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
+#
+# DRAFT LICENSE: source incoai/GLM-5.3-Flash-DFlash2 is CC BY-NC-ND 4.0
+# (non-commercial, no derivatives) — this image is a quantization of it; the
+# MXFP8 conversion itself is the "qualified" upstream artifact. Keep to
+# personal/non-commercial use.
+#
+# WHY THIS IMAGE: the card's qualified image (voipmonitor/vllm, TP=4 B12X on
+# 4× RTX PRO 6000) is amd64-only and its B12X attention/moe/linear/allreduce
+# backends do not exist on GB10. Neither DFlash2DraftModel nor the DFlash2
+# serialized-MXFP8 projection path was in our day-0 base. So we PORTED the
+# three DFlash2 source files from local-inference-lab/vllm
+# dev/jovian-judgement @ da4d7be6 (Apache-2.0, see docker-images/.../UPSTREAM.md)
+# onto our proven lab image → glm53-flash-dflash2:local. The day-0 base
+# already shipped W4A16_NVFP4 / MIXED_PRECISION / ModelOptMxFp8LinearMethod /
+# the dflash proposer, so the whole port is those 3 files + 1 registry line.
+#
+# DEVIATIONS from the card's qualified config (forced by hardware):
+#   * TP=4 → TP=2; no B12X attention/moe/linear/allreduce (fall back to the
+#     lab image's proven SM121 backends).
+#   * --enforce-eager ON: CUDA-graph+spec-decode quality regression is
+#     build-specific to this day-0 base (our 2026-08-28 A/B: graphs+MTP
+#     89 vs eager 92). Card runs CUDAGRAPH_MODE=FULL on a newer build.
+#   * TP=2 keeps draft KV in BF16 (auto), same as the card's config.
+# Adopted verbatim from the card: --speculative-config {method:dflash,
+# model:...DFlash2-MXFP8, num_speculative_tokens:7, kv_cache_dtype:auto}.
+#
+# =====================================================================
+# STATUS 2026-08-29: BLOCKED on GB10 — boots to weight-load + draft load
+# (both fine, MXFP8 draft kernel binds), then dies in KV-cache setup:
+#
+#   NotImplementedError: layers.3.self_attn.indexer.k_cache: page size is
+#   not divisible by the maximum page size and cannot be padded
+#   (indexes_kv_by_block_stride is False).
+#
+# MATH (block size cancels, so no --block-size / draft kv dtype fixes it):
+# per-token pages on this base are indexer 132 B, MLA fp8_ds_mla 660 B,
+# DFlash2 draft (8 kv heads x 128) bf16 4096 B / fp8 2048 B. vLLM unifies
+# target+draft pages by integer block-size scaling, which needs divisio-
+# nility: 4096%660 != 0, 2048%660 != 0. Tried and eliminated: draft
+# kv_cache_dtype fp8, target kv-cache-dtype variants, --disable-hybrid-
+# kv-cache-manager (then: "failed to convert the KV cache specs to one
+# unified type" — the kpool indexer spec is not coercible either).
+#
+# WHY THE FORK DOES NOT HIT THIS: its qualified run (voipmonitor image,
+# sm_120) uses B12X attention/indexer backends which opt into
+# indexes_kv_by_block_stride, so unify PADS the strided pages instead.
+# The b12x kernels are not part of the arm64 GB10 base and are compiled
+# for sm_120 consumer Blackwell.
+#
+# PATHS FORWARD (pick before spending boot time on this again):
+#   a) build local-inference-lab/vllm dev/jovian-judgement for arm64/cu130
+#      incl. b12x kernels (hours; sm_120->sm_121 compat unknown);
+#   b) wait for a GB10 indexer/MLA backend that sets
+#      indexes_kv_by_block_stride (upstream);
+#   c) give up — acceptance-rate comparison falls back to MTP (already
+#      characterized in specs/tech-architecture/glm53-recipe-state.md).
+#
+# What IS done and committed regardless: glm53-flash-dflash2:local image
+# (DFlash2DraftModel port + registry), patch_eagle3.py (target-side
+# EAGLE3/aux-hidden-state support for Glm5Next, fork parity), verified
+# config plumbing (spec JSON parsed, aux layers 5/14/24/33/42 extracted
+# from the draft config, FlashInferCutlassMxfp8LinearKernel bound).
+#
+# PREREQUISITES: BOTH models cached on both nodes:
+#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
+#   hf download local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
+
+engine = "vllm"
+name = "GLM-5.3-Flash + DFlash2 draft (vLLM, 2x Spark, spec-k7)"
+docker_image = "glm53-flash-dflash2:local"
+docker_image_dir = "glm53-flash-dflash2"
+# TARGET model — the DFlash2 draft model is referenced only via
+# --speculative-config below, never as the top-level model.
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail (KV pinned below): 0.90 demands
+    # 109.46 GiB free on GB10's 121.63 and driver reserve + page cache
+    # routinely leave ~108.5 -> flaky boot deaths. 0.82 always fits.
+    "--gpu-memory-utilization", "0.82",
+    # Pinned 10 GiB KV; the draft adds 5 small sliding-window layers on top
+    "--kv-cache-memory-bytes", "10737418240",
+    "--kv-cache-dtype", "fp8_ds_mla",
+    "--block-size", "256",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    "--disable-custom-all-reduce",
+    # DFlash2 draft from local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
+    # (block-diffusion drafter, proposal length 7 — verbatim from the card)
+    # Card: kv_cache_dtype auto (bf16 draft pages) — kept faithful.
+    "--speculative-config", """
+'{"method":"dflash","model":"local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8","num_speculative_tokens":7,"kv_cache_dtype":"auto"}'""",
+    "--language-model-only",
+    # See header: eager is the safe control on the day-0 base (graphs+spec
+    # quality regression). The card's newer build runs FULL graphs.
+    "--enforce-eager",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "4"

recipes/glm53-flash-nvfp4-4p67/recipe.toml

@@ -0,0 +1,110 @@
+# GLM-5.3-Flash NVFP4-4p67 — local-inference-lab HIGHER-FIDELITY quant (vLLM)
+# — 2× DGX Spark, TP=2, MTP-3.
+#
+# "4p67" ≈ 4.67 bits/weight, KLD ~0.04 vs bf16 — the lab's fidelity-grade
+# variant of GLM-5.3-Flash-NVFP4 (our glm53-flash-nvfp4-lab recipe is the
+# workhorse). Quant recipe per the model card:
+#   main model: routed experts NVFP4 (calibrated, ships A4 input scales)
+#               + attention/shared/indexer MXFP8
+#   MTP:        routed experts NVFP4 A16-only + attention/shared MXFP8
+# hf_quant_config declares the routed experts as quant_algo=W4A16_NVFP4 under
+# MIXED_PRECISION — i.e. they execute via the Marlin W4A16 GEMM (bf16
+# activations), not the A4 path.
+# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4-4p67
+#
+# IMAGE: the card says "should be used with local-inference-lab/vllm branch
+# dev/jovian-judgement" — every image they publish is amd64-only (RTX PRO
+# 6000 world, and the fork's model.py hard-imports the b12x backend). Our
+# day-0 lab base already carries the W4A16/MIXED_PRECISION/MXFP8 modelopt
+# methods; the ONE thing 4p67 additionally needs is fused-projection shard
+# resolution: it is the first glm5next checkpoint with quantized attention
+# projections, and without `packed_modules_mapping` on the model classes the
+# fused KDA/MLA/indexer linears load unquantized (KeyError
+# in_proj_qkvbfg_a.weight_scale after ~12 min of weight loading). So:
+# glm53-flash-nvfp4-4p67:local = lab image + patch_4p67.py (5-line class
+# attr on model.py + mtp.py, mirrors the fork's resolution path).
+#
+# Boot markers: weight load ~93 GiB/node (≈700 s) → "GPU KV cache size:
+# 1,164,369 tokens" → "Application startup complete" with the MTP speculator
+# attached. quantprobe lines here show algo=MXFP8 for attention projections
+# and W4A16_NVFP4 for routed experts (lab quant showed algo=None for attn —
+# that difference is the whole point of this image). Watch SpecDecoding
+# acceptance (MTP-3, target len ~3).
+#
+# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
+# :"low"} — omitted/other values select MAX reasoning effort (empty content on
+# short max_tokens).
+#
+# PREREQUISITES: weights cached on both nodes:
+#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4-4p67
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4-4p67 (vLLM, 2x Spark, MTP-3)"
+docker_image = "glm53-flash-nvfp4-4p67:local"
+docker_image_dir = "glm53-flash-nvfp4-4p67"
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4-4p67"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
+    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
+    # page cache routinely leave ~108.5 -> flaky
+    # "Free memory ... less than desired" boot deaths (this was the class
+    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
+    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
+    # allocated (weights ~93 + pinned KV 10 + activations).
+    "--gpu-memory-utilization", "0.82",
+    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
+    # ~93 GiB weights/node; pinning also keeps warmup off the swap path
+    "--kv-cache-memory-bytes", "10737418240",
+    "--kv-cache-dtype", "fp8_ds_mla",
+    "--block-size", "256",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    "--disable-custom-all-reduce",
+    # Proven lab boot: no flashinfer autotune, no CuTe-DSL warmup
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    # MTP-3 draft (MTP experts are W4A16_NVFP4 in this checkpoint; needs the
+    # modelopt.patch MTP-namespace fix baked into the image)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":3}'""",
+    "--language-model-only",
+    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
+    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
+    # graphs on = 89/100, eager = 92/100). Same build → same guard here;
+    # re-A/B only if the base image is ever refreshed past the fix.
+    "--enforce-eager",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "4"

recipes/glm53-flash-nvfp4-lab/recipe.toml

@@ -0,0 +1,116 @@
+# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
+#
+# local-inference-lab mixed quant: NVFP4 routed experts (layers 3–44,
+# group-16) + MXFP8 MTP experts (layer 45) + W4A4 input scales. Best agent
+# quant so far: tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8).
+# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
+#        (~186 GB → 92.72 GiB weights/node; KV pinned at 10 GiB, see below)
+#
+# Image: glm53-flash-nvfp4-lab:local (docker-images/glm53-flash-nvfp4-lab/ —
+# day-0 arm64 image + NoPE-MLA mod + the four FujitsuPolycom lab-checkpoint
+# patches: model naming shim, modelopt MTP namespace fix, persistent_topk
+# CC-12.x guards). Their launch config is mirrored below.
+# PROVENANCE: https://github.com/FujitsuPolycom/glm53-flash-tp2-spark
+#             (Apache-2.0; see patches/UPSTREAM.md) + forum 381350 #41.
+#
+# Deviations from their launch script (intentional):
+#   * no --kda-prefill-backend flashkda (needs their base-image patch — skipped)
+#   * --kv-cache-dtype fp8_ds_mla instead of `fp8` (same packed layout; this
+#     spelling is what the NoPE-MLA mod booted with on the LibertAIDAI stack)
+#   * --language-model-only + no mm-renderer patch/env (text-only)
+#   * no bundled NCCL/LD_PRELOAD, no NCCL_* fabric block (our switched fabric
+#     is configured via [nccl.ds4] + launcher GID auto-detect)
+#   * no --moe-backend marlin: their proven stack uses the default backend for
+#     THIS quant and reports clean evals. Sanity-check output for token loops
+#     anyway (the silent FP4 MoE corruption gotcha) before trusting a new node.
+#   * flashinfer autotune disabled via --kernel-config (their proven boot);
+#     VLLM_ENGINE_READY_TIMEOUT_S stays high regardless.
+#
+# Boot success markers (engine log, in order):
+#   "Model loading took 92.72 GiB" → "[quantprobe] ... algo=MXFP8" (MTP fix
+#   active; algo=None means broken) → "GPU KV cache size: 1,164,369 tokens" →
+#   "Application startup complete". ~700 s sequential weight load.
+#
+# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
+# :"low"} — omitted/other values select MAX reasoning effort (empty content on
+# short max_tokens).
+#
+# Optional: instanttensor loader (~30 s weight load vs ~700 s):
+#   FROM glm53-flash-nvfp4-lab:local + pip install --no-deps --no-build-isolation
+#   instanttensor==0.1.9, then add --load-format instanttensor (--no-deps is
+#   required: resolution would downgrade the image's NCCL).
+#
+# PREREQUISITES: weights cached on both nodes:
+#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4-lab (vLLM, 2x Spark, MTP-3)"
+docker_image = "glm53-flash-nvfp4-lab:local"
+docker_image_dir = "glm53-flash-nvfp4-lab"
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
+    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
+    # page cache routinely leave ~108.5 -> flaky
+    # "Free memory ... less than desired" boot deaths (this was the class
+    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
+    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
+    # allocated (weights ~93 + pinned KV 10 + activations).
+    "--gpu-memory-utilization", "0.82",
+    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
+    # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
+    "--kv-cache-memory-bytes", "10737418240",
+    "--kv-cache-dtype", "fp8_ds_mla",
+    "--block-size", "256",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    "--disable-custom-all-reduce",
+    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    # MTP-3 draft from the MXFP8 layer-45 shard (needs modelopt.patch in image)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":3}'""",
+    "--language-model-only",
+    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
+    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
+    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
+    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
+    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
+    "--enforce-eager",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+# Upstream launch-script env (memory + compile knobs)
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "4"

recipes/glm53-flash-nvfp4-redhat/recipe.toml

@@ -0,0 +1,97 @@
+# GLM-5.3-Flash NVFP4 — RedHatAI quant (vLLM) — 2× DGX Spark, TP=2.
+#
+# RedHatAI/LLM-Compressor quant: MoE experts W4A4 NVFP4 (group-16, local
+# per-group FP8 scales, compressed-tensors nvfp4-pack-quantized). Vision /
+# embeddings / lm_head kept original precision. Quantizes via regex targets,
+# so no name-prefix matching bugs.
+# Model: https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4
+#        (~177 GB → ~88 GB/node on 2 Sparks)
+# KV: pinned --kv-cache-memory-bytes 10 GiB (same as -lab, identical MLA
+#   geometry) → ~1.16 M-token pool @ fp8_ds_mla = 2.22× concurrency at the
+#   524288 ctx. GMU 0.90 matches the proven lab config; the pin overrides
+#   GMU-based KV sizing.
+#
+# ⚠ NO MTP DRAFTER IN THIS CHECKPOINT — the index has 146,745 tensors and
+#   zero layer-45/MTP weights. `--speculative-config mtp` WILL fail to load.
+#   This recipe is baseline decode only (~15 tok/s class on this arch —
+#   spec decode buys 1.3–2×; use glm53-flash-nvfp4[-lab] when MTP matters).
+#
+# Image: the -lab overlay (day-0 vLLM image + NoPE-MLA mod + FujitsuPolycom
+# arch patches). The sparse_attn_indexer CC-12.0 guards are REQUIRED at the
+# 512K ctx: at 262K the indexer topk fit 48 SMs, at 512K it wants 62 CTAs and
+# the fallback needs 128 KB smem (GB10 has 99 KB) → hard boot abort
+# (launch_persistent_topk oversubscribe, 2026-08-28). The patches are
+# arch-level, not quant-level: the modelopt quantprobe is a no-op for this
+# compressed-tensors checkpoint, the model.py naming shim anchors on
+# arch-internal names.
+#
+# GB10 gotchas handled (see glm53-flash-nvfp4 header for detail):
+#   --moe-backend marlin → if vLLM rejects marlin for W4A4 at startup (marlin
+#   is mainly W4A16), remove the flag — but then verify output is not
+#   token-loop garbage (the silent FLASHINFER_CUTLASS corruption) before
+#   trusting it; `--moe-backend triton` is the other candidate.
+#
+# GB10 gotcha (MEASURED): at 524288 ctx WITH flashinfer autotune on, rank 0
+# dies mid-boot on a driver OOM (NVRM NV_ERR_NO_MEMORY _memdescAllocInternal,
+# 2026-08-28 05:01) — autotune/graph scratch at the 512K verify shapes doesn't
+# fit the unified-memory headroom. The 262144 ctx + autotune combo booted
+# fine. Fix = the lab-proven boot: autotune + cutedsl warmup off (a perf
+# micro-opt only; the sparse-MLA kernels have a non-autotuned fallback) and
+# expandable_segments to cut fragmentation.
+#
+# PREREQUISITES: weights cached on both nodes:
+#   huggingface-cli download RedHatAI/GLM-5.3-Flash-NVFP4
+#
+# REFERENCE:
+#   https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4 (vendor vLLM cmd)
+#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark (mod)
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4-redhat (vLLM, 2x Spark, no MTP)"
+docker_image = "glm53-flash-nvfp4-lab:local"
+docker_image_dir = "glm53-flash-nvfp4-lab"
+model = "RedHatAI/GLM-5.3-Flash-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
+    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
+    # page cache routinely leave ~108.5 (and ~106.6 after a crashed boot)
+    # -> flaky "Free memory ... less than desired" boot deaths. 0.82 is
+    # below any realistic dip and does NOT change what gets allocated.
+    "--gpu-memory-utilization", "0.82",
+    "--kv-cache-memory-bytes", "10737418240",
+    "--max-model-len", "524288",
+    # Scan/graph scratch scales with the prefill chunk: 8192 OOMs the driver
+    # at 512K ctx (05:38 NVRM OOM), 4096 is the lab-proven ceiling here.
+    "--max-num-batched-tokens", "4096",
+    "--block-size", "256",
+    "--max-num-seqs", "1",
+    "--kv-cache-dtype", "fp8_ds_mla",
+    "--language-model-only",
+    "--moe-backend", "marlin",
+    # NO --speculative-config: checkpoint ships no MTP weights (verified index)
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"

recipes/glm53-flash-nvfp4/recipe.toml

@@ -0,0 +1,96 @@
+# GLM-5.3-Flash NVFP4 (vLLM) — 2× DGX Spark / GB10 (sm_121), TP=2.
+#
+# 320B/18B-active, hybrid 34 KDA linear-attention + 11 sparse-MLA layers,
+# glm5_next arch (NoPE-MLA, qk_rope_head_dim=0), in-checkpoint MTP head.
+# Text-only here (--language-model-only — the multimodal processor balloons
+# the API front-end to ~15.7 GB and the GB10 kernel OOM-kills it).
+#
+# WHY NVFP4 on 2 nodes (not the official FP8 on 4): LibertAIDAI's weight-only
+# NVFP4-A16 quant halves the footprint (~155 GB) so it fits 2 Sparks; the
+# sparse+linear attention keeps KV nearly free (KV pool ~738K tok at GMU 0.85).
+# Model: https://huggingface.co/LibertAIDAI/GLM-5.3-Flash-NVFP4
+#
+# Engine: vLLM day-0 per-model image (glm5_next support baked in). The arch
+# fails every sm_121 attention backend out of the box; a 4-part build-time
+# mod (docker-images/glm53-flash-nvfp4/patches/patch_mla.py) dresses GLM's
+# NoPE-MLA in DeepSeek's fp8_ds_mla 512+64 layout and compacts the sm120
+# sparse-MLA topk table 2176->2048. Applied at image build; this recipe
+# activates it at runtime via VLLM_MLA_NOPE_PAD_ROPE=1.
+#
+# TWO SILENT GB10 GOTCHAS (from the reference — both already handled here):
+#   1. SILENT FP4 MoE CORRUPTION: the auto-selected FLASHINFER_CUTLASS NVFP4
+#      MoE backend emits degenerate token loops with ZERO errors (model loads,
+#      /health green, every completion garbage). --moe-backend marlin fixes it.
+#      If a GB10 NVFP4 serve "works" but loops, check which MoE backend engaged
+#      before blaming the quant.
+#   2. A 60+ MIN "No available shared memory broadcast block found" loop at
+#      boot is NOT a hang — the worker is FlashInfer-autotuning a kernel flavor
+#      (tune persists to ~/.cache/vllm, pays once). VLLM_ENGINE_READY_TIMEOUT_S
+#      is raised to 3600 so the launcher doesn't give up first.
+#
+# Measured (median of 3, temp 0, single stream, 2x Spark, MTP-5 acceptance ~2.55):
+#   24.74 code / 30.30 structured / 19.58 prose tok/s; no-spec baseline 14.6 flat.
+#   256K needle-exact (30K/100K/200K deep); ~1150 tok/s prefill.
+#
+# Image: glm53-flash-nvfp4:local (built from vllm/vllm-openai:glm53-flash-arm64-cu130
+#   + the patch). `sm121 serve` builds it on the build node if the checksum label
+#   is stale (docker_image_dir below).
+#
+# PREREQUISITES:
+#   Weights cached on both nodes (NAS ~/.cache/huggingface):
+#       huggingface-cli download LibertAIDAI/GLM-5.3-Flash-NVFP4
+#
+# REFERENCE (provenance):
+#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark
+#   https://forums.developer.nvidia.com/t/381433
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4 (vLLM, 2x Spark)"
+docker_image = "glm53-flash-nvfp4:local"
+docker_image_dir = "glm53-flash-nvfp4"
+model = "LibertAIDAI/GLM-5.3-Flash-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    # GLM tokenizer
+    "--trust-remote-code",
+    # Executor (mp, no ray)
+    "--distributed-executor-backend", "mp",
+    # Memory — weights ~77.5 GiB of 119.6 GiB/node; sparse+linear KV nearly free
+    "--gpu-memory-utilization", "0.85",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "1",
+    # DeepSeek-MLA KV layout the NoPE-pad mod dresses the attention in
+    "--kv-cache-dtype", "fp8_ds_mla",
+    # Text-only (multimodal processor OOM-kills the GB10 front-end)
+    "--language-model-only",
+    # GOTCHA 1: marlin avoids the silent FP4 MoE corruption
+    "--moe-backend", "marlin",
+    # MTP=5 (in-checkpoint drafter; acceptance length ~2.55)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":5}'""",
+    # Tokenizer / reasoning / tool calling (GLM family)
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+
+[env]
+# Offline — weights must be cached on each node
+HF_HUB_OFFLINE = "1"
+# Activates the build-time NoPE-MLA rope-pad mod (mathematically a no-op:
+# zeros are rotation-free and add exactly 0 to every logit)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+# GOTCHA 2: survive the 60+ min FlashInfer-autotune boot window
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"

recipes/laguna-m1-nvfp4/recipe.toml

@@ -0,0 +1,37 @@
+engine = "vllm"
+name = "Laguna M.1 NVFP4 (vLLM)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
+model = "poolside/Laguna-M.1-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+
+# 225B total / 23B active MoE — NVFP4 quantized (~131B stored).
+# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
+# TP=2 → ~66 GB/node weights, room for 262K KV cache.
+# Quantization auto-detected from model config.
+
+entrypoint = ["vllm", "serve"]
+
+args = """
+--gpu-memory-utilization 0.8
+--max-model-len 262144
+--load-format instanttensor
+--enable-prefix-caching
+--tool-call-parser poolside_v1
+--reasoning-parser poolside_v1
+--enable-auto-tool-choice
+--trust-remote-code
+--kv-cache-dtype fp8
+--enable-flashinfer-autotune
+--served-model-name laguna
+"""
+
+[env]
+SAFETENSORS_FAST_GPU = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_FLOAT32_MATMUL_PRECISION = "high"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"

recipes/laguna-s2.1-nvfp4/recipe.toml

@@ -0,0 +1,87 @@
+# Laguna S 2.1 NVFP4 — 1× DGX Spark (GB10), TP=1.
+#
+# Model: poolside/Laguna-S-2.1-NVFP4
+#   → https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
+#
+# 117.6B total / 8.5B active MoE — NVFP4 quantized (~71 GB stored).
+# Fits on a single DGX Spark (128 GB unified memory).
+# 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
+# KV cache in FP8.
+#
+# DFlash speculative decoding with quantization-matched draft model:
+#   poolside/Laguna-S-2.1-DFlash-NVFP4
+#   → 15 speculative tokens, method dflash.
+#
+# REFERENCE:
+#   https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
+#
+# DEPLOYMENT: tmux (host venv). The pre-built Docker image
+# (ghcr.io/timothystewart6/vllm-gb10:v0.26.0-gb10.1) ships the wrong
+# flashinfer version, so Docker is non-functional. The sustainable path
+# is a host venv with the exact deps from the model card.
+#
+# ACTIVE (risky): vLLM 0.26.0 + flashinfer 0.6.16.dev20260729.
+#   0.6.16 adds SM121 (GB10/DGX Spark) NVFP4 attention support
+#   (flashinfer PR #3897) and Expert Parallelism, but introduces
+#   API changes that vLLM 0.26.0 (which requires flashinfer 0.6.14)
+#   may not fully track. Tested on dgx1.lan — monitor for MoE cubin
+#   validation regressions ("reject incompatible output-scale cubins").
+#
+# SAFE (commented out): vLLM 0.25.1 + flashinfer 0.6.15.dev20260728.
+#   Same minor as the original 0.6.15.dev20260712 — just 16 days of
+#   bug fixes. No API changes. Drop-in replacement.
+#
+#   uv venv ~/venvs/vllm025 -p 3.12
+#   uv pip install vllm==0.25.1 --torch-backend=cu130
+#   uv pip install flashinfer-python==0.6.15.dev20260728 flashinfer-cubin==0.6.15.dev20260728 flashinfer-jit-cache==0.6.15.dev20260728 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
+#   hf download poolside/Laguna-S-2.1-NVFP4
+#   hf download poolside/Laguna-S-2.1-DFlash-NVFP4
+#
+# The `venv` field below is used by tmux to activate the venv before
+# running the entrypoint.
+
+engine = "vllm"
+name = "Laguna S 2.1 NVFP4 (vLLM)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.26.0-gb10.1"
+model = "poolside/Laguna-S-2.1-NVFP4"
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+deploy = "tmux"
+venv = "~/venvs/vllm026"
+venv_setup = """
+    uv venv ~/venvs/vllm026 -p 3.12
+    uv pip install vllm==0.26.0 --torch-backend=cu130
+    uv pip install flashinfer-python==0.6.16.dev20260729 flashinfer-cubin==0.6.16.dev20260729 flashinfer-jit-cache==0.6.16.dev20260729 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
+    hf download poolside/Laguna-S-2.1-NVFP4
+    hf download poolside/Laguna-S-2.1-DFlash-NVFP4
+"""
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--gpu-memory-utilization", "0.85",
+    "--max-model-len", "262144",
+    "--load-format", "instanttensor",
+    "--enable-prefix-caching",
+    "--tool-call-parser", "poolside_v1",
+    "--reasoning-parser", "poolside_v1",
+    "--enable-auto-tool-choice",
+    "--trust-remote-code",
+    "--kv-cache-dtype", "fp8",
+    "--enable-flashinfer-autotune",
+    "--max-num-seqs", "32",
+    "--override-generation-config", "'{\"temperature\":0.7,\"top_p\":0.95}'",
+    # DFlash speculative decoding with quantization-matched draft
+    "--speculative-config", "'{\"model\":\"poolside/Laguna-S-2.1-DFlash-NVFP4\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}'",
+    "--served-model-name", "laguna-s-2.1", "laguna", "default",
+]
+
+[env]
+SAFETENSORS_FAST_GPU = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_FLOAT32_MATMUL_PRECISION = "high"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+MAX_JOBS = "4"

recipes/mimo-v2.5-nvfp4/recipe.toml

@@ -0,0 +1,58 @@
+engine = "vllm"
+name = "MiMo-V2.5 Omni NVFP4 (vLLM)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
+model = "lukealonso/MiMo-V2.5-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+
+# 1M context · NVFP4 4-bit KV · MTP1 · ~30 tok/s
+# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
+# REQUIRES patched vLLM container with mods (nvfp4-kv-diffkv, fix-mimo-v2-vllm, etc.)
+# Stock vLLM will reject NVFP4 KV or OOM.
+
+entrypoint = ["vllm", "serve"]
+
+args = """
+--served-model-name MiMo-V2.5-NVFP4
+--trust-remote-code
+--dtype auto
+--load-format safetensors
+--hf-overrides {"architectures":["MiMoV2OmniForCausalLM"]}
+--limit-mm-per-prompt {"image":4,"video":1,"audio":1}
+--mm-encoder-tp-mode data
+--attention-backend triton_attn_diffkv
+--kv-cache-dtype nvfp4
+--gpu-memory-utilization 0.84
+--max-model-len 1000000
+--max-num-batched-tokens 4096
+--max-num-seqs 4
+--block-size 32
+--enable-prefix-caching
+--enable-chunked-prefill
+--enable-auto-tool-choice
+--tool-call-parser mimo
+--reasoning-parser mimo
+--speculative-config {"method":"mtp","num_speculative_tokens":1}
+--enforce-eager
+"""
+
+# TP/PP/nnodes/node-rank/master-addr are
+# auto-generated by the launcher from the recipe's top-level fields.
+# mp backend auto-detected by vLLM >0.20, no --distributed-executor-backend flag needed.
+
+[env]
+# Memory stability
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+
+# NVFP4 weights + KV + WMMA decode
+VLLM_NVFP4_GEMM_BACKEND = "flashinfer-cutlass"
+VLLM_USE_FLASHINFER_MOE_FP4 = "1"
+VLLM_FLASHINFER_MOE_BACKEND = "throughput"
+VLLM_NVFP4_INLINE = "1"
+VLLM_WMMA_DECODE = "1"
+VLLM_WMMA_INSPECT = "0"
+VLLM_WMMA_COMPARE = "0"
+VLLM_WMMA_AUTHCOMPARE = "0"

recipes/minimax-m3-mxfp8/recipe.toml

@@ -0,0 +1,37 @@
+engine = "vllm"
+name = "MiniMax M3 MXFP8 (vLLM)"
+docker_image = "vllm/vllm-openai:minimax-m3"
+model = "MiniMaxAI/MiniMax-M3-MXFP8"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+
+# MXFP8 quantized MiniMax M3 (~440B params, ~23B active).
+# 4 nodes × 1 GPU (DGX Spark GB10 Blackwell), TP=4 across nodes.
+# MSA sparse attention requires --block-size 128.
+# Text-only mode skips the vision encoder to save VRAM.
+
+#entrypoint = ["vllm", "serve"]
+
+args = [
+    "--block-size", "128",
+    "--kv-cache-dtype", "fp8",
+    "--gpu-memory-utilization", "0.85",
+    "--load-format", "safetensors",
+    "--enable-prefix-caching",
+    "--tool-call-parser", "minimax_m3",
+    "--reasoning-parser", "minimax_m3",
+    "--enable-auto-tool-choice",
+    "--language-model-only",
+    "--enable-expert-parallel",
+    "--trust-remote-code",
+    "--served-model-name", "minimax-m3", "default",
+]
+
+[env]
+# M3 supports thinking via chat_template_kwargs.thinking_mode
+# enabled / adaptive / disabled — pass per-request in extra_body
+SAFETENSORS_FAST_GPU = "1"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_USE_B12X_MOE = "1"

recipes/minimax-m3-nvfp4/recipe.toml

@@ -0,0 +1,37 @@
+engine = "vllm"
+name = "MiniMax M3 NVFP (vLLM)"
+docker_image = "vllm/vllm-openai:minimax-m3"
+model = "Mapika/MiniMax-M3-NVFP4"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+
+# NVFP4 quantized MiniMax M3 (~440B params, ~23B active).
+# 4 nodes × 1 GPU (DGX Spark GB10 Blackwell), TP=4 across nodes.
+# MSA sparse attention requires --block-size 128.
+# Text-only mode skips the vision encoder to save VRAM.
+
+#entrypoint = ["vllm", "serve"]
+
+args = [
+    "--block-size", "128",
+    "--kv-cache-dtype", "fp8",
+    "--gpu-memory-utilization", "0.85",
+    "--load-format", "safetensors",
+    "--enable-prefix-caching",
+    "--tool-call-parser", "minimax_m3",
+    "--reasoning-parser", "minimax_m3",
+    "--enable-auto-tool-choice",
+    "--language-model-only",
+    "--enable-expert-parallel",
+    "--trust-remote-code",
+    "--served-model-name", "minimax-m3", "default",
+]
+
+[env]
+# M3 supports thinking via chat_template_kwargs.thinking_mode
+# enabled / adaptive / disabled — pass per-request in extra_body
+SAFETENSORS_FAST_GPU = "1"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_USE_B12X_MOE = "1"

recipes/minimax-m3/recipe.toml

@@ -0,0 +1,29 @@
+engine = "sglang"
+name = "Minimax-M3-v0 NVFP4"
+docker_image = "scitrera/dgx-spark-sglang-mm:v0"
+#model = "sparkarena/Minimax-M3-v0-NVFP4"
+model = "Mapika/MiniMax-M3-NVFP4"
+required_nodes = 4
+tensor_parallel_size = 4
+distributed_backend = "mp"
+
+entrypoint = ["sglang", "serve"]
+
+args = [
+    "--mem-fraction-static", "0.8",
+    "--context-length", "524288",
+    "--chunked-prefill-size", "8192",
+    "--fp4-gemm-backend", "cutlass",
+    "--load-format", "instanttensor",
+    "--kv-cache-dtype", "auto",
+    "--reasoning-parser", "minimax-m3",
+    "--tool-call-parser", "minimax-m3",
+    "--moe-runner-backend", "triton",
+    "--trust-remote-code",
+    "--cuda-graph-max-bs", "16",
+    "--cuda-graph-bs", "1", "2", "4", "8", "12", "16",
+    "--served-model-name", "minimax-m3",
+]
+
+[env]
+TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"

recipes/qwen-agentworld-35b/recipe.toml

@@ -0,0 +1,25 @@
+engine = "vllm"
+name = "Qwen-AgentWorld-35B-A3B (vLLM)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
+model = "Qwen/Qwen-AgentWorld-35B-A3B"
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+
+# Solo: ~70 GB BF16 weights, only 10 GA layers need KV cache (DeltaNet is linear).
+# Use: sm121-launcher serve --recipe qwen-agentworld-35b --node dgx1.lan
+# Nodes: sm121-launcher serve --recipe qwen-agentworld-35b --cluster 2nodes-dac
+# Requires --language-model-only (no vision weights in checkpoint).
+
+entrypoint = ["vllm", "serve"]
+
+args = """
+--max-model-len 262144
+--reasoning-parser qwen3
+--language-model-only
+--trust-remote-code
+--gpu-memory-utilization 0.90
+--enable-prefix-caching
+--enable-auto-tool-choice
+--tool-call-parser qwen3_xml
+--served-model-name qwen-agentworld-35b
+"""

recipes/qwen38-27b-fp8-dspark/recipe.toml

@@ -0,0 +1,92 @@
+# Qwen3.8-27B (FP8 + DSpark draft) — 1× DGX Spark / GB10 (sm_121), TP=1.
+#
+# DSpark: a dedicated semi-autoregressive draft model (~2.6 GB, 5 layers,
+# block_size 7) proposes blocks of tokens that the full model verifies in one
+# weight-read. 3× cheaper per guess than the in-checkpoint MTP head, so it wins
+# on edit-heavy / structured workloads (blog: 45-48 t/s edit, 17 t/s free-gen,
+# 66 t/s aggregate @ c10 on the FP8 target). DSpark blocks are verifier-rejected,
+# so no quality risk.
+#
+# Target: Qwen/Qwen3.8-27B-FP8 (official FP8 quant, 28.75 GB).
+# Draft:  RadixArk/Qwen3.8-27B-DSpark (arch DSparkDraftModel, bf16, 5 layers,
+#         dspark_block_size 7). Downloaded via hfrs into the node HF cache.
+#
+# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
+#   vLLM 0.27.1 (installed 0.27.2.dev0+g6e448d0ea), supports method "dspark"
+#   (SpeculativeMethod Literal includes dspark) + Qwen3DSparkModel draft.
+#
+# PREREQUISITES:
+#   Target + draft cached on the node (dgx1 ~/.cache/huggingface → NAS):
+#       hfrs download Qwen/Qwen3.8-27B-FP8
+#       hfrs download RadixArk/Qwen3.8-27B-DSpark
+#
+# REFERENCE:
+#   https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark  (DSpark section)
+#   https://huggingface.co/RadixArk/Qwen3.8-27B-DSpark
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
+
+engine = "vllm"
+name = "Qwen3.8-27B (FP8 + DSpark)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
+model = "Qwen/Qwen3.8-27B-FP8"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "6",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.75",
+    "--served-model-name", "qwen3.8-27b",
+    # Engine backends (dense GQA — FlashInfer)
+    "--attention-backend", "flashinfer",
+    # Compilation / perf
+    "--optimization-level", "3",
+    "--performance-mode", "throughput",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # DSpark speculative decoding — dedicated draft model (not the in-checkpoint
+    # MTP head). num_speculative_tokens must equal the draft's block size (7).
+    # The draft lives in a local dir (arch-patched copy of RadixArk/Qwen3.8-27B-DSpark
+    # re-labelled `Qwen3DSparkModel` so vLLM routes it to qwen3_dspark, not the
+    # DeepSeek-V4 DSpark code path) bind-mounted at /dspark-draft below.
+    "--speculative-config", """
+'{"method":"dspark","model":"/dspark-draft","num_speculative_tokens":7}'""",
+    # Tokenizer / reasoning (Qwen3 family)
+    "--tool-call-parser", "qwen3_xml",
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--generation-config", "auto",
+    "--default-chat-template-kwargs.preserve_thinking=true",
+    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
+    # practices)
+    "--override-generation-config", """
+'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
+    "--trust-remote-code",
+]
+
+# DSpark draft model — arch-patched local copy of RadixArk/Qwen3.8-27B-DSpark,
+# mounted at /dspark-draft so vLLM loads it via the qwen3_dspark draft class.
+[[extra_mounts]]
+host = "/home/kilork/dspark-draft"
+container = "/dspark-draft"
+
+
+[env]
+HF_HUB_OFFLINE = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-fp8/recipe.toml

@@ -0,0 +1,74 @@
+# Qwen3.8-27B (FP8) — 1× DGX Spark / GB10 (sm_121), TP=1.
+#
+# Official Qwen FP8 quant (29 GB). Dense 27B, GQA, hybrid DeltaNet+Attention,
+# MTP-1. Text + vision. As fast as NVFP4 typically, with full FP8 quality.
+#
+# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
+#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
+#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
+#
+# Model: Qwen/Qwen3.8-27B-FP8
+#   https://huggingface.co/Qwen/Qwen3.8-27B-FP8
+#
+# PREREQUISITES:
+#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
+#       huggingface-cli download Qwen/Qwen3.8-27B-FP8
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
+
+engine = "vllm"
+name = "Qwen3.8-27B (FP8)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
+model = "Qwen/Qwen3.8-27B-FP8"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "6",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.75",
+    "--served-model-name", "qwen3.8-27b",
+    # Engine backends (dense GQA — FlashInfer)
+    "--attention-backend", "flashinfer",
+    # Compilation / perf
+    "--optimization-level", "3",
+    "--performance-mode", "throughput",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # MTP speculative decoding (in-checkpoint draft)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":5}'""",
+    # Tokenizer / reasoning (Qwen3 family)
+    "--tool-call-parser", "qwen3_xml",
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--generation-config", "auto",
+    "--default-chat-template-kwargs.preserve_thinking=true",
+    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
+    # practices)
+    "--override-generation-config", """
+'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
+    "--trust-remote-code",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-int4/recipe.toml

@@ -0,0 +1,96 @@
+# Qwen3.8-27B (Pilcothink Int4 AutoRound + MTP) — 1× DGX Spark / GB10 (sm_121),
+# TP=1.
+#
+# Mixed 4-bit AutoRound quant (20 GB) tuned for a single DGX Spark. Sensitive
+# layers kept FP8/FP16; vision layers unquantized. MTP quantized + included.
+# Dense 27B, GQA, hybrid DeltaNet+Attention. Text + vision. The community has
+# heavily tuned vLLM flags for this exact build (thread 380248 post #39).
+#
+# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
+#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
+#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
+#
+# Model: Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound
+#   https://huggingface.co/Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound
+#
+# PREREQUISITES:
+#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
+#       huggingface-cli download Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
+
+engine = "vllm"
+name = "Qwen3.8-27B (Int4 AutoRound)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
+model = "Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "6",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.75",
+    "--served-model-name", "qwen3.8-27b",
+    # Engine backends (dense GQA — FlashInfer)
+    "--attention-backend", "flashinfer",
+    # Compilation / perf (AutoRound community tuning)
+    "--optimization-level", "3",
+    "--performance-mode", "throughput",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # MTP speculative decoding (quantized draft)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":4}'""",
+    # Tokenizer / reasoning (Qwen3 family)
+    "--tool-call-parser", "qwen3_xml",
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--generation-config", "auto",
+    "--default-chat-template-kwargs.preserve_thinking=true",
+    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
+    # practices)
+    "--override-generation-config", """
+'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
+    "--trust-remote-code",
+]
+
+[env]
+# AutoRound community tuning (thread 380248 post #39): faster load + higher
+# compile level for the 4-bit kernels.
+VLLM_TORCH_COMPILE_LEVEL = "3"
+VLLM_TRITON_COMPILE_ON_LOAD = "1"
+VLLM_WARMUP_BEFORE_SERVING = "1"
+VLLM_SKIP_WARMUP = "false"
+TORCH_MATMUL_PRECISION = "high"
+NVIDIA_FORWARD_COMPAT = "1"
+NVIDIA_DISABLE_REQUIRE = "1"
+CUDA_DEVICE_MAX_CONNECTIONS = "1"
+VLLM_MARLIN_USE_ATOMIC_ADD = "1"
+FLASHINFER_DISABLE_VERSION_CHECK = "1"
+
+# Offline mode
+HF_HUB_OFFLINE = "1"
+
+# Architecture
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+
+# GPU memory management
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+
+# GPU device isolation
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-nvfp4-sglang-dflash2/recipe.toml

@@ -0,0 +1,75 @@
+# Qwen3.8-27B (RadixArk NVFP4 + DFlash2 on SGLang) — 1× DGX Spark / GB10,
+# TP=1.
+#
+# SGLang + DFlash2 block-diffusion speculative decoding. Per the community
+# (MiaAI / forum 380350), DFlash2 on NVFP4 ties DSpark on code and *beats it*
+# on prose/essay and short-chat once counted right. This is the fork/engine
+# the forum's high tok/s numbers (52-61 code single, 87 TP=2) came from — NOT
+# the stock DSpark path.
+#
+# Image: lmsysorg/sglang:qwen38-27b-dflash2-minoverlay — a locally-built overlay
+#   of the 5 DFlash2 modules onto the cookbook qwen38-27b image. The cookbook
+#   tag itself predates DFlash2 (upstream merged commit c14312a66 on
+#   2026-08-19, after every published tag), so the modules are overlaid.
+#   The minoverlay modules already bake in the quantized-head selector fix
+#   (should_apply_lm_head_quant_method) so DFLASH runs the NVFP4 lm_head in
+#   place — no dense dequant.
+#
+# Draft: z-lab/Qwen3.8-27B-DFlash2 @ 50307d4 (block-diffusion drafter, 3.6 GB).
+#
+# Model: RadixArk/Qwen3.8-27B-NVFP4 (modelopt)
+#
+# PREREQUISITES (on the node):
+#   hfrs download RadixArk/Qwen3.8-27B-NVFP4
+#   hfrs download z-lab/Qwen3.8-27B-DFlash2
+#   docker build the minoverlay image (or use the recipe's docker_image as-is)
+#
+# REFERENCE:
+#   https://github.com/MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark  (start-dflash.sh)
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
+
+engine = "sglang"
+name = "Qwen3.8-27B (RadixArk NVFP4, SGLang DFlash2)"
+docker_image = "lmsysorg/sglang:qwen38-27b-dflash2-minoverlay"
+model = "RadixArk/Qwen3.8-27B-NVFP4"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["python3", "-m", "sglang.launch_server"]
+
+args = [
+    # Memory — NVFP4 needs 0.90 (0.95 hard-rebooted the box at graph capture)
+    "--mem-fraction-static", "0.90",
+    "--max-running-requests", "10",
+    "--max-mamba-cache-size", "30",
+    # Attention / KV
+    "--attention-backend", "flashinfer",
+    "--kv-cache-dtype", "fp8_e4m3",
+    # Mamba GDN hybrid — DFLASH requires extra_buffer (not extra_buffer_lazy)
+    "--mamba-ssm-dtype", "bfloat16",
+    "--mamba-full-memory-ratio", "4.21",
+    "--mamba-radix-cache-strategy", "extra_buffer",
+    # Chunked prefill
+    "--chunked-prefill-size", "8192",
+    # DFlash2 block-diffusion speculative decoding
+    "--speculative-algorithm", "DFLASH",
+    "--speculative-draft-model-path", "z-lab/Qwen3.8-27B-DFlash2",
+    "--speculative-draft-model-revision", "50307d4c4cde6860d4eee73e2547cd786fe8e8a4",
+    "--speculative-num-draft-tokens", "8",
+    # Tokenizer / reasoning / tool calling (Qwen3 family)
+    "--reasoning-parser", "qwen3",
+    "--tool-call-parser", "qwen3_coder",
+    # Observability
+    "--enable-metrics",
+    "--enable-cache-report",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"

recipes/qwen38-27b-nvfp4-sglang-dspark/recipe.toml

@@ -0,0 +1,89 @@
+# Qwen3.8-27B (RadixArk NVFP4 + DSpark on SGLang) — 1× DGX Spark / GB10 (sm_121),
+# TP=1.
+#
+# SGLang + DSpark block-diffusion speculative decoding. The community (MiaAI)
+# measured DSpark on this box at ~51.5 tok/s on code vs vLLM+MTP's 22.8 — the
+# drafter is 3× cheaper per guess and edits/rewrites much faster. Uses the
+# pre-built cookbook image (no source build): lmsysorg/sglang:qwen38-27b.
+#
+# TARGET MUST BE RadixArk MODELOPT NVFP4: the cookbook image accelerates
+# modelopt-quantized NVFP4. The unsloth/RedHatAI compressed-tensors builds
+# fall back to a slow unquantized path ("Falling back to UnquantizedLinear-
+# Method") — prefill crawls at ~0.5 tok/s. Use RadixArk here.
+#
+# Draft: RadixArk/Qwen3.8-27B-DSpark (bf16, block 7). SGLang loads it natively
+# from the HF cache — no arch patch needed (unlike the vLLM qwen3_dspark path).
+#
+# Image: lmsysorg/sglang:qwen38-27b  (multi-arch, incl. arm64; cookbook's
+#   DGX Spark Qwen3.8 cell, pinned image digest febfb971). Pre-built — no
+#   source build.
+#
+# Model: RadixArk/Qwen3.8-27B-NVFP4  (modelopt, cached locally on the node)
+#   https://huggingface.co/RadixArk/Qwen3.8-27B-NVFP4
+#
+# PREREQUISITES:
+#   Target + draft cached on the node (dgx1 ~/.cache/huggingface → NAS):
+#       hfrs download RadixArk/Qwen3.8-27B-NVFP4
+#       hfrs download RadixArk/Qwen3.8-27B-DSpark
+#
+# REFERENCE:
+#   https://github.com/MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark  (start-dspark.sh)
+#   https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark
+
+engine = "sglang"
+name = "Qwen3.8-27B (RadixArk NVFP4, SGLang DSpark)"
+docker_image = "lmsysorg/sglang:qwen38-27b"
+model = "RadixArk/Qwen3.8-27B-NVFP4"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["python3", "-m", "sglang.launch_server"]
+
+args = [
+    # Memory — larger KV pool (mamba GDN state pool sized from concurrency)
+    "--mem-fraction-static", "0.90",
+    "--max-running-requests", "10",
+    "--max-mamba-cache-size", "30",
+    # Attention / KV
+    "--attention-backend", "flashinfer",
+    "--kv-cache-dtype", "fp8_e4m3",
+    # Mamba GDN hybrid
+    "--mamba-ssm-dtype", "bfloat16",
+    "--mamba-full-memory-ratio", "4.21",
+    "--mamba-radix-cache-strategy", "extra_buffer_lazy",
+    # Chunked prefill
+    "--chunked-prefill-size", "8192",
+    # DSpark speculative decoding (block 7 / 8 draft tokens, unquant bf16 draft)
+    "--speculative-algorithm", "DSPARK",
+    "--speculative-draft-model-path", "RadixArk/Qwen3.8-27B-DSpark",
+    "--speculative-dspark-block-size", "7",
+    "--speculative-draft-model-quantization", "unquant",
+    "--speculative-num-draft-tokens", "8",
+    # Compile + decode graphs
+    "--enable-torch-compile", "--torch-compile-max-bs", "4",
+    "--cuda-graph-max-bs-decode", "4",
+    "--num-continuous-decode-steps", "2",
+    # Tokenizer / reasoning / tool calling (Qwen3 family)
+    "--reasoning-parser", "qwen3",
+    "--tool-call-parser", "qwen3_coder",
+    # Observability
+    "--enable-metrics",
+    "--enable-cache-report",
+]
+
+[env]
+# Offline — target + draft are both cached on the node
+HF_HUB_OFFLINE = "1"
+
+# Pin to GB10's Cortex-X5 big cores (scheduler/tokenizer off the A725 efficiency
+# cores — +2-7% decode per the MiaAI build)
+# CPUSET handled at the container layer; this is the SGLang-side hint.
+SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = "1"
+
+# Architecture
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"

recipes/qwen38-27b-nvfp4-unsloth-dspark/recipe.toml

@@ -0,0 +1,90 @@
+# Qwen3.8-27B (unsloth NVFP4 + DSpark) — 1× DGX Spark / GB10 (sm_121), TP=1.
+#
+# NVFP4 compressed-tensors quant (22 GB) with a dedicated DSpark draft model
+# (RadixArk, 2.6 GB, block_size 7) for speculative decoding. Dense 27B, GQA,
+# hybrid DeltaNet+Attention. Text + vision. Unsloth Dynamic V3.0 (preview)
+# quant — SOTA quality for NVFP4. vLLM-only (lm_head FP8).
+#
+# DSpark vs the in-checkpoint MTP head (qwen38-27b-nvfp4-unsloth recipe): the
+# draft is 3× cheaper per guess and edits/rewrites much faster, at the cost of
+# a 2.6 GB draft + a local arch-patched copy. Benchmark both.
+#
+# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
+#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
+#   instanttensor. Stock vLLM — supports qwen3_5 arch + method "dspark".
+#
+# Model: unsloth/Qwen3.8-27B-NVFP4
+#   https://huggingface.co/unsloth/Qwen3.8-27B-NVFP4
+#
+# PREREQUISITES:
+#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
+#       huggingface-cli download unsloth/Qwen3.8-27B-NVFP4
+#       hfrs download RadixArk/Qwen3.8-27B-DSpark  (then cp -aL + patch arch → /home/kilork/dspark-draft)
+#
+# REFERENCE:
+#   https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark  (DSpark section)
+
+engine = "vllm"
+name = "Qwen3.8-27B (NVFP4 unsloth + DSpark)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
+model = "unsloth/Qwen3.8-27B-NVFP4"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "6",
+    "--max-num-batched-tokens", "16384",
+    "--gpu-memory-utilization", "0.75",
+    "--served-model-name", "qwen3.8-27b",
+    # Engine backends (dense GQA — FlashInfer; NVFP4 route via compressed-tensors)
+    "--attention-backend", "flashinfer",
+    # Compilation / perf
+    "--optimization-level", "3",
+    "--performance-mode", "throughput",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # DSpark speculative decoding — dedicated draft model (arch-patched local
+    # copy of RadixArk/Qwen3.8-27B-DSpark, re-labelled `Qwen3DSparkModel` so
+    # vLLM routes it to qwen3_dspark not the DeepSeek-V4 path). block_size 7.
+    # batched-tokens raised to 16384 to fit the draft token slots (warn:
+    # max_num_scheduled_tokens=8156 at 8192).
+    "--speculative-config", """
+'{"method":"dspark","model":"/dspark-draft","num_speculative_tokens":7}'""",
+    # Tokenizer / reasoning (Qwen3 family)
+    "--tool-call-parser", "qwen3_xml",
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--generation-config", "auto",
+    "--default-chat-template-kwargs.preserve_thinking=true",
+    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
+    # practices)
+    "--override-generation-config", """
+'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
+    "--trust-remote-code",
+]
+
+# DSpark draft model — arch-patched local copy of RadixArk/Qwen3.8-27B-DSpark.
+[[extra_mounts]]
+host = "/home/kilork/dspark-draft"
+container = "/dspark-draft"
+
+
+[env]
+HF_HUB_OFFLINE = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-nvfp4-unsloth/recipe.toml

@@ -0,0 +1,80 @@
+# Qwen3.8-27B (unsloth NVFP4 + MTP) — 1× DGX Spark / GB10 (sm_121), TP=1.
+#
+# NVFP4 compressed-tensors quant (22 GB) with a separate MTP draft in
+# `model_mtp.safetensors`. Dense 27B, GQA, hybrid DeltaNet+Attention. Text +
+# vision. Unsloth Dynamic V3.0 (preview) quant — SOTA quality for NVFP4.
+# vLLM-only (lm_head FP8).
+#
+# Nearly identical to RedHatAI's NVFP4 build (qwen38-27b-nvfp4 recipe);
+# benchmark both to pick the faster / higher-quality one.
+#
+# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
+#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
+#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
+#
+# Model: unsloth/Qwen3.8-27B-NVFP4
+#   https://huggingface.co/unsloth/Qwen3.8-27B-NVFP4
+#
+# PREREQUISITES:
+#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
+#       huggingface-cli download unsloth/Qwen3.8-27B-NVFP4
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-nvfp4-on-a-single-dgx-spark-up-to-1m-context-vllm-mtp-measurements/380244
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
+
+engine = "vllm"
+name = "Qwen3.8-27B (NVFP4, unsloth)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
+model = "unsloth/Qwen3.8-27B-NVFP4"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "6",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.75",
+    "--served-model-name", "qwen3.8-27b",
+    # Engine backends (dense GQA — FlashInfer; NVFP4 route via compressed-tensors)
+    "--attention-backend", "flashinfer",
+    # Compilation / perf
+    "--optimization-level", "3",
+    "--performance-mode", "throughput",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # MTP speculative decoding (in-checkpoint draft, model_mtp.safetensors)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":5}'""",
+    # Tokenizer / reasoning (Qwen3 family)
+    "--tool-call-parser", "qwen3_xml",
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--generation-config", "auto",
+    "--default-chat-template-kwargs.preserve_thinking=true",
+    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
+    # practices)
+    "--override-generation-config", """
+'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
+    "--trust-remote-code",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-nvfp4/recipe.toml

@@ -0,0 +1,81 @@
+# Qwen3.8-27B (RedHatAI NVFP4 + MTP) — 1× DGX Spark / GB10 (sm_121), TP=1.
+#
+# NVFP4 compressed-tensors quant (22 GB) with a separate MTP draft in
+# `model_mtp.safetensors`. Dense 27B, GQA, hybrid DeltaNet+Attention. Text +
+# vision. The vLLM-only quant (lm_head FP8) — SGLang can't load it.
+#
+# Two nearly-identical NVFP4 builds are available:
+#   RedHatAI/Qwen3.8-27B-NVFP4  (this recipe)
+#   unsloth/Qwen3.8-27B-NVFP4   (qwen38-27b-nvfp4-unsloth recipe)
+# Speed/quality are close; benchmark both in the experiment.
+#
+# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
+#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
+#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
+#
+# Model: RedHatAI/Qwen3.8-27B-NVFP4
+#   https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4
+#
+# PREREQUISITES:
+#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
+#       huggingface-cli download RedHatAI/Qwen3.8-27B-NVFP4
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-nvfp4-on-a-single-dgx-spark-up-to-1m-context-vllm-mtp-measurements/380244
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
+
+engine = "vllm"
+name = "Qwen3.8-27B (NVFP4, RedHatAI)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
+model = "RedHatAI/Qwen3.8-27B-NVFP4"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "6",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.75",
+    "--served-model-name", "qwen3.8-27b",
+    # Engine backends (dense GQA — FlashInfer; NVFP4 route via compressed-tensors)
+    "--attention-backend", "flashinfer",
+    # Compilation / perf
+    "--optimization-level", "3",
+    "--performance-mode", "throughput",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # MTP speculative decoding (in-checkpoint draft, model_mtp.safetensors)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":5}'""",
+    # Tokenizer / reasoning (Qwen3 family)
+    "--tool-call-parser", "qwen3_xml",
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--generation-config", "auto",
+    "--default-chat-template-kwargs.preserve_thinking=true",
+    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
+    # practices)
+    "--override-generation-config", """
+'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
+    "--trust-remote-code",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b/recipe.toml

@@ -0,0 +1,88 @@
+# Qwen3.8-27B (BF16) — 1× DGX Spark / GB10 (sm_121), TP=1.
+#
+# Native BF16 dense 27B (52 GB). Quality reference baseline; the quantized
+# variants (FP8 / NVFP4 / Int4) are the speed candidates. Runs text + vision.
+#
+# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
+#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
+#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
+#
+# Model: Qwen/Qwen3.8-27B
+#   https://huggingface.co/Qwen/Qwen3.8-27B
+#   Dense 27B, 64 layers, hidden 5120, GQA (4 KV heads × 256), Gated
+#   DeltaNet + Gated Attention hybrid. MTP-1. Native 262,144 ctx (YaRN to 1M).
+#   Multimodal (vision tower, `language_model_only: False`).
+#
+# PREREQUISITES:
+#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
+#       huggingface-cli download Qwen/Qwen3.8-27B
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-at-34-38-tok-s-on-dgx-spark-open-source-one-command-setup-sglang-nvfp4-dspark/380257
+
+engine = "vllm"
+name = "Qwen3.8-27B (BF16)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
+model = "Qwen/Qwen3.8-27B"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "6",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.75",
+    "--served-model-name", "qwen3.8-27b",
+    # Engine backends (dense GQA — FlashInfer, not the DeepSeek MLA path)
+    "--attention-backend", "flashinfer",
+    # Compilation / perf
+    "--optimization-level", "3",
+    "--performance-mode", "throughput",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # MTP speculative decoding (in-checkpoint draft, mtp_num_hidden_layers=1
+    # cycled across steps)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":5}'""",
+    # Tokenizer / reasoning (Qwen3 family)
+    "--tool-call-parser", "qwen3_xml",
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--generation-config", "auto",
+    "--default-chat-template-kwargs.preserve_thinking=true",
+    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
+    # practices). repetition_penalty 1.0 = identity: accepted but never
+    # deviates from no-op during req handling (DSpark/MLA bug caveat — see
+    # findings doc for the DeepSeek path; irrelevant to dense Qwen).
+    "--override-generation-config", """
+'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
+    "--trust-remote-code",
+]
+
+[env]
+# Offline mode — weights must be cached on each node
+HF_HUB_OFFLINE = "1"
+
+# Architecture
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+
+# GPU memory management
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+
+# GPU device isolation
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-flash-next-fp8/recipe.toml

@@ -0,0 +1,79 @@
+# Qwen3.8-Flash-Next-FP8 (vLLM expert-parallel) — 2× DGX Spark / GB10 (sm_121), TP=2.
+#
+# Official FP8 quant of the Qwen4-architecture preview (176B total / 6B
+# active, GDN + QSA hybrid MoE, 512 experts, 51B n-gram "engram" PLE table,
+# in-checkpoint MTP). Text + vision. Native 262144 ctx.
+#
+# WHY vLLM + expert-parallel: the stock vLLM MoE path shards each expert's
+# intermediate by TP, and this FP8 checkpoint (weight_block_size=[128,128],
+# moe_intermediate_size=640) violates (intermediate/tp) % block_n == 0 at
+# tp=2 -> boot crash. --enable-expert-parallel splits the 512 *experts*
+# across the ranks instead (256 each, full 640-wide intermediate), so the
+# divisibility rule never fires. This is the verified 2× Spark FP8 path.
+#
+# Image: vllm/vllm-openai:qwen38-flash-next — vLLM's day-0 model tag (arm64).
+#
+# Model: Qwen/Qwen3.8-Flash-Next-FP8 (~186 GB)
+#   https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8
+#
+# REFERENCE (provenance — verified ~30-45 tok/s on 2× Sparks):
+#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228/97  (foogitiff)
+#   answers #106 / #113 (m-emm): ENFORCE_EAGER=1, disable MM, MTP=3 -> ~45 tok/s
+#
+# NOTE: day-0 config. foogitiff's base boots; m-emm's ENFORCE_EAGER + no-MM
+# gives the better throughput. Expect to tune max-num-seqs / mm limits.
+
+engine = "vllm"
+name = "Qwen3.8-Flash-Next FP8 (vLLM expert-parallel, 2x Spark)"
+docker_image = "vllm/vllm-openai:qwen38-flash-next"
+model = "Qwen/Qwen3.8-Flash-Next-FP8"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Stable served name — quant-agnostic, shared with qwen38-flash-next-nvfp4
+    "--served-model-name", "qwen3.8-flash-next",
+    # Expert parallelism — the fix for the FP8 MoE block-divisibility crash
+    "--distributed-executor-backend", "mp",
+    "--enable-expert-parallel",
+    "--all2all-backend", "allgather_reducescatter",
+    # Weight loading (no fastsafetensors day-0; lazy strategy keeps boot sane)
+    "--load-format", "safetensors",
+    "--safetensors-load-strategy", "lazy",
+    # Context / batching (verified; raise max-num-seqs once stable)
+    "--max-model-len", "262144",
+    "--max-num-seqs", "1",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.83",
+    "--enable-chunked-prefill",
+    # MTP speculative decoding (in-checkpoint draft)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":3}'""",
+    "--compilation-config", """
+'{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}'""",
+    # Tokenizer / reasoning / tools (Qwen3 family)
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "qwen3_coder",
+    # m-emm perf (post #113): disable multimodal to save VRAM/bandwidth
+    "--limit-mm-per-prompt", """
+'{"image":0,"video":0}'""",
+]
+
+[env]
+# Offline — weights must be cached on each node
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+# m-emm (post #113): eager mode for the 45 tok/s config (overrides the
+# cudagraph request above; day-0 stability knob)
+ENFORCE_EAGER = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"

recipes/qwen38-flash-next-nvfp4/recipe.toml

@@ -0,0 +1,89 @@
+# Qwen3.8-Flash-Next (NVFP4, SGLang) — 2× DGX Spark / GB10 (sm_121), TP=2.
+#
+# Qwen4-architecture preview: 176B total (6B active) GDN + QSA hybrid MoE,
+# 512 routed experts (top-10) + shared expert, 51B n-gram "engram" table
+# (PLE, offloaded to host RAM), in-checkpoint MTP draft. Text + vision.
+# Native 262144 ctx (YaRN to 1M). Always-thinking.
+#
+# QUANT: NVFP4 (RadixArk), NOT the official FP8. FP8 cannot serve on 2×
+# Spark: its MoE weight_block_size=[128,128] + moe_intermediate_size=640
+# violates SGLang's (intermediate/tp) % block_n == 0 at tp=2 (sgl-project/
+# sglang #21810) — only tp in {1,5} align, and tp=1 can't hold 125B. NVFP4
+# is the community-verified 2× Spark path. SGLang's own cookbook lists the
+# FP8 quant's supported HW as H200/B200/B300/GB300 — not GB10.
+#
+# Image: qwen38-flashnext-nvfp4:local — built by the launcher from
+#   docker-images/qwen38-flashnext-nvfp4/ (base lmsysorg/sglang:qwen38flashnext
+#   + a SM121 QSA decode kernel fix). The stock day-0 image cannot serve
+#   qwen4_exp on GB10: QSA's packed varlen decode resolves to flash-attn-4
+#   CuTe-DSL kernels that fail MLIR compilation on SM121; the overlay swaps in
+#   a CUDA-graph-safe Triton fallback. Provenance/validation:
+#     https://github.com/MiaAI-Lab/Qwen3.8-Flash-Next-Dual-DGX-Sparks
+#
+# Model: RadixArk/Qwen3.8-Flash-Next-NVFP4 (~135 GB)
+#   https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228
+
+engine = "sglang"
+name = "Qwen3.8-Flash-Next NVFP4 (SGLang, 2× Spark)"
+docker_image = "qwen38-flashnext-nvfp4:local"
+docker_image_dir = "qwen38-flashnext-nvfp4"
+model = "RadixArk/Qwen3.8-Flash-Next-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["python3", "-m", "sglang.launch_server"]
+
+args = [
+    # Stable served name — quant-agnostic, shared with qwen38-flash-next-fp8 so
+    # swapping quants never changes what agents/clients pin against.
+    "--served-model-name", "qwen3.8-flash-next",
+    "--trust-remote-code",
+    # NVFP4 (ModelOpt) weights
+    "--quantization", "modelopt_fp4",
+    # flashinfer is required on SM120/SM121 (trtllm_mha is SM100-only)
+    "--attention-backend", "flashinfer",
+    # page 64 is pinned by SGLang for compressed QSA (full//ratio addressing)
+    "--page-size", "64",
+    # GDN (DeltaNet) hybrid state pool
+    "--mamba-ssm-dtype", "bfloat16",
+    "--mamba-radix-cache-strategy", "extra_buffer",
+    "--mamba-track-interval", "64",
+    # Memory — 0.70 leaves DRAM for the ~26 GB/rank pinned PLE (engram) table
+    "--mem-fraction-static", "0.70",
+    "--chunked-prefill-size", "4096",
+    "--max-running-requests", "16",
+    "--context-length", "262144",
+    # NEXTN speculative decoding — draft = the in-checkpoint MTP layer
+    "--speculative-algorithm", "NEXTN",
+    "--speculative-num-steps", "3",
+    "--speculative-eagle-topk", "1",
+    "--speculative-num-draft-tokens", "4",
+    # Parsers (auto = resolve from the checkpoint chat template)
+    "--reasoning-parser", "auto",
+    "--tool-call-parser", "auto",
+    "--allow-auto-truncate",
+    # Observability
+    "--enable-metrics",
+    "--enable-cache-report",
+    # SM121 house rule: prefill CUDA graphs unreliable on GB10
+    "--disable-prefill-cuda-graph",
+    # Pin decode-graph batch list (default list OOMs the graph pool on GB10)
+    "--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16",
+]
+
+[env]
+# Offline — weights must be cached on each node
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"

recipes/solar-open2-nvfp4/recipe.toml

@@ -0,0 +1,85 @@
+# Solar Open2 250B — Nota NVFP4 (compressed-tensors, W4A4)
+# 2× DGX Spark (GB10, aarch64/sm_121a), TP=2 over RoCE.
+#
+# Model: nota-ai/Solar-Open2-250B-Nota-NVFP4
+#   → https://huggingface.co/nota-ai/Solar-Open2-250B-Nota-NVFP4
+#   145B params (MoE), 153.3 GB NVFP4 weights (vs 500.6 GB BF16).
+#   NVFP4 is auto-detected from config (compressed-tensors) — no --quantization.
+#
+# Image: solar-open2-nvfp4:probe (built locally from the UpstageAI vLLM fork
+#   v0.22.0-solar-open2 — carries the solar_open2 reasoning/tool parsers and
+#   the SolarOpen2TemplateLogitsProcessor). cu130 / aarch64 / sm_121a, same
+#   pinned base stack as glm52-nf3-hybrid (no b12x suite — stock vLLM serve).
+#
+# Memory: GB10 ≈ 121 GB usable (128 GB advertised − 4 GB OS). TP=2 →
+#   ~77 GB weights/node, ~40 GB/node for KV cache + activations.
+#
+# PREREQUISITES:
+#   - Weights cached on every node:
+#       huggingface-cli download nota-ai/Solar-Open2-250B-Nota-NVFP4
+#     (153 GB, NVFP4 compressed-tensors shards)
+#   - Image built (sm121 serve builds it automatically on the build node):
+#       cd docker-images/solar-open2-nvfp4 && bash build.sh
+
+engine = "vllm"
+name = "Solar Open2 250B Nota NVFP4 (vLLM)"
+docker_image = "solar-open2-nvfp4:probe"
+model = "nota-ai/Solar-Open2-250B-Nota-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+docker_image_dir = "solar-open2-nvfp4"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--load-format", "auto",
+    # Context length (128K; model supports up to 131584)
+    "--max-model-len", "131072",
+    # Batch & memory
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--max-cudagraph-capture-size", "16",
+    "--gpu-memory-utilization", "0.88",
+    # Scheduling
+    "--async-scheduling",
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    # Solar Open2 parsers + logits processor (ships in the UpstageAI fork)
+    "--reasoning-parser", "solar_open2",
+    "--tool-call-parser", "solar_open2",
+    "--enable-auto-tool-choice",
+    "--logits-processors", "vllm.v1.sample.logits_processor.solar_open2:SolarOpen2TemplateLogitsProcessor",
+    "--default-chat-template-kwargs", """
+'{"think_render_option":"preserved"}'""",
+    # General
+    "--trust-remote-code",
+    "--served-model-name", "solar",
+]
+
+[env]
+# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+
+# ── Offline mode (weights must be cached on each node) ────────────────────
+HF_HOME = "/root/.cache/huggingface"
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+
+# ── Timeout safety net (cold boot: cudagraph + JIT) ───────────────────────
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "3600"
+
+# ── NCCL / RoCE (cluster config ib_hca/socket_ifname used for most) ──────
+NCCL_CROSS_NIC = "1"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IGNORE_CPU_AFFINITY = "1"
+NCCL_DEBUG = "WARN"
+
+# ── Compute ────────────────────────────────────────────────────────────────
+OMP_NUM_THREADS = "4"

recipes/test-composed/recipe.toml

@@ -0,0 +1,50 @@
+# Test Composed Model — verifies the [composed_model] mechanism (symlink
+# composition + bind-mount + model rewrite) without launching heavy vLLM.
+#
+# Uses alpine (like test-sleep/test-mods) with a heartbeat entrypoint that
+# first inspects the composed dir, then loops. Requires both model repos
+# cached on every node — the composed dir's symlinks point into the HF cache
+# (container-side /root/.cache/huggingface), so they only resolve inside the
+# container where the HF cache is mounted.
+#
+# What this verifies:
+#   1. The composed dir is built on the node (/tmp/sm121_test-composed_<rank>/)
+#   2. MXFP4 base files are symlinked (config.json, tokenizer, etc.)
+#   3. Lloyd NF4 experts overlay the MXFP4 experts (L*.safetensors)
+#   4. The generated nf4_layers.json file is present
+#   5. The bind-mount makes /root/model/ accessible inside the container
+#   6. model= is rewritten to /root/model (visible in the docker run command)
+#
+# Usage:
+#   cargo run -- -d serve --recipe test-composed    # dry-run: inspect cmd
+#   cargo run serve --recipe test-composed           # real: starts alpine
+#   cargo run logs --recipe test-composed            # see the composed dir listing
+
+engine = "vllm"
+name = "Test Composed Model"
+docker_image = "alpine:latest"
+model = "noop"
+
+[composed_model]
+mount = "/root/model"
+
+[[composed_model.sources]]
+repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
+
+[[composed_model.sources]]
+repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ"
+glob = "L*.safetensors"
+
+[[composed_model.sources]]
+repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ"
+glob = "grids/*.json"
+
+[[composed_model.files]]
+name = "nf4_layers.json"
+content = '{"layers":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77]}'
+
+entrypoint = ["sh", "-c", "echo '=== composed model test ===' && echo 'dir contents (first 15):' && ls /root/model/ | head -15 && echo '--- nf4_layers.json ---' && cat /root/model/nf4_layers.json && echo '' && echo '=== test OK ===' && i=0; while true; do echo \"$(date -u +%H:%M:%S) [$(hostname)] composed $i\"; i=$((i+1)); sleep 2; done"]
+
+args = []
+
+[env]

recipes/test-mods/mods/local-patch/patch.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+# Simulates a vLLM patch: write a marker that the engine can read
+echo "alpine:3.20" > /tmp/.vllm-patched
+echo "MODS_PATCHED=1" >> /tmp/.vllm-patched
+echo "[mods] Applied vLLM compatibility patch at $(date)" >&2

recipes/test-mods/recipe.toml

@@ -0,0 +1,16 @@
+engine = "vllm"
+name = "Test Mods"
+docker_image = "alpine:latest"
+model = "noop"
+port = 30000
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+
+# Entrypoint must be set when mods are present.
+entrypoint = ["sh", "-c", "i=0; while true; do echo \"[$(hostname)] heartbeat $i\"; i=$((i+1)); sleep 2; done"]
+
+[mods]
+commands = [
+    "sh /mods/local-patch/patch.sh",
+]

recipes/test-sleep/recipe.toml

@@ -0,0 +1,19 @@
+engine = "vllm"
+name = "Test Heartbeat Container"
+docker_image = "alpine:latest"
+model = "noop"
+
+# Heartbeat loop: prints a timestamped line every 2s with UTC time and
+# the container hostname so you can tell logs from different nodes apart.
+#
+# Use it to verify the logging pipeline:
+#   cargo run serve --recipe test-sleep     # spins up the cluster
+#   cargo run logs --recipe test-sleep      # last 100 lines per node
+#   cargo run logs --recipe test-sleep --follow   # live stream from head
+#
+# Each line looks like:  10:42:13 [n0] heartbeat 42
+entrypoint = ["sh", "-c", "i=0; while true; do echo \"$(date -u +%H:%M:%S) [$(hostname)] heartbeat $i\"; i=$((i+1)); sleep 2; done"]
+
+args = []
+
+[env]