move recipe data to git.kilork.org/sm121/recipes (registry data repo)

recipes/ (40 recipe.toml + generated run.sh/README/install.sh bundles) now
lives in its own repo — it is the storage backend of the recipe registry,
and every push there is a registry entry. The launcher keeps code + image
build contexts (docker-images/); run `sm121 serve|doctor` from a clone of
the recipes repo (or `sm121 registry install <name>`).

Tests no longer read repo recipe data: tmux discover + web api/raw seed
their own ./recipes fixtures and clean up; recipes_endpoint asserts shape
only. ui.html raw-window select always offers config/effective (previously
empty select with zero local recipes — fill() and its call site both bailed
on an empty list).

AGENTS.md

@@ -89,6 +89,9 @@ per node instead of running it.
 
 ## Recipe format (`recipes/<name>/recipe.toml`)
 
+Recipe data lives in <https://git.kilork.org/sm121/recipes>; this repo
+keeps only the image build contexts (`docker-images/`).
+
 ```toml
 engine = "vllm"                     # or "sglang"
 docker_image = "img:tag"

CLAUDE.md

@@ -51,7 +51,7 @@ See `specs/tech-architecture/tech-stack.md` for the full architectural map.
 
 | Glob | Source of truth |
 |------|-----------------|
-| `recipes/**`, `docker-images/**`, `host-scripts/**` | `AGENTS.md` |
+| `docker-images/**`, `host-scripts/**` | `AGENTS.md` (recipe data itself: git.kilork.org/sm121/recipes) |
 | `src/**`, `locales/**` | `CONVENTIONS.md` + `specs/` |
 | Cluster/recipe/config format questions | `AGENTS.md` (cluster ops SoT) |
 | Dev workflow / planning / lifecycle | `specs/` (bigpowers SoT) |

README.md

@@ -60,19 +60,13 @@ Prefix must be unambiguous — error if multiple clusters match.
 
 ## Recipes
 
-Recipes are TOML files in `recipes/<name>/recipe.toml`.
-The launcher looks for them in the current directory first,
-then in `~/.config/sm121-launcher/recipes/<name>/recipe.toml`.
-
-### Available
-
-| Recipe | Engine | Nodes | Model |
-|--------|--------|-------|-------|
-| `deepseek-v4-flash` | vLLM | 2 | DeepSeek V4 Flash (MXFP8, 1M ctx, MTP) |
-| `minimax-m3-mxfp8` | vLLM | 4 | MiniMax M3 (MXFP8, 428B, MSA sparse attn) |
-| `glm51` | vLLM | 4 | GLM-5.1 NVFP8 |
-| `minimax-m3` | SGLang | 4 | MiniMax-M3-v0 NVFP4 |
-| `test-sleep` | vLLM | any | Heartbeat test container |
+Recipes are TOML files in `recipes/<name>/recipe.toml` — resolved relative
+to the current directory (then `~/.config/sm121-launcher/recipes/`). The
+canonical collection lives in its own repo:
+<https://git.kilork.org/sm121/recipes> (one flat bundle per recipe; also
+the storage repo of the recipe registry). Clone it, drop your
+`config.toml` beside `recipes/`, and run `sm121` from there — or fetch
+single recipes with `sm121 registry list|install`.
 
 ### Creating a recipe
 

recipes/deepseek-v4-flash-dspark38/recipe.toml

@@ -1,131 +0,0 @@
-# DeepSeek-V4-Flash-0731 DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
-#
-# Variant targeting aidendle94's production-3.8 image ("dsv4/apostolic" b12x
-# harness). This is the BUG-2026-08-03T204037 bring-up recipe — the two images
-# are *different b12x forks*, not a version bump from production-3.75, so the
-# launch contract differs:
-#
-#   - production-3.8 is a clean-overlay base (null ENTRYPOINT, CMD sleep
-#     infinity) whose intended launcher is /usr/local/bin/dsv4-vllm-entrypoint.
-#     We set that as the recipe entrypoint so the JIT toolchain env
-#     (DG_JIT_NVCC_COMPILER, NVCC_PREPEND_FLAGS + CCCL include path, CUDAHOSTCXX)
-#     and the /cache JIT dir layout are active. The 3.75 recipe ran vLLM directly
-#     because that image was self-contained with everything baked in.
-#   - VLLM_USE_V2_MODEL_RUNNER is dropped (experimental in this fork; the
-#     working glm52 recipe does not set it).
-#   - tiny-decode (VLLM_B12X_TINY_DECODE) is enabled at GMU 0.78 + CHUNK=8
-#     (the load-time repack needs headroom; it OOM'd at GMU 0.9).
-#   - /cache is mounted so the wrapper's JIT caches persist across restarts.
-
-engine = "vllm"
-name = "DeepSeek V4 Flash DSpark (production-3.8)"
-# Local overlay of the production-3.8 base built from
-# docker-images/deepseek-v4-flash-dspark38 (root-cause patch for the b12x
-# cutlass from_dlpack grad bug that blocked boot; see the Dockerfile).
-docker_image = "sparkrun-vllm-ds4-gb10:dspark38"
-model = "deepseek-ai/DeepSeek-V4-Flash-0731"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-# production-3.8 launch contract: the clean-base image must be started via the
-# dsv4-vllm-entrypoint wrapper, which sets the JIT toolchain + /cache dirs and
-# `exec vllm "$@"`. `serve <recipe-args/auto flags>` land as "$@".
-entrypoint = ["/usr/local/bin/dsv4-vllm-entrypoint", "serve"]
-
-args = [
-    "--kv-cache-dtype", "fp8",
-    "--block-size", "256",
-    "--load-format", "auto",
-    "--max-model-len", "524288",
-    "--max-num-seqs", "8",
-    "--max-num-batched-tokens", "8192",
-    "--max-cudagraph-capture-size", "64",
-    # 0.78 is the safe GMU ceiling on GB10 (weights ~77.7 GiB of ~119.6 GiB);
-    # higher boots but dies under traffic. Also leaves headroom for the tiny
-    # decode kernel's GPU load-time repack.
-    "--gpu-memory-utilization", "0.78",
-    "--served-model-name", "deepseek-v4-flash",
-    # Engine backends — b12x across attention / MoE / linear (this fork's path)
-    "--attention-backend", "B12X_MLA_SPARSE",
-    "--moe-backend", "b12x",
-    "--linear-backend", "b12x",
-    # Prefix caching
-    "--enable-prefix-caching",
-    # Compilation
-    "--compilation-config", """
-'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
-    # DSpark speculative decoding — k=5 (DeepSeek-V4-Flash native
-    # dspark_block_size). draft_sample_method greedy.
-    "--speculative-config", """
-'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"greedy"}'""",
-    # Tokenizer / reasoning
-    "--tokenizer-mode", "deepseek_v4",
-    "--tool-call-parser", "deepseek_v4",
-    "--reasoning-parser", "deepseek_v4",
-    "--enable-auto-tool-choice",
-    "--generation-config", "vllm",
-    "--default-chat-template-kwargs.thinking=true",
-    "--default-chat-template-kwargs.reasoning_effort=high",
-    "--disable-custom-all-reduce",
-    "--trust-remote-code",
-]
-
-# Persistent JIT/compile caches. The wrapper `mkdir -p`s under /cache
-# (DG_JIT_CACHE_DIR=/cache/deep_gemm_jit, VLLM_CACHE_ROOT=/cache/vllm,
-# TRITON_CACHE_DIR=/cache/triton, etc.), so /cache must be a mounted volume to
-# survive restarts. The HF cache is already mounted at /root/.cache/huggingface
-# by the launcher.
-[[extra_mounts]]
-host = "~/.cache/ds4-38-vllm"
-container = "/cache"
-
-[env]
-# --- b12x fork env family (shared with the proven eugr b12x recipe) ---
-VLLM_ATTENTION_BACKEND = "B12X_MLA_SPARSE"
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_MHC = "1"
-VLLM_USE_B12X_WO_PROJECTION = "1"
-VLLM_USE_B12X_FP8_GEMM = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_USE_FLASHINFER_SAMPLER = "1"
-VLLM_USE_AOT_COMPILE = "1"
-VLLM_USE_MEGA_AOT_ARTIFACT = "0"
-VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
-VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
-VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
-B12X_MLA_SM120_UNIFIED = "1"
-B12X_DENSE_SPLITK_TURBO = "1"
-B12X_W4A16_TC_DECODE = "1"
-# Tiny A8 decode kernel: DISABLED — its GPU load-time MoE-weight repack, on
-# top of the tight host RAM (~32 GiB free) + dual-node 155 GiB load, OOM'd both
-# hosts (had to power-cycle via HA). Revisit separately if host RAM headroom is
-# available. Keep the CHUNK pin for reference.
-# VLLM_B12X_TINY_DECODE = "1"
-# VLLM_B12X_TINY_CHUNK = "8"
-
-# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
-CUTE_DSL_ARCH = "sm_121a"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-OMP_NUM_THREADS = "4"
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"
-
-# --- offline ---
-HF_HUB_OFFLINE = "1"
-
-# --- NCCL overrides (image-agnostic; launcher injects IB/HCA/socket from
-#     cluster config, these are recipe-level extras) ---
-NCCL_P2P_LEVEL = "SYS"
-NCCL_CUMEM_ENABLE = "0"
-NCCL_IB_TIMEOUT = "24"
-NCCL_IB_QPS_PER_CONNECTION = "8"
-NCCL_BUFFERSIZE = "16777216"
-NCCL_SOCKET_FAMILY = "inet"
-
-VLLM_ENABLE_PCIE_ALLREDUCE = "0"
-VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"

recipes/deepseek-v4-flash-eugr-agent/recipe.toml

@@ -1,167 +0,0 @@
-# DeepSeek-V4-Flash-0731 DSpark — AGENT-SERVING profile (2× GB10, TP=2).
-#
-# Fork of deepseek-v4-flash-eugr tuned for interactive agent traffic, applying
-# the validated "agent-2048" profile from the SvangenStudios DGX-Spark
-# agent-serving benchmarks (v0.1.1, measured 2026-08-02 on this exact stack:
-# DeepSeek-V4-Flash-0731, DSpark k=5, 2× DGX Spark, TP=2, RoCE):
-#   https://github.com/SvangenStudios/dgx-spark-agent-serving-benchmarks
-#   configs/agent-2048.env  /  results/RESULTS.md §4
-#
-# The benchmark's headline finding: --max-num-batched-tokens is a FAIRNESS
-# lever for agent workloads, not just a jitter lever. Against an ongoing
-# decode stream, a 262K-token prefill at 2048 delivers 2.9× the decode share
-# (5.0% vs 1.7%), 3.1× the decode tokens, 73% lower p95 output-chunk jitter,
-# and +67% KV-pool capacity — for 7.7% prefill throughput. The launcher's
-# whole purpose is agent serving (a model may be serving this very agent), so
-# 2048 is the more correct default here.
-#
-# Mental model (benchmark §3): an LLM server is three nearly-independent
-# engines — prefill (compute-bound, serializes), decode (bandwidth-bound,
-# scales to ~N=4), admission (scheduler-bound, BLOCKED during long prefill).
-# Chunk size governs how often decode gets a scheduling slot between prefill
-# chunks; smaller = more slots. It does NOT fix new-request admission (a
-# short request submitted during a large prefill waits until prefill
-# completes — 221× normal TTFT in both profiles).
-#
-# Knob changes vs deepseek-v4-flash-eugr (everything else identical — same
-# eugr image, same backends, same env):
-#   --max-num-batched-tokens  8192 -> 2048   (the fairness lever)
-#   --max-num-seqs            8    -> 6      (decode saturates ~N=4; 6 gives
-#                                             headroom without inflating
-#                                             worst-case latency)
-#   --max-cudagraph-capture-size 64 -> 36    (= max_num_seqs * (k+1) = 6*6;
-#                                             the k+1 multiple rule — the
-#                                             benchmark notes k=5 rejects
-#                                             MAX_NUM_SEQS=1 at boot)
-#   --gpu-memory-utilization  0.85 -> 0.78   (benchmark: "0.80 boots but dies
-#                                             under traffic"; GMU is a KV
-#                                             lever, not a safety lever, when
-#                                             weights dominate memory)
-#   --max-model-len           auto -> 1048576  (benchmark's 1M working capacity
-#                                             — verified 12/12 needles at
-#                                             32K/128K/512K/900K)
-#
-# Trade-off: this profile costs ~7.7% prefill throughput vs the 8192 batch
-# profile. Use deepseek-v4-flash-eugr (8192) for exclusive long-context batch
-# jobs (512K–1M) with no concurrent agent traffic. See the batch-8192 profile
-# in the benchmark repo for the opposite extreme.
-#
-# 3× Sparks (TP=3): bump tensor_parallel_size + required_nodes to 3 (the eugr
-# image supports it per forum comment #28). k=5 unchanged (model-native).
-
-engine = "vllm"
-name = "DeepSeek V4 Flash DSpark (eugr b12x, agent 2048)"
-docker_image = "eugr/spark-vllm-b12x:sm121"
-model = "deepseek-ai/DeepSeek-V4-Flash-0731"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-docker_image_dir = "deepseek-v4-flash-eugr"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    "--kv-cache-dtype", "fp8",
-    "--block-size", "256",
-    "--load-format", "instanttensor",
-    # 1M working capacity (benchmark §2: 12/12 needles at 32K–900K). A 900K
-    # prefill takes ~15 min and starves everything — batch capacity, not an
-    # interactive mode.
-    "--max-model-len", "1048576",
-    # Decode saturates around N≈4 (benchmark §3); 6 gives headroom without
-    # inflating worst-case latency. cudagraph capture = max_num_seqs*(k+1)=36.
-    "--max-num-seqs", "6",
-    # FAIRNESS LEVER (benchmark §4): 2048 gives 2.9× the decode share under a
-    # concurrent prefill vs 8192, +67% KV pool, 73% lower jitter, for 7.7%
-    # prefill throughput. Use 8192 only for exclusive long-context batch jobs.
-    "--max-num-batched-tokens", "2048",
-    # = max_num_seqs * (k+1) = 6 * 6 (the k+1 multiple rule).
-    "--max-cudagraph-capture-size", "36",
-    # Benchmark finding: "0.80 boots but dies under traffic" — 0.78 is the
-    # safe ceiling. GMU is a KV lever, not a safety lever, when weights
-    # dominate memory (~77.7 GiB of ~119.6 GiB node).
-    "--gpu-memory-utilization", "0.78",
-    "--served-model-name", "deepseek-v4-flash",
-    # Engine backends — b12x across attention / MoE / linear
-    "--attention-backend", "B12X_MLA_SPARSE",
-    "--moe-backend", "b12x",
-    "--linear-backend", "b12x",
-    # Prefix caching — agent frameworks build prompts append-only, so reuse is
-    # 95.7–98.4% per turn (benchmark §9: divergence at the last token of the
-    # previous prompt; only ~270–660 tokens re-prefilled per agent step).
-    "--enable-prefix-caching",
-    # Compilation
-    "--compilation-config", """
-'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
-    # DSpark speculative decoding — k=5 (DeepSeek-V4-Flash native
-    # dspark_block_size). draft_sample_method per eugr recipe; draft attention
-    # backend B12X_MLA_SPARSE. The baked mod makes the embedded draft load via
-    # lazy safetensors while the target stays on InstantTensor.
-    "--speculative-config", """
-'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"B12X_MLA_SPARSE"}'""",
-    # Tokenizer / reasoning — reasoning-config pins the parser + empty
-    # reasoning markers (deepseek_v4 tokenizer mode emits them itself).
-    "--tokenizer-mode", "deepseek_v4",
-    "--tool-call-parser", "deepseek_v4",
-    "--reasoning-parser", "deepseek_v4",
-    "--reasoning-config", """
-'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
-    "--enable-auto-tool-choice",
-    "--generation-config", "vllm",
-    "--default-chat-template-kwargs.thinking=true",
-    "--default-chat-template-kwargs.reasoning_effort=high",
-    "--disable-custom-all-reduce",
-    "--trust-remote-code",
-]
-
-# Persistent vLLM compile cache (speeds up restarts). Mounted at vLLM's
-# default cache root for root (/root/.cache/vllm); the HF cache is already
-# mounted at /root/.cache/huggingface by the launcher. Shares the eugr
-# recipe's cache dir (same image/backends — compile artifacts are reusable).
-[[extra_mounts]]
-host = "~/.cache/vllm-ds4-eugr"
-container = "/root/.cache/vllm"
-
-[env]
-# --- b12x + compile (from eugr's recipe env, verbatim) ---
-CUTE_DSL_ARCH = "sm_121a"
-VLLM_USE_AOT_COMPILE = "1"
-VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
-VLLM_USE_MEGA_AOT_ARTIFACT = "-1"
-VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
-VLLM_USE_FLASHINFER_SAMPLER = "1"
-VLLM_USE_B12X_WO_PROJECTION = "1"
-VLLM_USE_B12X_MHC = "1"
-VLLM_USE_B12X_FP8_GEMM = "1"
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_USE_V2_MODEL_RUNNER = "1"
-B12X_MLA_SM120_UNIFIED = "1"
-B12X_MOE_FORCE_A8 = "1"
-
-# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
-TORCH_CUDA_ARCH_LIST = "12.1a"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-OMP_NUM_THREADS = "4"
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"
-
-# --- offline ---
-HF_HUB_OFFLINE = "1"
-
-# --- NCCL overrides (GB10 single-NIC tuning; image-agnostic — mirrors
-#     recipes/deepseek-v4-flash. The launcher still injects IB/HCA/socket
-#     vars from the cluster config; these are recipe-level extras.) ---
-NCCL_P2P_LEVEL = "SYS"
-NCCL_CUMEM_ENABLE = "0"
-NCCL_IB_TIMEOUT = "24"
-NCCL_IB_QPS_PER_CONNECTION = "8"
-NCCL_BUFFERSIZE = "16777216"
-NCCL_SOCKET_FAMILY = "inet"
-
-# PCIe allreduce not needed on GB10 single-NIC
-VLLM_ENABLE_PCIE_ALLREDUCE = "0"
-VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"

recipes/deepseek-v4-flash-eugr/recipe.toml

@@ -1,149 +0,0 @@
-# DeepSeek-V4-Flash-0731 DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
-#
-# This variant uses eugr's prebuilt b12x image (eugr/spark-vllm-b12x) and
-# mirrors eugr's recipes/deepseek-v4-flash-0731.yaml on the b12x branch — the
-# stack published in NVIDIA forum comment #28:
-#   https://forums.developer.nvidia.com/t/instructions-for-running-deepseek-v4-flash-with-dspark-using-eugrs-repo/376220/28
-# ("While I'm working on integrating b12x into the build/test pipeline and
-# merging into main, I pushed prebuilt container as eugr/spark-vllm-b12x …
-# this version supports 3x Sparks with TP=3! To use, just add --tp 3.")
-#
-# Image: eugr/spark-vllm-b12x:sm121 — the prebuilt base with the
-# instanttensor-hybrid-draft-loader mod baked in at build time (see
-# docker-images/deepseek-v4-flash-eugr). eugr's recipe applies that mod via
-# run-recipe.sh --apply-mod; sm121 has no runtime mod-upload step for Docker
-# recipes, so the overlay patches vLLM's model_loader get_model() at image
-# build time instead. The recipe therefore carries no [mods] block.
-#
-# The mod keeps InstantTensor for the primary model while letting the embedded
-# dspark/MTP draft use lazy safetensors (avoids a second InstantTensor
-# GPU-streaming pass over the checkpoint). It activates only when
-# --load-format instanttensor is set (which this recipe does).
-#
-# Model: deepseek-ai/DeepSeek-V4-Flash-0731 — official release; native
-# dspark_block_size = 5, so num_speculative_tokens = 5.
-#
-# Differences vs recipes/deepseek-v4-flash (aidendle94 production-3.76.1 image):
-#   - attention backend B12X_MLA_SPARSE (eugr) vs FLASHINFER_MLA_SPARSE_DSV4
-#   - --linear-backend b12x + --load-format instanttensor (the mod's trigger)
-#   - --reasoning-config pinned to deepseek_v4 parser w/ empty start/end strs
-#   - draft_sample_method probabilistic + draft attention_backend B12X_MLA_SPARSE
-#   - max_num_seqs=8 / batched=8192 / cudagraph=64 (eugr defaults)
-#   - VLLM_USE_MEGA_AOT_ARTIFACT="-1" (eugr) vs "0"
-#
-# 3× Sparks (TP=3): bump tensor_parallel_size + required_nodes to 3 (the eugr
-# image supports it per comment #28). k=5 is unchanged (model-native).
-
-engine = "vllm"
-name = "DeepSeek V4 Flash DSpark (eugr b12x)"
-docker_image = "eugr/spark-vllm-b12x:sm121"
-model = "deepseek-ai/DeepSeek-V4-Flash-0731"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-docker_image_dir = "deepseek-v4-flash-eugr"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    "--kv-cache-dtype", "fp8",
-    "--kv-cache-memory-bytes", "21474836480",
-    "--block-size", "256",
-    "--load-format", "instanttensor",
-    "--max-model-len", "524288",
-    "--max-num-seqs", "8",
-    "--max-num-batched-tokens", "4096",
-    "--long-prefill-token-threshold", "1024",
-    "--max-cudagraph-capture-size", "64",
-    # 0.78, not higher: the SvangenStudios agent-serving benchmark (v0.1.1) on
-    # this exact stack found "0.80 boots but dies under traffic". GMU is a KV
-    # lever, not a safety lever, when weights dominate memory (~77.7 GiB of
-    # ~119.6 GiB node). For interactive agent traffic prefer the
-    # deepseek-v4-flash-eugr-agent profile (2048 chunk). See
-    # specs/tech-architecture/dgx-spark-agent-serving-findings.md.
-    "--gpu-memory-utilization", "0.78",
-    "--served-model-name", "deepseek-v4-flash",
-    # Engine backends — b12x across attention / MoE / linear
-    "--attention-backend", "B12X_MLA_SPARSE",
-    "--moe-backend", "b12x",
-    "--linear-backend", "b12x",
-    # Prefix caching
-    "--enable-prefix-caching",
-    # Compilation
-    "--compilation-config", """
-'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
-    # DSpark speculative decoding — k=5 (DeepSeek-V4-Flash native
-    # dspark_block_size). draft_sample_method per eugr recipe; draft attention
-    # backend B12X_MLA_SPARSE. The baked mod makes the embedded draft load via
-    # lazy safetensors while the target stays on InstantTensor.
-    "--speculative-config", """
-'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"B12X_MLA_SPARSE"}'""",
-    # Tokenizer / reasoning — reasoning-config pins the parser + empty
-    # reasoning markers (deepseek_v4 tokenizer mode emits them itself).
-    "--tokenizer-mode", "deepseek_v4",
-    "--tool-call-parser", "deepseek_v4",
-    "--reasoning-parser", "deepseek_v4",
-    "--reasoning-config", """
-'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
-    "--enable-auto-tool-choice",
-    "--generation-config", "vllm",
-    "--default-chat-template-kwargs.thinking=true",
-    "--default-chat-template-kwargs.reasoning_effort=max",
-    "--disable-custom-all-reduce",
-    "--trust-remote-code",
-]
-
-# Persistent vLLM compile cache (speeds up restarts). Mounted at vLLM's
-# default cache root for root (/root/.cache/vllm); the HF cache is already
-# mounted at /root/.cache/huggingface by the launcher.
-[[extra_mounts]]
-host = "~/.cache/vllm-ds4-eugr"
-container = "/root/.cache/vllm"
-
-[env]
-# --- b12x + compile (from eugr's recipe env, verbatim) ---
-CUTE_DSL_ARCH = "sm_121a"
-VLLM_USE_AOT_COMPILE = "1"
-VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
-VLLM_USE_MEGA_AOT_ARTIFACT = "-1"
-VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
-VLLM_USE_FLASHINFER_SAMPLER = "1"
-VLLM_USE_B12X_WO_PROJECTION = "1"
-VLLM_USE_B12X_MHC = "1"
-VLLM_USE_B12X_FP8_GEMM = "1"
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_USE_V2_MODEL_RUNNER = "1"
-B12X_MLA_SM120_UNIFIED = "1"
-B12X_MOE_FORCE_A8 = "1"
-
-VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
-
-# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
-TORCH_CUDA_ARCH_LIST = "12.1a"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-OMP_NUM_THREADS = "4"
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"
-
-# --- offline ---
-HF_HUB_OFFLINE = "1"
-
-# --- NCCL overrides (GB10 single-NIC tuning; image-agnostic — mirrors
-#     recipes/deepseek-v4-flash. The launcher still injects IB/HCA/socket
-#     vars from the cluster config; these are recipe-level extras.) ---
-NCCL_P2P_LEVEL = "SYS"
-NCCL_CUMEM_ENABLE = "0"
-NCCL_IB_TIMEOUT = "24"
-NCCL_IB_QPS_PER_CONNECTION = "8"
-NCCL_BUFFERSIZE = "16777216"
-NCCL_SOCKET_FAMILY = "inet"
-
-# PCIe allreduce not needed on GB10 single-NIC
-VLLM_ENABLE_PCIE_ALLREDUCE = "0"
-VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"
-
-VLLM_DSV4_REASONING_CONTENT = "1"

recipes/deepseek-v4-flash-src/recipe.toml

@@ -1,97 +0,0 @@
-# EXPERIMENTAL — DeepSeek-V4-Flash-0731 DSpark, built FROM SOURCE from the
-# newest local-inference-lab b12x fork ("eldritch"), instead of the prebuilt
-# aidendle94 image.
-#
-#   fork branch: feat/gg-ds4-0731-launcher (e63a190ee) — torch 2.12.0,
-#                flashinfer 0.6.15, cutlass-dsl 4.6.0, sm_121a
-#   image:       vllm-ds4-dspark-src:probe (built on demand via docker_image_dir)
-#
-# Serving flags/env mirror the fork's authoritative serve-ds4-flash-spark.sh at
-# that commit (DSpark k=5, draft attention backend B12X_MLA_SPARSE, b12x MoE/MLA,
-# cudagraph capture size = max_num_seqs * (k+1)). Adapt cluster-side: TP=2,
-# 1 node per rank, no DCP.
-#
-# ⚠️ NOT production. The image is a multi-hour from-source build. Boot may need
-# tuning; verify acceptance/throughput before any real traffic.
-
-engine = "vllm"
-name = "DeepSeek V4 Flash DSpark (EXPERIMENTAL, from source)"
-docker_image = "vllm-ds4-dspark-src:probe"
-model = "deepseek-ai/DeepSeek-V4-Flash-0731"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-docker_image_dir = "deepseek-v4-flash-dspark-src"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    "--kv-cache-dtype", "fp8",
-    "--block-size", "256",
-    "--load-format", "auto",
-    "--max-model-len", "524288",
-    "--max-num-seqs", "4",
-    "--max-num-batched-tokens", "8192",
-    # fork rule: max_num_seqs * (k+1) = 4 * 6
-    "--max-cudagraph-capture-size", "24",
-    # 0.78, not 0.80: the SvangenStudios agent-serving benchmark (v0.1.1) on
-    # this exact stack found "0.80 boots but dies under traffic". GMU is a KV
-    # lever, not a safety lever, when weights dominate memory. See
-    # specs/tech-architecture/dgx-spark-agent-serving-findings.md.
-    "--gpu-memory-utilization", "0.78",
-    "--served-model-name", "deepseek-v4-flash", "default",
-    "--attention-backend", "B12X_MLA_SPARSE",
-    "--moe-backend", "b12x",
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    "--compilation-config", """
-'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
-    "--async-scheduling",
-    "--no-scheduler-reserve-full-isl",
-    # DSpark k=5 (dspark_block_size == 5); draft attention backend per fork
-    "--speculative-config", """
-'{"method":"dspark","num_speculative_tokens":5,"attention_backend":"B12X_MLA_SPARSE"}'""",
-    "--generation-config", "vllm",
-    "--default-chat-template-kwargs.thinking=true",
-    "--default-chat-template-kwargs.reasoning_effort=high",
-    "--default-chat-template-kwargs.temperature=1",
-    "--default-chat-template-kwargs.top_p=1",
-    "--disable-custom-all-reduce",
-    "--trust-remote-code",
-]
-
-[env]
-# --- from the fork's serve-ds4-flash-spark.sh (e63a190ee) ---
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-VLLM_USE_AOT_COMPILE = "1"
-VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
-VLLM_USE_MEGA_AOT_ARTIFACT = "1"
-VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS = "1"
-VLLM_USE_FLASHINFER_SAMPLER = "1"
-VLLM_USE_B12X_WO_PROJECTION = "1"
-VLLM_USE_B12X_MHC = "1"
-VLLM_USE_B12X_FP8_GEMM = "1"
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_USE_V2_MODEL_RUNNER = "1"
-VLLM_PCIE_ALLREDUCE_BACKEND = "b12x"
-VLLM_ENABLE_PCIE_ALLREDUCE = "1"
-B12X_MLA_SM120_UNIFIED = "1"
-B12X_DENSE_SPLITK_TURBO = "1"
-B12X_W4A16_TC_DECODE = "1"
-B12X_MOE_FORCE_A8 = "1"
-
-# --- build/arch + memory ---
-CUTE_DSL_ARCH = "sm_121a"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-DG_JIT_USE_NVRTC = "0"
-USE_CUDNN = "1"
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-OMP_NUM_THREADS = "4"
-
-# --- offline ---
-HF_HUB_OFFLINE = "1"

recipes/deepseek-v4-flash/recipe.toml

@@ -1,153 +0,0 @@
-# DeepSeek-V4-Flash-DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
-# DSpark speculative decoding on the official DeepSeek-V4-Flash-0731 release
-# via the local :fixed overlay on aidendle94's production-3.76.1 image (see
-# docker-images/deepseek-v4-flash-dspark).
-#
-# Image: sparkrun-vllm-ds4-gb10:fixed
-#   production-3.76.1 base (DSpark, b12x MoE, patched NCCL 2.30.4, multi-node
-#   fixes) + instanttensor for faster model loading.
-# Model: deepseek-ai/DeepSeek-V4-Flash-0731 — official release.
-#   0731 (and the -DSpark preview) both have dspark_block_size = 5. This image's
-#   speculator hard-rejects num_speculative_tokens > block_size
-#   (v1/worker/gpu/spec_decode/dspark/speculator.py, "must be <= dspark_block_size"),
-#   so k=7 (as suggested by the 0731 model card) FAILS to boot. k=5 is the
-#   model's native draft block and the correct value here.
-#
-# Note: this image lineage (vLLM 0.2x with the upstream DSpark speculator) does
-# NOT have the shared-expert draft-loader bug reported for the older fork
-# (tonyd2wild Patch 4). Its loader already maps mtp.*.shared_experts.w1/w3 into
-# gate_up_proj (dspark.py load_weights), so no patch is needed.
-#
-# Reference: Nvidia Developer Forum — DeepSeek-V4-Flash-0731 recipe
-#   https://forums.developer.nvidia.com/t/deepseek-v4-flash-0731-dspark-1m-nvfp4-kv-2x-dgx-spark/378824
-# Encoder reasoning-effort fix source: 0rand's serving stack (thread 372268/708)
-#   https://forums.developer.nvidia.com/t/deepseek-v4-flash-aiden-recipe-from-reddit-1m-token-session-operational-cuda-12-1-tailored-for-dgx-spark-gb10/372268/708
-#   (three-level low/high/max ladder; bundled into the :fixed overlay here)
-
-engine = "vllm"
-name = "DeepSeek V4 Flash DSpark (vLLM)"
-docker_image = "sparkrun-vllm-ds4-gb10:fixed"
-model = "deepseek-ai/DeepSeek-V4-Flash-0731"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-docker_image_dir = "deepseek-v4-flash-dspark"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    "--kv-cache-dtype", "fp8",
-    "--block-size", "256",
-    "--load-format", "auto",
-    "--max-model-len", "524288",
-    "--max-num-seqs", "16",
-    "--max-num-batched-tokens", "4096",
-    "--max-cudagraph-capture-size", "128",
-    "--gpu-memory-utilization", "0.78",
-    "--served-model-name", "deepseek-v4-flash",
-    # Engine backends
-    "--attention-backend", "FLASHINFER_MLA_SPARSE_DSV4",
-    "--moe-backend", "b12x",
-    # Chunked prefill + prefix caching
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    # Compilation
-    "--compilation-config", """
-'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
-    # Async scheduling
-    "--async-scheduling",
-    "--no-scheduler-reserve-full-isl",
-    # DSpark speculative decoding — k=5 is DeepSeek-V4-Flash's native
-    # dspark_block_size (0731 == preview). Do NOT raise to k=7 (0731 model-card
-    # rec): this image hard-rejects k > block_size at boot. draft_sample_method
-    # matches the image CMD (probabilistic).
-    "--speculative-config", """
-'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic"}'""",
-    # Tokenizer / reasoning
-    "--tokenizer-mode", "deepseek_v4",
-    "--tool-call-parser", "deepseek_v4",
-    "--reasoning-parser", "deepseek_v4",
-    "--enable-auto-tool-choice",
-    # Proven generation defaults (thread 372268/708 serving stack; with
-    # --generation-config vllm only the override params are applied, so these
-    # set the served defaults). repetition_penalty 1.0 = identity: the field is
-    # accepted by the spec but never deviates from no-op during req handling.
-    "--override-generation-config", """
-'{"temperature":0.8,"top_p":0.25,"repetition_penalty":1.0}'""",
-    "--generation-config", "vllm",
-    "--default-chat-template-kwargs.thinking=true",
-    "--default-chat-template-kwargs.reasoning_effort=max",
-    # Disable custom all-reduce (not needed on GB10 single-NIC)
-    "--disable-custom-all-reduce",
-    "--trust-remote-code",
-]
-
-# Persistent compile caches (speed up restarts — see guide §2/§7). The HF
-# cache is already mounted at /root/.cache/huggingface by the launcher.
-[[extra_mounts]]
-host = "~/.cache/vllm-ds4-dspark"   # compiled vLLM/attention kernels
-container = "/cache"
-
-[[extra_mounts]]
-host = "~/.cache/tilelang-ds4"      # DSpark speculative-decode kernels
-container = "/root/.tilelang"
-
-[env]
-# DSpark speculative decoding
-VLLM_DSPARK_REPLICATE_MARKOV_W1 = "1"
-# b12x MoE (this image's primary path)
-VLLM_USE_B12X_MOE = "1"
-# VLLM_USE_B12X_MOE = "0"              # b12x MoE OFF -> cutlass W4A8 (b12x MoE prefill stalls on GB10)
-
-# b12x suite = REQUIRED by this image
-VLLM_USE_B12X_MHC = "1"
-VLLM_USE_B12X_WO_PROJECTION = "1"
-VLLM_USE_B12X_FP8_GEMM = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_USE_V2_MODEL_RUNNER = "1"
-VLLM_USE_FLASHINFER_SAMPLER = "1"
-VLLM_USE_AOT_COMPILE = "1"
-VLLM_USE_MEGA_AOT_ARTIFACT = "0"
-VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
-VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
-VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
-B12X_MLA_SM120_UNIFIED = "1"
-B12X_DENSE_SPLITK_TURBO = "1"
-B12X_W4A16_TC_DECODE = "1"
-B12X_MHC_MAX_TOKENS = "16384"
-
-# GPU memory management
-CUTE_DSL_ARCH = "sm_121a"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-OMP_NUM_THREADS = "4"
-
-# Offline mode
-HF_HUB_OFFLINE = "1"
-
-# Compile-time arch hints
-TORCH_CUDA_ARCH_LIST = "12.1a"
-
-# NCCL overrides (cluster config ib_hca/socket_ifname used for most; recipe wins last)
-# NCCL_CROSS_NIC / NCCL_PROTO come from the cluster [nccl] section (2nodes-dac:
-# cross_nic=0, proto=LL,LL128,Simple) — matches the reference compose except
-# cross_nic (reference runs 1; our cluster config deliberately uses 0 for this
-# single-RoCE topology). NCCL_NVLS_ENABLE mirrors the reference compose.
-NCCL_P2P_LEVEL = "SYS"
-NCCL_CUMEM_ENABLE = "0"
-NCCL_NVLS_ENABLE = "0"
-NCCL_IB_TIMEOUT = "24"
-NCCL_IB_QPS_PER_CONNECTION = "8"
-NCCL_BUFFERSIZE = "16777216"
-NCCL_SOCKET_FAMILY = "inet"
-
-# PCIe allreduce not needed on GB10 single-NIC
-VLLM_ENABLE_PCIE_ALLREDUCE = "0"
-VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"
-
-# GPU device isolation (single GPU per node)
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"

recipes/gemma4-12b/recipe.toml

@@ -1,38 +0,0 @@
-engine = "vllm"
-name = "Gemma 4 12B IT (MTP)"
-docker_image = "vllm/vllm-openai:gemma4-unified"
-model = "google/gemma-4-12B-it"
-# docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
-# model = "RedHatAI/gemma-4-12B-it-NVFP4"
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-
-# Single-node recipe: Gemma 4 12B with BF16, FP8 KV cache,
-# MTP speculative decoding (5 tokens), 196K context.
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    "--dtype", "bfloat16",
-    "--kv-cache-dtype", "fp8",
-    "--gpu-memory-utilization", "0.75",
-    "--max-model-len", "196608",
-    "--max-num-seqs", "2",
-    "--max-num-batched-tokens", "8192",
-    "--enable-prefix-caching",
-    "--enable-chunked-prefill",
-    "--enable-auto-tool-choice",
-    "--tool-call-parser", "gemma4",
-    "--reasoning-parser", "gemma4",
-    "--generation-config", "vllm",
-    "--safetensors-load-strategy", "prefetch",
-    "--speculative-config", """
-'{"method":"mtp","model":"google/gemma-4-12B-it-assistant","num_speculative_tokens":5}'""",
-    "--limit-mm-per-prompt", """
-'{"image":4,"audio":1,"video":1}'""",
-    "--trust-remote-code",
-    "--served-model-name", "gemma4-12b",
-]
-
-[env]
-TRITON_CACHE_DIR = "/root/.cache/triton"

recipes/glm51/recipe.toml

@@ -1,34 +0,0 @@
-engine = "vllm"
-name = "GLM-5.1 NVFP8"
-docker_image = "vllm/vllm-openai:glm51-cu130"
-model = "lukealonso/GLM-5.1-NVFP4"
-
-required_nodes = 4
-distributed_backend = "mp"
-pipeline_parallel_size = 0
-
-args = [
-    "--speculative-config.method", "mtp",
-    "--speculative-config.num_speculative_tokens", "3",
-    "--tool-call-parser", "glm47",
-    "--reasoning-parser", "glm45",
-    "--enable-auto-tool-choice",
-    "--chat-template-content-format", "string",
-    "--attention-backend", "TRITON_MLA",
-    "--disable-flashinfer",
-    "--disable-custom-all-reduce",
-    "--enable-prefix-caching",
-    "--enable-chunked-prefill",
-    "--kv-cache-dtype", "fp8",
-    "--max-model-len", "32768",
-    "--max-num-batched-tokens", "6144",
-    "--max-num-seqs", "48",
-    "--gpu-memory-utilization", "0.88",
-    "--trust-remote-code",
-    "--served-model-name", "glm51", "default",
-]
-
-[env]
-NCCL_DEBUG = "WARN"
-NCCL_IB_MERGE_NICS = "1"
-NCCL_MAX_NCHANNELS = "12"

recipes/glm52-hybrid-fp8-mxfp4/recipe.toml

@@ -1,151 +0,0 @@
-# GLM-5.2 Hybrid FP8+NVFP4+MXFP4 — 4× DGX Spark (GB10, sm_121a), TP=4, DCP=4.
-#
-# Model: aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
-#   → https://huggingface.co/aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
-#
-# Mixed-precision GLM-5.2 checkpoint:
-#   - Attention (MLA projections, indexer):  block-FP8
-#   - Dense MLPs & shared experts:           NVFP4 (group-16)
-#   - Routed experts, layers 3–77:           MXFP4 (e2m1 + ue8m0 per-32)
-#   - MTP draft experts (layer 78):          MXFP4
-#   - Norms, gates, lm_head, MTP dense:      BF16
-#
-# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.2
-#   vLLM + b12x GB10 kernels with hybrid_mxfp4_ct quant config,
-#   index-authoritative loader, NVFP4 KV cache with per-token scaling,
-#   sparse-MLA CKV gather, boot-time kernel prewarm.
-#
-# PREREQUISITES:
-#   - Models cached on every node via HF_HOME:
-#       huggingface-cli download aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
-#       (382 GiB, requires free disk on each node)
-#
-# REFERENCE:
-#   https://forums.developer.nvidia.com/t/glm-5-2-hybrid-fp8-nvfp4-mxfp4-optimal-runtime-recipe/377598
-#   https://huggingface.co/aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
-
-engine = "vllm"
-name = "GLM-5.2 Hybrid FP8-MXFP4 (vLLM)"
-docker_image = "sparkrun-vllm-ds4-gb10:glm52"
-model = "aidendle94/GLM-5.2-Hybrid-FP8-MXFP4"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "10gb"
-docker_image_dir = "glm52"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # Model name
-    "--served-model-name", "glm52",
-    # General
-    "--trust-remote-code",
-    "--reasoning-parser", "glm45",
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--enable-prefix-caching",
-    # DCP (Decode Context Parallelism)
-    "--decode-context-parallel-size", "4",
-    "--dcp-comm-backend", "ag_rs",
-    # Engine backends
-    "--attention-backend", "B12X_MLA_SPARSE",
-    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
-    "--hf-overrides", """
-'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
-    # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
-    "--kv-cache-dtype", "nvfp4_ds_mla",
-    "--kv-cache-memory-bytes", "5500000000",
-    # Context & batch config
-    "--max-model-len", "262144",
-    "--max-num-batched-tokens", "2048",
-    "--max-num-seqs", "3",
-    "--gpu-memory-utilization", "0.89",
-    # Speculative decode — MTP k=4 with adaptive depths
-    # {{mtp_draft}} is substituted per-node with the resolved HF cache path
-    # (refs/main → latest snapshot) so symlinks resolve correctly.
-    "--speculative-config", """
-'{"model":"{{mtp_draft}}","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
-    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
-    "--compilation-config", """
-'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
-    # Timeout safety net (cold boot: ranks finish minutes apart)
-    "--distributed-timeout-seconds", "3600",
-    "--cpu-distributed-timeout-seconds", "3600",
-]
-
-# Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot).
-# `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent,
-# falls back to host "~/.cache/jit" (expanded to /home/<ssh_user>/.cache/jit).
-# `env` injects -e XDG_CACHE_HOME=/root/.cache/jit so vLLM/torch write there.
-[[extra_mounts]]
-alias = "jit_cache"
-host = "~/.cache/jit"
-container = "/root/.cache/jit"
-env = "XDG_CACHE_HOME"
-
-# MTP draft model — a subfolder of the main GLM-5.2 repo. Resolved at serve
-# time, per node, from the HF cache (refs/main → latest snapshot), so the
-# path survives `huggingface-cli download` updates without a fragile symlink.
-# The recipe references {{mtp_draft}} in --speculative-config; the launcher
-# substitutes the resolved full HF cache path per-node (no bind-mount — the
-# HF cache is already mounted at /root/.cache/huggingface, so symlinks
-# resolve correctly).
-[[extra_mounts]]
-repo = "aidendle94/GLM-5.2-Hybrid-FP8-MXFP4"
-subfolder = "mtp-draft"
-container = "mtp_draft"
-
-
-[env]
-# ── b12x suite ────────────────────────────────────────────────────────────
-VLLM_USE_V2_MODEL_RUNNER = "1"
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_DCP_SHARD_DRAFT = "1"
-VLLM_DCP_GLOBAL_TOPK = "1"
-VLLM_DSV4_INDEXER_SP = "1"
-
-# ── MLA CKV gather (sparse gather for decode) ─────────────────────────────
-VLLM_B12X_MLA_CKV_GATHER = "1"
-VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
-VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
-VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
-
-# ── Speculative decode ────────────────────────────────────────────────────
-VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
-
-# ── NVFP4 KV cache ────────────────────────────────────────────────────────
-VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
-
-# ── Timeout safety net ────────────────────────────────────────────────────
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-
-# ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ─────────────────
-# The image bakes a value that GLM-5.2's KV coordinator rejects.
-# Empty string = unset at runtime.
-VLLM_PREFIX_CACHE_RETENTION_INTERVAL = ""
-
-# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
-CUTE_DSL_ARCH = "sm_121a"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-
-# ── JIT cache ──────────────────────────────────────────────────────────────
-# Persisted via the [[extra_mounts]] jit_cache entry above: XDG_CACHE_HOME is
-# injected from that mount, pointing at /root/.cache/jit (no hardcoded dupe).
-
-# ── NCCL overrides ─────────────────────────────────────────────────────────
-# Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME
-# Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN
-NCCL_IB_DISABLE = "0"             # enable IB (override launcher default)
-NCCL_MAX_NCHANNELS = "4"
-NCCL_MIN_NCHANNELS = "4"
-NCCL_CUMEM_ENABLE = "0"
-
-# ── Compute ────────────────────────────────────────────────────────────────
-OMP_NUM_THREADS = "4"
-
-# Volumes not needed — model is a Hugging Face Hub ID, downloaded via HF_HOME

recipes/glm52-int4-int8/recipe.toml

@@ -1,109 +0,0 @@
-# GLM-5.2 QuantTrio Int4-Int8Mix — 4× DGX Spark / GB10 (sm_121), TP=4.
-#
-# EXPERIMENT: Using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6
-# instead of locally-built sparkrun-vllm-ds4-gb10:glm52.
-# Pre-built on Docker Hub with CUDA 13.2.1, custom NCCL, b12x suite.
-#
-# Model: QuantTrio/GLM-5.2-Int4-Int8Mix (mixed Int4/Int8, 10% expert-pruned)
-#   → https://huggingface.co/QuantTrio/GLM-5.2-Int4-Int8Mix
-#   Includes in-checkpoint MTP draft model.
-#
-# PREREQUISITES:
-#   - Weights cached on every node:
-#       huggingface-cli download QuantTrio/GLM-5.2-Int4-Int8Mix
-#
-# REFERENCE:
-#   https://forums.developer.nvidia.com/t/glm-5-2-int4-int8-on-8x-gb10-1200-t-s-prefill-33-54-t-s-avg-decode-generic-coding-structured/376831
-#   https://github.com/ciprianveg/gb10-glm-5.2
-
-engine = "vllm"
-name = "GLM-5.2 Int4-Int8Mix (vLLM, DS4 image)"
-docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6"
-model = "QuantTrio/GLM-5.2-Int4-Int8Mix"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "64gb"
-# docker_image_dir = "glm52"             # removed for experiment
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # KV cache & memory
-    "--kv-cache-dtype", "fp8",
-    "--max-model-len", "150000",
-    "--max-num-seqs", "1",
-    "--max-num-batched-tokens", "4096",
-    "--gpu-memory-utilization", "0.91",
-    # Engine backends
-    "--attention-backend", "B12X_MLA_SPARSE",
-    "--moe-backend", "b12x",
-    # Compilation
-    "--compilation-config", """
-'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
-    # Chunked prefill + prefix caching
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    # MTP speculative decoding (in-checkpoint draft model)
-#    "--speculative-config", """
-#'{"method":"mtp","num_speculative_tokens":4,"moe_backend":"b12x","draft_sample_method":"probabilistic"}'""",
-    # Tokenizer / reasoning (GLM-5.2 family)
-    "--tool-call-parser", "glm47",
-    "--reasoning-parser", "glm45",
-    "--enable-auto-tool-choice",
-    "--trust-remote-code",
-    "--served-model-name", "glm52",
-]
-
-[env]
-# b12x suite (image requires this)
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_MHC = "1"
-VLLM_USE_B12X_WO_PROJECTION = "1"
-VLLM_USE_B12X_FP8_GEMM = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_USE_V2_MODEL_RUNNER = "1"
-VLLM_USE_FLASHINFER_SAMPLER = "1"
-VLLM_USE_AOT_COMPILE = "1"
-VLLM_USE_MEGA_AOT_ARTIFACT = "0"
-VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
-VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
-B12X_MLA_SM120_UNIFIED = "1"
-B12X_DENSE_SPLITK_TURBO = "1"
-B12X_W4A16_TC_DECODE = "1"
-B12X_MHC_MAX_TOKENS = "16384"
-
-# GLM-5.2 support flags
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "256"
-
-# Timeout safety net for cold boot (cudagraph + JIT)
-VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
-
-# Offline mode — weights must be cached on each node
-HF_HUB_OFFLINE = "1"
-
-# Architecture
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"
-
-# GPU memory management
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-
-# NCCL overrides
-NCCL_CROSS_NIC = "1"
-NCCL_CUMEM_ENABLE = "0"
-NCCL_IGNORE_CPU_AFFINITY = "1"
-NCCL_DEBUG = "WARN"
-
-# PCIe allreduce not needed on GB10 single-NIC
-VLLM_ENABLE_PCIE_ALLREDUCE = "0"
-VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"
-
-# GPU device isolation
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"

recipes/glm52-mxfp4-experts-gptq/recipe.toml

@@ -1,148 +0,0 @@
-# GLM-5.2 MXFP4-Experts-GPTQ — 4× DGX Spark (GB10, sm_121a), TP=4.
-#
-# Model: aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
-#   → https://huggingface.co/aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
-#
-# GPTQ-quantized MXFP4 experts for GLM-5.2 (744B MoE, 256 experts/layer,
-# 8 activated per token, 78 layers, sparse-MLA). Experts quantized to
-# MXFP4 via GPTQ; non-expert layers (attention, indexer, router, lm_head)
-# kept in BF16.
-#
-# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.2
-#   vLLM + b12x suite with hybrid_mxfp4_ct quant config, index-authoritative
-#   loader, NVFP4 KV cache with per-token scaling, sparse-MLA CKV gather.
-#
-# PREREQUISITES:
-#   - Weights cached on every node via HF_HOME:
-#       huggingface-cli download aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
-#     (requires free disk on each node)
-#
-# REFERENCE:
-#   https://forums.developer.nvidia.com/t/glm-5-2-hybrid-fp8-nvfp4-mxfp4-optimal-runtime-recipe/377598
-#   https://huggingface.co/aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
-
-engine = "vllm"
-name = "GLM-5.2 MXFP4-Experts-GPTQ (vLLM)"
-docker_image = "sparkrun-vllm-ds4-gb10:glm52"
-model = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "10gb"
-docker_image_dir = "glm52"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # Model name
-    "--served-model-name", "glm52",
-    # General
-    "--trust-remote-code",
-    "--reasoning-parser", "glm45",
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--enable-prefix-caching",
-    # DCP (Decode Context Parallelism)
-    "--decode-context-parallel-size", "4",
-    "--dcp-comm-backend", "ag_rs",
-    # Engine backends
-    "--attention-backend", "B12X_MLA_SPARSE",
-    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
-    "--hf-overrides", """
-'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
-    # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
-    "--kv-cache-dtype", "nvfp4_ds_mla",
-    "--kv-cache-memory-bytes", "5500000000",
-    # Context & batch config
-    "--max-model-len", "262144",
-    "--max-num-batched-tokens", "2048",
-    "--max-num-seqs", "3",
-    "--gpu-memory-utilization", "0.89",
-    # Speculative decode — MTP k=4 with adaptive depths
-    # {{mtp_draft}} is substituted per-node with the resolved HF cache path
-    # (refs/main → latest snapshot) so symlinks resolve correctly.
-    "--speculative-config", """
-'{"model":"{{mtp_draft}}","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
-    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
-    "--compilation-config", """
-'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
-    # Timeout safety net (cold boot: ranks finish minutes apart)
-    "--distributed-timeout-seconds", "3600",
-    "--cpu-distributed-timeout-seconds", "3600",
-]
-
-# Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot).
-# `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent,
-# falls back to host "~/.cache/jit" (expanded to /home/<ssh_user>/.cache/jit).
-# `env` injects -e XDG_CACHE_HOME=/root/.cache/jit so vLLM/torch write there.
-[[extra_mounts]]
-alias = "jit_cache"
-host = "~/.cache/jit"
-container = "/root/.cache/jit"
-env = "XDG_CACHE_HOME"
-
-# MTP draft model — a subfolder of the main GLM-5.2 repo. Resolved at serve
-# time, per node, from the HF cache (refs/main → latest snapshot), so the
-# path survives `huggingface-cli download` updates without a fragile symlink.
-# The recipe references {{mtp_draft}} in --speculative-config; the launcher
-# substitutes the resolved full HF cache path per-node (no bind-mount — the
-# HF cache is already mounted at /root/.cache/huggingface, so symlinks
-# resolve correctly).
-[[extra_mounts]]
-repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
-subfolder = "mtp-draft"
-container = "mtp_draft"
-
-
-[env]
-# ── b12x suite ────────────────────────────────────────────────────────────
-VLLM_USE_V2_MODEL_RUNNER = "1"
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_DCP_SHARD_DRAFT = "1"
-VLLM_DCP_GLOBAL_TOPK = "1"
-VLLM_DSV4_INDEXER_SP = "1"
-
-# ── MLA CKV gather (sparse gather for decode) ─────────────────────────────
-VLLM_B12X_MLA_CKV_GATHER = "1"
-VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
-VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
-VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
-
-# ── Speculative decode ────────────────────────────────────────────────────
-VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
-
-# ── NVFP4 KV cache ────────────────────────────────────────────────────────
-VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
-
-# ── Timeout safety net ────────────────────────────────────────────────────
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-
-# ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ─────────────────
-# The image bakes a value that GLM-5.2's KV coordinator rejects.
-# Empty string = unset at runtime.
-VLLM_PREFIX_CACHE_RETENTION_INTERVAL = ""
-
-# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
-CUTE_DSL_ARCH = "sm_121a"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-
-# ── JIT cache ──────────────────────────────────────────────────────────────
-# Persisted via the [[extra_mounts]] jit_cache entry above: XDG_CACHE_HOME is
-# injected from that mount, pointing at /root/.cache/jit (no hardcoded dupe).
-
-# ── NCCL overrides ─────────────────────────────────────────────────────────
-# Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME
-# Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN
-NCCL_IB_DISABLE = "0"             # enable IB (override launcher default)
-NCCL_MAX_NCHANNELS = "4"
-NCCL_MIN_NCHANNELS = "4"
-NCCL_CUMEM_ENABLE = "0"
-
-# ── Compute ────────────────────────────────────────────────────────────────
-OMP_NUM_THREADS = "4"
-
-# Volumes not needed — model is a Hugging Face Hub ID, downloaded via HF_HOME

recipes/glm52-nf3-hybrid-context/recipe.toml

@@ -1,140 +0,0 @@
-# GLM-5.2 NF3-Hybrid — 🧠 CONTEXT LANE (DCP4, 800K max-len)
-# 4× DGX Spark (GB10, aarch64/sm_121a), TP=4, DCP=4.
-#
-# Model: madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
-#   → https://huggingface.co/madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
-#
-# Unpruned GLM-5.2 (753B/40B MoE, all 256 experts) in 3-format hybrid quant:
-#   327GB — top-64 experts/layer in NVFP4, remaining 192 in NF3 (3-bit),
-#   non-expert layers in MXFP8.
-#
-# EXPERIMENT: Using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6
-# instead of locally-built vllm-nf3-hybrid:probe. Pre-built on Docker Hub.
-# NOTE: production-hybrid-1.6 lacks NF3 hybrid overlay files
-# (hybrid_loader.py, nf3_kernel.py, nvfp4_kernel.py, mxfp8_tier.json).
-# If it fails, revert to vllm-nf3-hybrid:probe.
-#
-# DCP4 lane: KV sharded across all 4 nodes → 877K-token pool.
-#   Every decode step pays ag_rs allreduce on all 78 layers.
-#   MTP k=3 (needs fuse_allreduce_rms) — near-perfect acceptance ~3.0
-#   beat k=4/5 by ~8% on long network-taxed steps.
-#
-# Performance (measured 2026-07-07, 512-tok gens, temp 0, shallow ctx):
-#   c1: 21.4–21.6 tok/s  |  c2: 32.9  |  c3: 38.9
-#   c4: 45.4  |  c5: 37.4  |  c6: 40.1 aggregate
-#   KV pool: 877,056 tokens  |  max-model-len: 800,000
-#   Boot ~20 min, MTP k=3 acceptance ~3.0
-#
-# TUNING NOTES:
-#   - gpu-memory-utilization 0.88 (GB10 unified memory; author's 0.96 is
-#     dedicated-VRAM territory)
-#   - max-num-batched-tokens ≥2048 or NF3 graph capture breaks
-#   - max-cudagraph-capture-size must cover seqs×(1+k) = 4×4 = 16
-#   - VLLM_DCP_SHARD_DRAFT=1 mandatory for MTP draft KV under DCP
-#   - k is lane-specific: CONTEXT lane runs k=3 + fuse_allreduce_rms
-#
-# PREREQUISITES:
-#   - Weights cached on every node:
-#       huggingface-cli download madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
-#     (327GB, 334GB on disk, 96 safetensors shards)
-
-engine = "vllm"
-name = "GLM-5.2 NF3-Hybrid CONTEXT LANE (vLLM)"
-docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6"
-model = "madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-# docker_image_dir = "glm52-nf3-hybrid"             # removed for experiment
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # Quantization & KV cache
-    "--quantization", "modelopt_fp4",
-    "--kv-cache-dtype", "fp8",
-#    "--load-format", "auto",
-    "--load-format", "fastsafetensors",
-    # Context length (800K max-len, 877K KV pool)
-    "--max-model-len", "200000",
-    # Batch & memory
-    "--max-num-seqs", "4",
-    "--max-num-batched-tokens", "4096",
-    "--max-cudagraph-capture-size", "32",
-    "--gpu-memory-utilization", "0.84",
-    # Engine backends
-    #"--attention-backend", "B12X_MLA_SPARSE",
-    "--moe-backend", "b12x",
-    # Scheduling
-    "--async-scheduling",
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    # Compilation (CONTEXT: fuse_allreduce_rms for DCP network tax)
-    "--compilation-config", """
-'{"pass_config":{"fuse_allreduce_rms":true}}'""",
-    # Speculative decode — CONTEXT: MTP k=3 (near-perfect acceptance ~3.0)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":3,"moe_backend":"b12x","draft_sample_method":"probabilistic"}'""",
-    # Tool calling & reasoning (verified working from first boot)
-    "--tool-call-parser", "glm47",
-    "--reasoning-parser", "glm45",
-    "--enable-auto-tool-choice",
-    # General
-    "--trust-remote-code",
-    "--enable-flashinfer-autotune",
-    "--served-model-name", "glm-5.2",
-]
-
-[env]
-# Memory stability
-#PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-#VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-
-# ── DCP (Decode Context Parallelism) — CONTEXT: DCP4 ──────────────────────
-# KV sharded across all 4 nodes — 4× pool, decode pays ag_rs gather
-DECODE_CONTEXT_PARALLEL_SIZE = "4"
-DCP_COMM_BACKEND = "ag_rs"
-DCP_KV_CACHE_INTERLEAVE_SIZE = "1"
-
-# ── b12x suite (required) ─────────────────────────────────────────────────
-VLLM_USE_V2_MODEL_RUNNER = "1"
-VLLM_USE_B12X_FP8_GEMM = "1"
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_DCP_GLOBAL_TOPK = "1"
-VLLM_DCP_SHARD_DRAFT = "1"
-B12X_W4A16_TC_DECODE = "1"
-B12X_MOE_FORCE_A16 = "1"
-B12X_DENSE_SPLITK_TURBO = "1"
-
-# ── Hybrid loader knobs (v2) ──────────────────────────────────────────────
-HYBRID_TIER = "both"
-HYBRID_KEPT = "b12x_nf3"
-HYBRID_NF3 = "b12x_nf3"
-HYBRID_B12X_MAX_TOKENS = "4096"
-HYBRID_MXFP8_NATIVE = "1"
-# Real path to tier json inside the image (loader hardcodes /opt/venv layout)
-HYBRID_MXFP8_TIER_JSON = "/usr/local/lib/python3.12/dist-packages/mxfp8_tier.json"
-
-# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1a"
-
-# ── Offline mode (weights must be cached on each node) ────────────────────
-HF_HOME = "/root/.cache/huggingface"
-HF_HUB_OFFLINE = "1"
-TRANSFORMERS_OFFLINE = "1"
-
-# ── Timeout safety net (cold boot: cudagraph + JIT) ───────────────────────
-VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "3600"
-
-# ── NCCL / RoCE (cluster config ib_hca/socket_ifname used for most) ──────
-NCCL_CROSS_NIC = "1"
-NCCL_CUMEM_ENABLE = "0"
-NCCL_IGNORE_CPU_AFFINITY = "1"
-NCCL_DEBUG = "WARN"
-
-# ── Compute ────────────────────────────────────────────────────────────────
-OMP_NUM_THREADS = "4"

recipes/glm52-nf3-hybrid-fast/recipe.toml

@@ -1,133 +0,0 @@
-# GLM-5.2 NF3-Hybrid — 🏎️ FAST LANE (DCP1, 200K max-len)
-# 4× DGX Spark (GB10, aarch64/sm_121a), TP=4, DCP=1.
-#
-# Model: madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
-#   → https://huggingface.co/madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
-#
-# Unpruned GLM-5.2 (753B/40B MoE, all 256 experts) in 3-format hybrid quant:
-#   327GB — top-64 experts/layer in NVFP4, remaining 192 in NF3 (3-bit),
-#   non-expert layers in MXFP8.
-#
-# EXPERIMENT: Using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6
-# instead of locally-built vllm-nf3-hybrid:probe. Pre-built on Docker Hub.
-# NOTE: production-hybrid-1.6 lacks NF3 hybrid overlay files
-# (hybrid_loader.py, nf3_kernel.py, nvfp4_kernel.py, mxfp8_tier.json).
-# If it fails, revert to vllm-nf3-hybrid:probe.
-#
-# DCP1 lane: each sequence's KV cache lives entirely on one node → zero
-#   cross-node attention traffic per decode step. ~1/4 the KV pool of DCP4.
-#   Best for interactive chat, agents, throughput-sensitive workloads.
-#
-# Performance (measured 2026-07-07, 512-tok gens, temp 0, MTP acceptance 3.3–4.0):
-#   c1: 24–29 tok/s (content-dependent, peaks to 29.1 @ accept 4.02)
-#   c2: 36.6  |  c3: 45.7  |  c4: 54.2  |  c5: 59.2  |  c6: 67.4
-#   KV pool: 219,264 tokens  |  max-model-len: 200,000
-#
-# TUNING NOTES:
-#   - gpu-memory-utilization 0.88 (GB10 unified memory; author's 0.96 is
-#     dedicated-VRAM territory; 0.88 = safe for 200K + 12e9 KV allocation)
-#   - kv-cache-memory-bytes 12e9 per node caps KV pool at 219K tokens
-#   - max-num-batched-tokens ≥2048 or NF3 graph capture breaks
-#   - max-cudagraph-capture-size must cover seqs×(1+k) = 6×5 = 30
-#   - FAST lane uses MTP k=4, NO fuse_allreduce_rms (no benefit without
-#     DCP network tax)
-#   - k is lane-specific: FAST runs k=4, no fuse
-#
-# PREREQUISITES:
-#   - Weights cached on every node:
-#       huggingface-cli download madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
-#     (327GB, 334GB on disk, 96 safetensors shards)
-
-engine = "vllm"
-name = "GLM-5.2 NF3-Hybrid FAST LANE (vLLM)"
-docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.6"
-model = "madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "64gb"
-# docker_image_dir = "glm52-nf3-hybrid"             # removed for experiment
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # Quantization & KV cache
-    "--quantization", "modelopt_fp4",
-    "--kv-cache-dtype", "fp8",
-    "--load-format", "fastsafetensors",
-    # Context length (200K max-len, 219K KV pool)
-    "--max-model-len", "200000",
-    # Batch & memory — FAST lane: 6 concurrent, 12e9 KV bytes
-    "--max-num-seqs", "6",
-    "--max-num-batched-tokens", "4096",
-    "--max-cudagraph-capture-size", "32",
-    "--gpu-memory-utilization", "0.88",
-    "--kv-cache-memory-bytes", "12000000000",
-    # Engine backends
-    "--attention-backend", "B12X_MLA_SPARSE",
-    "--moe-backend", "b12x",
-    # Scheduling
-    "--async-scheduling",
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    # Speculative decode — FAST: MTP k=4 (accept 3.3–4.0, best aggregate)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":4,"moe_backend":"b12x","draft_sample_method":"probabilistic"}'""",
-    # Tool calling & reasoning (verified working from first boot)
-    "--tool-call-parser", "glm47",
-    "--reasoning-parser", "glm45",
-    "--enable-auto-tool-choice",
-    # General
-    "--trust-remote-code",
-    "--enable-flashinfer-autotune",
-    "--served-model-name", "glm-5.2",
-]
-
-[env]
-# ── DCP (Decode Context Parallelism) — FAST: DCP1 ────────────────────────
-# KV lives locally — zero cross-node attention traffic, ~1/4 pool of DCP4
-DECODE_CONTEXT_PARALLEL_SIZE = "1"
-DCP_COMM_BACKEND = "ag_rs"
-DCP_KV_CACHE_INTERLEAVE_SIZE = "1"
-
-# ── b12x suite (required) ─────────────────────────────────────────────────
-VLLM_USE_V2_MODEL_RUNNER = "1"
-VLLM_USE_B12X_FP8_GEMM = "1"
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_DCP_GLOBAL_TOPK = "1"
-VLLM_DCP_SHARD_DRAFT = "1"
-B12X_W4A16_TC_DECODE = "1"
-B12X_MOE_FORCE_A16 = "1"
-B12X_DENSE_SPLITK_TURBO = "1"
-
-# ── Hybrid loader knobs (v2) ──────────────────────────────────────────────
-HYBRID_TIER = "both"
-HYBRID_KEPT = "b12x_nf3"
-HYBRID_NF3 = "b12x_nf3"
-HYBRID_B12X_MAX_TOKENS = "4096"
-HYBRID_MXFP8_NATIVE = "1"
-HYBRID_MXFP8_TIER_JSON = "/usr/local/lib/python3.12/dist-packages/mxfp8_tier.json"
-
-# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1a"
-
-# ── Offline mode (weights must be cached on each node) ────────────────────
-HF_HOME = "/root/.cache/huggingface"
-HF_HUB_OFFLINE = "1"
-TRANSFORMERS_OFFLINE = "1"
-
-# ── Timeout safety net (cold boot: cudagraph + JIT) ───────────────────────
-VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "3600"
-
-# ── NCCL / RoCE (cluster config ib_hca/socket_ifname used for most) ──────
-NCCL_CROSS_NIC = "1"
-NCCL_CUMEM_ENABLE = "0"
-NCCL_IGNORE_CPU_AFFINITY = "1"
-NCCL_DEBUG = "WARN"
-
-# ── Compute ────────────────────────────────────────────────────────────────
-OMP_NUM_THREADS = "4"

recipes/glm52-nf4-lloyd-experts-gptq/recipe.toml

@@ -1,205 +0,0 @@
-# GLM-5.2 NF4 Lloyd-Experts-GPTQ — 4× DGX Spark (GB10, sm_121a), TP=4, DCP=4.
-#
-# Model: aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
-#   → https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
-#
-# Lloyd-codebook NF4 routed-expert overlay for GLM-5.2 (744B MoE, 256
-# experts/layer, 8 activated/token, 78 layers, sparse-MLA). Expert weights
-# quantized to 4-bit with a fitted Lloyd–Max codebook (−16.8% expert
-# reconstruction error vs E2M1/MXFP4 baseline on the same GPTQ pipeline).
-# Codes are indices into per-layer grids/L{n}.json — NOT the E2M1 table.
-#
-# LOADING via [composed_model]: the Lloyd repo is experts-only (no config,
-# no tokenizer, no non-expert weights). The launcher builds a /tmp dir per
-# node with symlinks into the HF cache and bind-mounts it at /root/model:
-#   1. aidendle94/GLM-5.2-MXFP4-Experts-GPTQ (base checkpoint — config.json
-#      with quant_method="hybrid_mxfp4_ct", hybrid-ct-*.safetensors non-expert
-#      weights, tokenizer, mtp-draft) → all files symlinked.
-#   2. aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ (Lloyd NF4 experts) →
-#      L*.safetensors symlinked, OVERWRITING the MXFP4 E2M1 experts
-#      (later sources win). Same filenames, same tensor names — the Lloyd
-#      codes replace the E2M1 codes.
-#   3. nf4_layers.json (generated) — tells the kernel which layers use the
-#      nf4_e8m0_k32 decode path vs the default fp4_e8m0_k32 E2M1 bit-twiddle.
-#
-# The symlinks point to container-side paths (/root/.cache/huggingface/...)
-# so they resolve inside the container where the HF cache is mounted. The
-# _NF4_CODEBOOK is GLOBAL and hardcoded in the b12x binary (fitted as the
-# mean of 71 per-layer Lloyd–Max tables) — the per-layer grids/L{n}.json
-# files are reference artifacts, NOT loaded at runtime.
-#
-# Image: sparkrun-vllm-ds4-gb10:glm52
-#   Built from docker-images/glm52/Dockerfile, which FROMs
-#   aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.4 (bakes the
-#   nf4_e8m0_k32 / nf4_packed kernel — inert without VLLM_NF4_LAYERS).
-#
-# PREREQUISITES:
-#   - Both repos cached on every node via HF_HOME:
-#       huggingface-cli download aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
-#       huggingface-cli download aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
-#     (MXFP4 base ~390 GB, Lloyd experts ~385 GB)
-#   No manual overlay or JSON creation — [composed_model] handles everything.
-
-engine = "vllm"
-name = "GLM-5.2 NF4 Lloyd-Experts-GPTQ (vLLM)"
-docker_image = "sparkrun-vllm-ds4-gb10:glm52"
-model = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "10gb"
-docker_image_dir = "glm52"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # Model name
-    "--served-model-name", "glm52",
-    # General
-    "--trust-remote-code",
-    "--reasoning-parser", "glm45",
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--enable-prefix-caching",
-    # DCP (Decode Context Parallelism)
-    "--decode-context-parallel-size", "4",
-    "--dcp-comm-backend", "ag_rs",
-    # Engine backends
-    "--attention-backend", "B12X_MLA_SPARSE",
-    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
-    "--hf-overrides", """
-'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
-    # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
-    "--kv-cache-dtype", "nvfp4_ds_mla",
-    "--kv-cache-memory-bytes", "5500000000",
-    # Context & batch config
-    "--max-model-len", "262144",
-    "--max-num-batched-tokens", "2048",
-    "--max-num-seqs", "3",
-    "--gpu-memory-utilization", "0.89",
-    # Speculative decode — MTP k=4 with adaptive depths
-    # /root/model/mtp-draft is symlinked from the MXFP4 base checkpoint
-    # (the composed dir includes it via source #1).
-    #
-    # NOTE: MTP + reasoning parser (glm45) + response_format (guided decoding)
-    # requires the vLLM structured-output reasoning-boundary fix (upstream
-    # PRs #44297 + #44993, forward-ported in docker-images/glm52/). Without
-    # it, the grammar FSM fails to advance across the reasoning-end marker
-    # (|◊|) under MTP, so the content channel is unconstrained and the model
-    # emits prose/markdown-fence prefixes before JSON. The fix is baked into
-    # the sparkrun-vllm-ds4-gb10:glm52 image.
-    "--speculative-config", """
-'{"model":"/root/model/mtp-draft","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
-    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
-    "--compilation-config", """
-'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
-    # Timeout safety net (cold boot: ranks finish minutes apart)
-    "--distributed-timeout-seconds", "3600",
-    "--cpu-distributed-timeout-seconds", "3600",
-]
-
-[composed_model]
-mount = "/root/model"
-
-[[composed_model.sources]]
-repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
-
-[[composed_model.sources]]
-repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ"
-glob = "L*.safetensors"
-# Exclude the 5 layers with mis-packed experts (per model card known-defect
-# workaround). These layers' Lloyd repo L{n}.safetensors contain 255 Lloyd-
-# coded experts + 1 E2M1 expert — decoding as E2M1 (which nf4_layers.json
-# does) would garbage-decode the 255 Lloyd experts. Instead, skip the Lloyd
-# overlay for these files so the MXFP4 base's all-E2M1 L{n}.safetensors
-# survives, and the excluded-from-NF4 E2M1 decode path matches their bytes.
-exclude = [
-    "L5.safetensors",
-    "L8.safetensors",
-    "L9.safetensors",
-    "L32.safetensors",
-    "L46.safetensors",
-]
-
-# NF4 layer list — all 75 MoE layers (3–77) EXCEPT the 5 layers with a
-# known packing defect (per model card 2026-07-27). Layers 5, 8, 9, 32, 46
-# each contain 1 expert whose bytes were packed with E2M1/MXFP4 nibbles
-# (quantizer arm won per-expert, but decoding is per-layer). These layers
-# are ALSO excluded from the Lloyd source above (exclude = [...]) so the
-# MXFP4 base's all-E2M1 L{n}.safetensors survives — decoding as E2M1
-# (which this list does) matches their bytes. Layer 10 had the same defect
-# but was already caught + fixed upstream by the packing guard, so it
-# stays in the NF4 set.
-# Ref: https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
-# ("Known defect: 5 experts are packed on the wrong grid")
-[[composed_model.files]]
-name = "nf4_layers.json"
-content = '{"layers":[3,4,6,7,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,33,34,35,36,37,38,39,40,41,42,43,44,45,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77]}'
-
-# Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot).
-# `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent,
-# falls back to host "~/.cache/jit" (expanded to /home/<ssh_user>/.cache/jit).
-# `env` injects -e XDG_CACHE_HOME=/root/.cache/jit so vLLM/torch write there.
-[[extra_mounts]]
-alias = "jit_cache"
-host = "~/.cache/jit"
-container = "/root/.cache/jit"
-env = "XDG_CACHE_HOME"
-
-
-[env]
-# ── NF4 Lloyd expert overlay ──────────────────────────────────────────────
-# Path to a JSON file listing which layers use the nf4_e8m0_k32 decode path
-# (vs the default fp4_e8m0_k32 E2M1 bit-twiddle). Generated by [composed_model]
-# at /root/model/nf4_layers.json — no placeholder needed (known container path).
-VLLM_NF4_LAYERS = "/root/model/nf4_layers.json"
-
-# ── b12x suite ────────────────────────────────────────────────────────────
-VLLM_USE_V2_MODEL_RUNNER = "1"
-VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_SPARSE_INDEXER = "1"
-VLLM_DCP_SHARD_DRAFT = "1"
-VLLM_DCP_GLOBAL_TOPK = "1"
-VLLM_DSV4_INDEXER_SP = "1"
-
-# ── MLA CKV gather (sparse gather for decode) ─────────────────────────────
-VLLM_B12X_MLA_CKV_GATHER = "1"
-VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
-VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
-VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
-
-# ── Speculative decode ────────────────────────────────────────────────────
-VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
-
-# ── NVFP4 KV cache ────────────────────────────────────────────────────────
-VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
-
-# ── Timeout safety net ────────────────────────────────────────────────────
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-
-# ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ─────────────────
-# The image bakes a value that GLM-5.2's KV coordinator rejects.
-# Empty string = unset at runtime.
-VLLM_PREFIX_CACHE_RETENTION_INTERVAL = ""
-
-# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
-CUTE_DSL_ARCH = "sm_121a"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-
-# ── JIT cache ──────────────────────────────────────────────────────────────
-# Persisted via the [[extra_mounts]] jit_cache entry above: XDG_CACHE_HOME is
-# injected from that mount, pointing at /root/.cache/jit (no hardcoded dupe).
-
-# ── NCCL overrides ─────────────────────────────────────────────────────────
-# Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME
-# Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN
-NCCL_IB_DISABLE = "0"             # enable IB (override launcher default)
-NCCL_MAX_NCHANNELS = "4"
-NCCL_MIN_NCHANNELS = "4"
-NCCL_CUMEM_ENABLE = "0"
-
-# ── Compute ────────────────────────────────────────────────────────────────
-OMP_NUM_THREADS = "4"

recipes/glm52/recipe.toml

@@ -1,35 +0,0 @@
-engine = "vllm"
-name = "GLM-5.2 NVFP4 (vLLM)"
-docker_image = "vllm/vllm-openai:glm52"
-model = "Mapika/GLM-5.2-NVFP4"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-
-# 410B NVFP4 model on 4× DGX Spark (GB10 Blackwell), 128 GB shared each.
-# TP=4 → ~103 GB/GPU for weights, ~25 GB for KV cache at 0.90 util.
-#entrypoint = ["vllm", "serve"]
-
-args = [
-#    "--kv-cache-dtype", "fp8",
-    "--gpu-memory-utilization", "0.85",
-    "--load-format", "safetensors",
-    "--enable-prefix-caching",
-#    "--speculative-config", """
-#'{"method":"mtp","num_speculative_tokens":5}'""",
-    "--tool-call-parser", "glm47",
-    "--reasoning-parser", "glm45",
-    "--enable-auto-tool-choice",
-#    "--enable-expert-parallel",
-    "--trust-remote-code",
-    "--served-model-name", "glm52",
-]
-
-[env]
-# Fast safetensors loading (GPU‑side decompress)
-SAFETENSORS_FAST_GPU = "1"
-# Allow long context lengths
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-# Blackwell MoE kernels
-#VLLM_USE_B12X_MOE = "1"

recipes/glm53-flash-dflash2-mxfp8/recipe.toml

@@ -1,137 +0,0 @@
-# GLM-5.3-Flash + DFlash2 draft (MXFP8) — speculative decoding — 2× Spark TP=2
-#
-# DFlash2-MXFP8 (local-inference-lab) is a ~1B BLOCK-DIFFUSION DRAFT model,
-# NOT a standalone LLM. It is loaded beside the TARGET
-# `local-inference-lab/GLM-5.3-Flash-NVFP4` (the lab quant) and every drafted
-# token is verified against it. Draft: 5 sliding-attention layers, block_size
-# 8, proposal length 7, reads target hidden states at layers 5/14/24/33/42.
-# Model card: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
-#
-# DRAFT LICENSE: source incoai/GLM-5.3-Flash-DFlash2 is CC BY-NC-ND 4.0
-# (non-commercial, no derivatives) — this image is a quantization of it; the
-# MXFP8 conversion itself is the "qualified" upstream artifact. Keep to
-# personal/non-commercial use.
-#
-# WHY THIS IMAGE: the card's qualified image (voipmonitor/vllm, TP=4 B12X on
-# 4× RTX PRO 6000) is amd64-only and its B12X attention/moe/linear/allreduce
-# backends do not exist on GB10. Neither DFlash2DraftModel nor the DFlash2
-# serialized-MXFP8 projection path was in our day-0 base. So we PORTED the
-# three DFlash2 source files from local-inference-lab/vllm
-# dev/jovian-judgement @ da4d7be6 (Apache-2.0, see docker-images/.../UPSTREAM.md)
-# onto our proven lab image → glm53-flash-dflash2:local. The day-0 base
-# already shipped W4A16_NVFP4 / MIXED_PRECISION / ModelOptMxFp8LinearMethod /
-# the dflash proposer, so the whole port is those 3 files + 1 registry line.
-#
-# DEVIATIONS from the card's qualified config (forced by hardware):
-#   * TP=4 → TP=2; no B12X attention/moe/linear/allreduce (fall back to the
-#     lab image's proven SM121 backends).
-#   * --enforce-eager ON: CUDA-graph+spec-decode quality regression is
-#     build-specific to this day-0 base (our 2026-08-28 A/B: graphs+MTP
-#     89 vs eager 92). Card runs CUDAGRAPH_MODE=FULL on a newer build.
-#   * TP=2 keeps draft KV in BF16 (auto), same as the card's config.
-# Adopted verbatim from the card: --speculative-config {method:dflash,
-# model:...DFlash2-MXFP8, num_speculative_tokens:7, kv_cache_dtype:auto}.
-#
-# =====================================================================
-# STATUS 2026-08-29: BLOCKED on GB10 — boots to weight-load + draft load
-# (both fine, MXFP8 draft kernel binds), then dies in KV-cache setup:
-#
-#   NotImplementedError: layers.3.self_attn.indexer.k_cache: page size is
-#   not divisible by the maximum page size and cannot be padded
-#   (indexes_kv_by_block_stride is False).
-#
-# MATH (block size cancels, so no --block-size / draft kv dtype fixes it):
-# per-token pages on this base are indexer 132 B, MLA fp8_ds_mla 660 B,
-# DFlash2 draft (8 kv heads x 128) bf16 4096 B / fp8 2048 B. vLLM unifies
-# target+draft pages by integer block-size scaling, which needs divisio-
-# nility: 4096%660 != 0, 2048%660 != 0. Tried and eliminated: draft
-# kv_cache_dtype fp8, target kv-cache-dtype variants, --disable-hybrid-
-# kv-cache-manager (then: "failed to convert the KV cache specs to one
-# unified type" — the kpool indexer spec is not coercible either).
-#
-# WHY THE FORK DOES NOT HIT THIS: its qualified run (voipmonitor image,
-# sm_120) uses B12X attention/indexer backends which opt into
-# indexes_kv_by_block_stride, so unify PADS the strided pages instead.
-# The b12x kernels are not part of the arm64 GB10 base and are compiled
-# for sm_120 consumer Blackwell.
-#
-# PATHS FORWARD (pick before spending boot time on this again):
-#   a) build local-inference-lab/vllm dev/jovian-judgement for arm64/cu130
-#      incl. b12x kernels (hours; sm_120->sm_121 compat unknown);
-#   b) wait for a GB10 indexer/MLA backend that sets
-#      indexes_kv_by_block_stride (upstream);
-#   c) give up — acceptance-rate comparison falls back to MTP (already
-#      characterized in specs/tech-architecture/glm53-recipe-state.md).
-#
-# What IS done and committed regardless: glm53-flash-dflash2:local image
-# (DFlash2DraftModel port + registry), patch_eagle3.py (target-side
-# EAGLE3/aux-hidden-state support for Glm5Next, fork parity), verified
-# config plumbing (spec JSON parsed, aux layers 5/14/24/33/42 extracted
-# from the draft config, FlashInferCutlassMxfp8LinearKernel bound).
-#
-# PREREQUISITES: BOTH models cached on both nodes:
-#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
-#   hf download local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
-
-engine = "vllm"
-name = "GLM-5.3-Flash + DFlash2 draft (vLLM, 2x Spark, spec-k7)"
-docker_image = "glm53-flash-dflash2:local"
-docker_image_dir = "glm53-flash-dflash2"
-# TARGET model — the DFlash2 draft model is referenced only via
-# --speculative-config below, never as the top-level model.
-model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-args = [
-    # Stable served name — quant-agnostic (shared across glm53 quants)
-    "--served-model-name", "glm53",
-    "--trust-remote-code",
-    "--distributed-executor-backend", "mp",
-    # GMU is ONLY a startup guardrail (KV pinned below): 0.90 demands
-    # 109.46 GiB free on GB10's 121.63 and driver reserve + page cache
-    # routinely leave ~108.5 -> flaky boot deaths. 0.82 always fits.
-    "--gpu-memory-utilization", "0.82",
-    # Pinned 10 GiB KV; the draft adds 5 small sliding-window layers on top
-    "--kv-cache-memory-bytes", "10737418240",
-    "--kv-cache-dtype", "fp8_ds_mla",
-    "--block-size", "256",
-    "--max-model-len", "262144",
-    "--max-num-seqs", "8",
-    "--max-num-batched-tokens", "4096",
-    "--enable-chunked-prefill",
-    "--async-scheduling",
-    "--enable-prefix-caching",
-    "--disable-custom-all-reduce",
-    # DFlash2 draft from local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8
-    # (block-diffusion drafter, proposal length 7 — verbatim from the card)
-    # Card: kv_cache_dtype auto (bf16 draft pages) — kept faithful.
-    "--speculative-config", """
-'{"method":"dflash","model":"local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8","num_speculative_tokens":7,"kv_cache_dtype":"auto"}'""",
-    "--language-model-only",
-    # See header: eager is the safe control on the day-0 base (graphs+spec
-    # quality regression). The card's newer build runs FULL graphs.
-    "--enforce-eager",
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--reasoning-parser", "glm45",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-VLLM_MLA_NOPE_PAD_ROPE = "1"
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-VLLM_USE_AOT_COMPILE = "0"
-TORCH_USE_RTLD_GLOBAL = "1"
-VLLM_NO_USAGE_STATS = "1"
-
-# Architecture (aarch64 / sm_121a DGX Spark)
-TORCH_CUDA_ARCH_LIST = "12.1a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1f"
-CUTE_DSL_ARCH = "sm_121a"
-CMAKE_CUDA_ARCHITECTURES = "121"
-OMP_NUM_THREADS = "4"

recipes/glm53-flash-nvfp4-4p67/recipe.toml

@@ -1,110 +0,0 @@
-# GLM-5.3-Flash NVFP4-4p67 — local-inference-lab HIGHER-FIDELITY quant (vLLM)
-# — 2× DGX Spark, TP=2, MTP-3.
-#
-# "4p67" ≈ 4.67 bits/weight, KLD ~0.04 vs bf16 — the lab's fidelity-grade
-# variant of GLM-5.3-Flash-NVFP4 (our glm53-flash-nvfp4-lab recipe is the
-# workhorse). Quant recipe per the model card:
-#   main model: routed experts NVFP4 (calibrated, ships A4 input scales)
-#               + attention/shared/indexer MXFP8
-#   MTP:        routed experts NVFP4 A16-only + attention/shared MXFP8
-# hf_quant_config declares the routed experts as quant_algo=W4A16_NVFP4 under
-# MIXED_PRECISION — i.e. they execute via the Marlin W4A16 GEMM (bf16
-# activations), not the A4 path.
-# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4-4p67
-#
-# IMAGE: the card says "should be used with local-inference-lab/vllm branch
-# dev/jovian-judgement" — every image they publish is amd64-only (RTX PRO
-# 6000 world, and the fork's model.py hard-imports the b12x backend). Our
-# day-0 lab base already carries the W4A16/MIXED_PRECISION/MXFP8 modelopt
-# methods; the ONE thing 4p67 additionally needs is fused-projection shard
-# resolution: it is the first glm5next checkpoint with quantized attention
-# projections, and without `packed_modules_mapping` on the model classes the
-# fused KDA/MLA/indexer linears load unquantized (KeyError
-# in_proj_qkvbfg_a.weight_scale after ~12 min of weight loading). So:
-# glm53-flash-nvfp4-4p67:local = lab image + patch_4p67.py (5-line class
-# attr on model.py + mtp.py, mirrors the fork's resolution path).
-#
-# Boot markers: weight load ~93 GiB/node (≈700 s) → "GPU KV cache size:
-# 1,164,369 tokens" → "Application startup complete" with the MTP speculator
-# attached. quantprobe lines here show algo=MXFP8 for attention projections
-# and W4A16_NVFP4 for routed experts (lab quant showed algo=None for attn —
-# that difference is the whole point of this image). Watch SpecDecoding
-# acceptance (MTP-3, target len ~3).
-#
-# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
-# :"low"} — omitted/other values select MAX reasoning effort (empty content on
-# short max_tokens).
-#
-# PREREQUISITES: weights cached on both nodes:
-#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4-4p67
-
-engine = "vllm"
-name = "GLM-5.3-Flash NVFP4-4p67 (vLLM, 2x Spark, MTP-3)"
-docker_image = "glm53-flash-nvfp4-4p67:local"
-docker_image_dir = "glm53-flash-nvfp4-4p67"
-model = "local-inference-lab/GLM-5.3-Flash-NVFP4-4p67"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-args = [
-    # Stable served name — quant-agnostic (shared across glm53 quants)
-    "--served-model-name", "glm53",
-    "--trust-remote-code",
-    "--distributed-executor-backend", "mp",
-    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
-    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
-    # page cache routinely leave ~108.5 -> flaky
-    # "Free memory ... less than desired" boot deaths (this was the class
-    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
-    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
-    # allocated (weights ~93 + pinned KV 10 + activations).
-    "--gpu-memory-utilization", "0.82",
-    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
-    # ~93 GiB weights/node; pinning also keeps warmup off the swap path
-    "--kv-cache-memory-bytes", "10737418240",
-    "--kv-cache-dtype", "fp8_ds_mla",
-    "--block-size", "256",
-    "--max-model-len", "524288",
-    "--max-num-seqs", "8",
-    "--max-num-batched-tokens", "4096",
-    "--enable-chunked-prefill",
-    "--async-scheduling",
-    "--enable-prefix-caching",
-    "--disable-custom-all-reduce",
-    # Proven lab boot: no flashinfer autotune, no CuTe-DSL warmup
-    "--kernel-config", """
-'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
-    # MTP-3 draft (MTP experts are W4A16_NVFP4 in this checkpoint; needs the
-    # modelopt.patch MTP-namespace fix baked into the image)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":3}'""",
-    "--language-model-only",
-    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
-    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
-    # graphs on = 89/100, eager = 92/100). Same build → same guard here;
-    # re-A/B only if the base image is ever refreshed past the fix.
-    "--enforce-eager",
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--reasoning-parser", "glm45",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-# NoPE-MLA rope-pad runtime gate (mod baked into the image)
-VLLM_MLA_NOPE_PAD_ROPE = "1"
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-VLLM_USE_AOT_COMPILE = "0"
-TORCH_USE_RTLD_GLOBAL = "1"
-VLLM_NO_USAGE_STATS = "1"
-
-# Architecture (aarch64 / sm_121a DGX Spark)
-TORCH_CUDA_ARCH_LIST = "12.1a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1f"
-CUTE_DSL_ARCH = "sm_121a"
-CMAKE_CUDA_ARCHITECTURES = "121"
-OMP_NUM_THREADS = "4"

recipes/glm53-flash-nvfp4-lab/recipe.toml

@@ -1,116 +0,0 @@
-# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
-#
-# local-inference-lab mixed quant: NVFP4 routed experts (layers 3–44,
-# group-16) + MXFP8 MTP experts (layer 45) + W4A4 input scales. Best agent
-# quant so far: tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8).
-# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
-#        (~186 GB → 92.72 GiB weights/node; KV pinned at 10 GiB, see below)
-#
-# Image: glm53-flash-nvfp4-lab:local (docker-images/glm53-flash-nvfp4-lab/ —
-# day-0 arm64 image + NoPE-MLA mod + the four FujitsuPolycom lab-checkpoint
-# patches: model naming shim, modelopt MTP namespace fix, persistent_topk
-# CC-12.x guards). Their launch config is mirrored below.
-# PROVENANCE: https://github.com/FujitsuPolycom/glm53-flash-tp2-spark
-#             (Apache-2.0; see patches/UPSTREAM.md) + forum 381350 #41.
-#
-# Deviations from their launch script (intentional):
-#   * no --kda-prefill-backend flashkda (needs their base-image patch — skipped)
-#   * --kv-cache-dtype fp8_ds_mla instead of `fp8` (same packed layout; this
-#     spelling is what the NoPE-MLA mod booted with on the LibertAIDAI stack)
-#   * --language-model-only + no mm-renderer patch/env (text-only)
-#   * no bundled NCCL/LD_PRELOAD, no NCCL_* fabric block (our switched fabric
-#     is configured via [nccl.ds4] + launcher GID auto-detect)
-#   * no --moe-backend marlin: their proven stack uses the default backend for
-#     THIS quant and reports clean evals. Sanity-check output for token loops
-#     anyway (the silent FP4 MoE corruption gotcha) before trusting a new node.
-#   * flashinfer autotune disabled via --kernel-config (their proven boot);
-#     VLLM_ENGINE_READY_TIMEOUT_S stays high regardless.
-#
-# Boot success markers (engine log, in order):
-#   "Model loading took 92.72 GiB" → "[quantprobe] ... algo=MXFP8" (MTP fix
-#   active; algo=None means broken) → "GPU KV cache size: 1,164,369 tokens" →
-#   "Application startup complete". ~700 s sequential weight load.
-#
-# Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
-# :"low"} — omitted/other values select MAX reasoning effort (empty content on
-# short max_tokens).
-#
-# Optional: instanttensor loader (~30 s weight load vs ~700 s):
-#   FROM glm53-flash-nvfp4-lab:local + pip install --no-deps --no-build-isolation
-#   instanttensor==0.1.9, then add --load-format instanttensor (--no-deps is
-#   required: resolution would downgrade the image's NCCL).
-#
-# PREREQUISITES: weights cached on both nodes:
-#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
-
-engine = "vllm"
-name = "GLM-5.3-Flash NVFP4-lab (vLLM, 2x Spark, MTP-3)"
-docker_image = "glm53-flash-nvfp4-lab:local"
-docker_image_dir = "glm53-flash-nvfp4-lab"
-model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-args = [
-    # Stable served name — quant-agnostic (shared across glm53 quants)
-    "--served-model-name", "glm53",
-    "--trust-remote-code",
-    "--distributed-executor-backend", "mp",
-    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
-    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
-    # page cache routinely leave ~108.5 -> flaky
-    # "Free memory ... less than desired" boot deaths (this was the class
-    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
-    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
-    # allocated (weights ~93 + pinned KV 10 + activations).
-    "--gpu-memory-utilization", "0.82",
-    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
-    # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
-    "--kv-cache-memory-bytes", "10737418240",
-    "--kv-cache-dtype", "fp8_ds_mla",
-    "--block-size", "256",
-    "--max-model-len", "524288",
-    "--max-num-seqs", "8",
-    "--max-num-batched-tokens", "4096",
-    "--enable-chunked-prefill",
-    "--async-scheduling",
-    "--enable-prefix-caching",
-    "--disable-custom-all-reduce",
-    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
-    "--kernel-config", """
-'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
-    # MTP-3 draft from the MXFP8 layer-45 shard (needs modelopt.patch in image)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":3}'""",
-    "--language-model-only",
-    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
-    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
-    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
-    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
-    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
-    "--enforce-eager",
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--reasoning-parser", "glm45",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-# NoPE-MLA rope-pad runtime gate (mod baked into the image)
-VLLM_MLA_NOPE_PAD_ROPE = "1"
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-# Upstream launch-script env (memory + compile knobs)
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-VLLM_USE_AOT_COMPILE = "0"
-TORCH_USE_RTLD_GLOBAL = "1"
-VLLM_NO_USAGE_STATS = "1"
-
-# Architecture (aarch64 / sm_121a DGX Spark)
-TORCH_CUDA_ARCH_LIST = "12.1a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1f"
-CUTE_DSL_ARCH = "sm_121a"
-CMAKE_CUDA_ARCHITECTURES = "121"
-OMP_NUM_THREADS = "4"

recipes/glm53-flash-nvfp4-redhat/recipe.toml

@@ -1,92 +0,0 @@
-# GLM-5.3-Flash NVFP4 — RedHatAI quant (vLLM) — 2× DGX Spark, TP=2.
-#
-# RedHatAI/LLM-Compressor quant: MoE experts W4A4 NVFP4 (group-16, local
-# per-group FP8 scales, compressed-tensors nvfp4-pack-quantized). Vision /
-# embeddings / lm_head kept original precision. Quantizes via regex targets,
-# so no name-prefix matching bugs.
-# Model: https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4
-#        (~177 GB → ~88 GB/node on 2 Sparks)
-# KV: pinned --kv-cache-memory-bytes 10 GiB (same as -lab, identical MLA
-#   geometry) → ~1.16 M-token pool @ fp8_ds_mla = 2.22× concurrency at the
-#   524288 ctx. GMU 0.90 matches the proven lab config; the pin overrides
-#   GMU-based KV sizing.
-#
-# ⚠ NO MTP DRAFTER IN THIS CHECKPOINT — the index has 146,745 tensors and
-#   zero layer-45/MTP weights. `--speculative-config mtp` WILL fail to load.
-#   This recipe is baseline decode only (~15 tok/s class on this arch —
-#   spec decode buys 1.3–2×; use glm53-flash-nvfp4[-lab] when MTP matters).
-#
-# Image: the -lab overlay (day-0 vLLM image + NoPE-MLA mod + FujitsuPolycom
-# arch patches). The sparse_attn_indexer CC-12.0 guards are REQUIRED at the
-# 512K ctx: at 262K the indexer topk fit 48 SMs, at 512K it wants 62 CTAs and
-# the fallback needs 128 KB smem (GB10 has 99 KB) → hard boot abort
-# (launch_persistent_topk oversubscribe, 2026-08-28). The patches are
-# arch-level, not quant-level: the modelopt quantprobe is a no-op for this
-# compressed-tensors checkpoint, the model.py naming shim anchors on
-# arch-internal names.
-#
-# GB10 gotchas handled (see glm53-flash-nvfp4 header for detail):
-#   --moe-backend marlin → if vLLM rejects marlin for W4A4 at startup (marlin
-#   is mainly W4A16), remove the flag — but then verify output is not
-#   token-loop garbage (the silent FLASHINFER_CUTLASS corruption) before
-#   trusting it; `--moe-backend triton` is the other candidate.
-#
-# GB10 gotcha (MEASURED): at 524288 ctx WITH flashinfer autotune on, rank 0
-# dies mid-boot on a driver OOM (NVRM NV_ERR_NO_MEMORY _memdescAllocInternal,
-# 2026-08-28 05:01) — autotune/graph scratch at the 512K verify shapes doesn't
-# fit the unified-memory headroom. The 262144 ctx + autotune combo booted
-# fine. Fix = the lab-proven boot: autotune + cutedsl warmup off (a perf
-# micro-opt only; the sparse-MLA kernels have a non-autotuned fallback) and
-# expandable_segments to cut fragmentation.
-#
-# PREREQUISITES: weights cached on both nodes:
-#   huggingface-cli download RedHatAI/GLM-5.3-Flash-NVFP4
-#
-# REFERENCE:
-#   https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4 (vendor vLLM cmd)
-#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark (mod)
-
-engine = "vllm"
-name = "GLM-5.3-Flash NVFP4-redhat (vLLM, 2x Spark, no MTP)"
-docker_image = "glm53-flash-nvfp4-lab:local"
-docker_image_dir = "glm53-flash-nvfp4-lab"
-model = "RedHatAI/GLM-5.3-Flash-NVFP4"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-args = [
-    # Stable served name — quant-agnostic (shared across glm53 quants)
-    "--served-model-name", "glm53",
-    "--trust-remote-code",
-    "--distributed-executor-backend", "mp",
-    "--gpu-memory-utilization", "0.90",
-    "--kv-cache-memory-bytes", "10737418240",
-    "--max-model-len", "524288",
-    # Scan/graph scratch scales with the prefill chunk: 8192 OOMs the driver
-    # at 512K ctx (05:38 NVRM OOM), 4096 is the lab-proven ceiling here.
-    "--max-num-batched-tokens", "4096",
-    "--block-size", "256",
-    "--max-num-seqs", "1",
-    "--kv-cache-dtype", "fp8_ds_mla",
-    "--language-model-only",
-    "--moe-backend", "marlin",
-    # NO --speculative-config: checkpoint ships no MTP weights (verified index)
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--reasoning-parser", "glm45",
-    "--kernel-config", """
-'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-VLLM_MLA_NOPE_PAD_ROPE = "1"
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"

recipes/glm53-flash-nvfp4/recipe.toml

@@ -1,96 +0,0 @@
-# GLM-5.3-Flash NVFP4 (vLLM) — 2× DGX Spark / GB10 (sm_121), TP=2.
-#
-# 320B/18B-active, hybrid 34 KDA linear-attention + 11 sparse-MLA layers,
-# glm5_next arch (NoPE-MLA, qk_rope_head_dim=0), in-checkpoint MTP head.
-# Text-only here (--language-model-only — the multimodal processor balloons
-# the API front-end to ~15.7 GB and the GB10 kernel OOM-kills it).
-#
-# WHY NVFP4 on 2 nodes (not the official FP8 on 4): LibertAIDAI's weight-only
-# NVFP4-A16 quant halves the footprint (~155 GB) so it fits 2 Sparks; the
-# sparse+linear attention keeps KV nearly free (KV pool ~738K tok at GMU 0.85).
-# Model: https://huggingface.co/LibertAIDAI/GLM-5.3-Flash-NVFP4
-#
-# Engine: vLLM day-0 per-model image (glm5_next support baked in). The arch
-# fails every sm_121 attention backend out of the box; a 4-part build-time
-# mod (docker-images/glm53-flash-nvfp4/patches/patch_mla.py) dresses GLM's
-# NoPE-MLA in DeepSeek's fp8_ds_mla 512+64 layout and compacts the sm120
-# sparse-MLA topk table 2176->2048. Applied at image build; this recipe
-# activates it at runtime via VLLM_MLA_NOPE_PAD_ROPE=1.
-#
-# TWO SILENT GB10 GOTCHAS (from the reference — both already handled here):
-#   1. SILENT FP4 MoE CORRUPTION: the auto-selected FLASHINFER_CUTLASS NVFP4
-#      MoE backend emits degenerate token loops with ZERO errors (model loads,
-#      /health green, every completion garbage). --moe-backend marlin fixes it.
-#      If a GB10 NVFP4 serve "works" but loops, check which MoE backend engaged
-#      before blaming the quant.
-#   2. A 60+ MIN "No available shared memory broadcast block found" loop at
-#      boot is NOT a hang — the worker is FlashInfer-autotuning a kernel flavor
-#      (tune persists to ~/.cache/vllm, pays once). VLLM_ENGINE_READY_TIMEOUT_S
-#      is raised to 3600 so the launcher doesn't give up first.
-#
-# Measured (median of 3, temp 0, single stream, 2x Spark, MTP-5 acceptance ~2.55):
-#   24.74 code / 30.30 structured / 19.58 prose tok/s; no-spec baseline 14.6 flat.
-#   256K needle-exact (30K/100K/200K deep); ~1150 tok/s prefill.
-#
-# Image: glm53-flash-nvfp4:local (built from vllm/vllm-openai:glm53-flash-arm64-cu130
-#   + the patch). `sm121 serve` builds it on the build node if the checksum label
-#   is stale (docker_image_dir below).
-#
-# PREREQUISITES:
-#   Weights cached on both nodes (NAS ~/.cache/huggingface):
-#       huggingface-cli download LibertAIDAI/GLM-5.3-Flash-NVFP4
-#
-# REFERENCE (provenance):
-#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark
-#   https://forums.developer.nvidia.com/t/381433
-
-engine = "vllm"
-name = "GLM-5.3-Flash NVFP4 (vLLM, 2x Spark)"
-docker_image = "glm53-flash-nvfp4:local"
-docker_image_dir = "glm53-flash-nvfp4"
-model = "LibertAIDAI/GLM-5.3-Flash-NVFP4"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-args = [
-    # Stable served name — quant-agnostic (shared across glm53 quants)
-    "--served-model-name", "glm53",
-    # GLM tokenizer
-    "--trust-remote-code",
-    # Executor (mp, no ray)
-    "--distributed-executor-backend", "mp",
-    # Memory — weights ~77.5 GiB of 119.6 GiB/node; sparse+linear KV nearly free
-    "--gpu-memory-utilization", "0.85",
-    "--max-model-len", "262144",
-    "--max-num-seqs", "1",
-    # DeepSeek-MLA KV layout the NoPE-pad mod dresses the attention in
-    "--kv-cache-dtype", "fp8_ds_mla",
-    # Text-only (multimodal processor OOM-kills the GB10 front-end)
-    "--language-model-only",
-    # GOTCHA 1: marlin avoids the silent FP4 MoE corruption
-    "--moe-backend", "marlin",
-    # MTP=5 (in-checkpoint drafter; acceptance length ~2.55)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":5}'""",
-    # Tokenizer / reasoning / tool calling (GLM family)
-    "--tool-call-parser", "glm47",
-    "--enable-auto-tool-choice",
-    "--reasoning-parser", "glm45",
-]
-
-[env]
-# Offline — weights must be cached on each node
-HF_HUB_OFFLINE = "1"
-# Activates the build-time NoPE-MLA rope-pad mod (mathematically a no-op:
-# zeros are rotation-free and add exactly 0 to every logit)
-VLLM_MLA_NOPE_PAD_ROPE = "1"
-# GOTCHA 2: survive the 60+ min FlashInfer-autotune boot window
-VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-
-# Architecture (aarch64 / sm_121a DGX Spark)
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"

recipes/laguna-m1-nvfp4/recipe.toml

@@ -1,37 +0,0 @@
-engine = "vllm"
-name = "Laguna M.1 NVFP4 (vLLM)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
-model = "poolside/Laguna-M.1-NVFP4"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-
-# 225B total / 23B active MoE — NVFP4 quantized (~131B stored).
-# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
-# TP=2 → ~66 GB/node weights, room for 262K KV cache.
-# Quantization auto-detected from model config.
-
-entrypoint = ["vllm", "serve"]
-
-args = """
---gpu-memory-utilization 0.8
---max-model-len 262144
---load-format instanttensor
---enable-prefix-caching
---tool-call-parser poolside_v1
---reasoning-parser poolside_v1
---enable-auto-tool-choice
---trust-remote-code
---kv-cache-dtype fp8
---enable-flashinfer-autotune
---served-model-name laguna
-"""
-
-[env]
-SAFETENSORS_FAST_GPU = "1"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1a"
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-VLLM_FLOAT32_MATMUL_PRECISION = "high"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"

recipes/laguna-s2.1-nvfp4/recipe.toml

@@ -1,87 +0,0 @@
-# Laguna S 2.1 NVFP4 — 1× DGX Spark (GB10), TP=1.
-#
-# Model: poolside/Laguna-S-2.1-NVFP4
-#   → https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
-#
-# 117.6B total / 8.5B active MoE — NVFP4 quantized (~71 GB stored).
-# Fits on a single DGX Spark (128 GB unified memory).
-# 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
-# KV cache in FP8.
-#
-# DFlash speculative decoding with quantization-matched draft model:
-#   poolside/Laguna-S-2.1-DFlash-NVFP4
-#   → 15 speculative tokens, method dflash.
-#
-# REFERENCE:
-#   https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
-#
-# DEPLOYMENT: tmux (host venv). The pre-built Docker image
-# (ghcr.io/timothystewart6/vllm-gb10:v0.26.0-gb10.1) ships the wrong
-# flashinfer version, so Docker is non-functional. The sustainable path
-# is a host venv with the exact deps from the model card.
-#
-# ACTIVE (risky): vLLM 0.26.0 + flashinfer 0.6.16.dev20260729.
-#   0.6.16 adds SM121 (GB10/DGX Spark) NVFP4 attention support
-#   (flashinfer PR #3897) and Expert Parallelism, but introduces
-#   API changes that vLLM 0.26.0 (which requires flashinfer 0.6.14)
-#   may not fully track. Tested on dgx1.lan — monitor for MoE cubin
-#   validation regressions ("reject incompatible output-scale cubins").
-#
-# SAFE (commented out): vLLM 0.25.1 + flashinfer 0.6.15.dev20260728.
-#   Same minor as the original 0.6.15.dev20260712 — just 16 days of
-#   bug fixes. No API changes. Drop-in replacement.
-#
-#   uv venv ~/venvs/vllm025 -p 3.12
-#   uv pip install vllm==0.25.1 --torch-backend=cu130
-#   uv pip install flashinfer-python==0.6.15.dev20260728 flashinfer-cubin==0.6.15.dev20260728 flashinfer-jit-cache==0.6.15.dev20260728 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
-#   hf download poolside/Laguna-S-2.1-NVFP4
-#   hf download poolside/Laguna-S-2.1-DFlash-NVFP4
-#
-# The `venv` field below is used by tmux to activate the venv before
-# running the entrypoint.
-
-engine = "vllm"
-name = "Laguna S 2.1 NVFP4 (vLLM)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.26.0-gb10.1"
-model = "poolside/Laguna-S-2.1-NVFP4"
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-deploy = "tmux"
-venv = "~/venvs/vllm026"
-venv_setup = """
-    uv venv ~/venvs/vllm026 -p 3.12
-    uv pip install vllm==0.26.0 --torch-backend=cu130
-    uv pip install flashinfer-python==0.6.16.dev20260729 flashinfer-cubin==0.6.16.dev20260729 flashinfer-jit-cache==0.6.16.dev20260729 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
-    hf download poolside/Laguna-S-2.1-NVFP4
-    hf download poolside/Laguna-S-2.1-DFlash-NVFP4
-"""
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    "--gpu-memory-utilization", "0.85",
-    "--max-model-len", "262144",
-    "--load-format", "instanttensor",
-    "--enable-prefix-caching",
-    "--tool-call-parser", "poolside_v1",
-    "--reasoning-parser", "poolside_v1",
-    "--enable-auto-tool-choice",
-    "--trust-remote-code",
-    "--kv-cache-dtype", "fp8",
-    "--enable-flashinfer-autotune",
-    "--max-num-seqs", "32",
-    "--override-generation-config", "'{\"temperature\":0.7,\"top_p\":0.95}'",
-    # DFlash speculative decoding with quantization-matched draft
-    "--speculative-config", "'{\"model\":\"poolside/Laguna-S-2.1-DFlash-NVFP4\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}'",
-    "--served-model-name", "laguna-s-2.1", "laguna", "default",
-]
-
-[env]
-SAFETENSORS_FAST_GPU = "1"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-VLLM_FLOAT32_MATMUL_PRECISION = "high"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-MAX_JOBS = "4"

recipes/mimo-v2.5-nvfp4/recipe.toml

@@ -1,58 +0,0 @@
-engine = "vllm"
-name = "MiMo-V2.5 Omni NVFP4 (vLLM)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
-model = "lukealonso/MiMo-V2.5-NVFP4"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-
-# 1M context · NVFP4 4-bit KV · MTP1 · ~30 tok/s
-# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
-# REQUIRES patched vLLM container with mods (nvfp4-kv-diffkv, fix-mimo-v2-vllm, etc.)
-# Stock vLLM will reject NVFP4 KV or OOM.
-
-entrypoint = ["vllm", "serve"]
-
-args = """
---served-model-name MiMo-V2.5-NVFP4
---trust-remote-code
---dtype auto
---load-format safetensors
---hf-overrides {"architectures":["MiMoV2OmniForCausalLM"]}
---limit-mm-per-prompt {"image":4,"video":1,"audio":1}
---mm-encoder-tp-mode data
---attention-backend triton_attn_diffkv
---kv-cache-dtype nvfp4
---gpu-memory-utilization 0.84
---max-model-len 1000000
---max-num-batched-tokens 4096
---max-num-seqs 4
---block-size 32
---enable-prefix-caching
---enable-chunked-prefill
---enable-auto-tool-choice
---tool-call-parser mimo
---reasoning-parser mimo
---speculative-config {"method":"mtp","num_speculative_tokens":1}
---enforce-eager
-"""
-
-# TP/PP/nnodes/node-rank/master-addr are
-# auto-generated by the launcher from the recipe's top-level fields.
-# mp backend auto-detected by vLLM >0.20, no --distributed-executor-backend flag needed.
-
-[env]
-# Memory stability
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-
-# NVFP4 weights + KV + WMMA decode
-VLLM_NVFP4_GEMM_BACKEND = "flashinfer-cutlass"
-VLLM_USE_FLASHINFER_MOE_FP4 = "1"
-VLLM_FLASHINFER_MOE_BACKEND = "throughput"
-VLLM_NVFP4_INLINE = "1"
-VLLM_WMMA_DECODE = "1"
-VLLM_WMMA_INSPECT = "0"
-VLLM_WMMA_COMPARE = "0"
-VLLM_WMMA_AUTHCOMPARE = "0"

recipes/minimax-m3-mxfp8/recipe.toml

@@ -1,37 +0,0 @@
-engine = "vllm"
-name = "MiniMax M3 MXFP8 (vLLM)"
-docker_image = "vllm/vllm-openai:minimax-m3"
-model = "MiniMaxAI/MiniMax-M3-MXFP8"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-
-# MXFP8 quantized MiniMax M3 (~440B params, ~23B active).
-# 4 nodes × 1 GPU (DGX Spark GB10 Blackwell), TP=4 across nodes.
-# MSA sparse attention requires --block-size 128.
-# Text-only mode skips the vision encoder to save VRAM.
-
-#entrypoint = ["vllm", "serve"]
-
-args = [
-    "--block-size", "128",
-    "--kv-cache-dtype", "fp8",
-    "--gpu-memory-utilization", "0.85",
-    "--load-format", "safetensors",
-    "--enable-prefix-caching",
-    "--tool-call-parser", "minimax_m3",
-    "--reasoning-parser", "minimax_m3",
-    "--enable-auto-tool-choice",
-    "--language-model-only",
-    "--enable-expert-parallel",
-    "--trust-remote-code",
-    "--served-model-name", "minimax-m3", "default",
-]
-
-[env]
-# M3 supports thinking via chat_template_kwargs.thinking_mode
-# enabled / adaptive / disabled — pass per-request in extra_body
-SAFETENSORS_FAST_GPU = "1"
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-VLLM_USE_B12X_MOE = "1"

recipes/minimax-m3-nvfp4/recipe.toml

@@ -1,37 +0,0 @@
-engine = "vllm"
-name = "MiniMax M3 NVFP (vLLM)"
-docker_image = "vllm/vllm-openai:minimax-m3"
-model = "Mapika/MiniMax-M3-NVFP4"
-required_nodes = 4
-tensor_parallel_size = 4
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-
-# NVFP4 quantized MiniMax M3 (~440B params, ~23B active).
-# 4 nodes × 1 GPU (DGX Spark GB10 Blackwell), TP=4 across nodes.
-# MSA sparse attention requires --block-size 128.
-# Text-only mode skips the vision encoder to save VRAM.
-
-#entrypoint = ["vllm", "serve"]
-
-args = [
-    "--block-size", "128",
-    "--kv-cache-dtype", "fp8",
-    "--gpu-memory-utilization", "0.85",
-    "--load-format", "safetensors",
-    "--enable-prefix-caching",
-    "--tool-call-parser", "minimax_m3",
-    "--reasoning-parser", "minimax_m3",
-    "--enable-auto-tool-choice",
-    "--language-model-only",
-    "--enable-expert-parallel",
-    "--trust-remote-code",
-    "--served-model-name", "minimax-m3", "default",
-]
-
-[env]
-# M3 supports thinking via chat_template_kwargs.thinking_mode
-# enabled / adaptive / disabled — pass per-request in extra_body
-SAFETENSORS_FAST_GPU = "1"
-VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-VLLM_USE_B12X_MOE = "1"

recipes/minimax-m3/recipe.toml

@@ -1,29 +0,0 @@
-engine = "sglang"
-name = "Minimax-M3-v0 NVFP4"
-docker_image = "scitrera/dgx-spark-sglang-mm:v0"
-#model = "sparkarena/Minimax-M3-v0-NVFP4"
-model = "Mapika/MiniMax-M3-NVFP4"
-required_nodes = 4
-tensor_parallel_size = 4
-distributed_backend = "mp"
-
-entrypoint = ["sglang", "serve"]
-
-args = [
-    "--mem-fraction-static", "0.8",
-    "--context-length", "524288",
-    "--chunked-prefill-size", "8192",
-    "--fp4-gemm-backend", "cutlass",
-    "--load-format", "instanttensor",
-    "--kv-cache-dtype", "auto",
-    "--reasoning-parser", "minimax-m3",
-    "--tool-call-parser", "minimax-m3",
-    "--moe-runner-backend", "triton",
-    "--trust-remote-code",
-    "--cuda-graph-max-bs", "16",
-    "--cuda-graph-bs", "1", "2", "4", "8", "12", "16",
-    "--served-model-name", "minimax-m3",
-]
-
-[env]
-TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"

recipes/qwen-agentworld-35b/recipe.toml

@@ -1,25 +0,0 @@
-engine = "vllm"
-name = "Qwen-AgentWorld-35B-A3B (vLLM)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
-model = "Qwen/Qwen-AgentWorld-35B-A3B"
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-
-# Solo: ~70 GB BF16 weights, only 10 GA layers need KV cache (DeltaNet is linear).
-# Use: sm121-launcher serve --recipe qwen-agentworld-35b --node dgx1.lan
-# Nodes: sm121-launcher serve --recipe qwen-agentworld-35b --cluster 2nodes-dac
-# Requires --language-model-only (no vision weights in checkpoint).
-
-entrypoint = ["vllm", "serve"]
-
-args = """
---max-model-len 262144
---reasoning-parser qwen3
---language-model-only
---trust-remote-code
---gpu-memory-utilization 0.90
---enable-prefix-caching
---enable-auto-tool-choice
---tool-call-parser qwen3_xml
---served-model-name qwen-agentworld-35b
-"""

recipes/qwen38-27b-fp8-dspark/recipe.toml

@@ -1,92 +0,0 @@
-# Qwen3.8-27B (FP8 + DSpark draft) — 1× DGX Spark / GB10 (sm_121), TP=1.
-#
-# DSpark: a dedicated semi-autoregressive draft model (~2.6 GB, 5 layers,
-# block_size 7) proposes blocks of tokens that the full model verifies in one
-# weight-read. 3× cheaper per guess than the in-checkpoint MTP head, so it wins
-# on edit-heavy / structured workloads (blog: 45-48 t/s edit, 17 t/s free-gen,
-# 66 t/s aggregate @ c10 on the FP8 target). DSpark blocks are verifier-rejected,
-# so no quality risk.
-#
-# Target: Qwen/Qwen3.8-27B-FP8 (official FP8 quant, 28.75 GB).
-# Draft:  RadixArk/Qwen3.8-27B-DSpark (arch DSparkDraftModel, bf16, 5 layers,
-#         dspark_block_size 7). Downloaded via hfrs into the node HF cache.
-#
-# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
-#   vLLM 0.27.1 (installed 0.27.2.dev0+g6e448d0ea), supports method "dspark"
-#   (SpeculativeMethod Literal includes dspark) + Qwen3DSparkModel draft.
-#
-# PREREQUISITES:
-#   Target + draft cached on the node (dgx1 ~/.cache/huggingface → NAS):
-#       hfrs download Qwen/Qwen3.8-27B-FP8
-#       hfrs download RadixArk/Qwen3.8-27B-DSpark
-#
-# REFERENCE:
-#   https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark  (DSpark section)
-#   https://huggingface.co/RadixArk/Qwen3.8-27B-DSpark
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
-
-engine = "vllm"
-name = "Qwen3.8-27B (FP8 + DSpark)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
-model = "Qwen/Qwen3.8-27B-FP8"
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # KV cache & memory
-    "--kv-cache-dtype", "fp8",
-    "--max-model-len", "262144",
-    "--max-num-seqs", "6",
-    "--max-num-batched-tokens", "8192",
-    "--gpu-memory-utilization", "0.75",
-    "--served-model-name", "qwen3.8-27b",
-    # Engine backends (dense GQA — FlashInfer)
-    "--attention-backend", "flashinfer",
-    # Compilation / perf
-    "--optimization-level", "3",
-    "--performance-mode", "throughput",
-    # Chunked prefill + prefix caching
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    # DSpark speculative decoding — dedicated draft model (not the in-checkpoint
-    # MTP head). num_speculative_tokens must equal the draft's block size (7).
-    # The draft lives in a local dir (arch-patched copy of RadixArk/Qwen3.8-27B-DSpark
-    # re-labelled `Qwen3DSparkModel` so vLLM routes it to qwen3_dspark, not the
-    # DeepSeek-V4 DSpark code path) bind-mounted at /dspark-draft below.
-    "--speculative-config", """
-'{"method":"dspark","model":"/dspark-draft","num_speculative_tokens":7}'""",
-    # Tokenizer / reasoning (Qwen3 family)
-    "--tool-call-parser", "qwen3_xml",
-    "--reasoning-parser", "qwen3",
-    "--enable-auto-tool-choice",
-    "--generation-config", "auto",
-    "--default-chat-template-kwargs.preserve_thinking=true",
-    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
-    # practices)
-    "--override-generation-config", """
-'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
-    "--trust-remote-code",
-]
-
-# DSpark draft model — arch-patched local copy of RadixArk/Qwen3.8-27B-DSpark,
-# mounted at /dspark-draft so vLLM loads it via the qwen3_dspark draft class.
-[[extra_mounts]]
-host = "/home/kilork/dspark-draft"
-container = "/dspark-draft"
-
-
-[env]
-HF_HUB_OFFLINE = "1"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-fp8/recipe.toml

@@ -1,74 +0,0 @@
-# Qwen3.8-27B (FP8) — 1× DGX Spark / GB10 (sm_121), TP=1.
-#
-# Official Qwen FP8 quant (29 GB). Dense 27B, GQA, hybrid DeltaNet+Attention,
-# MTP-1. Text + vision. As fast as NVFP4 typically, with full FP8 quality.
-#
-# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
-#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
-#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
-#
-# Model: Qwen/Qwen3.8-27B-FP8
-#   https://huggingface.co/Qwen/Qwen3.8-27B-FP8
-#
-# PREREQUISITES:
-#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
-#       huggingface-cli download Qwen/Qwen3.8-27B-FP8
-#
-# REFERENCE:
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
-
-engine = "vllm"
-name = "Qwen3.8-27B (FP8)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
-model = "Qwen/Qwen3.8-27B-FP8"
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # KV cache & memory
-    "--kv-cache-dtype", "fp8",
-    "--max-model-len", "262144",
-    "--max-num-seqs", "6",
-    "--max-num-batched-tokens", "8192",
-    "--gpu-memory-utilization", "0.75",
-    "--served-model-name", "qwen3.8-27b",
-    # Engine backends (dense GQA — FlashInfer)
-    "--attention-backend", "flashinfer",
-    # Compilation / perf
-    "--optimization-level", "3",
-    "--performance-mode", "throughput",
-    # Chunked prefill + prefix caching
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    # MTP speculative decoding (in-checkpoint draft)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":5}'""",
-    # Tokenizer / reasoning (Qwen3 family)
-    "--tool-call-parser", "qwen3_xml",
-    "--reasoning-parser", "qwen3",
-    "--enable-auto-tool-choice",
-    "--generation-config", "auto",
-    "--default-chat-template-kwargs.preserve_thinking=true",
-    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
-    # practices)
-    "--override-generation-config", """
-'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
-    "--trust-remote-code",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-int4/recipe.toml

@@ -1,96 +0,0 @@
-# Qwen3.8-27B (Pilcothink Int4 AutoRound + MTP) — 1× DGX Spark / GB10 (sm_121),
-# TP=1.
-#
-# Mixed 4-bit AutoRound quant (20 GB) tuned for a single DGX Spark. Sensitive
-# layers kept FP8/FP16; vision layers unquantized. MTP quantized + included.
-# Dense 27B, GQA, hybrid DeltaNet+Attention. Text + vision. The community has
-# heavily tuned vLLM flags for this exact build (thread 380248 post #39).
-#
-# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
-#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
-#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
-#
-# Model: Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound
-#   https://huggingface.co/Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound
-#
-# PREREQUISITES:
-#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
-#       huggingface-cli download Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound
-#
-# REFERENCE:
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
-
-engine = "vllm"
-name = "Qwen3.8-27B (Int4 AutoRound)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
-model = "Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound"
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # KV cache & memory
-    "--kv-cache-dtype", "fp8",
-    "--max-model-len", "262144",
-    "--max-num-seqs", "6",
-    "--max-num-batched-tokens", "8192",
-    "--gpu-memory-utilization", "0.75",
-    "--served-model-name", "qwen3.8-27b",
-    # Engine backends (dense GQA — FlashInfer)
-    "--attention-backend", "flashinfer",
-    # Compilation / perf (AutoRound community tuning)
-    "--optimization-level", "3",
-    "--performance-mode", "throughput",
-    # Chunked prefill + prefix caching
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    # MTP speculative decoding (quantized draft)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":4}'""",
-    # Tokenizer / reasoning (Qwen3 family)
-    "--tool-call-parser", "qwen3_xml",
-    "--reasoning-parser", "qwen3",
-    "--enable-auto-tool-choice",
-    "--generation-config", "auto",
-    "--default-chat-template-kwargs.preserve_thinking=true",
-    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
-    # practices)
-    "--override-generation-config", """
-'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
-    "--trust-remote-code",
-]
-
-[env]
-# AutoRound community tuning (thread 380248 post #39): faster load + higher
-# compile level for the 4-bit kernels.
-VLLM_TORCH_COMPILE_LEVEL = "3"
-VLLM_TRITON_COMPILE_ON_LOAD = "1"
-VLLM_WARMUP_BEFORE_SERVING = "1"
-VLLM_SKIP_WARMUP = "false"
-TORCH_MATMUL_PRECISION = "high"
-NVIDIA_FORWARD_COMPAT = "1"
-NVIDIA_DISABLE_REQUIRE = "1"
-CUDA_DEVICE_MAX_CONNECTIONS = "1"
-VLLM_MARLIN_USE_ATOMIC_ADD = "1"
-FLASHINFER_DISABLE_VERSION_CHECK = "1"
-
-# Offline mode
-HF_HUB_OFFLINE = "1"
-
-# Architecture
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"
-
-# GPU memory management
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-
-# GPU device isolation
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-nvfp4-sglang-dflash2/recipe.toml

@@ -1,75 +0,0 @@
-# Qwen3.8-27B (RadixArk NVFP4 + DFlash2 on SGLang) — 1× DGX Spark / GB10,
-# TP=1.
-#
-# SGLang + DFlash2 block-diffusion speculative decoding. Per the community
-# (MiaAI / forum 380350), DFlash2 on NVFP4 ties DSpark on code and *beats it*
-# on prose/essay and short-chat once counted right. This is the fork/engine
-# the forum's high tok/s numbers (52-61 code single, 87 TP=2) came from — NOT
-# the stock DSpark path.
-#
-# Image: lmsysorg/sglang:qwen38-27b-dflash2-minoverlay — a locally-built overlay
-#   of the 5 DFlash2 modules onto the cookbook qwen38-27b image. The cookbook
-#   tag itself predates DFlash2 (upstream merged commit c14312a66 on
-#   2026-08-19, after every published tag), so the modules are overlaid.
-#   The minoverlay modules already bake in the quantized-head selector fix
-#   (should_apply_lm_head_quant_method) so DFLASH runs the NVFP4 lm_head in
-#   place — no dense dequant.
-#
-# Draft: z-lab/Qwen3.8-27B-DFlash2 @ 50307d4 (block-diffusion drafter, 3.6 GB).
-#
-# Model: RadixArk/Qwen3.8-27B-NVFP4 (modelopt)
-#
-# PREREQUISITES (on the node):
-#   hfrs download RadixArk/Qwen3.8-27B-NVFP4
-#   hfrs download z-lab/Qwen3.8-27B-DFlash2
-#   docker build the minoverlay image (or use the recipe's docker_image as-is)
-#
-# REFERENCE:
-#   https://github.com/MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark  (start-dflash.sh)
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
-
-engine = "sglang"
-name = "Qwen3.8-27B (RadixArk NVFP4, SGLang DFlash2)"
-docker_image = "lmsysorg/sglang:qwen38-27b-dflash2-minoverlay"
-model = "RadixArk/Qwen3.8-27B-NVFP4"
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["python3", "-m", "sglang.launch_server"]
-
-args = [
-    # Memory — NVFP4 needs 0.90 (0.95 hard-rebooted the box at graph capture)
-    "--mem-fraction-static", "0.90",
-    "--max-running-requests", "10",
-    "--max-mamba-cache-size", "30",
-    # Attention / KV
-    "--attention-backend", "flashinfer",
-    "--kv-cache-dtype", "fp8_e4m3",
-    # Mamba GDN hybrid — DFLASH requires extra_buffer (not extra_buffer_lazy)
-    "--mamba-ssm-dtype", "bfloat16",
-    "--mamba-full-memory-ratio", "4.21",
-    "--mamba-radix-cache-strategy", "extra_buffer",
-    # Chunked prefill
-    "--chunked-prefill-size", "8192",
-    # DFlash2 block-diffusion speculative decoding
-    "--speculative-algorithm", "DFLASH",
-    "--speculative-draft-model-path", "z-lab/Qwen3.8-27B-DFlash2",
-    "--speculative-draft-model-revision", "50307d4c4cde6860d4eee73e2547cd786fe8e8a4",
-    "--speculative-num-draft-tokens", "8",
-    # Tokenizer / reasoning / tool calling (Qwen3 family)
-    "--reasoning-parser", "qwen3",
-    "--tool-call-parser", "qwen3_coder",
-    # Observability
-    "--enable-metrics",
-    "--enable-cache-report",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = "1"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"

recipes/qwen38-27b-nvfp4-sglang-dspark/recipe.toml

@@ -1,89 +0,0 @@
-# Qwen3.8-27B (RadixArk NVFP4 + DSpark on SGLang) — 1× DGX Spark / GB10 (sm_121),
-# TP=1.
-#
-# SGLang + DSpark block-diffusion speculative decoding. The community (MiaAI)
-# measured DSpark on this box at ~51.5 tok/s on code vs vLLM+MTP's 22.8 — the
-# drafter is 3× cheaper per guess and edits/rewrites much faster. Uses the
-# pre-built cookbook image (no source build): lmsysorg/sglang:qwen38-27b.
-#
-# TARGET MUST BE RadixArk MODELOPT NVFP4: the cookbook image accelerates
-# modelopt-quantized NVFP4. The unsloth/RedHatAI compressed-tensors builds
-# fall back to a slow unquantized path ("Falling back to UnquantizedLinear-
-# Method") — prefill crawls at ~0.5 tok/s. Use RadixArk here.
-#
-# Draft: RadixArk/Qwen3.8-27B-DSpark (bf16, block 7). SGLang loads it natively
-# from the HF cache — no arch patch needed (unlike the vLLM qwen3_dspark path).
-#
-# Image: lmsysorg/sglang:qwen38-27b  (multi-arch, incl. arm64; cookbook's
-#   DGX Spark Qwen3.8 cell, pinned image digest febfb971). Pre-built — no
-#   source build.
-#
-# Model: RadixArk/Qwen3.8-27B-NVFP4  (modelopt, cached locally on the node)
-#   https://huggingface.co/RadixArk/Qwen3.8-27B-NVFP4
-#
-# PREREQUISITES:
-#   Target + draft cached on the node (dgx1 ~/.cache/huggingface → NAS):
-#       hfrs download RadixArk/Qwen3.8-27B-NVFP4
-#       hfrs download RadixArk/Qwen3.8-27B-DSpark
-#
-# REFERENCE:
-#   https://github.com/MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark  (start-dspark.sh)
-#   https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark
-
-engine = "sglang"
-name = "Qwen3.8-27B (RadixArk NVFP4, SGLang DSpark)"
-docker_image = "lmsysorg/sglang:qwen38-27b"
-model = "RadixArk/Qwen3.8-27B-NVFP4"
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["python3", "-m", "sglang.launch_server"]
-
-args = [
-    # Memory — larger KV pool (mamba GDN state pool sized from concurrency)
-    "--mem-fraction-static", "0.90",
-    "--max-running-requests", "10",
-    "--max-mamba-cache-size", "30",
-    # Attention / KV
-    "--attention-backend", "flashinfer",
-    "--kv-cache-dtype", "fp8_e4m3",
-    # Mamba GDN hybrid
-    "--mamba-ssm-dtype", "bfloat16",
-    "--mamba-full-memory-ratio", "4.21",
-    "--mamba-radix-cache-strategy", "extra_buffer_lazy",
-    # Chunked prefill
-    "--chunked-prefill-size", "8192",
-    # DSpark speculative decoding (block 7 / 8 draft tokens, unquant bf16 draft)
-    "--speculative-algorithm", "DSPARK",
-    "--speculative-draft-model-path", "RadixArk/Qwen3.8-27B-DSpark",
-    "--speculative-dspark-block-size", "7",
-    "--speculative-draft-model-quantization", "unquant",
-    "--speculative-num-draft-tokens", "8",
-    # Compile + decode graphs
-    "--enable-torch-compile", "--torch-compile-max-bs", "4",
-    "--cuda-graph-max-bs-decode", "4",
-    "--num-continuous-decode-steps", "2",
-    # Tokenizer / reasoning / tool calling (Qwen3 family)
-    "--reasoning-parser", "qwen3",
-    "--tool-call-parser", "qwen3_coder",
-    # Observability
-    "--enable-metrics",
-    "--enable-cache-report",
-]
-
-[env]
-# Offline — target + draft are both cached on the node
-HF_HUB_OFFLINE = "1"
-
-# Pin to GB10's Cortex-X5 big cores (scheduler/tokenizer off the A725 efficiency
-# cores — +2-7% decode per the MiaAI build)
-# CPUSET handled at the container layer; this is the SGLang-side hint.
-SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = "1"
-
-# Architecture
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"

recipes/qwen38-27b-nvfp4-unsloth-dspark/recipe.toml

@@ -1,90 +0,0 @@
-# Qwen3.8-27B (unsloth NVFP4 + DSpark) — 1× DGX Spark / GB10 (sm_121), TP=1.
-#
-# NVFP4 compressed-tensors quant (22 GB) with a dedicated DSpark draft model
-# (RadixArk, 2.6 GB, block_size 7) for speculative decoding. Dense 27B, GQA,
-# hybrid DeltaNet+Attention. Text + vision. Unsloth Dynamic V3.0 (preview)
-# quant — SOTA quality for NVFP4. vLLM-only (lm_head FP8).
-#
-# DSpark vs the in-checkpoint MTP head (qwen38-27b-nvfp4-unsloth recipe): the
-# draft is 3× cheaper per guess and edits/rewrites much faster, at the cost of
-# a 2.6 GB draft + a local arch-patched copy. Benchmark both.
-#
-# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
-#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
-#   instanttensor. Stock vLLM — supports qwen3_5 arch + method "dspark".
-#
-# Model: unsloth/Qwen3.8-27B-NVFP4
-#   https://huggingface.co/unsloth/Qwen3.8-27B-NVFP4
-#
-# PREREQUISITES:
-#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
-#       huggingface-cli download unsloth/Qwen3.8-27B-NVFP4
-#       hfrs download RadixArk/Qwen3.8-27B-DSpark  (then cp -aL + patch arch → /home/kilork/dspark-draft)
-#
-# REFERENCE:
-#   https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark  (DSpark section)
-
-engine = "vllm"
-name = "Qwen3.8-27B (NVFP4 unsloth + DSpark)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
-model = "unsloth/Qwen3.8-27B-NVFP4"
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # KV cache & memory
-    "--kv-cache-dtype", "fp8",
-    "--max-model-len", "262144",
-    "--max-num-seqs", "6",
-    "--max-num-batched-tokens", "16384",
-    "--gpu-memory-utilization", "0.75",
-    "--served-model-name", "qwen3.8-27b",
-    # Engine backends (dense GQA — FlashInfer; NVFP4 route via compressed-tensors)
-    "--attention-backend", "flashinfer",
-    # Compilation / perf
-    "--optimization-level", "3",
-    "--performance-mode", "throughput",
-    # Chunked prefill + prefix caching
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    # DSpark speculative decoding — dedicated draft model (arch-patched local
-    # copy of RadixArk/Qwen3.8-27B-DSpark, re-labelled `Qwen3DSparkModel` so
-    # vLLM routes it to qwen3_dspark not the DeepSeek-V4 path). block_size 7.
-    # batched-tokens raised to 16384 to fit the draft token slots (warn:
-    # max_num_scheduled_tokens=8156 at 8192).
-    "--speculative-config", """
-'{"method":"dspark","model":"/dspark-draft","num_speculative_tokens":7}'""",
-    # Tokenizer / reasoning (Qwen3 family)
-    "--tool-call-parser", "qwen3_xml",
-    "--reasoning-parser", "qwen3",
-    "--enable-auto-tool-choice",
-    "--generation-config", "auto",
-    "--default-chat-template-kwargs.preserve_thinking=true",
-    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
-    # practices)
-    "--override-generation-config", """
-'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
-    "--trust-remote-code",
-]
-
-# DSpark draft model — arch-patched local copy of RadixArk/Qwen3.8-27B-DSpark.
-[[extra_mounts]]
-host = "/home/kilork/dspark-draft"
-container = "/dspark-draft"
-
-
-[env]
-HF_HUB_OFFLINE = "1"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-nvfp4-unsloth/recipe.toml

@@ -1,80 +0,0 @@
-# Qwen3.8-27B (unsloth NVFP4 + MTP) — 1× DGX Spark / GB10 (sm_121), TP=1.
-#
-# NVFP4 compressed-tensors quant (22 GB) with a separate MTP draft in
-# `model_mtp.safetensors`. Dense 27B, GQA, hybrid DeltaNet+Attention. Text +
-# vision. Unsloth Dynamic V3.0 (preview) quant — SOTA quality for NVFP4.
-# vLLM-only (lm_head FP8).
-#
-# Nearly identical to RedHatAI's NVFP4 build (qwen38-27b-nvfp4 recipe);
-# benchmark both to pick the faster / higher-quality one.
-#
-# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
-#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
-#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
-#
-# Model: unsloth/Qwen3.8-27B-NVFP4
-#   https://huggingface.co/unsloth/Qwen3.8-27B-NVFP4
-#
-# PREREQUISITES:
-#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
-#       huggingface-cli download unsloth/Qwen3.8-27B-NVFP4
-#
-# REFERENCE:
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-nvfp4-on-a-single-dgx-spark-up-to-1m-context-vllm-mtp-measurements/380244
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
-
-engine = "vllm"
-name = "Qwen3.8-27B (NVFP4, unsloth)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
-model = "unsloth/Qwen3.8-27B-NVFP4"
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # KV cache & memory
-    "--kv-cache-dtype", "fp8",
-    "--max-model-len", "262144",
-    "--max-num-seqs", "6",
-    "--max-num-batched-tokens", "8192",
-    "--gpu-memory-utilization", "0.75",
-    "--served-model-name", "qwen3.8-27b",
-    # Engine backends (dense GQA — FlashInfer; NVFP4 route via compressed-tensors)
-    "--attention-backend", "flashinfer",
-    # Compilation / perf
-    "--optimization-level", "3",
-    "--performance-mode", "throughput",
-    # Chunked prefill + prefix caching
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    # MTP speculative decoding (in-checkpoint draft, model_mtp.safetensors)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":5}'""",
-    # Tokenizer / reasoning (Qwen3 family)
-    "--tool-call-parser", "qwen3_xml",
-    "--reasoning-parser", "qwen3",
-    "--enable-auto-tool-choice",
-    "--generation-config", "auto",
-    "--default-chat-template-kwargs.preserve_thinking=true",
-    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
-    # practices)
-    "--override-generation-config", """
-'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
-    "--trust-remote-code",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b-nvfp4/recipe.toml

@@ -1,81 +0,0 @@
-# Qwen3.8-27B (RedHatAI NVFP4 + MTP) — 1× DGX Spark / GB10 (sm_121), TP=1.
-#
-# NVFP4 compressed-tensors quant (22 GB) with a separate MTP draft in
-# `model_mtp.safetensors`. Dense 27B, GQA, hybrid DeltaNet+Attention. Text +
-# vision. The vLLM-only quant (lm_head FP8) — SGLang can't load it.
-#
-# Two nearly-identical NVFP4 builds are available:
-#   RedHatAI/Qwen3.8-27B-NVFP4  (this recipe)
-#   unsloth/Qwen3.8-27B-NVFP4   (qwen38-27b-nvfp4-unsloth recipe)
-# Speed/quality are close; benchmark both in the experiment.
-#
-# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
-#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
-#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
-#
-# Model: RedHatAI/Qwen3.8-27B-NVFP4
-#   https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4
-#
-# PREREQUISITES:
-#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
-#       huggingface-cli download RedHatAI/Qwen3.8-27B-NVFP4
-#
-# REFERENCE:
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-nvfp4-on-a-single-dgx-spark-up-to-1m-context-vllm-mtp-measurements/380244
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
-
-engine = "vllm"
-name = "Qwen3.8-27B (NVFP4, RedHatAI)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
-model = "RedHatAI/Qwen3.8-27B-NVFP4"
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # KV cache & memory
-    "--kv-cache-dtype", "fp8",
-    "--max-model-len", "262144",
-    "--max-num-seqs", "6",
-    "--max-num-batched-tokens", "8192",
-    "--gpu-memory-utilization", "0.75",
-    "--served-model-name", "qwen3.8-27b",
-    # Engine backends (dense GQA — FlashInfer; NVFP4 route via compressed-tensors)
-    "--attention-backend", "flashinfer",
-    # Compilation / perf
-    "--optimization-level", "3",
-    "--performance-mode", "throughput",
-    # Chunked prefill + prefix caching
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    # MTP speculative decoding (in-checkpoint draft, model_mtp.safetensors)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":5}'""",
-    # Tokenizer / reasoning (Qwen3 family)
-    "--tool-call-parser", "qwen3_xml",
-    "--reasoning-parser", "qwen3",
-    "--enable-auto-tool-choice",
-    "--generation-config", "auto",
-    "--default-chat-template-kwargs.preserve_thinking=true",
-    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
-    # practices)
-    "--override-generation-config", """
-'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
-    "--trust-remote-code",
-]
-
-[env]
-HF_HUB_OFFLINE = "1"
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-27b/recipe.toml

@@ -1,88 +0,0 @@
-# Qwen3.8-27B (BF16) — 1× DGX Spark / GB10 (sm_121), TP=1.
-#
-# Native BF16 dense 27B (52 GB). Quality reference baseline; the quantized
-# variants (FP8 / NVFP4 / Int4) are the speed candidates. Runs text + vision.
-#
-# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
-#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
-#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
-#
-# Model: Qwen/Qwen3.8-27B
-#   https://huggingface.co/Qwen/Qwen3.8-27B
-#   Dense 27B, 64 layers, hidden 5120, GQA (4 KV heads × 256), Gated
-#   DeltaNet + Gated Attention hybrid. MTP-1. Native 262,144 ctx (YaRN to 1M).
-#   Multimodal (vision tower, `language_model_only: False`).
-#
-# PREREQUISITES:
-#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
-#       huggingface-cli download Qwen/Qwen3.8-27B
-#
-# REFERENCE:
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
-#   https://forums.developer.nvidia.com/t/qwen3-8-27b-at-34-38-tok-s-on-dgx-spark-open-source-one-command-setup-sglang-nvfp4-dspark/380257
-
-engine = "vllm"
-name = "Qwen3.8-27B (BF16)"
-docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
-model = "Qwen/Qwen3.8-27B"
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # KV cache & memory
-    "--kv-cache-dtype", "fp8",
-    "--max-model-len", "262144",
-    "--max-num-seqs", "6",
-    "--max-num-batched-tokens", "8192",
-    "--gpu-memory-utilization", "0.75",
-    "--served-model-name", "qwen3.8-27b",
-    # Engine backends (dense GQA — FlashInfer, not the DeepSeek MLA path)
-    "--attention-backend", "flashinfer",
-    # Compilation / perf
-    "--optimization-level", "3",
-    "--performance-mode", "throughput",
-    # Chunked prefill + prefix caching
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    "--enable-flashinfer-autotune",
-    # MTP speculative decoding (in-checkpoint draft, mtp_num_hidden_layers=1
-    # cycled across steps)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":5}'""",
-    # Tokenizer / reasoning (Qwen3 family)
-    "--tool-call-parser", "qwen3_xml",
-    "--reasoning-parser", "qwen3",
-    "--enable-auto-tool-choice",
-    "--generation-config", "auto",
-    "--default-chat-template-kwargs.preserve_thinking=true",
-    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
-    # practices). repetition_penalty 1.0 = identity: accepted but never
-    # deviates from no-op during req handling (DSpark/MLA bug caveat — see
-    # findings doc for the DeepSeek path; irrelevant to dense Qwen).
-    "--override-generation-config", """
-'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
-    "--trust-remote-code",
-]
-
-[env]
-# Offline mode — weights must be cached on each node
-HF_HUB_OFFLINE = "1"
-
-# Architecture
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"
-
-# GPU memory management
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
-
-# GPU device isolation
-CUDA_VISIBLE_DEVICES = "0"
-CUDA_DEVICE_ORDER = "PCI_BUS_ID"
-CUDA_CACHE_MAXSIZE = "4294967296"

recipes/qwen38-flash-next-fp8/recipe.toml

@@ -1,79 +0,0 @@
-# Qwen3.8-Flash-Next-FP8 (vLLM expert-parallel) — 2× DGX Spark / GB10 (sm_121), TP=2.
-#
-# Official FP8 quant of the Qwen4-architecture preview (176B total / 6B
-# active, GDN + QSA hybrid MoE, 512 experts, 51B n-gram "engram" PLE table,
-# in-checkpoint MTP). Text + vision. Native 262144 ctx.
-#
-# WHY vLLM + expert-parallel: the stock vLLM MoE path shards each expert's
-# intermediate by TP, and this FP8 checkpoint (weight_block_size=[128,128],
-# moe_intermediate_size=640) violates (intermediate/tp) % block_n == 0 at
-# tp=2 -> boot crash. --enable-expert-parallel splits the 512 *experts*
-# across the ranks instead (256 each, full 640-wide intermediate), so the
-# divisibility rule never fires. This is the verified 2× Spark FP8 path.
-#
-# Image: vllm/vllm-openai:qwen38-flash-next — vLLM's day-0 model tag (arm64).
-#
-# Model: Qwen/Qwen3.8-Flash-Next-FP8 (~186 GB)
-#   https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8
-#
-# REFERENCE (provenance — verified ~30-45 tok/s on 2× Sparks):
-#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228/97  (foogitiff)
-#   answers #106 / #113 (m-emm): ENFORCE_EAGER=1, disable MM, MTP=3 -> ~45 tok/s
-#
-# NOTE: day-0 config. foogitiff's base boots; m-emm's ENFORCE_EAGER + no-MM
-# gives the better throughput. Expect to tune max-num-seqs / mm limits.
-
-engine = "vllm"
-name = "Qwen3.8-Flash-Next FP8 (vLLM expert-parallel, 2x Spark)"
-docker_image = "vllm/vllm-openai:qwen38-flash-next"
-model = "Qwen/Qwen3.8-Flash-Next-FP8"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # Stable served name — quant-agnostic, shared with qwen38-flash-next-nvfp4
-    "--served-model-name", "qwen3.8-flash-next",
-    # Expert parallelism — the fix for the FP8 MoE block-divisibility crash
-    "--distributed-executor-backend", "mp",
-    "--enable-expert-parallel",
-    "--all2all-backend", "allgather_reducescatter",
-    # Weight loading (no fastsafetensors day-0; lazy strategy keeps boot sane)
-    "--load-format", "safetensors",
-    "--safetensors-load-strategy", "lazy",
-    # Context / batching (verified; raise max-num-seqs once stable)
-    "--max-model-len", "262144",
-    "--max-num-seqs", "1",
-    "--max-num-batched-tokens", "8192",
-    "--gpu-memory-utilization", "0.83",
-    "--enable-chunked-prefill",
-    # MTP speculative decoding (in-checkpoint draft)
-    "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":3}'""",
-    "--compilation-config", """
-'{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}'""",
-    # Tokenizer / reasoning / tools (Qwen3 family)
-    "--reasoning-parser", "qwen3",
-    "--enable-auto-tool-choice",
-    "--tool-call-parser", "qwen3_coder",
-    # m-emm perf (post #113): disable multimodal to save VRAM/bandwidth
-    "--limit-mm-per-prompt", """
-'{"image":0,"video":0}'""",
-]
-
-[env]
-# Offline — weights must be cached on each node
-HF_HUB_OFFLINE = "1"
-TRANSFORMERS_OFFLINE = "1"
-# m-emm (post #113): eager mode for the 45 tok/s config (overrides the
-# cudagraph request above; day-0 stability knob)
-ENFORCE_EAGER = "1"
-
-# Architecture (aarch64 / sm_121a DGX Spark)
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"

recipes/qwen38-flash-next-nvfp4/recipe.toml

@@ -1,89 +0,0 @@
-# Qwen3.8-Flash-Next (NVFP4, SGLang) — 2× DGX Spark / GB10 (sm_121), TP=2.
-#
-# Qwen4-architecture preview: 176B total (6B active) GDN + QSA hybrid MoE,
-# 512 routed experts (top-10) + shared expert, 51B n-gram "engram" table
-# (PLE, offloaded to host RAM), in-checkpoint MTP draft. Text + vision.
-# Native 262144 ctx (YaRN to 1M). Always-thinking.
-#
-# QUANT: NVFP4 (RadixArk), NOT the official FP8. FP8 cannot serve on 2×
-# Spark: its MoE weight_block_size=[128,128] + moe_intermediate_size=640
-# violates SGLang's (intermediate/tp) % block_n == 0 at tp=2 (sgl-project/
-# sglang #21810) — only tp in {1,5} align, and tp=1 can't hold 125B. NVFP4
-# is the community-verified 2× Spark path. SGLang's own cookbook lists the
-# FP8 quant's supported HW as H200/B200/B300/GB300 — not GB10.
-#
-# Image: qwen38-flashnext-nvfp4:local — built by the launcher from
-#   docker-images/qwen38-flashnext-nvfp4/ (base lmsysorg/sglang:qwen38flashnext
-#   + a SM121 QSA decode kernel fix). The stock day-0 image cannot serve
-#   qwen4_exp on GB10: QSA's packed varlen decode resolves to flash-attn-4
-#   CuTe-DSL kernels that fail MLIR compilation on SM121; the overlay swaps in
-#   a CUDA-graph-safe Triton fallback. Provenance/validation:
-#     https://github.com/MiaAI-Lab/Qwen3.8-Flash-Next-Dual-DGX-Sparks
-#
-# Model: RadixArk/Qwen3.8-Flash-Next-NVFP4 (~135 GB)
-#   https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4
-#
-# REFERENCE:
-#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228
-
-engine = "sglang"
-name = "Qwen3.8-Flash-Next NVFP4 (SGLang, 2× Spark)"
-docker_image = "qwen38-flashnext-nvfp4:local"
-docker_image_dir = "qwen38-flashnext-nvfp4"
-model = "RadixArk/Qwen3.8-Flash-Next-NVFP4"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-
-entrypoint = ["python3", "-m", "sglang.launch_server"]
-
-args = [
-    # Stable served name — quant-agnostic, shared with qwen38-flash-next-fp8 so
-    # swapping quants never changes what agents/clients pin against.
-    "--served-model-name", "qwen3.8-flash-next",
-    "--trust-remote-code",
-    # NVFP4 (ModelOpt) weights
-    "--quantization", "modelopt_fp4",
-    # flashinfer is required on SM120/SM121 (trtllm_mha is SM100-only)
-    "--attention-backend", "flashinfer",
-    # page 64 is pinned by SGLang for compressed QSA (full//ratio addressing)
-    "--page-size", "64",
-    # GDN (DeltaNet) hybrid state pool
-    "--mamba-ssm-dtype", "bfloat16",
-    "--mamba-radix-cache-strategy", "extra_buffer",
-    "--mamba-track-interval", "64",
-    # Memory — 0.70 leaves DRAM for the ~26 GB/rank pinned PLE (engram) table
-    "--mem-fraction-static", "0.70",
-    "--chunked-prefill-size", "4096",
-    "--max-running-requests", "16",
-    "--context-length", "262144",
-    # NEXTN speculative decoding — draft = the in-checkpoint MTP layer
-    "--speculative-algorithm", "NEXTN",
-    "--speculative-num-steps", "3",
-    "--speculative-eagle-topk", "1",
-    "--speculative-num-draft-tokens", "4",
-    # Parsers (auto = resolve from the checkpoint chat template)
-    "--reasoning-parser", "auto",
-    "--tool-call-parser", "auto",
-    "--allow-auto-truncate",
-    # Observability
-    "--enable-metrics",
-    "--enable-cache-report",
-    # SM121 house rule: prefill CUDA graphs unreliable on GB10
-    "--disable-prefill-cuda-graph",
-    # Pin decode-graph batch list (default list OOMs the graph pool on GB10)
-    "--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16",
-]
-
-[env]
-# Offline — weights must be cached on each node
-HF_HUB_OFFLINE = "1"
-TRANSFORMERS_OFFLINE = "1"
-
-# Architecture (aarch64 / sm_121a DGX Spark)
-TORCH_CUDA_ARCH_LIST = "12.1a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-OMP_NUM_THREADS = "4"

recipes/solar-open2-nvfp4/recipe.toml

@@ -1,85 +0,0 @@
-# Solar Open2 250B — Nota NVFP4 (compressed-tensors, W4A4)
-# 2× DGX Spark (GB10, aarch64/sm_121a), TP=2 over RoCE.
-#
-# Model: nota-ai/Solar-Open2-250B-Nota-NVFP4
-#   → https://huggingface.co/nota-ai/Solar-Open2-250B-Nota-NVFP4
-#   145B params (MoE), 153.3 GB NVFP4 weights (vs 500.6 GB BF16).
-#   NVFP4 is auto-detected from config (compressed-tensors) — no --quantization.
-#
-# Image: solar-open2-nvfp4:probe (built locally from the UpstageAI vLLM fork
-#   v0.22.0-solar-open2 — carries the solar_open2 reasoning/tool parsers and
-#   the SolarOpen2TemplateLogitsProcessor). cu130 / aarch64 / sm_121a, same
-#   pinned base stack as glm52-nf3-hybrid (no b12x suite — stock vLLM serve).
-#
-# Memory: GB10 ≈ 121 GB usable (128 GB advertised − 4 GB OS). TP=2 →
-#   ~77 GB weights/node, ~40 GB/node for KV cache + activations.
-#
-# PREREQUISITES:
-#   - Weights cached on every node:
-#       huggingface-cli download nota-ai/Solar-Open2-250B-Nota-NVFP4
-#     (153 GB, NVFP4 compressed-tensors shards)
-#   - Image built (sm121 serve builds it automatically on the build node):
-#       cd docker-images/solar-open2-nvfp4 && bash build.sh
-
-engine = "vllm"
-name = "Solar Open2 250B Nota NVFP4 (vLLM)"
-docker_image = "solar-open2-nvfp4:probe"
-model = "nota-ai/Solar-Open2-250B-Nota-NVFP4"
-required_nodes = 2
-tensor_parallel_size = 2
-pipeline_parallel_size = 1
-distributed_backend = "mp"
-shm_size = "32gb"
-docker_image_dir = "solar-open2-nvfp4"
-
-entrypoint = ["vllm", "serve"]
-
-args = [
-    # KV cache & memory
-    "--kv-cache-dtype", "fp8",
-    "--load-format", "auto",
-    # Context length (128K; model supports up to 131584)
-    "--max-model-len", "131072",
-    # Batch & memory
-    "--max-num-seqs", "8",
-    "--max-num-batched-tokens", "4096",
-    "--max-cudagraph-capture-size", "16",
-    "--gpu-memory-utilization", "0.88",
-    # Scheduling
-    "--async-scheduling",
-    "--enable-chunked-prefill",
-    "--enable-prefix-caching",
-    # Solar Open2 parsers + logits processor (ships in the UpstageAI fork)
-    "--reasoning-parser", "solar_open2",
-    "--tool-call-parser", "solar_open2",
-    "--enable-auto-tool-choice",
-    "--logits-processors", "vllm.v1.sample.logits_processor.solar_open2:SolarOpen2TemplateLogitsProcessor",
-    "--default-chat-template-kwargs", """
-'{"think_render_option":"preserved"}'""",
-    # General
-    "--trust-remote-code",
-    "--served-model-name", "solar",
-]
-
-[env]
-# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
-TORCH_CUDA_ARCH_LIST = "12.1a"
-CUTE_DSL_ARCH = "sm_121a"
-FLASHINFER_CUDA_ARCH_LIST = "12.1a"
-
-# ── Offline mode (weights must be cached on each node) ────────────────────
-HF_HOME = "/root/.cache/huggingface"
-HF_HUB_OFFLINE = "1"
-TRANSFORMERS_OFFLINE = "1"
-
-# ── Timeout safety net (cold boot: cudagraph + JIT) ───────────────────────
-VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "3600"
-
-# ── NCCL / RoCE (cluster config ib_hca/socket_ifname used for most) ──────
-NCCL_CROSS_NIC = "1"
-NCCL_CUMEM_ENABLE = "0"
-NCCL_IGNORE_CPU_AFFINITY = "1"
-NCCL_DEBUG = "WARN"
-
-# ── Compute ────────────────────────────────────────────────────────────────
-OMP_NUM_THREADS = "4"

recipes/test-composed/recipe.toml

@@ -1,50 +0,0 @@
-# Test Composed Model — verifies the [composed_model] mechanism (symlink
-# composition + bind-mount + model rewrite) without launching heavy vLLM.
-#
-# Uses alpine (like test-sleep/test-mods) with a heartbeat entrypoint that
-# first inspects the composed dir, then loops. Requires both model repos
-# cached on every node — the composed dir's symlinks point into the HF cache
-# (container-side /root/.cache/huggingface), so they only resolve inside the
-# container where the HF cache is mounted.
-#
-# What this verifies:
-#   1. The composed dir is built on the node (/tmp/sm121_test-composed_<rank>/)
-#   2. MXFP4 base files are symlinked (config.json, tokenizer, etc.)
-#   3. Lloyd NF4 experts overlay the MXFP4 experts (L*.safetensors)
-#   4. The generated nf4_layers.json file is present
-#   5. The bind-mount makes /root/model/ accessible inside the container
-#   6. model= is rewritten to /root/model (visible in the docker run command)
-#
-# Usage:
-#   cargo run -- -d serve --recipe test-composed    # dry-run: inspect cmd
-#   cargo run serve --recipe test-composed           # real: starts alpine
-#   cargo run logs --recipe test-composed            # see the composed dir listing
-
-engine = "vllm"
-name = "Test Composed Model"
-docker_image = "alpine:latest"
-model = "noop"
-
-[composed_model]
-mount = "/root/model"
-
-[[composed_model.sources]]
-repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
-
-[[composed_model.sources]]
-repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ"
-glob = "L*.safetensors"
-
-[[composed_model.sources]]
-repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ"
-glob = "grids/*.json"
-
-[[composed_model.files]]
-name = "nf4_layers.json"
-content = '{"layers":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77]}'
-
-entrypoint = ["sh", "-c", "echo '=== composed model test ===' && echo 'dir contents (first 15):' && ls /root/model/ | head -15 && echo '--- nf4_layers.json ---' && cat /root/model/nf4_layers.json && echo '' && echo '=== test OK ===' && i=0; while true; do echo \"$(date -u +%H:%M:%S) [$(hostname)] composed $i\"; i=$((i+1)); sleep 2; done"]
-
-args = []
-
-[env]

recipes/test-mods/mods/local-patch/patch.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-# Simulates a vLLM patch: write a marker that the engine can read
-echo "alpine:3.20" > /tmp/.vllm-patched
-echo "MODS_PATCHED=1" >> /tmp/.vllm-patched
-echo "[mods] Applied vLLM compatibility patch at $(date)" >&2

recipes/test-mods/recipe.toml

@@ -1,16 +0,0 @@
-engine = "vllm"
-name = "Test Mods"
-docker_image = "alpine:latest"
-model = "noop"
-port = 30000
-required_nodes = 1
-tensor_parallel_size = 1
-pipeline_parallel_size = 1
-
-# Entrypoint must be set when mods are present.
-entrypoint = ["sh", "-c", "i=0; while true; do echo \"[$(hostname)] heartbeat $i\"; i=$((i+1)); sleep 2; done"]
-
-[mods]
-commands = [
-    "sh /mods/local-patch/patch.sh",
-]

recipes/test-sleep/recipe.toml

@@ -1,19 +0,0 @@
-engine = "vllm"
-name = "Test Heartbeat Container"
-docker_image = "alpine:latest"
-model = "noop"
-
-# Heartbeat loop: prints a timestamped line every 2s with UTC time and
-# the container hostname so you can tell logs from different nodes apart.
-#
-# Use it to verify the logging pipeline:
-#   cargo run serve --recipe test-sleep     # spins up the cluster
-#   cargo run logs --recipe test-sleep      # last 100 lines per node
-#   cargo run logs --recipe test-sleep --follow   # live stream from head
-#
-# Each line looks like:  10:42:13 [n0] heartbeat 42
-entrypoint = ["sh", "-c", "i=0; while true; do echo \"$(date -u +%H:%M:%S) [$(hostname)] heartbeat $i\"; i=$((i+1)); sleep 2; done"]
-
-args = []
-
-[env]

src/tmux/discover.rs

@@ -145,20 +145,29 @@ mod tests {
 
     #[test]
     fn list_recipe_dirs_finds_directories() {
+        // Recipe data lives in git.kilork.org/sm121/recipes now — tests
+        // seed their own fixture instead of relying on repo data.
+        let fixture = "recipes/ztest-list-fixture";
+        std::fs::create_dir_all(fixture).unwrap();
         let dirs = list_recipe_dirs();
-        // Should find at least the laguna recipe
-        assert!(dirs.iter().any(|d| d == "laguna-s2.1-nvfp4"));
-        // Should be sorted
         let sorted = dirs.windows(2).all(|w| w[0] <= w[1]);
+        let _ = std::fs::remove_dir(fixture);
+        assert!(dirs.iter().any(|d| d == "ztest-list-fixture"));
         assert!(sorted);
     }
 
     #[tokio::test]
     async fn find_tmux_recipes_finds_tmux_deployed() {
-        // This test runs in the project directory, so it should find
-        // the laguna recipe which has deploy = "tmux"
+        let fixture = "recipes/ztest-tmux-fixture";
+        std::fs::create_dir_all(fixture).unwrap();
+        std::fs::write(
+            format!("{fixture}/recipe.toml"),
+            "name = \"ztest-tmux-fixture\"\ndeploy = \"tmux\"\n",
+        )
+        .unwrap();
         let recipes = find_tmux_recipes(&[]).await;
-        assert!(recipes.iter().any(|(dir, _)| dir == "laguna-s2.1-nvfp4"));
+        let _ = std::fs::remove_dir_all(fixture);
+        assert!(recipes.iter().any(|(dir, _)| dir == "ztest-tmux-fixture"));
         for (_dir, recipe) in &recipes {
             assert_eq!(recipe.deploy, DeployTarget::Tmux);
         }

src/web/api_tests.rs

@@ -30,13 +30,9 @@ async fn recipes_endpoint_lists_repo_recipes() {
     let res = call(addr, "/api/recipes").await;
     assert_eq!(res.status(), StatusCode::OK);
     let v: serde_json::Value = res.json().await.unwrap();
-    let arr = v["recipes"].as_array().unwrap();
-    assert!(!arr.is_empty(), "repo recipes/ should be discoverable");
-    let names: Vec<&str> = arr.iter().map(|r| r["name"].as_str().unwrap()).collect();
-    assert!(
-        names.contains(&"qwen38-27b-nvfp4-unsloth"),
-        "got: {names:?}"
-    );
+    // Recipe data lives in git.kilork.org/sm121/recipes now; the launcher
+    // repo only guarantees the endpoint shape, not any specific entry.
+    assert!(v["recipes"].is_array());
 }
 
 #[tokio::test]
@@ -204,13 +200,21 @@ async fn raw_endpoints_serve_config_recipe_effective() {
             .unwrap_or(true)
     );
 
-    // Recipe: a real recipe from the repo (tests run from the crate root).
+    // Recipe: seed our own fixture (recipe data itself lives in
+    // git.kilork.org/sm121/recipes).
+    std::fs::create_dir_all("recipes/ztest-raw-fixture").unwrap();
+    std::fs::write(
+        "recipes/ztest-raw-fixture/recipe.toml",
+        "name = \"ztest-raw-fixture\"\nengine = \"vllm\"\n",
+    )
+    .unwrap();
     let res = client
-        .get(format!("{base}/api/raw/recipe/deepseek-v4-flash"))
+        .get(format!("{base}/api/raw/recipe/ztest-raw-fixture"))
         .send()
         .await
         .unwrap();
-    assert_eq!(res.status(), 200, "repo recipe readable");
+    let _ = std::fs::remove_dir_all("recipes/ztest-raw-fixture");
+    assert_eq!(res.status(), 200, "fixture recipe readable");
     let text = res.text().await.unwrap();
     assert!(text.contains("engine"), "recipe.toml verbatim");
 

src/web/ui.html

@@ -938,7 +938,12 @@ async function openRaw() {
     try { await navigator.clipboard.writeText(pre.textContent); } catch {}
   };
   const fill = () => {
-    if (!state.recipes.length) return;
+    // rebuild only when the recipe set changed (works with zero recipes:
+    // config/effective are always selectable; recipe data may live in an
+    // external workspace).
+    const sig = state.recipes.map((r) => r.name).join(",");
+    if (sel.dataset.sig === sig && sel.options.length) return;
+    sel.dataset.sig = sig;
     const cur = sel.value;
     sel.innerHTML = "";
     for (const kind of ["config", "effective", "recipe:", ...state.recipes.map((r) => "recipe:" + r.name)]) {
@@ -966,8 +971,8 @@ async function openRaw() {
     } catch (e) { note.textContent = "error: " + e.message; }
   }
   sel.onchange = load;
-  if (state.recipes.length) fill();
-  else state.onRecipesReady = fill; // recipes may arrive after the first open
+  fill(); // config/effective are always available; recipes fill in later
+  state.onRecipesReady = fill; // recipes may arrive after the first open
 }
 
 // --- actions -----------------------------------------------------------------------