aiden lanes: 2x+4x recipes from aidenle.com guides (glm prod-1.1 DFlash2-7 + dsv4 vision prod-3.72 DSpark-6); 2x to verify, 4x deferred (FP8 ckpt downloading)

recipes/dsv4-vision-aiden-2x/recipe.toml

@@ -0,0 +1,105 @@
+# DeepSeek-V4-Flash-Vision-Exp on 2× GB10 — aidendle94 sparkrun vision image.
+#
+# aiden's vision guide (https://aidenle.com/recipes/deepseek-v4-flash-vision-4x-dgx-spark/)
+# translated to 2× Spark / DCP1 (his "DSpark, DCP1" profile: 73.8-75.2 tok/s at
+# TP4/DCP1; on 2x expect roughly half). His production lane ships DCP2 + DSpark
+# (66-71 tok/s, 3.5M KV); DCP flags omitted here — DSpark runs fine under DCP1.
+#
+# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-3.72-vision (published
+# vision stack: hand-ported tower/routing/CKV gather, cross-checked against
+# upstream vLLM PR #54566). Sibling comparison: dsv4-flash-vision:local (our
+# plugin overlay on his production-3.76.1 base) — the A/B pair for the vision
+# lane.
+#
+# Weights: deepseek-ai/DeepSeek-V4-Flash-Vision-Exp (cached; ~84 GiB/node at
+# TP2 via ~/dsv4vision-hf NVMe copies). Draft heads ship in the checkpoint
+# (mtp.*) — no extra download.
+#
+# DSpark: k=6, greedy draft, acceptance 3.5-3.8 tokens/step. Acceptance is the
+# DCP correctness signal; equal DSpark-DCP1 numbers here = draft sees full ctx.
+# max-cudagraph-capture-size = seqs × (spec tokens + 1) = 8 × 7 = 56.
+#
+# VISION_MHC=1 follows aiden's production profile; our plugin lane keeps
+# VLLM_USE_B12X_MHC=0 (checkpoint rms_norm_eps 1e-20 vs fused Gram kernel's
+# 1e-6 — upstream README). If this boot asserts on the mHC kernel, flip to 0.
+#
+# Boot markers: vision plugin registered → "Model loading took ~84 GiB" →
+# "GPU KV cache size" ≈ 1.25M → "Application startup complete".
+
+engine = "vllm"
+name = "DeepSeek V4 Flash Vision-Exp (aiden image, 2x Spark, DSpark-6)"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.72-vision"
+model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--served-model-name", "deepseek-v4-vision",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--gpu-memory-utilization", "0.80",
+    "--kv-cache-memory", "12884901888",
+    "--max-model-len", "393216",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "8192",
+    "--max-cudagraph-capture-size", "56",
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8]}'""",
+    "--async-scheduling",
+    "--no-scheduler-reserve-full-isl",
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "deepseek_v4",
+    "--reasoning-config", """
+'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
+    "--default-chat-template-kwargs", """
+'{"thinking":false,"reasoning_effort":"low"}'""",
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"greedy","attention_backend":"B12X_MLA_SPARSE"}'""",
+    "--disable-chunked-mm-input",
+    "--limit-mm-per-prompt", """
+'{"image":4}'""",
+]
+
+[env]
+# aidee runner envs (his guide, minus DCP2-only and fabric knobs)
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_ENABLE_DSPARK = "1"
+VLLM_ENABLE_MTP = "0"
+NUM_SPECULATIVE_TOKENS = "6"
+DSPARK_DRAFT_SAMPLE_METHOD = "greedy"
+DSPARK_SPS_CURVE = "auto"
+VLLM_CUDAGRAPH_MODE = "FULL_AND_PIECEWISE"
+VISION_MHC = "1"
+MAX_CUDAGRAPH_CAPTURE_SIZE = "56"
+# proven 2x-lane knobs carried over
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_MOE_FORCE_A8 = "1"
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Weights from per-node NVMe copies; extra_mounts stay AFTER args.
+[[extra_mounts]]
+host = "~/dsv4vision-hf"
+container = "/opt/local-hf"
+env = "HF_HOME"

recipes/dsv4-vision-aiden-4x/recipe.toml

@@ -0,0 +1,97 @@
+# DeepSeek-V4-Flash-Vision-Exp on 4× GB10 — aiden production profile. DEFERRED.
+#
+# Source: https://aidenle.com/recipes/deepseek-v4-flash-vision-4x-dgx-spark/
+# TP4 across 4 nodes with **DCP2 inside it** and DSpark-6 running together —
+# the combination stock vLLM refuses twice ("DSpark requires DCP_SIZE=1" +
+# "Non-causal DSpark SWA does not support DCP"); aiden's image resolves it by
+# marking the draft's sliding-window cache `dcp_replicated` (full ctx on every
+# rank, cost-free for a few hundred tokens). Shipped profile numbers:
+#   decode 66-71 tok/s @ acceptance 3.5-3.8; cold 334K prefill 2,088 tok/s;
+#   KV pool 3.51M tokens (8.9× full-length ctx); 393K ctx verified;
+#   image request ~1.2 s warm / ~6 s first.
+#
+# Profile trade (his table): DSpark+DCP1 73.8-75.2 / DCP2-no-spec 34 /
+# DSpark+DCP2 66-71 (DCP1 pool 1.77M vs DCP2 3.51M — KV doubled at ~7% decode).
+#
+# NOT READY TO DEPLOY: needs a 4-node cluster window. Fabric NCCL envs omitted
+# here (launcher-managed, [nccl.*]). His runner envs carrying operational
+# requirements: earlyoom tuned on every node, root for RDMA passthrough +
+# memlock + page-cache flushing, ~120 GB free per node for weights + JIT cache.
+#
+# Env/flag deltas vs our 2x variant (dsv4-vision-aiden-2x): DCP_SIZE=2,
+# --decode-context-parallel-size 2 --dcp-comm-backend ag_rs (ag_rs = +12% KV
+# over a2a), KV_CACHE_MEMORY_BYTES unpin (empty — the ~26 GB cap off),
+# GPU_MEMORY_UTILIZATION 0.75 (0.82 → earlyoom kills a worker on his kit),
+# VLLM_DSV4_CKV_GATHER=1 + CKV_GATHER_MAX_TOKENS + ALLOW_IMAGES (prefill
+# gather required for images under DCP).
+
+engine = "vllm"
+name = "DeepSeek V4 Flash Vision-Exp (aiden image, 4x Spark, DSpark-6 + DCP2) [DEFERRED]"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.72-vision"
+model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--served-model-name", "deepseek-v4-vision",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    "--decode-context-parallel-size", "2",
+    "--dcp-comm-backend", "ag_rs",
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--gpu-memory-utilization", "0.75",
+    "--max-model-len", "393216",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "8192",
+    "--max-cudagraph-capture-size", "56",
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    "--async-scheduling",
+    "--no-scheduler-reserve-full-isl",
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "deepseek_v4",
+    "--reasoning-config", """
+'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
+    "--default-chat-template-kwargs", """
+'{"thinking":true}'""",
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"greedy"}'""",
+    "--disable-chunked-mm-input",
+    "--limit-mm-per-prompt", """
+'{"image":4}'""",
+]
+
+[env]
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_ENABLE_DSPARK = "1"
+VLLM_ENABLE_MTP = "0"
+NUM_SPECULATIVE_TOKENS = "6"
+DSPARK_DRAFT_SAMPLE_METHOD = "greedy"
+DSPARK_SPS_CURVE = "auto"
+VLLM_CUDAGRAPH_MODE = "FULL_AND_PIECEWISE"
+VISION_MHC = "1"
+MAX_CUDAGRAPH_CAPTURE_SIZE = "56"
+# DCP prefill gather (images)
+VLLM_DSV4_CKV_GATHER = "1"
+VLLM_DSV4_CKV_GATHER_MAX_TOKENS = "393216"
+VLLM_DSV4_CKV_ALLOW_IMAGES = "1"
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+[[extra_mounts]]
+host = "~/dsv4vision-hf"
+container = "/opt/local-hf"
+env = "HF_HOME"

recipes/glm53-flash-aiden-2x/recipe.toml

@@ -0,0 +1,83 @@
+# GLM-5.3-Flash on 2× DGX Spark — aidendle94 sparkrun image (TP=2).
+#
+# aiden's 4x guide (https://aidenle.com/recipes/glm-5-3-flash-4x-dgx-spark)
+# translated to our 2-node ds4 launcher: same image (production-1.1 — SM90
+# NoPE-MLA-to-capability-12 route + three functional fixes + DFlash2 overlay),
+# same serve flags, TP2 instead of TP4.
+#
+# Image: aidendle94/sparkrun-vllm-glm53-gb10:production-1.1 (Docker Hub).
+#   production-1.0 = MTP-only fallback; 1.1 ships the DFlash2 drafter overlay
+#   (drafter package + aux-capture patch + KV-page LCM patch + backend-registry
+#   patch). Grep the log for "B12X PCIe|DFlash" style markers per his guide.
+#
+# Weights: LibertAIDAI/GLM-5.3-Flash-NVFP4 (194 GB, 97 GiB/node at TP2; cached
+# on both nodes). aiden: NVFP4 is ~23% faster than unsloth FP8 (which needs 4
+# nodes at 82 GB/rank). Drafter: incoai/GLM-5.3-Flash-DFlash2 (2.34 GB, cached).
+#   spec k MUST be 7 = block_size(2304? no: drafter block 8) − 1; acceptance
+#   0.5–0.8, ~0.15 = broken aux capture (silent).
+#
+# Memory plan (2x): 97 GiB weights/node + KV pin 4 GiB + eager activations
+# ≈ 103 GiB vs GMU 0.85 × 121.63 = 103.4 GiB. If a boot dies on free-memory,
+# drop KV pin to 3 GiB or GMU to 0.83. KV pool ≈ 400-600K tokens — fine for
+# agent-class benches, NOT for the 393K context the 4x profile serves.
+#
+# WARNING (from aiden's guide, untested on our 2x boot): the image bakes
+# VLLM_PREFIX_CACHE_RETENTION_INTERVAL; his launcher unsets it via bash -lc.
+# If the boot rejects the value in the KV coordinator, add a mods script that
+# unsets it (ours cannot env-unset across the exec boundary).
+#
+# Boot markers: "Loading safetensors" (~2-4 min sequential from NVMe) →
+# "GPU KV cache size" → "Application startup complete". DFlash2 acceptance
+# 0.5-0.8 in SpecDecoding metrics.
+
+engine = "vllm"
+name = "GLM-5.3-Flash aiden image (vLLM, 2x Spark, DFlash2-7)"
+docker_image = "aidendle94/sparkrun-vllm-glm53-gb10:production-1.1"
+model = "LibertAIDAI/GLM-5.3-Flash-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    # functional parity (silent client gaps in aiden's guide)
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    # aiden ships thinking ON (his default template kwargs)
+    "--default-chat-template-kwargs", """
+'{"enable_thinking": true, "reasoning_effort": "high"}'""",
+    # the SM90-NoPE + FA2 route: plain fp8 cache, real qk_rope_head_dim=0
+    "--kv-cache-dtype", "fp8_e4m3",
+    "--block-size", "2304",
+    "--moe-backend", "marlin",
+    "--max-model-len", "393216",
+    "--max-num-batched-tokens", "8192",
+    "--max-num-seqs", "16",
+    "--enable-chunked-prefill",
+    # deliberate per aiden (graphs neutral at TP4+collective-bound step;
+    # saves 0.6 GiB + 28 s capture — and we need every GiB on 2 nodes)
+    "--enforce-eager",
+    "--gpu-memory-utilization", "0.85",
+    "--kv-cache-memory-bytes", "4294967296",
+    "--speculative-config", """
+'{"method":"dflash","model":"incoai/GLM-5.3-Flash-DFlash2","num_speculative_tokens":7}'""",
+    "--language-model-only",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+TORCH_USE_RTLD_GLOBAL = "1"
+TORCHINDUCTOR_COMPILE_THREADS = "1"
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_NO_USAGE_STATS = "1"
+OMP_NUM_THREADS = "2"

recipes/glm53-flash-aiden-4x/recipe.toml

@@ -0,0 +1,72 @@
+# GLM-5.3-Flash on 4× DGX Spark — aidendle94 sparkrun image (TP=4). DEFERRED.
+#
+# aiden's production 4-node profile, verbatim translation of
+# https://aidenle.com/recipes/glm-5-3-flash-4x-dgx-spark into our 4x launcher
+# shape. Reference numbers (his, single stream, idle cluster, thinking off):
+#   ~84 tok/s decode (FP8, DFlash2 k=7) — NVFP4 +23% (110.7 structured);
+#   prefill ~2,100 cold; TTFT 4 s @ 8K → 40 s @ 94K; KV 2.25M tokens at 16 GiB
+#   pin; 393K context verified. Decode flat with context (34/45 KDA layers +
+#   11 sparse-MLA @ index_topk 2048).
+#
+# DFlash2: acceptance 0.5–0.8; k=7 (drafter block 8 − 1 target token); k=6
+# wins prose, loses structured. Native MTP fallback (production-1.0 image or
+# spec omitted) optimum k=5. ~15 min full relaunch on FP8 (963 s measured).
+# FP8-MLA + DFlash2 combination is STRUCTURALLY broken upstream (three trees
+# fail differently) — hence plain fp8_e4m3 KV.
+#
+# NOT READY TO DEPLOY: needs a 4-node cluster (ds4 + 2nodes-dac freed) and the
+# unsloth/GLM-5.3-Flash-FP8 checkpoint (328 GB — background download running
+# on dgx3, /tmp/dl_fp8.log; ~340 GB free per node required for weights + Triton
+# cache). Fabric: RoCE dual-HCA NCCL profile comes from the launcher (config
+# [nccl.*] / GID auto-detect) — his NCCL_* env block intentionally omitted.
+# NOTE: his launcher unsets VLLM_PREFIX_CACHE_RETENTION_INTERVAL via bash -lc
+# before exec; if our boot rejects it in the KV coordinator, add a mods script.
+
+engine = "vllm"
+name = "GLM-5.3-Flash aiden image (vLLM, 4x Spark, DFlash2-7, FP8) [DEFERRED]"
+docker_image = "aidendle94/sparkrun-vllm-glm53-gb10:production-1.1"
+model = "unsloth/GLM-5.3-Flash-FP8"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--default-chat-template-kwargs", """
+'{"enable_thinking": true, "reasoning_effort": "high"}'""",
+    "--kv-cache-dtype", "fp8_e4m3",
+    "--block-size", "2304",
+    "--moe-backend", "marlin",
+    "--max-model-len", "393216",
+    "--max-num-batched-tokens", "8192",
+    "--max-num-seqs", "16",
+    "--enable-chunked-prefill",
+    "--enforce-eager",
+    "--gpu-memory-utilization", "0.85",
+    "--kv-cache-memory-bytes", "17179869184",
+    "--speculative-config", """
+'{"method":"dflash","model":"incoai/GLM-5.3-Flash-DFlash2","num_speculative_tokens":7}'""",
+    "--language-model-only",
+    # image requests need the mm template shipped/in-wallpaper on aiden's rigs:
+    # "--chat-template", "<path-to>/chat_template_mm.jinja",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+TORCH_USE_RTLD_GLOBAL = "1"
+TORCHINDUCTOR_COMPILE_THREADS = "1"
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_NO_USAGE_STATS = "1"
+OMP_NUM_THREADS = "2"