launcher/recipes/dsv4-vision-aiden-2x/recipe.toml

# DeepSeek-V4-Flash-Vision-Exp on 2× GB10 — aidendle94 sparkrun vision image.
#
# aiden's vision guide (https://aidenle.com/recipes/deepseek-v4-flash-vision-4x-dgx-spark/)
# translated to 2× Spark / DCP1 (his "DSpark, DCP1" profile: 73.8-75.2 tok/s at
# TP4/DCP1; on 2x expect roughly half). His production lane ships DCP2 + DSpark
# (66-71 tok/s, 3.5M KV); DCP flags omitted here — DSpark runs fine under DCP1.
#
# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-3.72-vision (published
# vision stack: hand-ported tower/routing/CKV gather, cross-checked against
# upstream vLLM PR #54566). Sibling comparison: dsv4-flash-vision:local (our
# plugin overlay on his production-3.76.1 base) — the A/B pair for the vision
# lane.
#
# Weights: deepseek-ai/DeepSeek-V4-Flash-Vision-Exp (cached; ~84 GiB/node at
# TP2 via ~/dsv4vision-hf NVMe copies). Draft heads ship in the checkpoint
# (mtp.*) — no extra download.
#
# DSpark: k=6, greedy draft, acceptance 3.5-3.8 tokens/step. Acceptance is the
# DCP correctness signal; equal DSpark-DCP1 numbers here = draft sees full ctx.
# max-cudagraph-capture-size = seqs × (spec tokens + 1) = 8 × 7 = 56.
#
# VISION_MHC=1 follows aiden's production profile; our plugin lane keeps
# VLLM_USE_B12X_MHC=0 (checkpoint rms_norm_eps 1e-20 vs fused Gram kernel's
# 1e-6 — upstream README). If this boot asserts on the mHC kernel, flip to 0.
#
# Boot markers: vision plugin registered → "Model loading took ~84 GiB" →
# "GPU KV cache size" ≈ 1.25M → "Application startup complete".

engine = "vllm"
name = "DeepSeek V4 Flash Vision-Exp (aiden image, 2x Spark, DSpark-6)"
docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.72-vision"
model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
entrypoint = ["vllm", "serve"]

args = [
    "--served-model-name", "deepseek-v4-vision",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    "--attention-backend", "B12X_MLA_SPARSE",
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--kv-cache-dtype", "fp8",
    "--block-size", "256",
    "--gpu-memory-utilization", "0.80",
    "--kv-cache-memory", "12884901888",
    "--max-model-len", "393216",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "8192",
    "--max-cudagraph-capture-size", "56",
    "--compilation-config", """
'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8]}'""",
    "--async-scheduling",
    "--no-scheduler-reserve-full-isl",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    "--tokenizer-mode", "deepseek_v4",
    "--tool-call-parser", "deepseek_v4",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v4",
    "--reasoning-config", """
'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
    "--default-chat-template-kwargs", """
'{"thinking":false,"reasoning_effort":"low"}'""",
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"greedy","attention_backend":"B12X_MLA_SPARSE"}'""",
    "--disable-chunked-mm-input",
    "--limit-mm-per-prompt", """
'{"image":4}'""",
]

[env]
# aidee runner envs (his guide, minus DCP2-only and fabric knobs)
CUTE_DSL_ARCH = "sm_121a"
VLLM_ENABLE_DSPARK = "1"
VLLM_ENABLE_MTP = "0"
NUM_SPECULATIVE_TOKENS = "6"
DSPARK_DRAFT_SAMPLE_METHOD = "greedy"
DSPARK_SPS_CURVE = "auto"
VLLM_CUDAGRAPH_MODE = "FULL_AND_PIECEWISE"
VISION_MHC = "1"
MAX_CUDAGRAPH_CAPTURE_SIZE = "56"
# proven 2x-lane knobs carried over
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_B12X_WO_PROJECTION = "1"
VLLM_USE_B12X_FP8_GEMM = "1"
VLLM_USE_B12X_MOE = "1"
VLLM_USE_B12X_SPARSE_INDEXER = "1"
B12X_MLA_SM120_UNIFIED = "1"
B12X_MOE_FORCE_A8 = "1"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_NO_USAGE_STATS = "1"

# Weights from per-node NVMe copies; extra_mounts stay AFTER args.
[[extra_mounts]]
host = "~/dsv4vision-hf"
container = "/opt/local-hf"
env = "HF_HOME"