launcher/recipes/dsv4-vision-aiden-2x/recipe.toml

# DeepSeek-V4-Flash-Vision-Exp on 2× GB10 — aidendle94 sparkrun vision image.
#
# aiden's vision guide (https://aidenle.com/recipes/deepseek-v4-flash-vision-4x-dgx-spark/)
# translated to 2× Spark / DCP1 (his "DSpark, DCP1" profile: 73.8-75.2 tok/s at
# TP4/DCP1; on 2x expect roughly half). His production lane ships DCP2 + DSpark
# (66-71 tok/s, 3.5M KV); DCP flags omitted here — DSpark runs fine under DCP1.
#
# Image: dsv4-flash-vision:local (our plugin overlay on aiden's
# production-3.76.1 base — see dsv4-flash-vision/ UPSTREAM.md). Aiden's
# production-3.72-vision tag is the published 4x stack; running it under our
# launcher needs his env-script contract (NODE_RANK/MASTER_ADDR/NCCL_IB_*
# required by /usr/local/bin/deepseek-v4-flash-2x-spark-entrypoint →
# /opt/vllm-src/serve-ds4-flash-spark.sh) — a dedicated integration pass,
# tracked as follow-up. The overlay image carries the equivalent flags:
# draft greedy + FULL_AND_PIECEWISE graphs + capture cap via the envs below.
#
# Weights: deepseek-ai/DeepSeek-V4-Flash-Vision-Exp (cached; ~84 GiB/node at
# TP2 via ~/dsv4vision-hf NVMe copies). Draft heads ship in the checkpoint
# (mtp.*) — no extra download.
#
# DSpark: k=6, greedy draft, acceptance 3.5-3.8 tokens/step. Acceptance is the
# DCP correctness signal; equal DSpark-DCP1 numbers here = draft sees full ctx.
# max-cudagraph-capture-size = seqs × (spec tokens + 1) = 8 × 7 = 56.
#
# VISION_MHC=1 follows aiden's production profile; our plugin lane keeps
# VLLM_USE_B12X_MHC=0 (checkpoint rms_norm_eps 1e-20 vs fused Gram kernel's
# 1e-6 — upstream README). If this boot asserts on the mHC kernel, flip to 0.
#
# Boot markers: vision plugin registered → "Model loading took ~84 GiB" →
# "GPU KV cache size" ≈ 1.25M → "Application startup complete".

engine = "vllm"
name = "DeepSeek V4 Flash Vision-Exp (aiden image, 2x Spark, DSpark-6)"
docker_image = "dsv4-flash-vision:local"
model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# his image has TWO vllm wheels: dist-packages (plain, NO vision) and the
# compiled stack at /opt/vllm-src/.venv (multimodal-wrapped DeepseekV4). The
# image's own entrypoint script also injects a big env block (B12X_*,
# PCIE_ALLREDUCE, MEGA_AOT, JIT paths) — the flag-relevant subset is in [env]
# below. Point the launcher tail straight at their venv python.
entrypoint = ["/opt/vllm-src/.venv/bin/python", "-m", "vllm.entrypoints.cli.main", "serve"]

args = [
    "--served-model-name", "deepseek-v4-vision",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    "--attention-backend", "B12X_MLA_SPARSE",
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--kv-cache-dtype", "fp8",
    "--block-size", "256",
    "--gpu-memory-utilization", "0.80",
    "--kv-cache-memory", "12884901888",
    "--max-model-len", "393216",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "8192",
# capture cap comes from the sizes list (max 8) — the 56 value would need 56 in the list
    "--compilation-config", """
'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8]}'""",
    "--async-scheduling",
    "--no-scheduler-reserve-full-isl",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    "--tokenizer-mode", "deepseek_v4",
    "--tool-call-parser", "deepseek_v4",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v4",
    "--reasoning-config", """
'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
    "--default-chat-template-kwargs", """
'{"thinking":false,"reasoning_effort":"low"}'""",
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"greedy","attention_backend":"B12X_MLA_SPARSE"}'""",
    "--disable-chunked-mm-input",
    "--limit-mm-per-prompt", """
'{"image":4}'""",
]

[env]
# aidee runner envs (his guide, minus DCP2-only and fabric knobs)
CUTE_DSL_ARCH = "sm_121a"
TRITON_PTXAS_PATH = "/usr/local/cuda/bin/ptxas"
DG_JIT_USE_NVRTC = "0"
USE_CUDNN = "1"
VLLM_USE_MEGA_AOT_ARTIFACT = "1"
VLLM_ENABLE_PCIE_ALLREDUCE = "1"
VLLM_PCIE_ALLREDUCE_BACKEND = "b12x"
VLLM_USE_B12X_DENSE_SPLITK_TURBO = "1"
VLLM_USE_B12X_W4A16_TC_DECODE = "1"
VLLM_ENABLE_DSPARK = "1"
VLLM_ENABLE_MTP = "0"
NUM_SPECULATIVE_TOKENS = "6"
DSPARK_DRAFT_SAMPLE_METHOD = "greedy"
DSPARK_SPS_CURVE = "auto"
VLLM_CUDAGRAPH_MODE = "FULL_AND_PIECEWISE"
VISION_MHC = "1"
MAX_CUDAGRAPH_CAPTURE_SIZE = "56"
# proven 2x-lane knobs carried over
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_B12X_WO_PROJECTION = "1"
VLLM_USE_B12X_FP8_GEMM = "1"
VLLM_USE_B12X_MOE = "1"
VLLM_USE_B12X_SPARSE_INDEXER = "1"
B12X_MLA_SM120_UNIFIED = "1"
B12X_MOE_FORCE_A8 = "1"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_NO_USAGE_STATS = "1"

# Weights from per-node NVMe copies; extra_mounts stay AFTER args.
[[extra_mounts]]
host = "~/dsv4vision-hf"
container = "/opt/local-hf"
env = "HF_HOME"