launcher/recipes/dsv4-vision-aiden-4x/recipe.toml

# DeepSeek-V4-Flash-Vision-Exp on 4× GB10 — aiden production profile. DEFERRED.
#
# Source: https://aidenle.com/recipes/deepseek-v4-flash-vision-4x-dgx-spark/
# TP4 across 4 nodes with **DCP2 inside it** and DSpark-6 running together —
# the combination stock vLLM refuses twice ("DSpark requires DCP_SIZE=1" +
# "Non-causal DSpark SWA does not support DCP"); aiden's image resolves it by
# marking the draft's sliding-window cache `dcp_replicated` (full ctx on every
# rank, cost-free for a few hundred tokens). Shipped profile numbers:
#   decode 66-71 tok/s @ acceptance 3.5-3.8; cold 334K prefill 2,088 tok/s;
#   KV pool 3.51M tokens (8.9× full-length ctx); 393K ctx verified;
#   image request ~1.2 s warm / ~6 s first.
#
# Profile trade (his table): DSpark+DCP1 73.8-75.2 / DCP2-no-spec 34 /
# DSpark+DCP2 66-71 (DCP1 pool 1.77M vs DCP2 3.51M — KV doubled at ~7% decode).
#
# NOT READY TO DEPLOY: needs a 4-node cluster window. Fabric NCCL envs omitted
# here (launcher-managed, [nccl.*]). His runner envs carrying operational
# requirements: earlyoom tuned on every node, root for RDMA passthrough +
# memlock + page-cache flushing, ~120 GB free per node for weights + JIT cache.
#
# Env/flag deltas vs our 2x variant (dsv4-vision-aiden-2x): DCP_SIZE=2,
# --decode-context-parallel-size 2 --dcp-comm-backend ag_rs (ag_rs = +12% KV
# over a2a), KV_CACHE_MEMORY_BYTES unpin (empty — the ~26 GB cap off),
# GPU_MEMORY_UTILIZATION 0.75 (0.82 → earlyoom kills a worker on his kit),
# VLLM_DSV4_CKV_GATHER=1 + CKV_GATHER_MAX_TOKENS + ALLOW_IMAGES (prefill
# gather required for images under DCP).

engine = "vllm"
name = "DeepSeek V4 Flash Vision-Exp (aiden image, 4x Spark, DSpark-6 + DCP2) [DEFERRED]"
docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.72-vision"
model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# the image's BAKED-IN entrypoint is aiden's env-script (doubles our tail
# args if left in place). The entrypoint below + one mods line makes the
# launcher wrap in `--entrypoint sh -c` and exec `vllm serve <model> <args>`
# directly with the image's system vllm (his multimodal wheel).
entrypoint = ["vllm", "serve"]


args = [
    "--served-model-name", "deepseek-v4-vision",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    "--decode-context-parallel-size", "2",
    "--dcp-comm-backend", "ag_rs",
    "--attention-backend", "B12X_MLA_SPARSE",
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--kv-cache-dtype", "fp8",
    "--block-size", "256",
    "--gpu-memory-utilization", "0.75",
    "--max-model-len", "393216",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "8192",
    "--max-cudagraph-capture-size", "56",
    "--compilation-config", """
'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
    "--async-scheduling",
    "--no-scheduler-reserve-full-isl",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    "--tokenizer-mode", "deepseek_v4",
    "--tool-call-parser", "deepseek_v4",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v4",
    "--reasoning-config", """
'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
    "--default-chat-template-kwargs", """
'{"thinking":true}'""",
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"greedy"}'""",
    "--disable-chunked-mm-input",
    "--limit-mm-per-prompt", """
'{"image":4}'""",
]

[mods]
commands = [
    # (mods exist so the launcher takes the --entrypoint sh override path)
    "true",
]

[env]
CUTE_DSL_ARCH = "sm_121a"
VLLM_ENABLE_DSPARK = "1"
VLLM_ENABLE_MTP = "0"
NUM_SPECULATIVE_TOKENS = "6"
DSPARK_DRAFT_SAMPLE_METHOD = "greedy"
DSPARK_SPS_CURVE = "auto"
VLLM_CUDAGRAPH_MODE = "FULL_AND_PIECEWISE"
VISION_MHC = "1"
MAX_CUDAGRAPH_CAPTURE_SIZE = "56"
# DCP prefill gather (images)
VLLM_DSV4_CKV_GATHER = "1"
VLLM_DSV4_CKV_GATHER_MAX_TOKENS = "393216"
VLLM_DSV4_CKV_ALLOW_IMAGES = "1"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_NO_USAGE_STATS = "1"