launcher/recipes/dsv4-vision-aiden-2x/recipe.toml

# DeepSeek-V4-Flash-Vision-Exp on 2× GB10 — aiden image, flag route.
#
# aidendle94/sparkrun-vllm-ds4-gb10:production-3.72-vision, TP2 over ds4.
# His production 2x DSpark-6 profile translated to our launcher shape: his
# own serve line (moe/linear b12x, B12X_MLA_SPARSE attention, fp8 KV, block
# 256, DSpark-6 greedy, graphs FULL_AND_PIECEWISE) as plain flags — the
# image's BAKED-IN entrypoint is his env-script (which doubles our tail args
# if left in place), so the recipe sets entrypoint = ["vllm","serve"] and one
# mods line, which makes the launcher wrap everything in `--entrypoint sh -c`
# and exec our tail directly with the image's system vllm — which IS his
# multimodal wheel (the dist-packages vllm here carries the DeepseekV4 vision
# wrap; the earlier "not multimodal" 400s came from OUR overlay image, not
# this one).
#
# Weights from the SHARED NAS HF cache (all 4 nodes mount nas.lan:/sna as
# /mnt/nas; the repo's refs/main resolves offline). 2x reference: his 4x
# DCP1 DSpark profile is 73.8-75.2 tok/s; 2x should sit below that.
# Boot markers: "Model loading took ~82 GiB" → "GPU KV cache size" →
# "Application startup complete". Smoke: 47*83 → 3901 + image_url ask.

engine = "vllm"
name = "DeepSeek V4 Flash Vision-Exp (aiden image, 2x Spark, DSpark-6)"
docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.72-vision"
model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# required for the launcher's `--entrypoint sh -c` override path (mods),
# which also replaces the image's baked env-script entrypoint.
entrypoint = ["vllm", "serve"]


args = [
    "--served-model-name", "deepseek-v4-vision",
    "--trust-remote-code",
    "--attention-backend", "B12X_MLA_SPARSE",
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--kv-cache-dtype", "fp8",
    "--block-size", "256",
    # his verified 2x (TP2) values: 0.75 fails KV alloc, 0.82 leaves one rank
    # short, 0.85 works; KV pin left OFF (a pin caps the pool ~26 GB/rank and
    # hides the layout headroom)
    "--gpu-memory-utilization", "0.85",
    "--max-model-len", "393216",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "8192",
    "--max-cudagraph-capture-size", "56",
    "--compilation-config", """
'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
    "--async-scheduling",
    "--no-scheduler-reserve-full-isl",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    "--tokenizer-mode", "deepseek_v4",
    "--tool-call-parser", "deepseek_v4",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v4",
    "--reasoning-config", """
'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
    "--default-chat-template-kwargs", """
'{"thinking":true}'""",
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"greedy"}'""",
    "--disable-chunked-mm-input",
    "--limit-mm-per-prompt", """
'{"image":4}'""",
]

[mods]
commands = [
    # (mods exist so the launcher takes the --entrypoint sh override path;
    # the pre-serve work — page-cache flush — is done host-side by before_serve)
    "true",
]

[env]
CUTE_DSL_ARCH = "sm_121a"
VLLM_ENABLE_DSPARK = "1"
VLLM_ENABLE_MTP = "0"
NUM_SPECULATIVE_TOKENS = "6"
DSPARK_DRAFT_SAMPLE_METHOD = "greedy"
DSPARK_SPS_CURVE = "auto"
VLLM_CUDAGRAPH_MODE = "FULL_AND_PIECEWISE"
VISION_MHC = "1"
MAX_CUDAGRAPH_CAPTURE_SIZE = "56"
# (DCP prefill gather envs are DCP2-only; DCP1 here needs none)
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_NO_USAGE_STATS = "1"