launcher/recipes/dsv4-flash-vision/recipe.toml

# DeepSeek-V4-Flash-Vision-Exp — 2× GB10 / DGX Spark, TP=2 (vision lane).
#
# Model: deepseek-ai/DeepSeek-V4-Flash-Vision-Exp (MIT) — DeepSeek's first
# multimodal V4: 284B total / 13B active MoE, experts MXFP4, attention FP8,
# 32-layer ViT (~0.47B BF16) + aligner, native ctx 1,048,576; 43+3 MoE
# layers, DFlash attention, index_topk 512, fp8 block [128,128].
# Day-0 vision support lives in a vLLM PLUGIN (dsv4_vision) subclassing
# DeepseekV4ForCausalLM so DSpark speculative still works; the plugin is
# baked into our overlay image (docker-images/dsv4-flash-vision/, see its
# UPSTREAM.md for provenance and the sfxnz repo @7a958128b84a).
#
# Image: dsv4-flash-vision:local — built from eugr/spark-vllm-b12x:latest
# (digest sha256:7dc02f162929943ba2e14514066ed2a04bb7e9ed3592d4eb460ebcbb1f8376bd)
# by default; the aidendle94/sparkrun-vllm-ds4-gb10:production-3.76.1 base
# (sha256:39cda85c28a8795dc20a274fdabfece66f329d1e21c3f4552a7e75061ea8696a)
# is the comparison variant for the same plugin (fork A/B).
#
# Model revision: 86f746b36186f0e567729a5c06a8c918caba82a9 (sfxnz pin).
#
# DEVIATIONS from the sfxnz run.sh defaults — all forced by our launcher:
#   * no --host/--port/--nnodes/--node-rank/--master-* (launcher fabric),
#   * no NCCL/* / IFACE / HCA env (launcher resolves; RoCE on sw fabric),
#   * weights stream from the shared NAS hub (their kit uses per-node local
#     NVMe; if the loader balloons memory like it did on EXL3, flip to
#     --load-format instanttensor).
# VLLM_USE_B12X_MHC stays OFF (this checkpoint has rms_norm_eps 1e-20; the
# fused Gram mHC kernel only accepts 1e-6 — upstream README).
#
# Boot markers: "vision" plugin registered line → "Model loading took …"
# (~84 GiB/node) → "GPU KV cache size" ≈ 1.25M tokens → "Application
# startup complete". Vision smoke: image_url content with the
# <｜deepseek_image｜> placeholder (cap 384 tokens per image).
#
# PREREQUISITES: weights cached on the NAS hub once (168 GB):
#   docker run --rm --entrypoint python3 -v /mnt/nas/ai/huggingface:/root/.cache/huggingface \
#     glm53-flash-nvfp4-lab:local -c "from huggingface_hub import snapshot_download; \
#     snapshot_download('deepseek-ai/DeepSeek-V4-Flash-Vision-Exp')"

engine = "vllm"
name = "DeepSeek V4 Flash Vision-Exp (vLLM, 2x Spark, DSpark-6)"
docker_image = "dsv4-flash-vision:local"
docker_image_dir = "dsv4-flash-vision"
model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name (quant-agnostic convention for the ds4-vision lane)
    "--served-model-name", "deepseek-v4-flash-vision",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # sfxnz run.sh defaults (verbatim, minus launcher-managed fabric knobs)
    "--max-model-len", "1048576",
    "--kv-cache-dtype", "fp8",
    # their pin: "a 26 GiB community pin" is too big for this UMA leftover
    "--kv-cache-memory", "12884901888",
    "--gpu-memory-utilization", "0.80",
    "--max-num-seqs", "2",
    "--max-num-batched-tokens", "8192",
    "--block-size", "256",
    # Engine backends (B12X GB10 lane)
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--attention-backend", "B12X_MLA_SPARSE",
    # DSpark-6 (k must be divisible by n_predict=3; DSpark-5 is rejected)
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic","attention_backend":"B12X_MLA_SPARSE"}'""",
    "--tokenizer-mode", "deepseek_v4",
    "--tool-call-parser", "deepseek_v4",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v4",
    "--reasoning-config", """
'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
    "--default-chat-template-kwargs", """
'{"thinking":false,"reasoning_effort":"low"}'""",
    "--limit-mm-per-prompt", """
'{"image":8}'""",
    "--enable-prefix-caching",
    # If the stock loader balloons passenger anon buffers while streaming
    # from the NAS (the EXL3 lesson, 3 kernel kills on a sibling kit),
    # flip this on: the overlay ships instanttensor==0.1.9 (no-deps).
    # "--load-format", "instanttensor",
]

[env]
# Vision + engine lane knobs (sfxnz run.sh, verbatim minus fabric)
CUTE_DSL_ARCH = "sm_121a"
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_USE_MEGA_AOT_ARTIFACT = "-1"
VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_B12X_WO_PROJECTION = "1"
VLLM_USE_B12X_MHC = "0"
VLLM_USE_B12X_FP8_GEMM = "1"
VLLM_USE_B12X_MOE = "1"
VLLM_USE_B12X_SPARSE_INDEXER = "1"
VLLM_USE_V2_MODEL_RUNNER = "1"
B12X_MLA_SM120_UNIFIED = "1"
B12X_MOE_FORCE_A8 = "1"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_NO_USAGE_STATS = "1"

# Weights from the shared NAS hub; JIT caches persist per node.
# NOTE: extra_mounts stay AFTER args (TOML table headers would swallow the
# args array otherwise).
[[extra_mounts]]
host = "/mnt/nas/ai/huggingface"
container = "/mnt/nas/ai/huggingface"
env = "HF_HOME"

[[extra_mounts]]
host = "~/.cache/dsv4vision-jit/triton"
container = "/root/.triton"
[[extra_mounts]]
host = "~/.cache/dsv4vision-jit/flashinfer"
container = "/root/.cache/flashinfer"
[[extra_mounts]]
host = "~/.cache/dsv4vision-jit/vllm"
container = "/root/.cache/vllm"