launcher/recipes/dsv4-flash-vision-eugr/recipe.toml

# BLOCKED 2026-09-06 (night shift): the eugr 0904 image's DeepseekV4ForCausalLM
# has no `aligner` module (predates V4-Vision support) — the Vision-Exp
# checkpoint's vision weights fail to load. Re-check on a newer eugr image;
# the fork lane `dsv4-flash-vision` remains the only V4-Vision path.
#
# DeepSeek-V4-Flash-Vision-Exp — eugr B12X variant (vLLM) — 2× DGX Spark, TP=2.
#
# The eugr-stack A/B cell against our fork lane `dsv4-flash-vision` (same
# model, same node shape; their prebuilt image vs our dsv4-flash-vision:local
# overlay + dsv4_vision plugin). Port of eugr's deepseek-v4-flash-0731.yaml
# shape (eugr/spark-vllm-docker @ main) onto the Vision-Exp model.
#
# Image: eugr/spark-vllm-b12x:nightly-20260904 (digest c3b44d3f…, = :latest).
#   local-inference-lab/vllm dev/jovian-judgement 09-04 tip (ref switched from
#   dev/infernal-invocation by their e91bc76, 09-03) + lukealonso/b12x @ master
#   + their patch set; torch 2.13.0, CUDA 13.0.2, cutlass-dsl 4.7.0.
#   NOTE: image ships NO ENTRYPOINT → entrypoint = ["vllm","serve"] below.
#   Exact b12x pin baked in-image: /workspace/b12x-source-commit.
#
# Model: deepseek-ai/DeepSeek-V4-Flash-Vision-Exp (MIT; ~168 GB on disk,
#   experts MXFP4 / attention FP8, 32-layer ViT BF16). Reuse the per-node
#   local NVMe copies from the fork lane if present (~/dsv4vision-hf on
#   dgx3/dgx4, rsync'd 2026-09-01 — check the HF_HOME mapping); else
#   huggingface-cli download on BOTH nodes (HF_HUB_OFFLINE=1 below).
#
# Diffs vs the fork lane (dsv4-flash-vision) — deliberate parity choices:
#   - attention backend: plain B12X (their 09-03 switch away from
#     B12X_MLA_SPARSE) — the fork lane keeps B12X_MLA_SPARSE
#   - VLLM_USE_MEGA_AOT_ARTIFACT=1 (their 09-03 switch; fork lane: -1)
#   - max-cudagraph-capture-size 48 (their 09-03 value)
#   - spec: vendor card k=3 + enable_adaptive_verification (published:
#     2.99 mean acceptance, 66.3%); k=6 (fork lane's proven value) is the
#     fallback if adaptive verification misbehaves on this stack
#   - VLLM_USE_B12X_MHC forced 0 like the fork lane: this checkpoint has
#     rms_norm_eps 1e-20, the fused Gram mHC kernel only accepts 1e-6
#   - no dsv4_vision plugin (their image has stock vLLM vision path) — if
#     boot fails on the vision processor, that's the first suspect
#
# REFERENCE:
#   https://github.com/eugr/spark-vllm-docker (recipes/deepseek-v4-flash-0731.yaml)
#   https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp
#   https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp

engine = "vllm"
name = "DeepSeek V4 Flash Vision-Exp (vLLM, 2x Spark, eugr B12X, DSpark-3)"
docker_image = "eugr/spark-vllm-b12x:nightly-20260904"
model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# image ships no ENTRYPOINT — the launcher supplies the vllm command
entrypoint = ["vllm", "serve"]

args = [
    # Stable served name (shared with the fork lane — quant/stack-agnostic)
    "--served-model-name", "deepseek-v4-flash-vision",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # eugr 0731 shape: full b12x cluster, plain B12X attention (09-03 switch)
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--attention-backend", "B12X",
    "--max-model-len", "1048576",
    "--kv-cache-dtype", "fp8",
    # fork lane's proven pool for this checkpoint on 2 Sparks (1.25M tokens)
    "--kv-cache-memory-bytes", "12884901888",
    "--gpu-memory-utilization", "0.85",
    "--max-num-seqs", "2",
    "--max-num-batched-tokens", "8192",
    "--block-size", "256",
    "--max-cudagraph-capture-size", "48",
    # vendor card (published numbers): DSpark k=3, probabilistic, adaptive
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"B12X","enable_adaptive_verification":true}'""",
    "--tokenizer-mode", "deepseek_v4",
    "--tool-call-parser", "deepseek_v4",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v4",
    "--reasoning-config", """
'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
    "--default-chat-template-kwargs", """
'{"thinking":false,"reasoning_effort":"low"}'""",
    "--limit-mm-per-prompt", """
'{"image":8}'""",
    "--enable-prefix-caching",
    "--load-format", "instanttensor",
]

[env]
CUTE_DSL_ARCH = "sm_121a"
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_USE_MEGA_AOT_ARTIFACT = "1"
VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_B12X_WO_PROJECTION = "1"
# OFF for this checkpoint (rms_norm_eps 1e-20 — see header); their 0731
# text recipe sets it 1, but the model constraint wins
VLLM_USE_B12X_MHC = "0"
VLLM_USE_B12X_FP8_GEMM = "1"
VLLM_USE_B12X_MOE = "1"
VLLM_USE_B12X_SPARSE_INDEXER = "1"
VLLM_USE_V2_MODEL_RUNNER = "1"
B12X_MLA_SM120_UNIFIED = "1"
B12X_MOE_FORCE_A8 = "1"
HF_HUB_OFFLINE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"