launcher/recipes/dsv4-vision-exp-vavrusa/recipe.toml

# DeepSeek-V4-Flash-Vision-Exp — vavrusa sparkrun lane ported to the launcher.
#
# Working OpenAI image_url multimodal serving on DGX Spark from
# vavrusa/sparkrun-recipes `deepseek-v4-flash-vision-exp-eugr.yaml`
# (ghcr.io/spark-arena/dgx-vllm-eugr-nightly-b12x:20260810 + two runtime
# mods). This is the eugr A/B lane (todo #26) AND the image_url lane for
# the vision checkpoint — compare against recipes/dsv4-flash-vision
# (our plugin on dsv4-flash-vision:local).
#
# Mod provenance (vendored verbatim under mods/):
#   * instanttensor-hybrid-draft-loader — keeps InstantTensor for the target
#     while embedded-MTP drafts load via lazy safetensors.
#   * deepseek-v4-vision-exp — stages vision_exp + injects a fail-closed
#     import hook (port of MiaAI-Lab/DeepSeek-v4-Flash-DSpark-2x-DGX-Spark)
#     into the Anemll fork's deepseek_v4/nvidia/model.py: ViT + aligner,
#     weight mapping for vision./aligner./image_*/bias_vl, vLLM multimodal
#     processor registration, embed_input_ids embedding scatter, DSpark
#     draft ffn.gate.bias_vl remap.
#
# Checkpoint deltas vs text-only 0731 (from their README): vision keys;
# num_nextn_predict_layers 1→3 (DSpark k must be a multiple of 3 → k=6);
# rms_norm_eps 1e-6→1e-20 (mod overrides back to 1e-6 for the fused Gram
# mHC kernel); ffn.gate +bias_vl (loaded, not applied at routing).
# Images go in USER messages only; ≤384 image tokens per image.
#
# DEVIATIONS from the sparkrun recipe — all forced by our launcher:
#   * no --host/--port (launcher fabric), TP size from recipe math,
#   * --served-model-name pinned to our quant-agnostic lane alias
#     `deepseek-v4-flash-vision` (HF_HUB_OFFLINE makes a bare label carry
#     the cache path otherwise),
#   * default-chat-template-kwargs thinking=false / effort=low to match the
#     tool-eval c1 lane (their recipe ships thinking=true / high),
#   * weights from per-node local NVMe copies (`~/dsv4vision-hf` on
#     dgx3/dgx4, 167.8 GiB each). Fallback: swap the extra_mounts block to
#     /mnt/nas/ai/huggingface (see recipes/dsv4-flash-vision).

engine = "vllm"
# eugr nightly image carries no ENTRYPOINT — required by the mods contract.
entrypoint = ["vllm", "serve"]
name = "DeepSeek V4 Flash Vision-Exp (eugr nightly + vavrusa vision mods)"
docker_image = "ghcr.io/spark-arena/dgx-vllm-eugr-nightly-b12x:20260810"
model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    "--served-model-name", "deepseek-v4-flash-vision",
    "--trust-remote-code",
    "--tensor-parallel-size", "2",
    "--kv-cache-dtype", "fp8_ds_mla",
    # their picks: max-model-len auto (KV pool revalidated after first
    # launch — the vision tower replicates per rank), 5 seqs / 4096 tokens.
    "--max-model-len", "auto",
    "--max-num-seqs", "5",
    "--max-num-batched-tokens", "4096",
    "--gpu-memory-utilization", "0.82",
    "--block-size", "256",
    "--enable-prefix-caching",
    "--enable-chunked-prefill",
    "--enable-prompt-tokens-details",
    "--enable-flashinfer-autotune",
    "--async-scheduling",
    "--tokenizer-mode", "deepseek_v4",
    "--tool-call-parser", "deepseek_v4",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v4",
    "--reasoning-config", """
'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
    "--default-chat-template-kwargs", """
'{"thinking":false,"reasoning_effort":"low"}'""",
    "--override-generation-config", """
'{"temperature":1.0,"top_p":0.95}'""",
    "--limit-mm-per-prompt", """
'{"image":8}'""",
    "--load-format", "instanttensor",
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--attention-backend", "B12X_MLA_SPARSE",
    "--max-cudagraph-capture-size", "64",
    "--compilation-config", """
'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
    # num_nextn_predict_layers=3 → k multiple of 3; k=6 (smallest ≥ dspark
    # block size 5). Do NOT drop back to 5.
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"greedy","attention_backend":"B12X_MLA_SPARSE"}'""",
]

[env]
PYTHONHASHSEED = "0"
CUTE_DSL_ARCH = "sm_121a"
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_USE_MEGA_AOT_ARTIFACT = "-1"
VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_B12X_WO_PROJECTION = "1"
VLLM_USE_B12X_MHC = "1"
VLLM_USE_B12X_FP8_GEMM = "1"
VLLM_USE_B12X_MOE = "1"
VLLM_USE_B12X_SPARSE_INDEXER = "1"
VLLM_MOE_SKIP_PADDING = "0"
VLLM_USE_V2_MODEL_RUNNER = "1"
B12X_MLA_SM120_UNIFIED = "1"
B12X_MOE_FORCE_A8 = "1"
VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_NO_USAGE_STATS = "1"

# Weights from per-node local NVMe copies; JIT caches persist per node.
# NOTE: extra_mounts stay AFTER args (TOML table headers would swallow the
# args array otherwise).
[[extra_mounts]]
host = "~/dsv4vision-hf"
container = "/opt/local-hf"
env = "HF_HOME"

# Both mods run inside the container before exec'ing vllm serve. The
# vision mod stages to /opt/dspark-patches and rewrites site-packages —
# the container fs stays writable (only /mods is :ro).
[[mods.commands]]
command = "bash /mods/instanttensor-hybrid-draft-loader/run.sh"

[[mods.commands]]
command = "bash /mods/deepseek-v4-vision-exp/run.sh"