recipes: dsv4-flash-vision

recipes/dsv4-flash-vision/recipe.toml

@@ -0,0 +1,138 @@
+# DeepSeek-V4-Flash-Vision-Exp — 2× GB10 / DGX Spark, TP=2 (vision lane).
+#
+# Model: deepseek-ai/DeepSeek-V4-Flash-Vision-Exp (MIT) — DeepSeek's first
+# multimodal V4: 284B total / 13B active MoE, experts MXFP4, attention FP8,
+# 32-layer ViT (~0.47B BF16) + aligner, native ctx 1,048,576; 43+3 MoE
+# layers, DFlash attention, index_topk 512, fp8 block [128,128].
+# Day-0 vision support lives in a vLLM PLUGIN (dsv4_vision) subclassing
+# DeepseekV4ForCausalLM so DSpark speculative still works; the plugin is
+# baked into our overlay image (docker-images/dsv4-flash-vision/, see its
+# UPSTREAM.md for provenance and the sfxnz repo @7a958128b84a).
+#
+# Image: dsv4-flash-vision:local — built from eugr/spark-vllm-b12x:latest
+# (digest sha256:7dc02f162929943ba2e14514066ed2a04bb7e9ed3592d4eb460ebcbb1f8376bd)
+# by default; the aidendle94/sparkrun-vllm-ds4-gb10:production-3.76.1 base
+# (sha256:39cda85c28a8795dc20a274fdabfece66f329d1e21c3f4552a7e75061ea8696a)
+# is the comparison variant for the same plugin (fork A/B).
+#
+# Model revision: 86f746b36186f0e567729a5c06a8c918caba82a9 (sfxnz pin).
+#
+# DEVIATIONS from the sfxnz run.sh defaults — all forced by our launcher:
+#   * no --host/--port/--nnodes/--node-rank/--master-* (launcher fabric),
+#   * no NCCL/* / IFACE / HCA env (launcher resolves; RoCE on sw fabric),
+#   * weights from per-node local NVMe copies (`~/dsv4vision-hf`, 167 GiB
+#     each on dgx3/dgx4 — rsync'd from the NAS hub 2026-09-01); shard load
+#     drops from ~7 min (NAS) to well under a minute. Fallback: re-add the
+#     `[[extra_mounts]] host = "/mnt/nas/ai/huggingface" … env = "HF_HOME"`
+#     block below to stream from the NAS hub again.
+# VLLM_USE_B12X_MHC stays OFF (this checkpoint has rms_norm_eps 1e-20; the
+# fused Gram mHC kernel only accepts 1e-6 — upstream README).
+#
+# Boot markers: "vision" plugin registered line → "Model loading took …"
+# (~84 GiB/node) → "GPU KV cache size" ≈ 1.25M tokens → "Application
+# startup complete". Vision smoke: image_url content with the
+# <｜deepseek_image｜> placeholder (cap 384 tokens per image).
+#
+# PREREQUISITES: weights cached on the NAS hub once (168 GB):
+#   docker run --rm --entrypoint python3 -v /mnt/nas/ai/huggingface:/root/.cache/huggingface \
+#     glm53-flash-nvfp4-lab:local -c "from huggingface_hub import snapshot_download; \
+#     snapshot_download('deepseek-ai/DeepSeek-V4-Flash-Vision-Exp')"
+
+engine = "vllm"
+name = "DeepSeek V4 Flash Vision-Exp (vLLM, 2x Spark, DSpark-6)"
+docker_image = "dsv4-flash-vision:local"
+docker_image_dir = "dsv4-flash-vision"
+model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name (quant-agnostic convention for the ds4-vision lane)
+    "--served-model-name", "deepseek-v4-flash-vision",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # sfxnz run.sh defaults (verbatim, minus launcher-managed fabric knobs)
+    "--max-model-len", "1048576",
+    "--kv-cache-dtype", "fp8",
+    # their pin: "a 26 GiB community pin" is too big for this UMA leftover
+    "--kv-cache-memory", "12884901888",
+    "--gpu-memory-utilization", "0.80",
+    "--max-num-seqs", "2",
+    "--max-num-batched-tokens", "8192",
+    "--block-size", "256",
+    # Engine backends (B12X GB10 lane)
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    "--attention-backend", "B12X_MLA_SPARSE",
+    # DSpark-6 (k must be divisible by n_predict=3; DSpark-5 is rejected)
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic","attention_backend":"B12X_MLA_SPARSE"}'""",
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "deepseek_v4",
+    "--reasoning-config", """
+'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
+    "--default-chat-template-kwargs", """
+'{"thinking":false,"reasoning_effort":"low"}'""",
+    "--limit-mm-per-prompt", """
+'{"image":8}'""",
+    "--enable-prefix-caching",
+    # If the stock loader balloons passenger anon buffers while streaming
+    # from the NAS (the EXL3 lesson, 3 kernel kills on a sibling kit),
+    # flip this on: the overlay ships instanttensor==0.1.9 (no-deps).
+    # "--load-format", "instanttensor",
+]
+
+[env]
+# Vision + engine lane knobs (sfxnz run.sh, verbatim minus fabric)
+CUTE_DSL_ARCH = "sm_121a"
+# AOT compile stays ON for the text-only lane (verified). The experimental
+# VLLM_DSV4_MM=1 mode traces its embed path with a 2-D dummy and then rejects
+# the hc-stream 3-D form ("wrong number of dimensions3 for op: input") —
+# flip this to "0" when testing that mode.
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_USE_MEGA_AOT_ARTIFACT = "-1"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_MHC = "0"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_MOE_FORCE_A8 = "1"
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Weights from the shared NAS hub; JIT caches persist per node.
+# NOTE: extra_mounts stay AFTER args (TOML table headers would swallow the
+# args array otherwise).
+[[extra_mounts]]
+host = "~/dsv4vision-hf"
+container = "/opt/local-hf"
+env = "HF_HOME"
+
+# Weights from the shared NAS hub; JIT caches persist per node.
+# NOTE: extra_mounts stay AFTER args (TOML table headers would swallow the
+# args array otherwise).
+# [[extra_mounts]]
+# host = "/mnt/nas/ai/huggingface"
+# container = "/mnt/nas/ai/huggingface"
+# env = "HF_HOME"
+
+[[extra_mounts]]
+host = "~/.cache/dsv4vision-jit/triton"
+container = "/root/.triton"
+[[extra_mounts]]
+host = "~/.cache/dsv4vision-jit/flashinfer"
+container = "/root/.cache/flashinfer"
+[[extra_mounts]]
+host = "~/.cache/dsv4vision-jit/vllm"
+container = "/root/.cache/vllm"