recipes: dsv4-flash-vision-eugr

recipes/dsv4-flash-vision-eugr/recipe.toml

@@ -0,0 +1,109 @@
+# BLOCKED 2026-09-06 (night shift): the eugr 0904 image's DeepseekV4ForCausalLM
+# has no `aligner` module (predates V4-Vision support) — the Vision-Exp
+# checkpoint's vision weights fail to load. Re-check on a newer eugr image;
+# the fork lane `dsv4-flash-vision` remains the only V4-Vision path.
+#
+# DeepSeek-V4-Flash-Vision-Exp — eugr B12X variant (vLLM) — 2× DGX Spark, TP=2.
+#
+# The eugr-stack A/B cell against our fork lane `dsv4-flash-vision` (same
+# model, same node shape; their prebuilt image vs our dsv4-flash-vision:local
+# overlay + dsv4_vision plugin). Port of eugr's deepseek-v4-flash-0731.yaml
+# shape (eugr/spark-vllm-docker @ main) onto the Vision-Exp model.
+#
+# Image: eugr/spark-vllm-b12x:nightly-20260904 (digest c3b44d3f…, = :latest).
+#   local-inference-lab/vllm dev/jovian-judgement 09-04 tip (ref switched from
+#   dev/infernal-invocation by their e91bc76, 09-03) + lukealonso/b12x @ master
+#   + their patch set; torch 2.13.0, CUDA 13.0.2, cutlass-dsl 4.7.0.
+#   NOTE: image ships NO ENTRYPOINT → entrypoint = ["vllm","serve"] below.
+#   Exact b12x pin baked in-image: /workspace/b12x-source-commit.
+#
+# Model: deepseek-ai/DeepSeek-V4-Flash-Vision-Exp (MIT; ~168 GB on disk,
+#   experts MXFP4 / attention FP8, 32-layer ViT BF16). Reuse the per-node
+#   local NVMe copies from the fork lane if present (~/dsv4vision-hf on
+#   dgx3/dgx4, rsync'd 2026-09-01 — check the HF_HOME mapping); else
+#   huggingface-cli download on BOTH nodes (HF_HUB_OFFLINE=1 below).
+#
+# Diffs vs the fork lane (dsv4-flash-vision) — deliberate parity choices:
+#   - attention backend: plain B12X (their 09-03 switch away from
+#     B12X_MLA_SPARSE) — the fork lane keeps B12X_MLA_SPARSE
+#   - VLLM_USE_MEGA_AOT_ARTIFACT=1 (their 09-03 switch; fork lane: -1)
+#   - max-cudagraph-capture-size 48 (their 09-03 value)
+#   - spec: vendor card k=3 + enable_adaptive_verification (published:
+#     2.99 mean acceptance, 66.3%); k=6 (fork lane's proven value) is the
+#     fallback if adaptive verification misbehaves on this stack
+#   - VLLM_USE_B12X_MHC forced 0 like the fork lane: this checkpoint has
+#     rms_norm_eps 1e-20, the fused Gram mHC kernel only accepts 1e-6
+#   - no dsv4_vision plugin (their image has stock vLLM vision path) — if
+#     boot fails on the vision processor, that's the first suspect
+#
+# REFERENCE:
+#   https://github.com/eugr/spark-vllm-docker (recipes/deepseek-v4-flash-0731.yaml)
+#   https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp
+#   https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp
+
+engine = "vllm"
+name = "DeepSeek V4 Flash Vision-Exp (vLLM, 2x Spark, eugr B12X, DSpark-3)"
+docker_image = "eugr/spark-vllm-b12x:nightly-20260904"
+model = "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+# image ships no ENTRYPOINT — the launcher supplies the vllm command
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Stable served name (shared with the fork lane — quant/stack-agnostic)
+    "--served-model-name", "deepseek-v4-flash-vision",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # eugr 0731 shape: full b12x cluster, plain B12X attention (09-03 switch)
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    "--attention-backend", "B12X",
+    "--max-model-len", "1048576",
+    "--kv-cache-dtype", "fp8",
+    # fork lane's proven pool for this checkpoint on 2 Sparks (1.25M tokens)
+    "--kv-cache-memory-bytes", "12884901888",
+    "--gpu-memory-utilization", "0.85",
+    "--max-num-seqs", "2",
+    "--max-num-batched-tokens", "8192",
+    "--block-size", "256",
+    "--max-cudagraph-capture-size", "48",
+    # vendor card (published numbers): DSpark k=3, probabilistic, adaptive
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"B12X","enable_adaptive_verification":true}'""",
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "deepseek_v4",
+    "--reasoning-config", """
+'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
+    "--default-chat-template-kwargs", """
+'{"thinking":false,"reasoning_effort":"low"}'""",
+    "--limit-mm-per-prompt", """
+'{"image":8}'""",
+    "--enable-prefix-caching",
+    "--load-format", "instanttensor",
+]
+
+[env]
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+# OFF for this checkpoint (rms_norm_eps 1e-20 — see header); their 0731
+# text recipe sets it 1, but the model constraint wins
+VLLM_USE_B12X_MHC = "0"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_MOE_FORCE_A8 = "1"
+HF_HUB_OFFLINE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"