recipes: deepseek-v4-flash-dspark38

recipes/deepseek-v4-flash-dspark38/recipe.toml

@@ -0,0 +1,131 @@
+# DeepSeek-V4-Flash-0731 DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
+#
+# Variant targeting aidendle94's production-3.8 image ("dsv4/apostolic" b12x
+# harness). This is the BUG-2026-08-03T204037 bring-up recipe — the two images
+# are *different b12x forks*, not a version bump from production-3.75, so the
+# launch contract differs:
+#
+#   - production-3.8 is a clean-overlay base (null ENTRYPOINT, CMD sleep
+#     infinity) whose intended launcher is /usr/local/bin/dsv4-vllm-entrypoint.
+#     We set that as the recipe entrypoint so the JIT toolchain env
+#     (DG_JIT_NVCC_COMPILER, NVCC_PREPEND_FLAGS + CCCL include path, CUDAHOSTCXX)
+#     and the /cache JIT dir layout are active. The 3.75 recipe ran vLLM directly
+#     because that image was self-contained with everything baked in.
+#   - VLLM_USE_V2_MODEL_RUNNER is dropped (experimental in this fork; the
+#     working glm52 recipe does not set it).
+#   - tiny-decode (VLLM_B12X_TINY_DECODE) is enabled at GMU 0.78 + CHUNK=8
+#     (the load-time repack needs headroom; it OOM'd at GMU 0.9).
+#   - /cache is mounted so the wrapper's JIT caches persist across restarts.
+
+engine = "vllm"
+name = "DeepSeek V4 Flash DSpark (production-3.8)"
+# Local overlay of the production-3.8 base built from
+# docker-images/deepseek-v4-flash-dspark38 (root-cause patch for the b12x
+# cutlass from_dlpack grad bug that blocked boot; see the Dockerfile).
+docker_image = "sparkrun-vllm-ds4-gb10:dspark38"
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+# production-3.8 launch contract: the clean-base image must be started via the
+# dsv4-vllm-entrypoint wrapper, which sets the JIT toolchain + /cache dirs and
+# `exec vllm "$@"`. `serve <recipe-args/auto flags>` land as "$@".
+entrypoint = ["/usr/local/bin/dsv4-vllm-entrypoint", "serve"]
+
+args = [
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--load-format", "auto",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "8192",
+    "--max-cudagraph-capture-size", "64",
+    # 0.78 is the safe GMU ceiling on GB10 (weights ~77.7 GiB of ~119.6 GiB);
+    # higher boots but dies under traffic. Also leaves headroom for the tiny
+    # decode kernel's GPU load-time repack.
+    "--gpu-memory-utilization", "0.78",
+    "--served-model-name", "deepseek-v4-flash",
+    # Engine backends — b12x across attention / MoE / linear (this fork's path)
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    # Prefix caching
+    "--enable-prefix-caching",
+    # Compilation
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    # DSpark speculative decoding — k=5 (DeepSeek-V4-Flash native
+    # dspark_block_size). draft_sample_method greedy.
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"greedy"}'""",
+    # Tokenizer / reasoning
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--reasoning-parser", "deepseek_v4",
+    "--enable-auto-tool-choice",
+    "--generation-config", "vllm",
+    "--default-chat-template-kwargs.thinking=true",
+    "--default-chat-template-kwargs.reasoning_effort=high",
+    "--disable-custom-all-reduce",
+    "--trust-remote-code",
+]
+
+# Persistent JIT/compile caches. The wrapper `mkdir -p`s under /cache
+# (DG_JIT_CACHE_DIR=/cache/deep_gemm_jit, VLLM_CACHE_ROOT=/cache/vllm,
+# TRITON_CACHE_DIR=/cache/triton, etc.), so /cache must be a mounted volume to
+# survive restarts. The HF cache is already mounted at /root/.cache/huggingface
+# by the launcher.
+[[extra_mounts]]
+host = "~/.cache/ds4-38-vllm"
+container = "/cache"
+
+[env]
+# --- b12x fork env family (shared with the proven eugr b12x recipe) ---
+VLLM_ATTENTION_BACKEND = "B12X_MLA_SPARSE"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_MEGA_AOT_ARTIFACT = "0"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_DENSE_SPLITK_TURBO = "1"
+B12X_W4A16_TC_DECODE = "1"
+# Tiny A8 decode kernel: DISABLED — its GPU load-time MoE-weight repack, on
+# top of the tight host RAM (~32 GiB free) + dual-node 155 GiB load, OOM'd both
+# hosts (had to power-cycle via HA). Revisit separately if host RAM headroom is
+# available. Keep the CHUNK pin for reference.
+# VLLM_B12X_TINY_DECODE = "1"
+# VLLM_B12X_TINY_CHUNK = "8"
+
+# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+OMP_NUM_THREADS = "4"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"
+
+# --- offline ---
+HF_HUB_OFFLINE = "1"
+
+# --- NCCL overrides (image-agnostic; launcher injects IB/HCA/socket from
+#     cluster config, these are recipe-level extras) ---
+NCCL_P2P_LEVEL = "SYS"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IB_TIMEOUT = "24"
+NCCL_IB_QPS_PER_CONNECTION = "8"
+NCCL_BUFFERSIZE = "16777216"
+NCCL_SOCKET_FAMILY = "inet"
+
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"