recipes: deepseek-v4-flash-eugr

recipes/deepseek-v4-flash-eugr/recipe.toml

@@ -0,0 +1,149 @@
+# DeepSeek-V4-Flash-0731 DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
+#
+# This variant uses eugr's prebuilt b12x image (eugr/spark-vllm-b12x) and
+# mirrors eugr's recipes/deepseek-v4-flash-0731.yaml on the b12x branch — the
+# stack published in NVIDIA forum comment #28:
+#   https://forums.developer.nvidia.com/t/instructions-for-running-deepseek-v4-flash-with-dspark-using-eugrs-repo/376220/28
+# ("While I'm working on integrating b12x into the build/test pipeline and
+# merging into main, I pushed prebuilt container as eugr/spark-vllm-b12x …
+# this version supports 3x Sparks with TP=3! To use, just add --tp 3.")
+#
+# Image: eugr/spark-vllm-b12x:sm121 — the prebuilt base with the
+# instanttensor-hybrid-draft-loader mod baked in at build time (see
+# docker-images/deepseek-v4-flash-eugr). eugr's recipe applies that mod via
+# run-recipe.sh --apply-mod; sm121 has no runtime mod-upload step for Docker
+# recipes, so the overlay patches vLLM's model_loader get_model() at image
+# build time instead. The recipe therefore carries no [mods] block.
+#
+# The mod keeps InstantTensor for the primary model while letting the embedded
+# dspark/MTP draft use lazy safetensors (avoids a second InstantTensor
+# GPU-streaming pass over the checkpoint). It activates only when
+# --load-format instanttensor is set (which this recipe does).
+#
+# Model: deepseek-ai/DeepSeek-V4-Flash-0731 — official release; native
+# dspark_block_size = 5, so num_speculative_tokens = 5.
+#
+# Differences vs recipes/deepseek-v4-flash (aidendle94 production-3.76.1 image):
+#   - attention backend B12X_MLA_SPARSE (eugr) vs FLASHINFER_MLA_SPARSE_DSV4
+#   - --linear-backend b12x + --load-format instanttensor (the mod's trigger)
+#   - --reasoning-config pinned to deepseek_v4 parser w/ empty start/end strs
+#   - draft_sample_method probabilistic + draft attention_backend B12X_MLA_SPARSE
+#   - max_num_seqs=8 / batched=8192 / cudagraph=64 (eugr defaults)
+#   - VLLM_USE_MEGA_AOT_ARTIFACT="-1" (eugr) vs "0"
+#
+# 3× Sparks (TP=3): bump tensor_parallel_size + required_nodes to 3 (the eugr
+# image supports it per comment #28). k=5 is unchanged (model-native).
+
+engine = "vllm"
+name = "DeepSeek V4 Flash DSpark (eugr b12x)"
+docker_image = "eugr/spark-vllm-b12x:sm121"
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+docker_image_dir = "deepseek-v4-flash-eugr"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--kv-cache-dtype", "fp8",
+    "--kv-cache-memory-bytes", "21474836480",
+    "--block-size", "256",
+    "--load-format", "instanttensor",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--long-prefill-token-threshold", "1024",
+    "--max-cudagraph-capture-size", "64",
+    # 0.78, not higher: the SvangenStudios agent-serving benchmark (v0.1.1) on
+    # this exact stack found "0.80 boots but dies under traffic". GMU is a KV
+    # lever, not a safety lever, when weights dominate memory (~77.7 GiB of
+    # ~119.6 GiB node). For interactive agent traffic prefer the
+    # deepseek-v4-flash-eugr-agent profile (2048 chunk). See
+    # specs/tech-architecture/dgx-spark-agent-serving-findings.md.
+    "--gpu-memory-utilization", "0.78",
+    "--served-model-name", "deepseek-v4-flash",
+    # Engine backends — b12x across attention / MoE / linear
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    # Prefix caching
+    "--enable-prefix-caching",
+    # Compilation
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    # DSpark speculative decoding — k=5 (DeepSeek-V4-Flash native
+    # dspark_block_size). draft_sample_method per eugr recipe; draft attention
+    # backend B12X_MLA_SPARSE. The baked mod makes the embedded draft load via
+    # lazy safetensors while the target stays on InstantTensor.
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"B12X_MLA_SPARSE"}'""",
+    # Tokenizer / reasoning — reasoning-config pins the parser + empty
+    # reasoning markers (deepseek_v4 tokenizer mode emits them itself).
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--reasoning-parser", "deepseek_v4",
+    "--reasoning-config", """
+'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
+    "--enable-auto-tool-choice",
+    "--generation-config", "vllm",
+    "--default-chat-template-kwargs.thinking=true",
+    "--default-chat-template-kwargs.reasoning_effort=max",
+    "--disable-custom-all-reduce",
+    "--trust-remote-code",
+]
+
+# Persistent vLLM compile cache (speeds up restarts). Mounted at vLLM's
+# default cache root for root (/root/.cache/vllm); the HF cache is already
+# mounted at /root/.cache/huggingface by the launcher.
+[[extra_mounts]]
+host = "~/.cache/vllm-ds4-eugr"
+container = "/root/.cache/vllm"
+
+[env]
+# --- b12x + compile (from eugr's recipe env, verbatim) ---
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_USE_MEGA_AOT_ARTIFACT = "-1"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_MOE_FORCE_A8 = "1"
+
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
+
+# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+OMP_NUM_THREADS = "4"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"
+
+# --- offline ---
+HF_HUB_OFFLINE = "1"
+
+# --- NCCL overrides (GB10 single-NIC tuning; image-agnostic — mirrors
+#     recipes/deepseek-v4-flash. The launcher still injects IB/HCA/socket
+#     vars from the cluster config; these are recipe-level extras.) ---
+NCCL_P2P_LEVEL = "SYS"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IB_TIMEOUT = "24"
+NCCL_IB_QPS_PER_CONNECTION = "8"
+NCCL_BUFFERSIZE = "16777216"
+NCCL_SOCKET_FAMILY = "inet"
+
+# PCIe allreduce not needed on GB10 single-NIC
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"
+
+VLLM_DSV4_REASONING_CONTENT = "1"