recipes: deepseek-v4-flash-eugr-agent

recipes/deepseek-v4-flash-eugr-agent/recipe.toml

@@ -0,0 +1,167 @@
+# DeepSeek-V4-Flash-0731 DSpark — AGENT-SERVING profile (2× GB10, TP=2).
+#
+# Fork of deepseek-v4-flash-eugr tuned for interactive agent traffic, applying
+# the validated "agent-2048" profile from the SvangenStudios DGX-Spark
+# agent-serving benchmarks (v0.1.1, measured 2026-08-02 on this exact stack:
+# DeepSeek-V4-Flash-0731, DSpark k=5, 2× DGX Spark, TP=2, RoCE):
+#   https://github.com/SvangenStudios/dgx-spark-agent-serving-benchmarks
+#   configs/agent-2048.env  /  results/RESULTS.md §4
+#
+# The benchmark's headline finding: --max-num-batched-tokens is a FAIRNESS
+# lever for agent workloads, not just a jitter lever. Against an ongoing
+# decode stream, a 262K-token prefill at 2048 delivers 2.9× the decode share
+# (5.0% vs 1.7%), 3.1× the decode tokens, 73% lower p95 output-chunk jitter,
+# and +67% KV-pool capacity — for 7.7% prefill throughput. The launcher's
+# whole purpose is agent serving (a model may be serving this very agent), so
+# 2048 is the more correct default here.
+#
+# Mental model (benchmark §3): an LLM server is three nearly-independent
+# engines — prefill (compute-bound, serializes), decode (bandwidth-bound,
+# scales to ~N=4), admission (scheduler-bound, BLOCKED during long prefill).
+# Chunk size governs how often decode gets a scheduling slot between prefill
+# chunks; smaller = more slots. It does NOT fix new-request admission (a
+# short request submitted during a large prefill waits until prefill
+# completes — 221× normal TTFT in both profiles).
+#
+# Knob changes vs deepseek-v4-flash-eugr (everything else identical — same
+# eugr image, same backends, same env):
+#   --max-num-batched-tokens  8192 -> 2048   (the fairness lever)
+#   --max-num-seqs            8    -> 6      (decode saturates ~N=4; 6 gives
+#                                             headroom without inflating
+#                                             worst-case latency)
+#   --max-cudagraph-capture-size 64 -> 36    (= max_num_seqs * (k+1) = 6*6;
+#                                             the k+1 multiple rule — the
+#                                             benchmark notes k=5 rejects
+#                                             MAX_NUM_SEQS=1 at boot)
+#   --gpu-memory-utilization  0.85 -> 0.78   (benchmark: "0.80 boots but dies
+#                                             under traffic"; GMU is a KV
+#                                             lever, not a safety lever, when
+#                                             weights dominate memory)
+#   --max-model-len           auto -> 1048576  (benchmark's 1M working capacity
+#                                             — verified 12/12 needles at
+#                                             32K/128K/512K/900K)
+#
+# Trade-off: this profile costs ~7.7% prefill throughput vs the 8192 batch
+# profile. Use deepseek-v4-flash-eugr (8192) for exclusive long-context batch
+# jobs (512K–1M) with no concurrent agent traffic. See the batch-8192 profile
+# in the benchmark repo for the opposite extreme.
+#
+# 3× Sparks (TP=3): bump tensor_parallel_size + required_nodes to 3 (the eugr
+# image supports it per forum comment #28). k=5 unchanged (model-native).
+
+engine = "vllm"
+name = "DeepSeek V4 Flash DSpark (eugr b12x, agent 2048)"
+docker_image = "eugr/spark-vllm-b12x:sm121"
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+docker_image_dir = "deepseek-v4-flash-eugr"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--load-format", "instanttensor",
+    # 1M working capacity (benchmark §2: 12/12 needles at 32K–900K). A 900K
+    # prefill takes ~15 min and starves everything — batch capacity, not an
+    # interactive mode.
+    "--max-model-len", "1048576",
+    # Decode saturates around N≈4 (benchmark §3); 6 gives headroom without
+    # inflating worst-case latency. cudagraph capture = max_num_seqs*(k+1)=36.
+    "--max-num-seqs", "6",
+    # FAIRNESS LEVER (benchmark §4): 2048 gives 2.9× the decode share under a
+    # concurrent prefill vs 8192, +67% KV pool, 73% lower jitter, for 7.7%
+    # prefill throughput. Use 8192 only for exclusive long-context batch jobs.
+    "--max-num-batched-tokens", "2048",
+    # = max_num_seqs * (k+1) = 6 * 6 (the k+1 multiple rule).
+    "--max-cudagraph-capture-size", "36",
+    # Benchmark finding: "0.80 boots but dies under traffic" — 0.78 is the
+    # safe ceiling. GMU is a KV lever, not a safety lever, when weights
+    # dominate memory (~77.7 GiB of ~119.6 GiB node).
+    "--gpu-memory-utilization", "0.78",
+    "--served-model-name", "deepseek-v4-flash",
+    # Engine backends — b12x across attention / MoE / linear
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "b12x",
+    "--linear-backend", "b12x",
+    # Prefix caching — agent frameworks build prompts append-only, so reuse is
+    # 95.7–98.4% per turn (benchmark §9: divergence at the last token of the
+    # previous prompt; only ~270–660 tokens re-prefilled per agent step).
+    "--enable-prefix-caching",
+    # Compilation
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
+    # DSpark speculative decoding — k=5 (DeepSeek-V4-Flash native
+    # dspark_block_size). draft_sample_method per eugr recipe; draft attention
+    # backend B12X_MLA_SPARSE. The baked mod makes the embedded draft load via
+    # lazy safetensors while the target stays on InstantTensor.
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"B12X_MLA_SPARSE"}'""",
+    # Tokenizer / reasoning — reasoning-config pins the parser + empty
+    # reasoning markers (deepseek_v4 tokenizer mode emits them itself).
+    "--tokenizer-mode", "deepseek_v4",
+    "--tool-call-parser", "deepseek_v4",
+    "--reasoning-parser", "deepseek_v4",
+    "--reasoning-config", """
+'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
+    "--enable-auto-tool-choice",
+    "--generation-config", "vllm",
+    "--default-chat-template-kwargs.thinking=true",
+    "--default-chat-template-kwargs.reasoning_effort=high",
+    "--disable-custom-all-reduce",
+    "--trust-remote-code",
+]
+
+# Persistent vLLM compile cache (speeds up restarts). Mounted at vLLM's
+# default cache root for root (/root/.cache/vllm); the HF cache is already
+# mounted at /root/.cache/huggingface by the launcher. Shares the eugr
+# recipe's cache dir (same image/backends — compile artifacts are reusable).
+[[extra_mounts]]
+host = "~/.cache/vllm-ds4-eugr"
+container = "/root/.cache/vllm"
+
+[env]
+# --- b12x + compile (from eugr's recipe env, verbatim) ---
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_USE_MEGA_AOT_ARTIFACT = "-1"
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_MOE_FORCE_A8 = "1"
+
+# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+OMP_NUM_THREADS = "4"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"
+
+# --- offline ---
+HF_HUB_OFFLINE = "1"
+
+# --- NCCL overrides (GB10 single-NIC tuning; image-agnostic — mirrors
+#     recipes/deepseek-v4-flash. The launcher still injects IB/HCA/socket
+#     vars from the cluster config; these are recipe-level extras.) ---
+NCCL_P2P_LEVEL = "SYS"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_IB_TIMEOUT = "24"
+NCCL_IB_QPS_PER_CONNECTION = "8"
+NCCL_BUFFERSIZE = "16777216"
+NCCL_SOCKET_FAMILY = "inet"
+
+# PCIe allreduce not needed on GB10 single-NIC
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
+VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"