|
|
@@ -0,0 +1,91 @@
|
|
|
+# DeepSeek-V4.1-Flash (552B MoE, MXFP4 experts) — 8x DGX Spark, in-memory Engram.
|
|
|
+#
|
|
|
+# P2.3 of the 2026-09-18 night: the first 8x trial. Operator's design: at 8x
|
|
|
+# the Engram table fits in DRAM (406 GB / 8 = 50.7 GB/node) — no disk mode,
|
|
|
+# no per-node sparse staging, no /engram-local mounts. The engram rows load
|
|
|
+# from the NAS checkpoint into DRAM once at boot (one-off NFS cost), decode
|
|
|
+# reads them from memory (no per-token disk reads).
|
|
|
+#
|
|
|
+# Memory math per node (121.6 GiB): weights 307.3 GB / 8 = 38.4 GB + engram
|
|
|
+# rows 50.7 GB + process ~12 GB = ~101 GB. GMU 0.91 (the lab DS4.1 policy,
|
|
|
+# lil-catalog 09-15) leaves a ~9.5 GiB KV pool ≈ 1M tokens = ~1.9x 512K.
|
|
|
+#
|
|
|
+# Image: dsv41-flash:local (the tonyd2wild prod chain — engram.py patch
|
|
|
+# baked in; DSV41_ENGRAM_DISK simply NOT set here = memory mode). The
|
|
|
+# launcher syncs the image to dgx6-8 at serve time (dgx1-5 have it).
|
|
|
+#
|
|
|
+# This is also the natural "Boot D" test the 4x r37 bisect could not run:
|
|
|
+# engram WITHOUT the disk-read path. If the text is clean here while the r37
|
|
|
+# 4x disk-mode run garbles, the bisect narrows to the lab tree's engram
|
|
|
+# disk-read path.
|
|
|
+engine = "vllm"
|
|
|
+name = "DeepSeek-V4.1-Flash MXFP4 (dsv41 img, 8x, DSpark k=5, Engram in DRAM)"
|
|
|
+docker_image = "dsv41-flash:local"
|
|
|
+docker_image_dir = "dsv41-flash"
|
|
|
+model = "deepseek-ai/DeepSeek-V4.1-Flash"
|
|
|
+required_nodes = 8
|
|
|
+tensor_parallel_size = 8
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+# NO recipe entrypoint: the base image bakes ENTRYPOINT ["vllm","serve"]
|
|
|
+# (vllm/vllm-openai), so the launcher's CMD (model + flags) completes it.
|
|
|
+
|
|
|
+args = [
|
|
|
+ # Stable gateway alias — clients/gateways keep the DeepSeek-V4-Flash
|
|
|
+ # model name; no per-deployment config churn.
|
|
|
+ "--served-model-name", "deepseek-v4-flash",
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--distributed-executor-backend", "mp",
|
|
|
+ "--gpu-memory-utilization", "0.91",
|
|
|
+ "--max-model-len", "524288",
|
|
|
+ "--max-num-seqs", "8",
|
|
|
+ "--max-num-batched-tokens", "8192",
|
|
|
+ # boot-4 fix: the V4 indexer backend refuses the smallest-listed block
|
|
|
+ "--block-size", "128",
|
|
|
+ # boot-10: exact capture sizes for DSpark k=5, SEQS=8 (multiples of 5
|
|
|
+ # and 6 up to 48 — every decode batch gets an exact FULL graph, no padding)
|
|
|
+ "--compilation-config", """
|
|
|
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[5,6,10,12,15,18,20,24,25,30,35,36,40,42,48]}'""",
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}'""",
|
|
|
+ "--tool-call-parser", "deepseek_v41",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--reasoning-parser", "deepseek_v41",
|
|
|
+ "--default-chat-template-kwargs", """
|
|
|
+'{"thinking": true, "reasoning_effort": "max"}'""",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+# Engram: MEMORY mode — DSV41_ENGRAM_DISK deliberately NOT set (the baked
|
|
|
+# engram.py patch defaults to "0" = in-DRAM table). At 8x the table fits:
|
|
|
+# 406 GB / 8 = 50.7 GB/node.
|
|
|
+# CUDA graphs: the Engram rows are staged in prepare_inputs (baked
|
|
|
+# model_state.py prestage patch) so the whole decode step is capturable
|
|
|
+VLLM_USE_BREAKABLE_CUDAGRAPH = "1"
|
|
|
+VLLM_HAS_FLASHINFER_CUBIN = "1"
|
|
|
+VLLM_USE_RUST_FRONTEND = "0"
|
|
|
+# runtime JIT wedge countermeasure (their boot-3 lesson): kernels are
|
|
|
+# prebuilt in the image, but any residual JIT must not OOM the box
|
|
|
+MAX_JOBS = "2"
|
|
|
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
|
|
|
+OMP_NUM_THREADS = "2"
|
|
|
+
|
|
|
+# JIT caches persist per-node (self-namespace by engine version key; the
|
|
|
+# image prebakes the flashinfer 0.7.0rc1 SM120 kernels, first boots re-JIT
|
|
|
+# the rest)
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/flashinfer"
|
|
|
+container = "/root/.cache/flashinfer"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/torchinductor"
|
|
|
+container = "/root/.cache/torchinductor"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/vllm"
|
|
|
+container = "/root/.cache/vllm"
|
