recipes: dsv41-flash-eugr-0913-4x

recipes/dsv41-flash-eugr-0913-4x/recipe.toml

@@ -0,0 +1,131 @@
+# DeepSeek-V4.1-Flash — eugr b12x nightly-20260913 (lab `dev/infernal-invocation`), 4x.
+#
+# P1.2 of the 2026-09-18 night: the r37 typo A/B. r37 (09-13, vllmc687594b8a8)
+# garbled DS4.1 text in the morning bisect (A/B/C: flags excluded; D infeasible
+# — engram-off 4x OOMs, the table is part of the weights and only fits in DRAM
+# at 8x). This runs the lab's NEWER published b12x build (09-13 DockerHub, cut
+# from dev/infernal-invocation — newer than r37's 3fb88fdf tree) with the SAME
+# measured r37 contract (GMU 0.88, 512K, DCP1, block 256, B12X attn/linear/moe,
+# K7 greedy/adaptive, engram disk, instanttensor/lazy). Garbled -> the r37
+# garbling is a lab-tree trait (not our build); clean -> r37-specific
+# regression (file the exact bisect with the lab).
+#
+# Image already pulled on dgx1-4 (24.5 GB). Same model/weights/engram staging
+# as the r37 recipe (NAS snapshot + per-node /var/tmp/engram-local hardlinks).
+engine = "vllm"
+name = "DeepSeek-V4.1-Flash (eugr b12x 0913 = lab dev/infernal-invocation, 4x DCP1, DSpark K7)"
+docker_image = "eugr/spark-vllm-b12x:nightly-20260913"  # prebuilt (no docker_image_dir)
+model = "deepseek-ai/DeepSeek-V4.1-Flash"
+parent_model = "deepseek-ai/DeepSeek-V4.1-Flash"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+# Our r27 build dir images ship with ENTRYPOINT [] — set it explicitly.
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--served-model-name", "dsv41",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GB10 startup-gate value (see header); KV pool is engine-computed from
+    # the remainder (no pin — their profile pins nothing either).
+    "--gpu-memory-utilization", "0.88",
+    # 512K context (our floor — their qualified 131K is not usable for us;
+    # the KV pool is engine-computed from the memory budget, unchanged):
+    "--max-model-len", "524288",
+    "--decode-context-parallel-size", "1",
+    "--max-num-seqs", "4",
+    "--max-num-batched-tokens", "4096",
+    "--prefill-compute-share", "0.4",
+    "--block-size", "256",
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--kv-cache-dtype", "fp8",
+    "--attention-backend", "B12X",
+    "--linear-backend", "b12x",
+    "--moe-backend", "b12x",
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE"}'""",
+    "--max_cudagraph_capture_size", "128",
+    # Native DSpark K7 — the draft weights live inside the target checkpoint
+    # (no separate draft repo). Greedy draft, standard rejection, adaptive
+    # verification (the lab's r36 qualified trio vs our prod's K5
+    # probabilistic/block).
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":7,"draft_tensor_parallel_size":4,"attention_backend":"B12X","draft_sample_method":"greedy","rejection_sample_method":"standard","enable_adaptive_verification":true}'""",
+    # Engram: the model's 196B learned ngram lookup storage. disk mode =
+    # io_uring reads of the table rows from the checkpoint dir (bounded
+    # staging; NOT pinned RAM). cpu_offload stays false (no generic CPU
+    # model offload).
+    "--engram-config", """
+'{"cpu_offload":false,"table_memory":"disk"}'""",
+    "--load-format", "instanttensor",
+    "--safetensors-load-strategy", "lazy",
+    "--generation-config", "vllm",
+    "--override-generation-config", """
+'{"temperature":1.0,"top_p":0.95}'""",
+    "--limit-mm-per-prompt", """
+'{"image":2}'""",
+    "--reasoning-parser", "deepseek_v41",
+    "--tool-call-parser", "deepseek_v41",
+    "--enable-auto-tool-choice",
+    "--jit-monitor-mode", "error",
+]
+
+[env]
+
+# Their serve-ds41-jovian.sh runtime env (multi-node-adapted; the launcher
+# injects NCCL_SOCKET_IFNAME/IB_HCA/GID + VLLM_HOST_IP per node).
+# 2026-09-14 (night shift, operator's typo report): the lab tree's runtime
+# LM-head quantization (vocab_parallel_embedding.py) quantizes an unquantized
+# LM head to MXFP8 BY DEFAULT (envs.py: VLLM_MXFP8_LM_HEAD, default "1").
+# The eugr 2026-09-10 nightly (our prod image) has NO such feature and keeps
+# the BF16 head - the coarsened 8-bit vocab-logit layer is the suspected cause
+# of the typo-rate regression (close-token logits distorted). Opting out.
+VLLM_MXFP8_LM_HEAD = "0"
+NCCL_NCHANNELS = "16"
+NCCL_MIN_NCHANNELS = "16"
+NCCL_MAX_NCHANNELS = "16"
+NCCL_BUFFSIZE = "2097152"
+OMP_NUM_THREADS = "8"
+CUDA_DEVICE_MAX_CONNECTIONS = "32"
+# GB10 arch (theirs sm_120a on RTX PRO 6000):
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+# Their V2 model-runner + AOT compile pipeline (baked in their image env):
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_STANDALONE_COMPILE = "1"
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_WORKER_MULTIPROC_METHOD = "spawn"
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "900"
+HF_HUB_OFFLINE = "1"
+# Deliberately NOT set: VLLM_ENABLE_PCIE_ALLREDUCE (single-host PCIe in
+# their profile; cross-node on the switch fabric uses the launcher's NCCL),
+# VLLM_HOST_IP (launcher per-node), NCCL_SOCKET_IFNAME (launcher per cluster).
+
+# --- Engram from node-local NVMe, the rest from the shared NAS ---
+# [composed_model] (standard launcher feature) builds a per-node /tmp dir:
+# the whole DSV4.1 snapshot symlinked into the HF cache + the two 101.5 GB
+# Engram table shards (model-00047/00048) HARDLINKED from the pre-existing
+# per-rank sparse copies in /var/tmp/engram-local (48 GB real per node, built
+# for the eugr/aiden lanes — offset-preserving full-size files, only this
+# rank's rows materialized; hardlink = zero extra disk, same fs as /tmp).
+# The r36 io_uring PLE reader then reads this rank's table rows from local
+# NVMe (no NFS); weights stream from the NAS at load time (one-shot, same as
+# prod). Local files run after the repo symlinks and win at the same name.
+[composed_model]
+mount = "/model"
+
+[[composed_model.sources]]
+repo = "deepseek-ai/DeepSeek-V4.1-Flash"          # whole snapshot, symlinked into the HF cache
+
+[[composed_model.local_files]]                      # hardlinks win over the symlinks above
+host = "/var/tmp/engram-local/DeepSeek-V4.1-Flash/model-00047-of-00048.safetensors"
+
+[[composed_model.local_files]]
+host = "/var/tmp/engram-local/DeepSeek-V4.1-Flash/model-00048-of-00048.safetensors"