|
|
@@ -0,0 +1,157 @@
|
|
|
+# DeepSeek-V4.1-Flash — local-inference-lab r37 (jovian-judgement, 2026-09-13), 4x.
|
|
|
+#
|
|
|
+# Clone of dsv41-flash-jovian-r36-4x retargeted at the r37 source lock — the
|
|
|
+# lab's newest published DS4.1 qualification. r37's headline is a
|
|
|
+# kernel-dispatch overhead removal: no more repeated CPU argument-list
|
|
|
+# expansion, mutable-schema inspection, scalar dtype allocations or
|
|
|
+# tensor-equality calls per dispatch (no quantization or sampling change);
|
|
|
+# plus measured B12X projection tiles, four FP32 projection partials, visible
|
|
|
+# index-score tiles, a compact mHC prefill config, removed attention-output
|
|
|
+# copies and bounded short-index scratch. Their RTX-quartet qualification
|
|
|
+# (public-source control -> r37): uncached 32K prefill 14,896 -> 21,399 tok/s
|
|
|
+# (+43.65%), C1 output 238.80 -> 281.08 tok/s (+17.70%, budget-50 three-cell
|
|
|
+# median), C1 verifier +11.92%, Sieve +8.17%. r37 also corrects the named
|
|
|
+# `high` reasoning budget 50 -> 75 (behaviour, not speed — our benches pass
|
|
|
+# reasoning_effort explicitly). Carries the B12X small-tile barrier fix (#363)
|
|
|
+# and TP3 prefill-head support (TP3 is not serving-qualified here).
|
|
|
+#
|
|
|
+# Image: our own build for GB10 sm_121a from docker-images/glm53-jovian-r27
|
|
|
+# with profile jovian-r37-ds41 (vLLM c687594b8a8 + b12x f1c4e9dd5b1; LMCache
|
|
|
+# 29bc5a2e / FlashKDA 3b225bf2 / DeepGEMM 8b1392b9 unchanged; torch 2.13.0 +
|
|
|
+# flashinfer 0.6.17 = the same GB10 pins as r35/r36, dep gates re-checked
|
|
|
+# against the r37 tree; liburing 2.5 + dev headers already in the apt lock).
|
|
|
+# The published lab image is CUDA 13.3 x86 / sm_120 — ours is the documented
|
|
|
+# CUDA 13.2 aarch64 / sm_121a deviation.
|
|
|
+#
|
|
|
+# Serving profile, GB10 deviations and the Engram-on-node-NVMe mechanism are
|
|
|
+# unchanged from r36 (see that recipe's header + history/night-shifts/
|
|
|
+# 2026-09-12.md "Day follow-up 2/3"): TP4/DCP1, native DSpark K7 (greedy
|
|
|
+# draft, standard rejection, adaptive verification), 512K (our floor — their
|
|
|
+# 131K qualified profile is not usable for us), GMU 0.88, B12X attn/MoE/linear,
|
|
|
+# block 256, instanttensor, Engram table on disk via
|
|
|
+# [composed_model].local_files hardlinked from the eugr lane's per-rank sparse
|
|
|
+# copies in /var/tmp/engram-local (48 GB real per node, zero extra disk).
|
|
|
+#
|
|
|
+# Measured reference points (dev65/seed 42, d0 c1):
|
|
|
+# r36 + local Engram @512K 154/176 (88), 3,774 pp / 45.7 tg / 822 ms TTFT
|
|
|
+# r36 + local Engram @131K 149/176 (85), 3,654 pp / 43.9 tg / 826 ms
|
|
|
+# r36 + Engram over NFS 158/176 (90), 401 pp / 33.7 tg / 5,061 ms
|
|
|
+# eugr dsv41 prod (K5) 167/176 (95), 1,642 pp / 47.0 tg / 1,738 ms
|
|
|
+# What r37 targets: the same r36 profile with their +43% prefill fix. The A/B
|
|
|
+# vs the eugr prod decides (quality gap vs speed lead) — plan:
|
|
|
+# history/night-shifts/2026-09-13.md.
|
|
|
+#
|
|
|
+# PREREQUISITE: weights in the shared NAS HF cache; per-node local Engram
|
|
|
+# sparse copies under /var/tmp/engram-local/DeepSeek-V4.1-Flash.
|
|
|
+engine = "vllm"
|
|
|
+name = "DeepSeek-V4.1-Flash (jovian r37 img, 4x DCP1, DSpark K7 greedy + Engram disk)"
|
|
|
+docker_image = "jovian-r37-ds41:local"
|
|
|
+docker_image_dir = "glm53-jovian-r27"
|
|
|
+build_profile = "jovian-r37-ds41"
|
|
|
+model = "deepseek-ai/DeepSeek-V4.1-Flash"
|
|
|
+parent_model = "deepseek-ai/DeepSeek-V4.1-Flash"
|
|
|
+required_nodes = 4
|
|
|
+tensor_parallel_size = 4
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+# Our r27 build dir images ship with ENTRYPOINT [] — set it explicitly.
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+
|
|
|
+args = [
|
|
|
+ "--served-model-name", "dsv41",
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--distributed-executor-backend", "mp",
|
|
|
+ # GB10 startup-gate value (see header); KV pool is engine-computed from
|
|
|
+ # the remainder (no pin — their profile pins nothing either).
|
|
|
+ "--gpu-memory-utilization", "0.88",
|
|
|
+ # 512K context (our floor — their qualified 131K is not usable for us;
|
|
|
+ # the KV pool is engine-computed from the memory budget, unchanged):
|
|
|
+ "--max-model-len", "524288",
|
|
|
+ "--decode-context-parallel-size", "1",
|
|
|
+ "--max-num-seqs", "4",
|
|
|
+ "--max-num-batched-tokens", "4096",
|
|
|
+ "--prefill-compute-share", "0.4",
|
|
|
+ "--block-size", "256",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
+ "--attention-backend", "B12X",
|
|
|
+ "--linear-backend", "b12x",
|
|
|
+ "--moe-backend", "b12x",
|
|
|
+ "--compilation-config", """
|
|
|
+'{"cudagraph_mode":"FULL_AND_PIECEWISE"}'""",
|
|
|
+ "--max_cudagraph_capture_size", "128",
|
|
|
+ # Native DSpark K7 — the draft weights live inside the target checkpoint
|
|
|
+ # (no separate draft repo). Greedy draft, standard rejection, adaptive
|
|
|
+ # verification (the lab's r36 qualified trio vs our prod's K5
|
|
|
+ # probabilistic/block).
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"dspark","num_speculative_tokens":7,"draft_tensor_parallel_size":4,"attention_backend":"B12X","draft_sample_method":"greedy","rejection_sample_method":"standard","enable_adaptive_verification":true}'""",
|
|
|
+ # Engram: the model's 196B learned ngram lookup storage. disk mode =
|
|
|
+ # io_uring reads of the table rows from the checkpoint dir (bounded
|
|
|
+ # staging; NOT pinned RAM). cpu_offload stays false (no generic CPU
|
|
|
+ # model offload).
|
|
|
+ "--engram-config", """
|
|
|
+'{"cpu_offload":false,"table_memory":"disk"}'""",
|
|
|
+ "--load-format", "instanttensor",
|
|
|
+ "--safetensors-load-strategy", "lazy",
|
|
|
+ "--generation-config", "vllm",
|
|
|
+ "--override-generation-config", """
|
|
|
+'{"temperature":1.0,"top_p":0.95}'""",
|
|
|
+ "--limit-mm-per-prompt", """
|
|
|
+'{"image":2}'""",
|
|
|
+ "--reasoning-parser", "deepseek_v41",
|
|
|
+ "--tool-call-parser", "deepseek_v41",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--jit-monitor-mode", "error",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+
|
|
|
+# Their serve-ds41-jovian.sh runtime env (multi-node-adapted; the launcher
|
|
|
+# injects NCCL_SOCKET_IFNAME/IB_HCA/GID + VLLM_HOST_IP per node).
|
|
|
+NCCL_NCHANNELS = "16"
|
|
|
+NCCL_MIN_NCHANNELS = "16"
|
|
|
+NCCL_MAX_NCHANNELS = "16"
|
|
|
+NCCL_BUFFSIZE = "2097152"
|
|
|
+OMP_NUM_THREADS = "8"
|
|
|
+CUDA_DEVICE_MAX_CONNECTIONS = "32"
|
|
|
+# GB10 arch (theirs sm_120a on RTX PRO 6000):
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+# Their V2 model-runner + AOT compile pipeline (baked in their image env):
|
|
|
+VLLM_USE_V2_MODEL_RUNNER = "1"
|
|
|
+VLLM_USE_AOT_COMPILE = "1"
|
|
|
+VLLM_USE_STANDALONE_COMPILE = "1"
|
|
|
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
|
|
|
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
|
|
|
+VLLM_USE_FLASHINFER_SAMPLER = "1"
|
|
|
+VLLM_WORKER_MULTIPROC_METHOD = "spawn"
|
|
|
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "900"
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+# Deliberately NOT set: VLLM_ENABLE_PCIE_ALLREDUCE (single-host PCIe in
|
|
|
+# their profile; cross-node on the switch fabric uses the launcher's NCCL),
|
|
|
+# VLLM_HOST_IP (launcher per-node), NCCL_SOCKET_IFNAME (launcher per cluster).
|
|
|
+
|
|
|
+# --- Engram from node-local NVMe, the rest from the shared NAS ---
|
|
|
+# [composed_model] (standard launcher feature) builds a per-node /tmp dir:
|
|
|
+# the whole DSV4.1 snapshot symlinked into the HF cache + the two 101.5 GB
|
|
|
+# Engram table shards (model-00047/00048) HARDLINKED from the pre-existing
|
|
|
+# per-rank sparse copies in /var/tmp/engram-local (48 GB real per node, built
|
|
|
+# for the eugr/aiden lanes — offset-preserving full-size files, only this
|
|
|
+# rank's rows materialized; hardlink = zero extra disk, same fs as /tmp).
|
|
|
+# The r36 io_uring PLE reader then reads this rank's table rows from local
|
|
|
+# NVMe (no NFS); weights stream from the NAS at load time (one-shot, same as
|
|
|
+# prod). Local files run after the repo symlinks and win at the same name.
|
|
|
+[composed_model]
|
|
|
+mount = "/model"
|
|
|
+
|
|
|
+[[composed_model.sources]]
|
|
|
+repo = "deepseek-ai/DeepSeek-V4.1-Flash" # whole snapshot, symlinked into the HF cache
|
|
|
+
|
|
|
+[[composed_model.local_files]] # hardlinks win over the symlinks above
|
|
|
+host = "/var/tmp/engram-local/DeepSeek-V4.1-Flash/model-00047-of-00048.safetensors"
|
|
|
+
|
|
|
+[[composed_model.local_files]]
|
|
|
+host = "/var/tmp/engram-local/DeepSeek-V4.1-Flash/model-00048-of-00048.safetensors"
|
