launcher/recipes/dsv41-flash-jovian-r37-4x/recipe.toml

# DeepSeek-V4.1-Flash — local-inference-lab r37 (jovian-judgement, 2026-09-13), 4x.
#
# Clone of dsv41-flash-jovian-r36-4x retargeted at the r37 source lock — the
# lab's newest published DS4.1 qualification. r37's headline is a
# kernel-dispatch overhead removal: no more repeated CPU argument-list
# expansion, mutable-schema inspection, scalar dtype allocations or
# tensor-equality calls per dispatch (no quantization or sampling change);
# plus measured B12X projection tiles, four FP32 projection partials, visible
# index-score tiles, a compact mHC prefill config, removed attention-output
# copies and bounded short-index scratch. Their RTX-quartet qualification
# (public-source control -> r37): uncached 32K prefill 14,896 -> 21,399 tok/s
# (+43.65%), C1 output 238.80 -> 281.08 tok/s (+17.70%, budget-50 three-cell
# median), C1 verifier +11.92%, Sieve +8.17%. r37 also corrects the named
# `high` reasoning budget 50 -> 75 (behaviour, not speed — our benches pass
# reasoning_effort explicitly). Carries the B12X small-tile barrier fix (#363)
# and TP3 prefill-head support (TP3 is not serving-qualified here).
#
# Image: our own build for GB10 sm_121a from docker-images/jovian-judgement
# with profile r37-ds41 (vLLM c687594b8a8 + b12x f1c4e9dd5b1; LMCache
# 29bc5a2e / FlashKDA 3b225bf2 / DeepGEMM 8b1392b9 unchanged; torch 2.13.0 +
# flashinfer 0.6.17 = the same GB10 pins as r35/r36, dep gates re-checked
# against the r37 tree; liburing 2.5 + dev headers already in the apt lock).
# The published lab image is CUDA 13.3 x86 / sm_120 — ours is the documented
# CUDA 13.2 aarch64 / sm_121a deviation.
#
# Serving profile, GB10 deviations and the Engram-on-node-NVMe mechanism are
# unchanged from r36 (see that recipe's header + history/night-shifts/
# 2026-09-12.md "Day follow-up 2/3"): TP4/DCP1, native DSpark K7 (greedy
# draft, standard rejection, adaptive verification), 512K (our floor — their
# 131K qualified profile is not usable for us), GMU 0.88, B12X attn/MoE/linear,
# block 256, instanttensor, Engram table on disk via
# [composed_model].local_files hardlinked from the eugr lane's per-rank sparse
# copies in /var/tmp/engram-local (48 GB real per node, zero extra disk).
#
# Measured reference points (dev65/seed 42, d0 c1):
#   r36 + local Engram @512K   154/176 (88), 3,774 pp / 45.7 tg /  822 ms TTFT
#   r36 + local Engram @131K   149/176 (85), 3,654 pp / 43.9 tg /  826 ms
#   r36 + Engram over NFS      158/176 (90),   401 pp / 33.7 tg / 5,061 ms
#   eugr dsv41 prod (K5)       167/176 (95), 1,642 pp / 47.0 tg / 1,738 ms
# What r37 targets: the same r36 profile with their +43% prefill fix. The A/B
# vs the eugr prod decides (quality gap vs speed lead) — plan:
# history/night-shifts/2026-09-13.md.
#
# PREREQUISITE: weights in the shared NAS HF cache; per-node local Engram
# sparse copies under /var/tmp/engram-local/DeepSeek-V4.1-Flash.
engine = "vllm"
name = "DeepSeek-V4.1-Flash (jovian r37 img, 4x DCP1, DSpark K7 greedy + Engram disk)"
docker_image = "jovian-judgement:r37-ds41"
docker_image_dir = "jovian-judgement"
build_profile = "r37-ds41"
model = "deepseek-ai/DeepSeek-V4.1-Flash"
parent_model = "deepseek-ai/DeepSeek-V4.1-Flash"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# Our r27 build dir images ship with ENTRYPOINT [] — set it explicitly.
entrypoint = ["vllm", "serve"]

args = [
    "--served-model-name", "dsv41",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # Bisect C (2026-09-14 morning typo hunt): the FULL PROD CONTRACT on the
    # r37 image - prod's memory/compute flags (GMU 0.85, seqs 8, tokens 8192,
    # block 128, K5-tuned capture sizes), no B12X attention/linear backends,
    # no instanttensor, prod's K5/block spec. Bisect log:
    #   A: removed --linear-backend b12x -> typos PERSIST (not the cause)
    #   B: prod's K5 spec instead of K7 greedy/adaptive -> typos PERSIST
    #   C: if clean -> a contract flag is the cause (re-add one by one);
    #      if garbled -> the r37 tree's DeepSeek model code is the cause
    #      (file with the lab; the r37 tree is not qualified for DS4.1).
    "--gpu-memory-utilization", "0.85",
    "--max-model-len", "524288",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "8192",
    "--block-size", "128",
    "--compilation-config", """
'{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[5,6,10,12,15,18,20,24,25,30,35,36,40,42,48]}'""",
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}'""",
    "--moe-backend", "b12x",
    # Bisect D: engram disabled (no --engram-config). If the boot fails
    # (model requires engram) the numeric-tree hypothesis stands; if it boots
    # and the text is clean -> the lab's engram disk-read path is the cause.
    # (A/B/C above: all garbled - flags are not the cause.)
    "--tool-call-parser", "deepseek_v41",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v41",
    "--default-chat-template-kwargs", """
'{"thinking": true, "reasoning_effort": "max"}'""",
]

[env]

# Their serve-ds41-jovian.sh runtime env (multi-node-adapted; the launcher
# injects NCCL_SOCKET_IFNAME/IB_HCA/GID + VLLM_HOST_IP per node).
# 2026-09-14 (night shift, operator's typo report): the lab tree's runtime
# LM-head quantization (vocab_parallel_embedding.py) quantizes an unquantized
# LM head to MXFP8 BY DEFAULT (envs.py: VLLM_MXFP8_LM_HEAD, default "1").
# The eugr 2026-09-10 nightly (our prod image) has NO such feature and keeps
# the BF16 head - the coarsened 8-bit vocab-logit layer is the suspected cause
# of the typo-rate regression (close-token logits distorted). Opting out.
VLLM_MXFP8_LM_HEAD = "0"
NCCL_NCHANNELS = "16"
NCCL_MIN_NCHANNELS = "16"
NCCL_MAX_NCHANNELS = "16"
NCCL_BUFFSIZE = "2097152"
OMP_NUM_THREADS = "8"
CUDA_DEVICE_MAX_CONNECTIONS = "32"
# GB10 arch (theirs sm_120a on RTX PRO 6000):
CUTE_DSL_ARCH = "sm_121a"
TORCH_CUDA_ARCH_LIST = "12.1a"
# Their V2 model-runner + AOT compile pipeline (baked in their image env):
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_STANDALONE_COMPILE = "1"
VLLM_USE_MEGA_AOT_ARTIFACT = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "900"
HF_HUB_OFFLINE = "1"
# Deliberately NOT set: VLLM_ENABLE_PCIE_ALLREDUCE (single-host PCIe in
# their profile; cross-node on the switch fabric uses the launcher's NCCL),
# VLLM_HOST_IP (launcher per-node), NCCL_SOCKET_IFNAME (launcher per cluster).

# --- Engram from node-local NVMe, the rest from the shared NAS ---
# [composed_model] (standard launcher feature) builds a per-node /tmp dir:
# the whole DSV4.1 snapshot symlinked into the HF cache + the two 101.5 GB
# Engram table shards (model-00047/00048) HARDLINKED from the pre-existing
# per-rank sparse copies in /var/tmp/engram-local (48 GB real per node, built
# for the eugr/aiden lanes — offset-preserving full-size files, only this
# rank's rows materialized; hardlink = zero extra disk, same fs as /tmp).
# The r36 io_uring PLE reader then reads this rank's table rows from local
# NVMe (no NFS); weights stream from the NAS at load time (one-shot, same as
# prod). Local files run after the repo symlinks and win at the same name.
[composed_model]
mount = "/model"

[[composed_model.sources]]
repo = "deepseek-ai/DeepSeek-V4.1-Flash"          # whole snapshot, symlinked into the HF cache

[[composed_model.local_files]]                      # hardlinks win over the symlinks above
host = "/var/tmp/engram-local/DeepSeek-V4.1-Flash/model-00047-of-00048.safetensors"

[[composed_model.local_files]]
host = "/var/tmp/engram-local/DeepSeek-V4.1-Flash/model-00048-of-00048.safetensors"