launcher/recipes/dsv41-flash-jovian-r36-4x/recipe.toml

# DeepSeek-V4.1-Flash — local-inference-lab r36 (jovian-judgement), 4x DCP1.
#
# THE lab's first published DS4.1 lock (qualified 2026-09-12): their image
# localinferencelab/vllm:jovian-judgement-community-20260912-r36 serves GLM
# (r35 lane) + DS4.1 (this recipe) in one two-layer image. We build the same
# source composition for GB10 sm_121a from our own build dir
# (docker-images/glm53-jovian-r27, profile jovian-r36-ds41 — the r36 delta
# over our r35 pins is only vLLM 202a11a98eb + b12x 323107ff + liburing 2.5;
# LMCache/FlashKDA/DeepGEMM/FI/torch are the SAME pins as the r35 build).
#
# Lab DS4.1 serving profile (models/deepseek-v4.1-flash.md, compose +
# serve-ds41-jovian.sh/serve-ds41-flash.sh in the repo): TP4/DCP1, native
# DSpark K7 (draft weights INSIDE the target checkpoint — no separate draft
# download), greedy draft + standard rejection + adaptive verification,
# B12X attention/MoE/linear, 131K context, 4 seqs / 4096 batched, block 256,
# FULL_AND_PIECEWISE graphs capped at 128, InstantTensor, Engram ngram tables
# in disk mode (io_uring; their 188.83 GiB RAM mode cannot fit a GB10 node —
# 121.63 GiB unified pool).
#
# GB10 deviations from the RTX-qualified profile (documented):
#   - GMU 0.88 (theirs 0.95 on 96 GB RTX hosts; 0.95 fails GB10's startup
#     free-mem gate — the #48140 class, see the glm53 r35 recipe notes).
#   - CUTE_DSL_ARCH=sm_121a / arch 12.1a (theirs sm_120a; the published
#     image's native extension is sm_120 SASS — ours is built for GB10).
#   - NO VLLM_ENABLE_PCIE_ALLREDUCE (theirs is single-host PCIe; on our
#     4-node switch fabric the launcher's NCCL/RoCE env is the transport).
#   - Engram disk mode, table on node-local NVMe (see [composed_model]
#     below): the NFS read path was the 4.1x prefill collapse (day
#     follow-up 2); their 188.83 GiB RAM mode cannot fit a GB10 node
#     (121.63 GiB unified pool) either way.
#   - VLLM_USE_V2_MODEL_RUNNER/AOT/MEGA_AOT env from their serve script
#     (their image bakes them; ours must carry them explicitly).
#
# A/B results (2026-09-13, 4nodes-switch, dev65/seed 42, vs the eugr dsv41
# prod 167/176 / 1,642 pp / 47.0 tg / 1,738 ms TTFT):
#   - Engram on NFS: 158/176 (90), 401 pp, 33.7 tg, 5,061 ms TTFT — prefill
#     collapse (io_uring row reads over NFS).
#   - Engram on local NVMe (this recipe, composed_model.local_files + the
#     eugr lane's per-rank sparse copies): 149/176 (85), 3,654 pp (2.2x
#     prod), 43.9 tg, 826 ms TTFT (2.1x better than prod), KV 3.34M @131K.
#     Storage suspect CONFIRMED (prefill 9.1x faster than NFS run). Quality
#     149 = run variance vs the NFS run (11 tool-discipline wobbles); the
#     r36-vs-prod gap is the engine stack (K7 vs K5). Status: CANDIDATE lane
#     — fastest prefill/TTFT we have, -18 quality vs prod. Full write-up:
#     history/night-shifts/2026-09-12.md ("Day follow-up 2 09-13").
#
# A/B target: the current dsv41 prod (eugr image, DSpark K5 probabilistic,
# 167/176, 1,642 pp / 47.0 tg / 1,738 ms TTFT @d0 c1, 512K, dev65). Same
# bench (tool-eval-bench dev65, seed 42, hardmode, c1/c2) — the clean delta
# isolates the engine stack (K7+adaptive+V2-runner+AOT+b12x-moe vs K5).

engine = "vllm"
name = "DeepSeek-V4.1-Flash (jovian r36 img, 4x DCP1, DSpark K7 greedy + Engram disk)"
docker_image = "jovian-r36-ds41:local"
docker_image_dir = "glm53-jovian-r27"
build_profile = "jovian-r36-ds41"
model = "deepseek-ai/DeepSeek-V4.1-Flash"
parent_model = "deepseek-ai/DeepSeek-V4.1-Flash"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# Our r27 build dir images ship with ENTRYPOINT [] — set it explicitly.
entrypoint = ["vllm", "serve"]

args = [
    "--served-model-name", "dsv41",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # GB10 startup-gate value (see header); KV pool is engine-computed from
    # the remainder (no pin — their profile pins nothing either).
    "--gpu-memory-utilization", "0.88",
    # 512K context (our floor — their qualified 131K is not usable for us;
    # the KV pool is engine-computed from the memory budget, unchanged):
    "--max-model-len", "524288",
    "--decode-context-parallel-size", "1",
    "--max-num-seqs", "4",
    "--max-num-batched-tokens", "4096",
    "--prefill-compute-share", "0.4",
    "--block-size", "256",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    "--kv-cache-dtype", "fp8",
    "--attention-backend", "B12X",
    "--linear-backend", "b12x",
    "--moe-backend", "b12x",
    "--compilation-config", """
'{"cudagraph_mode":"FULL_AND_PIECEWISE"}'""",
    "--max_cudagraph_capture_size", "128",
    # Native DSpark K7 — the draft weights live inside the target checkpoint
    # (no separate draft repo). Greedy draft, standard rejection, adaptive
    # verification (the lab's r36 qualified trio vs our prod's K5
    # probabilistic/block).
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":7,"draft_tensor_parallel_size":4,"attention_backend":"B12X","draft_sample_method":"greedy","rejection_sample_method":"standard","enable_adaptive_verification":true}'""",
    # Engram: the model's 196B learned ngram lookup storage. disk mode =
    # io_uring reads of the table rows from the checkpoint dir (bounded
    # staging; NOT pinned RAM). cpu_offload stays false (no generic CPU
    # model offload).
    "--engram-config", """
'{"cpu_offload":false,"table_memory":"disk"}'""",
    "--load-format", "instanttensor",
    "--safetensors-load-strategy", "lazy",
    "--generation-config", "vllm",
    "--override-generation-config", """
'{"temperature":1.0,"top_p":0.95}'""",
    "--limit-mm-per-prompt", """
'{"image":2}'""",
    "--reasoning-parser", "deepseek_v41",
    "--tool-call-parser", "deepseek_v41",
    "--enable-auto-tool-choice",
    "--jit-monitor-mode", "error",
]

[env]

# Their serve-ds41-jovian.sh runtime env (multi-node-adapted; the launcher
# injects NCCL_SOCKET_IFNAME/IB_HCA/GID + VLLM_HOST_IP per node).
NCCL_NCHANNELS = "16"
NCCL_MIN_NCHANNELS = "16"
NCCL_MAX_NCHANNELS = "16"
NCCL_BUFFSIZE = "2097152"
OMP_NUM_THREADS = "8"
CUDA_DEVICE_MAX_CONNECTIONS = "32"
# GB10 arch (theirs sm_120a on RTX PRO 6000):
CUTE_DSL_ARCH = "sm_121a"
TORCH_CUDA_ARCH_LIST = "12.1a"
# Their V2 model-runner + AOT compile pipeline (baked in their image env):
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_STANDALONE_COMPILE = "1"
VLLM_USE_MEGA_AOT_ARTIFACT = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "900"
HF_HUB_OFFLINE = "1"
# Deliberately NOT set: VLLM_ENABLE_PCIE_ALLREDUCE (single-host PCIe in
# their profile; cross-node on the switch fabric uses the launcher's NCCL),
# VLLM_HOST_IP (launcher per-node), NCCL_SOCKET_IFNAME (launcher per cluster).

# --- Engram from node-local NVMe, the rest from the shared NAS ---
# [composed_model] (standard launcher feature) builds a per-node /tmp dir:
# the whole DSV4.1 snapshot symlinked into the HF cache + the two 101.5 GB
# Engram table shards (model-00047/00048) HARDLINKED from the pre-existing
# per-rank sparse copies in /var/tmp/engram-local (48 GB real per node, built
# for the eugr/aiden lanes — offset-preserving full-size files, only this
# rank's rows materialized; hardlink = zero extra disk, same fs as /tmp).
# The r36 io_uring PLE reader then reads this rank's table rows from local
# NVMe (no NFS); weights stream from the NAS at load time (one-shot, same as
# prod). Local files run after the repo symlinks and win at the same name.
[composed_model]
mount = "/model"

[[composed_model.sources]]
repo = "deepseek-ai/DeepSeek-V4.1-Flash"          # whole snapshot, symlinked into the HF cache

[[composed_model.local_files]]                      # hardlinks win over the symlinks above
host = "/var/tmp/engram-local/DeepSeek-V4.1-Flash/model-00047-of-00048.safetensors"

[[composed_model.local_files]]
host = "/var/tmp/engram-local/DeepSeek-V4.1-Flash/model-00048-of-00048.safetensors"