launcher/recipes/dsv41-flash-8x/recipe.toml

# DeepSeek-V4.1-Flash (552B MoE, MXFP4 experts) — 8x DGX Spark, in-memory Engram.
#
# P2.3 of the 2026-09-18 night: the first 8x trial. Operator's design: at 8x
# the Engram table fits in DRAM (406 GB / 8 = 50.7 GB/node) — no disk mode,
# no per-node sparse staging, no /engram-local mounts. The engram rows load
# from the NAS checkpoint into DRAM once at boot (one-off NFS cost), decode
# reads them from memory (no per-token disk reads).
#
# Memory math per node (121.6 GiB): weights 307.3 GB / 8 = 38.4 GB + engram
# rows ~25.4 GB (the real table data is ~203 GB — the two 101.5 GB sparse
# shards whose 4x /var/tmp copies of 48 GB real sum to 192 GB) + process
# ~12 GB = ~76 GB.
#
# GMU: the lab DS4.1 policy says 0.91, but the GB10 admission check needs
# free >= GMU x 121.63 and the baseline-reserved ~12.8 GiB leaves only
# 108.8 GiB free at startup — 0.91 (110.7) fails the admission on GB10
# (first 8x boot, 2026-09-18 16:02: "Free memory on device cuda:0
# (108.78/121.63 GiB) ... less than desired GPU memory utilization"). 0.85
# (103.4) is the GB10-qualified prod value and still leaves ~27.6 GiB of KV
# per rank (103.4 - 38.4 - 25.4 - 12), ≈ 3M tokens ≈ 6x 512K.
#
# Image: dsv41-flash:local (the tonyd2wild prod chain — engram.py patch
# baked in; DSV41_ENGRAM_DISK simply NOT set here = memory mode). The
# launcher syncs the image to dgx6-8 at serve time (dgx1-5 have it).
#
# This is also the natural "Boot D" test the 4x r37 bisect could not run:
# engram WITHOUT the disk-read path. If the text is clean here while the r37
# 4x disk-mode run garbles, the bisect narrows to the lab tree's engram
# disk-read path.
engine = "vllm"
name = "DeepSeek-V4.1-Flash MXFP4 (dsv41 img, 8x, DSpark k=5, Engram in DRAM)"
docker_image = "dsv41-flash:local"
docker_image_dir = "dsv41-flash"
model = "deepseek-ai/DeepSeek-V4.1-Flash"
required_nodes = 8
tensor_parallel_size = 8
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# NO recipe entrypoint: the base image bakes ENTRYPOINT ["vllm","serve"]
# (vllm/vllm-openai), so the launcher's CMD (model + flags) completes it.

args = [
    # Stable gateway alias — clients/gateways keep the DeepSeek-V4-Flash
    # model name; no per-deployment config churn.
    "--served-model-name", "deepseek-v4-flash",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    "--gpu-memory-utilization", "0.85",
    "--max-model-len", "524288",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "8192",
    # boot-4 fix: the V4 indexer backend refuses the smallest-listed block
    "--block-size", "128",
    # boot-10: exact capture sizes for DSpark k=5, SEQS=8 (multiples of 5
    # and 6 up to 48 — every decode batch gets an exact FULL graph, no padding)
    "--compilation-config", """
'{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[5,6,10,12,15,18,20,24,25,30,35,36,40,42,48]}'""",
    "--speculative-config", """
'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}'""",
    "--tool-call-parser", "deepseek_v41",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v41",
    "--default-chat-template-kwargs", """
'{"thinking": true, "reasoning_effort": "max"}'""",
]

[env]
HF_HUB_OFFLINE = "1"
# Engram: MEMORY mode — DSV41_ENGRAM_DISK deliberately NOT set (the baked
# engram.py patch defaults to "0" = in-DRAM table). At 8x the table fits:
# 406 GB / 8 = 50.7 GB/node.
# CUDA graphs: the Engram rows are staged in prepare_inputs (baked
# model_state.py prestage patch) so the whole decode step is capturable
VLLM_USE_BREAKABLE_CUDAGRAPH = "1"
VLLM_HAS_FLASHINFER_CUBIN = "1"
VLLM_USE_RUST_FRONTEND = "0"
# runtime JIT wedge countermeasure (their boot-3 lesson): kernels are
# prebuilt in the image, but any residual JIT must not OOM the box
MAX_JOBS = "2"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
OMP_NUM_THREADS = "2"

# JIT caches persist per-node (self-namespace by engine version key; the
# image prebakes the flashinfer 0.7.0rc1 SM120 kernels, first boots re-JIT
# the rest)
[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"