recipes/recipes/dsv41-flash-1017-4x-sglang/recipe.toml

# DeepSeek-V4.1-Flash-1017 on 4x DGX Spark with Aiden's SGLang production-1.7.1
# (aidenle.com/recipes/deepseek-v4-1-flash-sglang-4x-dgx-spark).
#
# DROP-IN of the 2026-10-17 re-cut `deepseek-ai/DeepSeek-V4.1-Flash-1017`
# (same arch DeepseekV41ForCausalLM, fp8, 48 shards) over the old
# `deepseek-ai/DeepSeek-V4.1-Flash` — same serving contract as
# dsv41-flash-4x-sglang-aiden16, only the model repo (+ staging src) changes.
#
# Image: aidendle94/sparkrun-sglang-dsv41-gb10:production-1.7.1 (Docker Hub,
# 2026-09-29 — Aiden's latest DSV4.1 SGLang production image) = lmsysorg
# SGLang DSV4.1 + Aiden's MIT overlay (Engram row store inside the CUDA graph,
# b12x SM12x FP8 dense kernels, SM120 prefill paging, RDMA collectives).
#
# Model: deepseek-ai/DeepSeek-V4.1-Flash-1017 (~476 GB, fp8) on the NAS HF
# cache (weights mmap-loaded from the snapshot). The Engram rows are staged
# node-locally by [[staging]] below and mounted at /engram-local —
# SPARK_ENGRAM_DIR points there. NOTE: the staging dst is a NEW dir
# (/home/kilork/engram-1017) — the old aiden-engram copies are packed from the
# OLD snapshot's tables and must not be reused. First serve packs ~95 GiB/node
# from the NAS; later serves verify sampled row hashes against the ledger.
# Unknown risk: the 1.7.1 overlay predates 1017 — if the re-cut changed the
# Engram table layout, staging/serve will say so (row ranges derive from the
# checkpoint's safetensors headers, nothing is hardcoded).

engine = "sglang"
name = "DeepSeek-V4.1-Flash-1017 (SGLang 4x, Aiden production-1.7.1)"
docker_image = "dsv41-sglang-1017:local"
docker_image_dir = "dsv41-sglang-1017"
model = "deepseek-ai/DeepSeek-V4.1-Flash-1017"
parent_model = "deepseek-ai/DeepSeek-V4.1-Flash"
port = 30000
required_nodes = 4
tensor_parallel_size = 4
distributed_backend = "mp"
shm_size = "32gb"

# The wrapper (mounted read-only from the host via extra_mounts below) sets
# SPARK_ENGRAM_ROWS (+ B12X_ROCE_GID_INDEX) from NODE_RANK and execs the
# engine with the launcher's full flag list in "$@".
entrypoint = ["sh", "/aiden4x-launch.sh"]

args = [
    "--trust-remote-code",
    "--served-model-name", "deepseek-v4.1-flash",
    "--load-format", "safetensors",
    # Aiden's measured shape (production-1.6, carried into 1.7.1); --tp 4 /
    # --nnodes 4 / --node-rank / --dist-init-addr are appended by the launcher
    "--ep-size", "4",
    "--attention-backend", "dsv4",
    "--moe-runner-backend", "flashinfer_mxfp4",
    # required for the MXFP8 route (selects the block-scaled path the hook wraps;
    # K=576 projections fall back to CUTLASS per shape)
    "--fp8-gemm-backend", "flashinfer_cutlass",
    "--mem-fraction-static", "0.80",
    "--chunked-prefill-size", "2048",
    "--context-length", "524288",
    "--max-running-requests", "16",
    "--cuda-graph-max-bs-decode", "16",
    "--random-seed", "0",
    "--enable-decoder-swa-bounded-replay",
    "--speculative-algorithm", "DSPARK",
    "--speculative-dspark-block-size", "5",
    "--tool-call-parser", "deepseekv41",
    "--reasoning-parser", "deepseek-v41",
    "--watchdog-timeout", "1800",
    "--min-free-slots-delay", "1",
]

# The launcher also appends: --model-path <resolved NAS snapshot>, --nnodes 4,
# --node-rank N, --dist-init-addr <head-ib>:25000, --host 0.0.0.0, --port 30000.

# NOTE: no --default-chat-template-kwargs '{"thinking":true}' — the bench
# harness controls thinking; Aiden's default is for his interactive fleet.

# The per-rank Engram range wrapper, mounted read-only (copied from the
# dsv41-flash-4x-sglang-aiden16 recipe dir; placed at this host path on the
# launch nodes).
[[extra_mounts]]
host = "/home/kilork/aiden4x-launch.sh"
container = "/aiden4x-launch.sh"
read_only = true


# Node-local sparse copy of the Engram tables (staging P2.5, sparse-copy —
# the aiden overlay's engram-local.json contract). Built/verified per serve by
# the [[staging]] block; the wrapper-computed SPARK_ENGRAM_ROWS and the
# manifest agree because sparse_rows.py tiles ranks with the wrapper's
# base+rem formula (the overlay refuses to start unless both are identical).
[[extra_mounts]]
host = "/home/kilork/engram-1017"
container = "/engram-local"
read_only = true

# Staging: sparse-copy packs the aiden contract (engram-local.json manifest +
# sparse shard copies at the original safetensors byte offsets) from the 1017
# snapshot. Ranges use the wrapper's base+rem tiling; first serve packs
# ~95 GiB/node from the NAS, later serves verify sampled row hashes against
# the ledger stored inside the dst tree.
[[staging]]
kind = "sparse-rows"
format = "sparse-copy"
id = "engram"
src = "{{hf}}/hub/models--deepseek-ai--DeepSeek-V4.1-Flash-1017/snapshots/1bae0e1088993b9b82de08f59de5be5a34e42170"
dst = "/home/kilork/engram-1017"
min_free = "55G"

[env]
# Engram row source: the node-local sparse copy staged by [[staging]] above
# (Aiden's measured production config — ~47 GiB of rows on node NVMe). The
# manifest (engram-local.json) carries the per-layer rank ranges;
# SPARK_ENGRAM_ROWS from the wrapper is the cross-check.
SPARK_ENGRAM_DIR = "/engram-local"
SPARK_ENGRAM_MODE = "hostnode"
SPARK_ENGRAM_EARLY = "1"
SPARK_ENGRAM_PREFETCH = "1"
SPARK_PAGE_CACHE_RELEASE = "1"
SPARK_LATE_TAIL_SKIP = "1"
SPARK_LATE_TAIL_SKIP_ALL = "1"
SPARK_SM120_REAL_HEADS = "1"
SPARK_ENGRAM_THREADS = "64"
SPARK_ENGRAM_MAX_IDS = "65536"
SPARK_MXFP8_BACKEND = "b12x"
SPARK_ROCE_AR = "1"
SPARK_ROCE_AR_MAX = "1MB"
SPARK_ROCE_AG_MAX = "16MB"
# b12x RoCE runtime (GID index set by the wrapper from the detected value)
B12X_ROCE_HCA = "rocep1s0f0,roceP2p1s0f0"
B12X_ROCE_SPIN_LIMIT = "300000000"
# image defaults Aiden overrides
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE = "0"
SGLANG_FLASHINFER_MOE_FUSED_FINALIZE = "0"
SGLANG_DSV41_REASONING_EFFORT = "100"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:False"
CUDA_DEVICE_ORDER = "PCI_BUS_ID"
# Aiden's NCCL pins (4x 200G RoCE fabric, 192.168.200.0/24) — [env] wins over
# the config's [nccl."4nodes-switch"] where they differ
NCCL_NET = "IB"
NCCL_IB_DISABLE = "0"
NCCL_IB_HCA = "rocep1s0f0,roceP2p1s0f0"
NCCL_SOCKET_IFNAME = "enp1s0f0np0"
GLOO_SOCKET_IFNAME = "enp1s0f0np0"
NCCL_IB_ROCE_VERSION_NUM = "2"
NCCL_IB_ADDR_FAMILY = "AF_INET"
NCCL_IB_ADDR_RANGE = "192.168.200.0/24"
NCCL_IB_SUBNET_PREFIX_LEN = "24"
NCCL_CROSS_NIC = "1"
NCCL_IB_MERGE_NICS = "0"
NCCL_IB_SUBNET_AWARE_ROUTING = "1"
NCCL_CUMEM_ENABLE = "0"
NCCL_P2P_DISABLE = "1"
NCCL_SHM_DISABLE = "1"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_PROTO = "^LL128"
NCCL_MAX_NCHANNELS = "8"