launcher/recipes/dsv41-flash-4x-sglang-aiden16/recipe.toml

# DeepSeek-V4.1-Flash on 4x DGX Spark with Aiden's SGLang production-1.6
# (aidenle.com/recipes/deepseek-v4-1-flash-sglang-4x-dgx-spark).
#
# Image: aidendle94/sparkrun-sglang-dsv41-gb10:production-1.6 (Docker Hub,
# arm64, 33.3 GB) = lmsysorg/sglang:dev-dsv41 (SGLang team's own DSV4.1
# image, unmodified) + Aiden's MIT overlay (import hooks: Engram row store
# from node-local/shard files inside the CUDA graph, b12x SM12x FP8 dense
# kernels, SM120 prefill paging, allocator flush, one-shot RDMA collectives,
# request guard). Lineage: replaced the vLLM stack as Aiden's production
# 2026-09-17 (prefill +30-45%, KV pool +23%); 1.2: 16 concurrent; 1.5: rank
# crash fix + 2 GB10 speedups; 1.6 current. Measured by Aiden on 4x: decode
# 107/87/36 tok/s (count/code/prose C1), prefill 3.3K tok/s @32K, KV pool
# ~4.2M, 524K context, ~10 min to healthy.
#
# Model: deepseek-ai/DeepSeek-V4.1-Flash (510 GB, fp8 native) on the NAS —
# serve with --hf-cache /mnt/nas/ai/huggingface. The checkpoint snapshot
# dba1be0a... is also the Engram row source (SPARK_ENGRAM_DIR): all four
# ranks read their rows directly from the NAS shards (Aiden's fallback path;
# his measured config uses 47.2 GiB node-local sparse copies, which needs the
# private engram_local.py — the NFS read costs ~3-5 ms/step extra, acceptable
# for a validation run).
#
# Per-rank Engram row ranges: the two tables have DIFFERENT row counts
# (layer 1: 384,006,168; layer 14: 384,016,682 rows for this snapshot) and the
# overlay refuses to start unless the ranks' ranges tile [0, N) exactly per
# layer. Computed per node by aiden4x-launch.sh (next to this recipe) from
# $NODE_RANK + the checkpoint's safetensors headers (floor/ceil tiling; the
# launcher exports NODE_RANK per rank). A full range on every rank would
# double-count row contributions in the reduce — the range is mandatory.
#
# B12X_ROCE_GID_INDEX reuses the launcher's per-node NCCL GID auto-detect
# (the wrapper exports it), so no pinned GID anywhere.
#
# Measured 2026-09-26 (4nodes-switch, NAS model, NAS Engram rows):
#   162/176 (92/100, ★★★★★) — best DSV4.1 score on this cluster; fails
#   TC-68/74/88, partials TC-03/14/50/51/57/58/62/81. Smoke: count-20 exact,
#   tools OK, 155K-token needle OK, 524K context, KV pool 9.5M full tokens.
#   PERF CAVEAT (engram/NAS, not the engine): every FRESH context pays a
#   2-10 s prefill penalty (llama-benchy: pp 154-359 t/s, tg 8.6-21.8 t/s)
#   while prefix-cached repeats run 0.3 s TTFT — the engram n-gram rows come
#   from the NAS shards (30 ms mean gather; the 4 GB page cache releases once
#   and misses re-fetch). Aiden's production uses 47.2 GiB node-local sparse
#   copies (private engram_local.py). Agent traffic (short, repeated system
#   prompts) sits in the cached regime and is fast; long fresh-context loads
#   are slow. The tool-eval-bench llama-benchy perf table for this lane
#   reflects the fresh-context penalty (see the shift notes 2026-09-26).

engine = "sglang"
name = "DeepSeek-V4.1-Flash (SGLang 4x, Aiden production-1.6)"
docker_image = "aidendle94/sparkrun-sglang-dsv41-gb10:production-1.6"
model = "deepseek-ai/DeepSeek-V4.1-Flash"
parent_model = "deepseek-ai/DeepSeek-V4.1-Flash"
port = 30000
required_nodes = 4
tensor_parallel_size = 4
distributed_backend = "mp"
shm_size = "32gb"

# The wrapper (mounted read-only from the host via extra_mounts below) sets
# SPARK_ENGRAM_ROWS (+ B12X_ROCE_GID_INDEX) from NODE_RANK and execs the
# engine with the launcher's full flag list in "$@".
entrypoint = ["sh", "/aiden4x-launch.sh"]

args = [
    "--trust-remote-code",
    "--served-model-name", "deepseek-v4.1-flash",
    "--load-format", "safetensors",
    # Aiden's measured shape (production-1.6); --tp 4 / --nnodes 4 /
    # --node-rank / --dist-init-addr are appended by the launcher
    "--ep-size", "4",
    "--attention-backend", "dsv4",
    "--moe-runner-backend", "flashinfer_mxfp4",
    # required for the MXFP8 route (selects the block-scaled path the hook wraps;
    # K=576 projections fall back to CUTLASS per shape)
    "--fp8-gemm-backend", "flashinfer_cutlass",
    "--mem-fraction-static", "0.80",
    "--chunked-prefill-size", "2048",
    "--context-length", "524288",
    "--max-running-requests", "16",
    "--cuda-graph-max-bs-decode", "16",
    "--random-seed", "0",
    "--enable-decoder-swa-bounded-replay",
    "--speculative-algorithm", "DSPARK",
    "--speculative-dspark-block-size", "5",
    "--tool-call-parser", "deepseekv41",
    "--reasoning-parser", "deepseek-v41",
    "--watchdog-timeout", "1800",
    "--min-free-slots-delay", "1",
]

# The launcher also appends: --model-path <resolved NAS snapshot>, --nnodes 4,
# --node-rank N, --dist-init-addr <head-ib>:25000, --host 0.0.0.0, --port 30000.

# NOTE: no --default-chat-template-kwargs '{"thinking":true}' — the bench
# harness controls thinking; Aiden's default is for his interactive fleet.

# The per-rank Engram range wrapper, mounted read-only (bare keys above; the
# table header must come after them or `args`/`[env]` would parse into it).
[[extra_mounts]]
host = "/home/kilork/aiden4x-launch.sh"
container = "/aiden4x-launch.sh"
read_only = true

# Engram row source: the full checkpoint snapshot on the NAS (container-side
# path — the launcher mounts --hf-cache /mnt/nas/ai/huggingface at
# /root/.cache/huggingface). No engram-local.json manifest in a full snapshot,
# so per-rank ranges come from SPARK_ENGRAM_ROWS (wrapper).
[env]
SPARK_ENGRAM_DIR = "/root/.cache/huggingface/hub/models--deepseek-ai--DeepSeek-V4.1-Flash/snapshots/dba1be0a40aa45a94ad051997016db3960a90277"
SPARK_ENGRAM_MODE = "hostnode"
SPARK_ENGRAM_EARLY = "1"
SPARK_ENGRAM_PREFETCH = "1"
SPARK_PAGE_CACHE_RELEASE = "1"
SPARK_LATE_TAIL_SKIP = "1"
SPARK_LATE_TAIL_SKIP_ALL = "1"
SPARK_SM120_REAL_HEADS = "1"
SPARK_ENGRAM_THREADS = "64"
SPARK_ENGRAM_MAX_IDS = "65536"
SPARK_MXFP8_BACKEND = "b12x"
SPARK_ROCE_AR = "1"
SPARK_ROCE_AR_MAX = "1MB"
SPARK_ROCE_AG_MAX = "16MB"
# b12x RoCE runtime (GID index set by the wrapper from the detected value)
B12X_ROCE_HCA = "rocep1s0f0,roceP2p1s0f0"
B12X_ROCE_SPIN_LIMIT = "300000000"
# image defaults Aiden overrides
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE = "0"
SGLANG_FLASHINFER_MOE_FUSED_FINALIZE = "0"
SGLANG_DSV41_REASONING_EFFORT = "100"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:False"
CUDA_DEVICE_ORDER = "PCI_BUS_ID"
# Aiden's NCCL pins (4x 200G RoCE fabric, 192.168.200.0/24) — [env] wins over
# the config's [nccl."4nodes-switch"] where they differ
NCCL_NET = "IB"
NCCL_IB_DISABLE = "0"
NCCL_IB_HCA = "rocep1s0f0,roceP2p1s0f0"
NCCL_SOCKET_IFNAME = "enp1s0f0np0"
GLOO_SOCKET_IFNAME = "enp1s0f0np0"
NCCL_IB_ROCE_VERSION_NUM = "2"
NCCL_IB_ADDR_FAMILY = "AF_INET"
NCCL_IB_ADDR_RANGE = "192.168.200.0/24"
NCCL_IB_SUBNET_PREFIX_LEN = "24"
NCCL_CROSS_NIC = "1"
NCCL_IB_MERGE_NICS = "0"
NCCL_IB_SUBNET_AWARE_ROUTING = "1"
NCCL_CUMEM_ENABLE = "0"
NCCL_P2P_DISABLE = "1"
NCCL_SHM_DISABLE = "1"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_PROTO = "^LL128"
NCCL_MAX_NCHANNELS = "8"