launcher/recipes/dsv41-flash-4x-sglang-aiden16/recipe.toml

# DeepSeek-V4.1-Flash on 4x DGX Spark with Aiden's SGLang production-1.6
# (aidenle.com/recipes/deepseek-v4-1-flash-sglang-4x-dgx-spark).
#
# Image: aidendle94/sparkrun-sglang-dsv41-gb10:production-1.6 (Docker Hub,
# arm64, 33.3 GB) = lmsysorg/sglang:dev-dsv41 (SGLang team's own DSV4.1
# image, unmodified) + Aiden's MIT overlay (import hooks: Engram row store
# from node-local/shard files inside the CUDA graph, b12x SM12x FP8 dense
# kernels, SM120 prefill paging, allocator flush, one-shot RDMA collectives,
# request guard). Lineage: replaced the vLLM stack as Aiden's production
# 2026-09-17 (prefill +30-45%, KV pool +23%); 1.2: 16 concurrent; 1.5: rank
# crash fix + 2 GB10 speedups; 1.6 current. Measured by Aiden on 4x: decode
# 107/87/36 tok/s (count/code/prose C1), prefill 3.3K tok/s @32K, KV pool
# ~4.2M, 524K context, ~10 min to healthy.
#
# Model: deepseek-ai/DeepSeek-V4.1-Flash (510 GB, fp8 native) on the NAS —
# serve with --hf-cache /mnt/nas/ai/huggingface (weights are mmap-loaded from
# the snapshot). The Engram rows are staged node-locally by [[staging]] below
# (Aiden's measured production config: 47.2 GiB sparse copies per node) and
# mounted at /engram-local — SPARK_ENGRAM_DIR points there.
#
# Per-rank Engram row ranges: the two tables have DIFFERENT row counts
# (layer 1: 384,006,168; layer 14: 384,016,682 rows for this snapshot) and the
# overlay refuses to start unless the ranks' ranges tile [0, N) exactly per
# layer. Computed per node by aiden4x-launch.sh (next to this recipe) from
# $NODE_RANK + the checkpoint's safetensors headers (floor/ceil tiling; the
# launcher exports NODE_RANK per rank). A full range on every rank would
# double-count row contributions in the reduce — the range is mandatory.
#
# B12X_ROCE_GID_INDEX reuses the launcher's per-node NCCL GID auto-detect
# (the wrapper exports it), so no pinned GID anywhere.
#
# MEASURED 2026-09-26 (4nodes-switch, NAS model, NAS Engram rows):
#   162/176 (92/100, ★★★★★) — best DSV4.1 score on this cluster.
#   PERF CAVEAT then: every FRESH context paid a 2-10 s prefill penalty —
#   engram rows came from the NAS shards (30 ms mean gather). The [[staging]]
#   block below now localizes them on node NVMe (Aiden's production config);
#   a re-bench on staged rows is pending — treat the 09-26 perf row as the
#   NAS-engram baseline, not this config.
#
# MEASURED 2026-09-27 (staged sparse-copy engram, harness v2.7.1.dev7):
#   167/184 (91/100, ★★★★★), Deployability 85, Responsiveness 70/100
#   (median turn 1.7 s — the NAS-engram regime is gone: pp 2.4-3.7K t/s
#   vs 154-359 baseline, tg 34-52 t/s c1/c2 vs 8.6-21.8). Fails TC-68/74/
#   92 (same safety families as every DS4.1 lane). Reports: reports/
#   2026-09-27_193307_engram-local.*.

engine = "sglang"
name = "DeepSeek-V4.1-Flash (SGLang 4x, Aiden production-1.6)"
docker_image = "aidendle94/sparkrun-sglang-dsv41-gb10:production-1.6"
model = "deepseek-ai/DeepSeek-V4.1-Flash"
parent_model = "deepseek-ai/DeepSeek-V4.1-Flash"
port = 30000
required_nodes = 4
tensor_parallel_size = 4
distributed_backend = "mp"
shm_size = "32gb"

# The wrapper (mounted read-only from the host via extra_mounts below) sets
# SPARK_ENGRAM_ROWS (+ B12X_ROCE_GID_INDEX) from NODE_RANK and execs the
# engine with the launcher's full flag list in "$@".
entrypoint = ["sh", "/aiden4x-launch.sh"]

args = [
    "--trust-remote-code",
    "--served-model-name", "deepseek-v4.1-flash",
    "--load-format", "safetensors",
    # Aiden's measured shape (production-1.6); --tp 4 / --nnodes 4 /
    # --node-rank / --dist-init-addr are appended by the launcher
    "--ep-size", "4",
    "--attention-backend", "dsv4",
    "--moe-runner-backend", "flashinfer_mxfp4",
    # required for the MXFP8 route (selects the block-scaled path the hook wraps;
    # K=576 projections fall back to CUTLASS per shape)
    "--fp8-gemm-backend", "flashinfer_cutlass",
    "--mem-fraction-static", "0.80",
    "--chunked-prefill-size", "2048",
    "--context-length", "524288",
    "--max-running-requests", "16",
    "--cuda-graph-max-bs-decode", "16",
    "--random-seed", "0",
    "--enable-decoder-swa-bounded-replay",
    "--speculative-algorithm", "DSPARK",
    "--speculative-dspark-block-size", "5",
    "--tool-call-parser", "deepseekv41",
    "--reasoning-parser", "deepseek-v41",
    "--watchdog-timeout", "1800",
    "--min-free-slots-delay", "1",
]

# The launcher also appends: --model-path <resolved NAS snapshot>, --nnodes 4,
# --node-rank N, --dist-init-addr <head-ib>:25000, --host 0.0.0.0, --port 30000.

# NOTE: no --default-chat-template-kwargs '{"thinking":true}' — the bench
# harness controls thinking; Aiden's default is for his interactive fleet.

# The per-rank Engram range wrapper, mounted read-only (bare keys above; the
# table header must come after them or `args`/`[env]` would parse into it).
[[extra_mounts]]
host = "/home/kilork/aiden4x-launch.sh"
container = "/aiden4x-launch.sh"
read_only = true

# Node-local sparse copy of the Engram tables (staging P2.5, sparse-copy —
# the aiden overlay's engram-local.json contract). Built/verified per serve by
# the launcher's [[staging]] below; the wrapper-computed SPARK_ENGRAM_ROWS and
# the manifest agree because sparse_rows.py tiles ranks with the wrapper's
# base+rem formula (the overlay refuses to start unless both are identical).
[[extra_mounts]]
host = "/home/kilork/aiden-engram"
container = "/engram-local"
read_only = true

# Staging (spec: specs/tech-architecture/staging-feature.md): the sparse-copy
# format writes the aiden16 contract directly (engram-local.json manifest +
# sparse shard copies at the original safetensors byte offsets) so the overlay
# reads rows from node-local NVMe. Ranges use the wrapper's base+rem tiling;
# first serve packs ~95 GiB/node from the NAS, later serves verify sampled
# row hashes against the ledger stored inside the dst tree.
[[staging]]
kind = "sparse-rows"
format = "sparse-copy"
id = "engram"
src = "{{hf}}/hub/models--deepseek-ai--DeepSeek-V4.1-Flash/snapshots/dba1be0a40aa45a94ad051997016db3960a90277"
dst = "/home/kilork/aiden-engram"
min_free = "55G"

# Engram row source: the node-local sparse copy staged by [[staging]] —
# Aiden's measured production config (47.2 GiB rows on node NVMe). The
# manifest (engram-local.json) carries the per-layer rank ranges; the
# wrapper's SPARK_ENGRAM_ROWS is the identity cross-check the overlay runs.
[env]
# Engram row source: the node-local sparse copy staged by [[staging]] below
# (Aiden's measured production config — 47.2 GiB of rows on node NVMe, ~0.3 ms
# gather; previously read straight from the NAS snapshot, which cost a 2-10 s
# fresh-context prefill penalty). The manifest (engram-local.json) carries the
# per-layer rank ranges; SPARK_ENGRAM_ROWS from the wrapper is the cross-check.
SPARK_ENGRAM_DIR = "/engram-local"
SPARK_ENGRAM_MODE = "hostnode"
SPARK_ENGRAM_EARLY = "1"
SPARK_ENGRAM_PREFETCH = "1"
SPARK_PAGE_CACHE_RELEASE = "1"
SPARK_LATE_TAIL_SKIP = "1"
SPARK_LATE_TAIL_SKIP_ALL = "1"
SPARK_SM120_REAL_HEADS = "1"
SPARK_ENGRAM_THREADS = "64"
SPARK_ENGRAM_MAX_IDS = "65536"
SPARK_MXFP8_BACKEND = "b12x"
SPARK_ROCE_AR = "1"
SPARK_ROCE_AR_MAX = "1MB"
SPARK_ROCE_AG_MAX = "16MB"
# b12x RoCE runtime (GID index set by the wrapper from the detected value)
B12X_ROCE_HCA = "rocep1s0f0,roceP2p1s0f0"
B12X_ROCE_SPIN_LIMIT = "300000000"
# image defaults Aiden overrides
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE = "0"
SGLANG_FLASHINFER_MOE_FUSED_FINALIZE = "0"
SGLANG_DSV41_REASONING_EFFORT = "100"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:False"
CUDA_DEVICE_ORDER = "PCI_BUS_ID"
# Aiden's NCCL pins (4x 200G RoCE fabric, 192.168.200.0/24) — [env] wins over
# the config's [nccl."4nodes-switch"] where they differ
NCCL_NET = "IB"
NCCL_IB_DISABLE = "0"
NCCL_IB_HCA = "rocep1s0f0,roceP2p1s0f0"
NCCL_SOCKET_IFNAME = "enp1s0f0np0"
GLOO_SOCKET_IFNAME = "enp1s0f0np0"
NCCL_IB_ROCE_VERSION_NUM = "2"
NCCL_IB_ADDR_FAMILY = "AF_INET"
NCCL_IB_ADDR_RANGE = "192.168.200.0/24"
NCCL_IB_SUBNET_PREFIX_LEN = "24"
NCCL_CROSS_NIC = "1"
NCCL_IB_MERGE_NICS = "0"
NCCL_IB_SUBNET_AWARE_ROUTING = "1"
NCCL_CUMEM_ENABLE = "0"
NCCL_P2P_DISABLE = "1"
NCCL_SHM_DISABLE = "1"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_PROTO = "^LL128"
NCCL_MAX_NCHANNELS = "8"