launcher/recipes/dsv41-flash-4x-mia/recipe.toml

# DeepSeek-V4.1-Flash on 4x DGX Spark — Mia AI-Lab TP4 production line
# (github.com/MiaAI-Lab/DeepSeek-v4.1-Flash-DGX-Sparks @ cad252b7d3,
# forum /t/382725 posts #136-#156). Their own TP4 profile, adapted to sm121:
# the image is our docker-images/mia-tp4 build (Dockerfile.canary-roce —
# SGLang dsv4.1 branch f80c91a4b + SG17 RoCEnante overlay + b12x/b12x_next,
# everything pinned in runtime/sources.manifest); the boot.py entrypoint is
# bypassed — we launch sglang.launch_server directly, so the flags boot.py
# would emit live in `args` and the gated adapters live in [env] (the image's
# PYTHONPATH/sitecustomize arms them from env alone).
#
# Their production shape (docs/tp4.md, measured 2026-09-25, switched fabric):
# decode prose c1 87.7 tok/s, prefill 4.1-5.9K tok/s cold, 1M context with a
# verified 1,011,084-token needle, qeval 72/75. Claims UNVERIFIED here.
#
# Model: NAS checkpoint via --hf-cache (same dba1be0a snapshot as the
# aiden16 lane). Engram: UNPACKED read path (DSV41_SOURCE = the checkpoint)
# for the v1 boot — their pack_engram.py packing (~48 GiB/rank) is exactly
# our staging P2 (sparse-rows) and waits for that feature; per the repo the
# unpacked worker read is a per-step penalty, so perf here is a floor.
#
# boot.py maps OFFLOAD_MODE=nvme + no DSV41_PACKED_DIR to "rows straight
# from the checkpoint" (adapter/engram_backend.py) — the equivalent of the
# aiden16 NAS fallback, but their engine (row store + prefetch + 4 GiB
# cache). Their served-model-name is kept identical to aiden16 so bench
# providers work unchanged.
#
# If a boot log line from docs/tp4.md's "must show" block is missing, the
# line is not active — grep it: shared-expert padding, indexer chunked
# ARMED, gamma=5/verify_num_draft_tokens=6, RoCEnante ready, moe_b12x_next
# armed/EP_SIZE=1, hc_fused bit-identical, max_running_requests=16.

engine = "sglang"
name = "DeepSeek-V4.1-Flash (SGLang TP4, Mia production line)"
docker_image = "dsv41-4x-spark:mia-tp4"
docker_image_dir = "mia-tp4"
model = "deepseek-ai/DeepSeek-V4.1-Flash"
parent_model = "deepseek-ai/DeepSeek-V4.1-Flash"
port = 30000
required_nodes = 4
tensor_parallel_size = 4
distributed_backend = "mp"
shm_size = "32gb"

entrypoint = ["python3", "-m", "sglang.launch_server"]

# = boot.py serve() + production EXTRA_SGLANG_ARGS, launcher appends
# --model-path/--nnodes/--node-rank/--dist-init-addr/--host/--port.
args = [
    "--served-model-name", "deepseek-v4.1-flash",
    "--trust-remote-code",
    "--load-format", "safetensors",
    "--ep-size", "1",                        # production: tensor-shard all 384 experts
    "--attention-backend", "dsv4",
    "--moe-runner-backend", "flashinfer_mxfp4",
    "--fp8-gemm-backend", "flashinfer_cutlass",   # required for the MXFP8 route
    "--mem-fraction-static", "0.80",
    "--chunked-prefill-size", "4096",
    "--context-length", "1048576",
    "--max-total-tokens", "8000000",         # pinned KV pool (their 0.80 budget)
    "--max-running-requests", "16",
    "--cuda-graph-max-bs-decode", "16",
    "--min-free-slots-delay", "1",           # 16 means 16 (delayer default holds ~13)
    "--random-seed", "0",
    "--enable-decoder-swa-bounded-replay",
    "--enable-cache-report",
    "--speculative-algorithm", "DSPARK",
    "--speculative-dspark-block-size", "5",
    "--tool-call-parser", "deepseekv41",
    "--reasoning-parser", "deepseek-v41",
    "--enable-deepseek-v4-fp4-indexer",      # kept to match their measured arg line
    "--watchdog-timeout", "1800",
]

[env]
# Master gate for every TP4 production adapter (sitecustomize's
# _tp4_launcher check — start-tp4.sh sets this; without it the gated
# installs (indexer v3, shared_pad_k, l2_prefetch, moe_b12x_next, …)
# silently stay off and the stock FlashInfer path rejects the 576-wide
# EP1 experts. Must be set before anything else arms.
DSV41_LAUNCHER = "tp4"
# boot.py exports these to the container; the adapters read them from env
# (not from sglang's flags). TP_SIZE is load-bearing: tp3_pad.py defaults to
# 3 when unset, pads heads 64->96 under our TP4 and the sparse-MLA kernel
# aborts on 24 heads/rank. DSV41_TP_PAD=0 is their own TP4 setting (docs/tp4.md:
# everything divides 4, the pad has nothing to fix).
TP_SIZE = "4"
EP_SIZE = "1"
DSV41_TP_PAD = "0"
# ---- image defaults our path must keep explicit ----
OFFLOAD_MODE = "nvme"                    # never ram (Engram would evict the model)
STATE_PATH = "/state"                    # boot.py's state dir (sparks tables, api key)
DSV41_SOURCE = "/root/.cache/huggingface/hub/models--deepseek-ai--DeepSeek-V4.1-Flash/snapshots/dba1be0a40aa45a94ad051997016db3960a90277"
SGLANG_FLASHINFER_MOE_FUSED_FINALIZE = "0"   # determinism gate
SGLANG_DSV41_REASONING_EFFORT = "100"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:False"   # True => NaN logits >64-token prefills
# Packed Engram read path (staging P2): [[staging]] below materializes this
# node's rank slice at /home/kilork/mia-state/engram, bind-mounted into the
# container at /state/engram (the mia-state mount). With the files present
# the row store attaches packed; DSV41_SOURCE stays as the unpacked
# fallback if a slice is ever missing.
DSV41_PACKED_DIR = "/state/engram"
# ---- production-line adapter gates (docs/tp4.md "What runs" / EXTRA_CONTAINER_ENV) ----
DSV41_INDEXER_CHUNKED = "1"
SGLANG_DSPARK_FOLDED_SAMPLING = "2"
SGLANG_ROCE_ALLREDUCE = "1"
SGLANG_ROCE_MAX_SIZE = "2097152"
DSV41_ROCE_GATHER = "2097152"
B12X_ROCE_HCA = "rocep1s0f0,roceP2p1s0f0"
B12X_ROCE_CACHE_DIR = "/state/b12x-roce"
B12X_COMPILE_CACHE_DIR = "/state/b12x-compile"
SPARK_PREFILL_TP_SPLIT = "1"
SPARK_PREFILL_TP_MIN_CONTEXT = "32768"
SPARK_PREFILL_TP_MIN_ROWS = "1024"
SGLANG_RUST_BUILD_MODE = "never"
DSV41_FAST_LOAD = "1"
DSV41_FAST_LOAD_EP_SIZE = "1"
DSV41_FAST_LOAD_TP_SLICE = "auto"
DSV41_FAST_LOAD_N_EXPERTS = "384"
DSV41_FAST_LOAD_INFLIGHT_GB = "6"
DSV41_ENGRAM_PREFETCH = "1"
DSV41_CACHE_GIB = "4"                    # engram row cache (67-76% hits)
DSV41_CACHE_WAYS = "16"
DSV41_SHARED_PAD_K = "1"
DSV41_WO_A_W8 = "1"
DSV41_WO_A_W8_MID = "1"
DSV41_WO_A_W8_DROP = "1"
DSV41_DRAFT_TAU = "0.7"
DSV41_DRAFT_HEAD_FP8 = "1"
DSV41_DRAFT_HEAD_FP8_IMPL = "tp4"
DSV41_BLOCK_VERIFY = "1"
DSV41_FOLDED_FENCE = "1"
DSV41_VERIFY_CAP = "conf:0.1"
DSV41_AUTOTUNE_KEEP = "1"
DSV41_REPLICATED_SPLIT = "wqkv_a,engram.wkv"
DSV41_DRAFT_MAIN_PROJ_SPLIT = "1"
DSV41_ROUTER_LIVE = "1"
DSV41_MOE_B12X_NEXT = "1"
DSV41_MOE_B12X_NEXT_DETERMINISTIC = "1"
DSV41_HC_FUSED = "1"
DSV41_PREFILL_SP = "1"
DSV41_PREFILL_SP_FP8 = "1"
DSV41_L2_PREFETCH = "1"
# ---- fabric / NCCL (their memory fix: 4.7 GiB pinned -> 139 MB; aiden16's
# measured pins for the same NICs, plus Mia's buffer reduction) ----
NCCL_NET = "IB"
NCCL_IB_DISABLE = "0"
NCCL_IB_HCA = "rocep1s0f0,roceP2p1s0f0"
NCCL_SOCKET_IFNAME = "enp1s0f0np0"
GLOO_SOCKET_IFNAME = "enp1s0f0np0"
NCCL_IB_ROCE_VERSION_NUM = "2"
NCCL_IB_ADDR_FAMILY = "AF_INET"
NCCL_IB_ADDR_RANGE = "192.168.200.0/24"
NCCL_IB_SUBNET_PREFIX_LEN = "24"
NCCL_CROSS_NIC = "1"
NCCL_IB_MERGE_NICS = "0"
NCCL_IB_SUBNET_AWARE_ROUTING = "1"
NCCL_CUMEM_ENABLE = "0"
NCCL_P2P_DISABLE = "1"
NCCL_SHM_DISABLE = "1"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_PROTO = "^LL128"
NCCL_MAX_NCHANNELS = "8"

[[extra_mounts]]
# boot.py's state dir: dspark tables (future), b12x jit/roce caches; per-node host dir
host = "/home/kilork/mia-state"
container = "/state"

# ---- staging (spec: specs/tech-architecture/staging-feature.md) ----
# Per-rank Engram rows packed onto node-local NVMe (staging P2, packed-bin):
# rows split by launch rank (rank r owns rows r*rows/tp .. (r+1)*rows/tp),
# derived from the checkpoint header at stage time. First serve packs from
# the NAS snapshot (~7 min/layer/node, all nodes in parallel); later serves
# verify sampled row hashes against the ledger and pack only what changed.
[[staging]]
kind = "sparse-rows"
id = "engram"
src = "{{hf}}/hub/models--deepseek-ai--DeepSeek-V4.1-Flash/snapshots/dba1be0a40aa45a94ad051997016db3960a90277"
dst = "/home/kilork/mia-state/engram"
min_free = "55G"