recipes/docker-images/mia-tp4/.env.example

# Copy to .env and edit. Never commit .env.

# ─── cluster ──────────────────────────────────────────────────────────────────
HEAD_IP=10.0.0.1
WORKER1_IP=10.0.0.2
WORKER2_IP=10.0.0.3
WORKER1_HOST=spark2
WORKER2_HOST=spark3
WORKER_USER=zurih
SSH_IDENTITY=$HOME/.ssh/id_ed25519_shared

# ─── fabric (ConnectX-7 RoCE triangle) ────────────────────────────────────────
# Gloo/NCCL TCP bootstrap on the LAN NIC. Data plane prefers RoCE (NCCL_NET=IB).
FABRIC_IFACE=enp1s0f1np1
GLOO_SOCKET_IFNAME=enP7s7
NCCL_SOCKET_IFNAME=enP7s7
IB_HCA=rocep1s0f0,rocep1s0f1
NCCL_NET=IB
NCCL_IB_DISABLE=0
NCCL_P2P_DISABLE=1
NCCL_SHM_DISABLE=1
NCCL_DEBUG=WARN
# NCCL connection buffers live in pinned host RAM, which on GB10 is GPU memory:
# the defaults (4 MiB Simple + 4.69 MiB LL128 + 0.5 MiB LL per connection, 512
# connections) cost 4.7 GiB per node. Decode all-reduces are 61 KB (LL protocol),
# prefill uses Simple; LL128 buys nothing here. Measured: 5.0 GB -> 0.14 GB pinned.
NCCL_BUFFSIZE=1048576
NCCL_LL128_BUFFSIZE=262144
NCCL_PROTO=^LL128
NCCL_MAX_NCHANNELS=8
# Set NCCL_DEBUG=INFO for one boot to see communicator/channel/protocol choices.
NCCL_DEBUG_SUBSYS=INIT,ENV
NCCL_HOST_DIR=$HOME/nccl-2.30.7

# ─── model / image ────────────────────────────────────────────────────────────
MODEL_DIR=$HOME/NewModels/DeepSeek-V4.1-Flash
COMMON_MODEL=/var/tmp/DeepSeek-V4.1-Flash
# Same digest as the Dockerfile FROM: the dev-dsv41 tag moves, the digest does not.
BASE_IMAGE=lmsysorg/sglang:dev-dsv41@sha256:3dbc313030a6ef2c5d7de8ecf48e9aece722694a82182cb618cc82b588816349
IMAGE=dsv41-3x-spark:local
PORT=8888

# ─── weights on spark2/spark3 ─────────────────────────────────────────────────
# Always from spark1 over NFSv4 on ConnectX (no rsync, no SSHFS).
# spark2 mounts 10.0.22.1; spark3 mounts 10.0.23.1 (pairwise CX7).
NFS_SHARE=1
NFS_VOLUME=dsv41-weights
NFS_EXPORT_NAME=dsv41-native
NFS_SERVER_IP_1=10.0.22.1
NFS_SERVER_IP_2=10.0.23.1
NFS_CLIENTS=10.0.0.2,10.0.0.3,10.0.22.0/24,10.0.23.0/24,10.0.33.0/24

# ─── parallelism (3 Sparks) ───────────────────────────────────────────────────
# Heads=64 / hidden=5120 / vocab=129280 are not divisible by 3; adapter/tp3_pad.py pads them.
# EP_SIZE=1: every rank holds a third of every expert's 2304-wide intermediate instead of
# whole experts. Balances the per-layer MoE work: NCCL 12.9 -> 5.6 ms/step, C1 +8-12 %,
# C4 +10 % over EP_SIZE=3 (2026-09-24, docs/overnight-results.md). Rollback: EP_SIZE=3.
NNODES=3
TP_SIZE=3
EP_SIZE=1
DIST_PORT=20000

# ─── serving ──────────────────────────────────────────────────────────────────
# NVMe Engram offload is required on Spark (host RAM == GPU unified memory).
OFFLOAD_MODE=nvme
# Per-host cache budget, split across the 2 Engram layers and the ranks sharing
# that host. 0 disables it: with the packed local shards a miss is ONE local
# read, and the measured miss rate costs ~145 reads/s against a drive that does
# ~112k IOPS. Host RAM is GPU memory here and rank 0 has ~0.3 GB available, so
# the GiB this used to hold is worth far more as headroom than as hit rate.
DSV41_CACHE_GIB=0
# GB10 NVMe: ~3.5k IOPS at QD=1, ~112k at QD=64. The Engram callback blocks the
# compute stream, so misses are serviced by a pool rather than one at a time.
DSV41_IO_THREADS=96
# Pinning the owned scale rows makes a miss cost one pread instead of two
# (~1.8x on the callback), but it needs ~1 GiB/layer/rank of EAGER host RAM, and
# host RAM here is GPU memory: with weights at ~104 GB the pin raised SGLang's
# "minimum viable" mem-fraction from ~0.940 to 0.9569 and left no KV cache.
# Off until the packed local shard lands, which gets the same single read from
# disk layout instead of RAM.
DSV41_RESIDENT_SCALES=0
DSV41_CACHE_WAYS=4
DSV41_STATS_SECONDS=60
# Per-request limit (model max 1,048,576); the KV pool below (750k tokens) covers it
# with room for 4 concurrent requests. The head's memory bounds what actually works
# (REPORT.md section 17b): with the prefill empty_cache hook and 1024-token chunks,
# single prompts to 208k and 4 concurrent 46k prompts are verified; a 256k prompt
# missed the 1.5 GB guard by 20 MB. Keep long prompts guarded (scripts/verify/memguard.py).
CONTEXT_LENGTH=262144
# Lowering the fraction does not free RAM; it only starves the KV pool.
# 0.95 is the working value.
MEM_FRACTION_STATIC=0.95
HEAD_MEM_FRACTION_STATIC=0.95
MAX_RUNNING_REQUESTS=4
# History: with the prefill empty_cache hook the per-chunk indexer buffers ([chunk, prefix]
# fp32 per low-ratio source layer) are the long-prompt peak on the head; 2048-token
# chunks fit a 100k prompt, 1024 fits 200k (2026-09-11 ramps, REPORT.md section 17b).
# Smaller chunks prefill slower (2.1k -> 1.5k tok/s from 2048 to 1024).
# 768 since 2026-09-24: with the decode stack below, a ~200k-token prompt at 1024 ran the head
# out of RAM and hung spark1 until a hard reset. 768 cuts that per-chunk peak by a quarter;
# decode speed is unchanged, prefill is somewhat slower. 3 x 256-token KV pages.
CHUNKED_PREFILL_SIZE=768
# Pin the KV pool. Unpinned, SGLang sizes it from whatever is free at load time
# and it swung 326k -> 593k across identical boots, making decode speed a lottery.
# This does NOT free host RAM (torch reserves mem-fraction up front and the pool
# is carved out of that reservation) -- it controls how many pages get touched,
# which is what drives reclaim stalls. Measured on this box, tok/s at conc 1 / 4:
#   200k -> 28.6 / 55.4   (pressure 0.00, but too small: batching starves)
#   416k -> 31.9 / 75.6   (pressure 0.00, best measured)
#   593k -> 12.6 / 33.9   (pressure 6.9 median, head thrashing)
# Pinned, SGLang demands exactly this and FAILS if it does not fit -- 400k died
# with "no room ... within the available 0.17 GB". Unpinned it auto-fits, and the
# smallest size it ever chose here was 326k, so the pin must sit below that.
MAX_TOTAL_TOKENS=750000
SPEC_ALGO=DSPARK
# 5 together with DSV41_VERIFY_CAP=conf:0.1 below: code +10 %, prose neutral vs 3.
# 5 without the cap loses 8-16 % on prose. Rollback: 3 with DSV41_VERIFY_CAP=0.
DSPARK_BLOCK_SIZE=5
# Profile with sglang.benchmark.dspark_sps_profiler into state/dspark_sps.json;
# until that file exists DSpark runs the verify-all schedule and gains nothing.
DSPARK_SPS_TABLE=/state/dspark_sps.json
DSPARK_STS_TABLE=/state/dspark_sts.json

# ─── engram local shards ──────────────────────────────────────────────────────
# ./start.sh pack repacks each rank's owned Engram rows (weight+scale adjacent)
# onto node-local NVMe: ~68 GiB/node, one read per miss instead of two, and no
# NFS hop on the workers. Serving picks them up automatically when present.
ENGRAM_DIR=$HOME/dsv41-engram
DSV41_PACKED_DIR=/engram
SKIP_PREPARE=1
SKIP_VERIFY=1
SKIP_SMOKE=0
SMOKE_QUICK=1

# ─── kernels (GB10 / SM121) ───────────────────────────────────────────────────
# The 32x32 ue8m0 block-FP8 dense projections only have a Triton kernel unless a
# FlashInfer backend is named, which routes them to the MXFP8 dense kernels.
# --watchdog-timeout covers the CuTe-DSL JIT compile of the b12x kernels on the
# first forward after a boot.
# --sleep-on-idle parks the scheduler when idle instead of spinning a CPU core.
EXTRA_SGLANG_ARGS="--fp8-gemm-backend flashinfer_cutlass --watchdog-timeout 1800 --sleep-on-idle"
# Which FlashInfer MXFP8 kernel serves those projections (adapter/mxfp8_b12x.py):
# b12x is FlashInfer's own SM120/121 choice (16|32-row tiles, ~3x the CUTLASS
# SM120 kernel at decode M); cudnn is the alternative; cutlass restores SGLang's
# default, which measured no faster than Triton (both ~50 ms of a decode step).
DSV41_MXFP8_BACKEND=b12x

# Decode adapters, each validated one boot at a time on TP3 at fraction 0.95 (2026-09-24,
# docs/overnight-results.md). wo_a: fp8 twin of the target's wo_a; W8 alone does not fit
# at 0.95 (+688 MiB), DROP releases the bf16 copies (net -640 MB/rank), MID serves the
# 9-192-row verifies (C4). The draft's wo_a stays bf16 (DSV41_WO_A_W8_DRAFT=1 tried: neutral).
DSV41_WO_A_W8=1
DSV41_WO_A_W8_MID=1
DSV41_WO_A_W8_DROP=1
DSV41_WO_A_W8_DRAFT=0
# fp8 copy of the draft LM head (+210 MiB): about -1 ms/step.
DSV41_DRAFT_HEAD_FP8=1
# Confidence cap for DSPARK_BLOCK_SIZE=5; needs DSV41_BLOCK_VERIFY=1 for sampled rows.
DSV41_VERIFY_CAP=conf:0.1
# 0.8 measured +1.2 % on sampled chat, inside noise: left at 1 (off).
DSV41_DRAFT_TAU=1
DSV41_BLOCK_VERIFY=1
# Engram rows fetched on a side stream: -2.3 to -2.9 ms/step. DSV41_ENGRAM_PREFETCH_CHECK=1
# re-runs the synchronous lookup and logs mismatches (validation only, not for timing).
DSV41_ENGRAM_PREFETCH=1
DSV41_FOLDED_FENCE=0
# Keep FlashInfer autotune caches across boots of the same launch (adapter/autotune_keep.py):
# same tactics, reproducible greedy output and tok/s across restarts. Delete
# ~/.cache/sglang/flashinfer/autotune/*/sm121/*/rank_*.{json,launch} on all nodes to force a retune.
DSV41_AUTOTUNE_KEEP=1
# FlashInfer's fused MoE finalize sums the top-k expert outputs with atomic bf16
# adds. The autotuner picks it for the 32-token bucket (17-32 token prefills,
# decode at concurrency 3-4), which made greedy outputs nondeterministic by up to
# ~1 nat on first-token logprobs. Off = separate finalize kernel, ~0.3 ms/step.
SGLANG_FLASHINFER_MOE_FUSED_FINALIZE=0

# Reasoning budget for thinking-mode requests that carry no reasoning_effort.
# SGLang's V4.1 encoder maps low/high/xhigh/max to 25/50/75/100 and defaults to
# "high" = 50; the publisher's encoding/encoding.py says low=50, high=75, max=100
# with default 75. adapter/sitecustomize.py remaps the named tiers to the
# publisher's table; this pins the default as a number so it holds even without
# the remap. Integer 1-100 or low/high/xhigh/max. Per request: OpenAI
# reasoning_effort (a tier, or a 0-0.99 float = budget/100) or
# chat_template_kwargs {"reasoning_effort": N}. Only rendered in thinking mode.
SGLANG_DSV41_REASONING_EFFORT=75

# When a chat request omits max_tokens / max_completion_tokens, fill this in and
# also clamp larger client values. 32768 would have stopped the 714k-token
# racing-animation loop while covering the visual/HTML trials (~15k). 0 restores
# SGLang's fill-the-remaining-context behaviour (1M on TP4). Raise it for long
# thinking-on traces.
DSV41_MAX_NEW_TOKENS=32768

# Decode-side loop abort. --watchdog-timeout only fires when a step hangs; a live
# token stream (VIS-04: 714k tokens of racing-animation HTML) is healthy as far
# as the GPU watchdog is concerned. Finish with stop/repetition after four
# copies of a ≥32-token n-gram, 64 identical tokens, or eight identical decoded
# lines. The 2× EXL3 recipe has no equivalent. 0 disables.
DSV41_LOOP_ABORT=1
DSV41_LOOP_NGRAM=32
DSV41_LOOP_REPEATS=4
DSV41_LOOP_LINE_REPEATS=8

# PyTorch caching allocator mode. NEVER set expandable_segments:True on this stack:
# with it, every prefill of more than 64 query tokens (a prompt of 65-~4000 tokens, or
# any prefill batch of 3+ short requests) returns NaN logits -> garbage text at
# temperature 0 and a scheduler crash ("probability tensor contains inf/nan") when
# sampling. Bisected 2026-09-11 over five boots (chunk size, context length and the
# warm-up were all ruled out); see logs/profile-2026-09-10/REPORT.md section 17.
# The native allocator does fragment on long prompts (REPORT.md section 15), which is
# why CONTEXT_LENGTH is capped below; expandable_segments:False is the explicit native mode.
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False

# adapter/prefill_empty_cache.py: after every prefill chunk of a sequence at least this
# long, return PyTorch's unused cached blocks to the driver, so a long prompt costs one
# chunk's transient instead of the sum over chunks (the native allocator cannot reuse
# the previous chunk's smaller indexer-logits buffer). Measured 2026-09-11 on the head:
# without it a 34k prompt kept 1-1.5 GB and 64k ran out of memory; with it 100k prompts
# complete and the memory comes back after each. 0 disables the hook.
DSV41_PREFILL_EMPTY_CACHE_TOKENS=8192