launcher/recipes/mimo26-flash-mopd-4x/recipe.toml

# MiMo-V2.6-Flash-MOPD (Xiaomi, ~310B total / 12B-active MoE) on 4x DGX Spark
# with SGLang + DFlash — the 4-node contingency lane for the 2x production.
#
# Rationale (2026-09-28): the 2x lane (recipes/mimo26-flash-mopd-2x) hit the
# 512K target at GMU 0.90 with a 570,752-token pool but only ~109K tokens of
# headroom and ~7 GB free host after a 515K probe. On 4 nodes each rank books
# ~44.5 GB of weights (178 GB / TP4) instead of ~89 GB, so the KV pool grows
# ~3x at equal GMU — a single 524K request needs only a fraction of it and
# many concurrent long requests fit.
#
# GMU trials (SGLang accounts mem-fraction-static more precisely than vLLM's
# ~0.84 ceiling on this stack — 2x booted clean at 0.90 with 8 GB GPU + 7 GB
# host headroom, so pushing further on TP4 is cheap): 0.90 -> 0.92 -> 0.94.
# Current: 0.88 (prod) + RoCEnante trial. Historical trial data: Measured 2026-09-28: 0.92 → pool 3,971,456, GPU 6.3
# GB free; 0.94 → pool 4,138,432, GPU 5.35 GB free, host ~4 GB during the
# 515K-token probe (reclaimable page cache, not an OOM signal — the probe
# served clean in 340 s). SGLang's static fraction is accounted tightly:
# both values boot and serve without a hitch, so the knob is free to push.
#
# Image: docker-images/mimo26-spark (a15) — same as the 2x lane; at TP4 the
# FP8 tile fallbacks of the 2x regime apply (RoCEnante one-shot all-reduce
# is TP8-only, stock NCCL over the 200G fabric is used).
#
# Cluster: 4nodes-switch (dgx1..dgx4). Serve with --cluster 4nodes-switch;
# NCCL HCA/ifaces come from [nccl."4nodes-switch"], the DAC-specific
# NCCL_PROTO override of the 2x lane is kept (reference deployment).

engine = "sglang"
name = "MiMo-V2.6-Flash-MOPD (SGLang DFlash, 4x Spark TP4)"
docker_image = "mimo26-spark:a15"
docker_image_dir = "mimo26-spark"
model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
parent_model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
port = 30000
required_nodes = 4
tensor_parallel_size = 4
distributed_backend = "mp"
shm_size = "16gb"

entrypoint = ["python3", "-m", "sglang.launch_server"]

args = [
    "--trust-remote-code",
    "--served-model-name", "mimo-v2.6-flash",
    # Attention / MoE backends (same as the 2x lane)
    "--attention-backend", "triton",
    "--prefill-attention-backend", "fa4",
    "--mm-attention-backend", "triton_attn",
    "--enable-multimodal",
    "--moe-runner-backend", "marlin",
    "--moe-a2a-backend", "none",
    "--mem-fraction-static", "0.88",
    "--context-length", "524288",
    "--max-running-requests", "8",
    "--chunked-prefill-size", "8192",
    "--page-size", "64",
    "--swa-full-tokens-ratio", "0.1",
    "--reasoning-parser", "mimo",
    "--tool-call-parser", "mimo",
    "--speculative-algorithm", "DFLASH",
    "--speculative-draft-model-path", "{{dflash}}",
    "--speculative-num-draft-tokens", "8",
    "--cuda-graph-max-bs-decode", "8",
    "--disable-prefill-cuda-graph",
    "--weight-loader-drop-cache-after-load",
    "--enable-metrics",
    "--enable-cache-report",
    "--watchdog-timeout", "1800",
]

# DFlash draft weights: a subfolder of the main model repo, resolved per node.
[[extra_mounts]]
repo = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
subfolder = "dflash"
container = "dflash"


# JIT/compile cache (verified against the live container, 2026-09-28): SGLang
# keeps everything under /root/.cache/sglang — triton .so (~35 hash dirs),
# inductor (aotautograd/ev/fxgraph), nv/, flashinfer .cache. Bind it to a
# node-local NVMe dir so boots reuse compiled artifacts instead of redoing
# JIT; keys are content/hash based, so mixing images/lanes on one dir is
# safe (and switching 2x<->4x reuses the other's cache). b12x-roce does not
# materialize on TP<8 — nothing else to mount.
[[extra_mounts]]
host = "~/.cache/mimo26-jit/sglang"
container = "/root/.cache/sglang"

[env]
# RoCEnante (b12x one-shot RoCE all-reduce): the sglang port hard-gated it to
# world=8 ("TP8 Gloo exchange group") while the b12x core supports 2..16 — the
# vLLM lanes already ran it on 4 nodes. a15 relaxes the sglang guard to the
# core range, so TP4 can try it; the two-active-HCA check is untouched.
SGLANG8_ROCE_ALLREDUCE = "1"
# a13/a14 tuning switches
MIMO26_FP8_OPROJ = "1"
MIMO26_FP8_DRAFT = "1"
MIMO26_DFLASH_ADAPTIVE = "1"
# NCCL extras from the reference deployment (HCAs/ifname from [nccl."4nodes-switch"])
NCCL_CUMEM_ENABLE = "0"
NCCL_NVLS_ENABLE = "0"
NCCL_PROTO = "^LL128"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_MAX_NCHANNELS = "8"
NCCL_DEBUG = "WARN"