launcher/recipes/mimo26-pro-rl-8x/recipe.toml

# MiMo-V2.6-Pro-RL (Xiaomi, 1.02T/42B-active MoE, MXFP4 experts, hybrid SWA)
# on 8x DGX Spark with SGLang + DFlash speculative decoding.
#
# Source: github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8 @ 2f3e5fd (2026-09-24,
# build a14) — the SGLang variant of the NVIDIA forum thread 384064
# ("MiMo V2.6 Pro RL on 8x DGX Spark: 17.8 -> 68.3 tok/s with DFlash").
# Image: docker-images/mimo26-spark (pinned base digest + 12 patches + b12x
# RoCEnante + GB10 FP8 tile tables).
#
# Flags are the repo's SERVE_ENV profile (a13/a14 numbers: C1 mean 69.1
# tok/s, C8 aggregate 190.8, long extraction C1 104.6 / C8 306.7).
# TP8 + the rendezvous flags are launcher-generated (distributed_backend
# "mp"); the model path is launcher-resolved.
#
# Model: XiaomiMiMo/MiMo-V2.6-Pro-RL @ 73875d00 (on the NAS at
# /mnt/nas/ai/huggingface). The dflash/ subfolder carries the DFlash draft
# weights — wired as a {{dflash}} path ref.
#
# PREREQUISITES: the model must be readable from every launch node's HF
# cache. It currently lives on the NAS only, so serve with
# `--hf-cache /mnt/nas/ai/huggingface` (all 8 nodes read from the NAS at
# load; ~535 GB each). For fast re-boots, fan the model out node-to-node
# first and drop the flag.

engine = "sglang"
name = "MiMo-V2.6-Pro-RL (SGLang DFlash, 8x Spark TP8)"
docker_image = "mimo26-spark:a15"
docker_image_dir = "mimo26-spark"
model = "XiaomiMiMo/MiMo-V2.6-Pro-RL"
parent_model = "XiaomiMiMo/MiMo-V2.6-Pro-RL"
port = 30000
required_nodes = 8
tensor_parallel_size = 8
distributed_backend = "mp"
shm_size = "16gb"

entrypoint = ["python3", "-m", "sglang.launch_server"]

args = [
    "--trust-remote-code",
    "--served-model-name", "mimo-v2.6-pro",
    # Attention / MoE backends (a13 profile)
    "--attention-backend", "triton",
    "--prefill-attention-backend", "fa4",
    "--mm-attention-backend", "triton_attn",
    "--enable-multimodal",
    "--moe-runner-backend", "marlin",
    "--moe-a2a-backend", "none",
    # Memory / scheduling (128 GB unified per node)
    "--mem-fraction-static", "0.85",
    # KV pool at a14 defaults: 2,197,568 tokens cluster-wide (TP8, DFlash
    # accounted) — a 512K request uses ~23% of it, so the context limit is a
    # declaration, not a memory wall. The model is natively 1M
    # (max_position_embeddings, no rope_scaling; 70 layers, mostly SWA-128).
    # 1M is also pool-feasible (45% per request) if ever needed.
    "--context-length", "524288",
    "--max-running-requests", "8",
    "--chunked-prefill-size", "8192",
    "--page-size", "64",
    # MiMo hybrid SWA attention
    "--swa-full-tokens-ratio", "0.1",
    # Reasoning / tool parsers
    "--reasoning-parser", "mimo",
    "--tool-call-parser", "mimo",
    # DFlash speculative decoding (draft weights ship inside the model repo)
    "--speculative-algorithm", "DFLASH",
    "--speculative-draft-model-path", "{{dflash}}",
    "--speculative-num-draft-tokens", "8",
    # CUDA graphs
    "--cuda-graph-max-bs-decode", "8",
    "--disable-prefill-cuda-graph",
    # Unified memory: page cache from weight loading competes with the KV pool
    "--weight-loader-drop-cache-after-load",
    # Observability
    "--enable-metrics",
    "--enable-cache-report",
    "--watchdog-timeout", "1800",
]

# DFlash draft weights: a subfolder of the main model repo, resolved per node
# (later in the file — bare keys must come before table headers).
[[extra_mounts]]
repo = "XiaomiMiMo/MiMo-V2.6-Pro-RL"
subfolder = "dflash"
container = "dflash"

[env]
# RoCEnante one-shot all-reduce over b12x (baked into the image, PYTHONPATH set)
SGLANG8_ROCE_ALLREDUCE = "1"
# a13/a14 tuning switches
MIMO26_FP8_OPROJ = "1"
MIMO26_FP8_DRAFT = "1"
MIMO26_DFLASH_ADAPTIVE = "1"
# NCCL extras from the reference deployment (HCAs/ifname from [nccl."8nodes-switch"])
NCCL_CUMEM_ENABLE = "0"
NCCL_NVLS_ENABLE = "0"
NCCL_PROTO = "^LL128"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_MAX_NCHANNELS = "8"
NCCL_DEBUG = "WARN"