mimo26-flash-mopd-4x: new 4-node contingency lane, GMU 0.94 prod (pool 4.14M)

SGLang accounts mem-fraction-static far more tightly than vLLM's ~0.84
ceiling — 0.92 and 0.94 both boot and serve clean on TP4, including a
515K-token probe at 0.94 (340 s). Production value: 0.94; +167K pool
tokens, GPU margin 5.35 GB. Trial data in the header.

recipes/mimo26-flash-mopd-4x/recipe.toml

@@ -0,0 +1,91 @@
+# MiMo-V2.6-Flash-MOPD (Xiaomi, ~310B total / 12B-active MoE) on 4x DGX Spark
+# with SGLang + DFlash — the 4-node contingency lane for the 2x production.
+#
+# Rationale (2026-09-28): the 2x lane (recipes/mimo26-flash-mopd-2x) hit the
+# 512K target at GMU 0.90 with a 570,752-token pool but only ~109K tokens of
+# headroom and ~7 GB free host after a 515K probe. On 4 nodes each rank books
+# ~44.5 GB of weights (178 GB / TP4) instead of ~89 GB, so the KV pool grows
+# ~3x at equal GMU — a single 524K request needs only a fraction of it and
+# many concurrent long requests fit.
+#
+# GMU trials (SGLang accounts mem-fraction-static more precisely than vLLM's
+# ~0.84 ceiling on this stack — 2x booted clean at 0.90 with 8 GB GPU + 7 GB
+# host headroom, so pushing further on TP4 is cheap): 0.90 -> 0.92 -> 0.94.
+# Current: 0.94 (prod). Measured 2026-09-28: 0.92 → pool 3,971,456, GPU 6.3
+# GB free; 0.94 → pool 4,138,432, GPU 5.35 GB free, host ~4 GB during the
+# 515K-token probe (reclaimable page cache, not an OOM signal — the probe
+# served clean in 340 s). SGLang's static fraction is accounted tightly:
+# both values boot and serve without a hitch, so the knob is free to push.
+#
+# Image: docker-images/mimo26-spark (a14) — same as the 2x lane; at TP4 the
+# FP8 tile fallbacks of the 2x regime apply (RoCEnante one-shot all-reduce
+# is TP8-only, stock NCCL over the 200G fabric is used).
+#
+# Cluster: 4nodes-switch (dgx1..dgx4). Serve with --cluster 4nodes-switch;
+# NCCL HCA/ifaces come from [nccl."4nodes-switch"], the DAC-specific
+# NCCL_PROTO override of the 2x lane is kept (reference deployment).
+
+engine = "sglang"
+name = "MiMo-V2.6-Flash-MOPD (SGLang DFlash, 4x Spark TP4)"
+docker_image = "mimo26-spark:a14"
+docker_image_dir = "mimo26-spark"
+model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
+parent_model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
+port = 30000
+required_nodes = 4
+tensor_parallel_size = 4
+distributed_backend = "mp"
+shm_size = "16gb"
+
+entrypoint = ["python3", "-m", "sglang.launch_server"]
+
+args = [
+    "--trust-remote-code",
+    "--served-model-name", "mimo-v2.6-flash",
+    # Attention / MoE backends (same as the 2x lane)
+    "--attention-backend", "triton",
+    "--prefill-attention-backend", "fa4",
+    "--mm-attention-backend", "triton_attn",
+    "--enable-multimodal",
+    "--moe-runner-backend", "marlin",
+    "--moe-a2a-backend", "none",
+    "--mem-fraction-static", "0.94",
+    "--context-length", "524288",
+    "--max-running-requests", "8",
+    "--chunked-prefill-size", "8192",
+    "--page-size", "64",
+    "--swa-full-tokens-ratio", "0.1",
+    "--reasoning-parser", "mimo",
+    "--tool-call-parser", "mimo",
+    "--speculative-algorithm", "DFLASH",
+    "--speculative-draft-model-path", "{{dflash}}",
+    "--speculative-num-draft-tokens", "8",
+    "--cuda-graph-max-bs-decode", "8",
+    "--disable-prefill-cuda-graph",
+    "--weight-loader-drop-cache-after-load",
+    "--enable-metrics",
+    "--enable-cache-report",
+    "--watchdog-timeout", "1800",
+]
+
+# DFlash draft weights: a subfolder of the main model repo, resolved per node.
+[[extra_mounts]]
+repo = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
+subfolder = "dflash"
+container = "dflash"
+
+[env]
+# RoCEnante overlay is TP8-only (needs the TP8 Gloo exchange group) — stock
+# NCCL on TP4.
+# a13/a14 tuning switches
+MIMO26_FP8_OPROJ = "1"
+MIMO26_FP8_DRAFT = "1"
+MIMO26_DFLASH_ADAPTIVE = "1"
+# NCCL extras from the reference deployment (HCAs/ifname from [nccl."4nodes-switch"])
+NCCL_CUMEM_ENABLE = "0"
+NCCL_NVLS_ENABLE = "0"
+NCCL_PROTO = "^LL128"
+NCCL_BUFFSIZE = "1048576"
+NCCL_LL128_BUFFSIZE = "262144"
+NCCL_MAX_NCHANNELS = "8"
+NCCL_DEBUG = "WARN"