|
@@ -11,13 +11,13 @@
|
|
|
# GMU trials (SGLang accounts mem-fraction-static more precisely than vLLM's
|
|
# GMU trials (SGLang accounts mem-fraction-static more precisely than vLLM's
|
|
|
# ~0.84 ceiling on this stack — 2x booted clean at 0.90 with 8 GB GPU + 7 GB
|
|
# ~0.84 ceiling on this stack — 2x booted clean at 0.90 with 8 GB GPU + 7 GB
|
|
|
# host headroom, so pushing further on TP4 is cheap): 0.90 -> 0.92 -> 0.94.
|
|
# host headroom, so pushing further on TP4 is cheap): 0.90 -> 0.92 -> 0.94.
|
|
|
-# Current: 0.94 (prod). Measured 2026-09-28: 0.92 → pool 3,971,456, GPU 6.3
|
|
|
|
|
|
|
+# Current: 0.88 (prod) + RoCEnante trial. Historical trial data: Measured 2026-09-28: 0.92 → pool 3,971,456, GPU 6.3
|
|
|
# GB free; 0.94 → pool 4,138,432, GPU 5.35 GB free, host ~4 GB during the
|
|
# GB free; 0.94 → pool 4,138,432, GPU 5.35 GB free, host ~4 GB during the
|
|
|
# 515K-token probe (reclaimable page cache, not an OOM signal — the probe
|
|
# 515K-token probe (reclaimable page cache, not an OOM signal — the probe
|
|
|
# served clean in 340 s). SGLang's static fraction is accounted tightly:
|
|
# served clean in 340 s). SGLang's static fraction is accounted tightly:
|
|
|
# both values boot and serve without a hitch, so the knob is free to push.
|
|
# both values boot and serve without a hitch, so the knob is free to push.
|
|
|
#
|
|
#
|
|
|
-# Image: docker-images/mimo26-spark (a14) — same as the 2x lane; at TP4 the
|
|
|
|
|
|
|
+# Image: docker-images/mimo26-spark (a15) — same as the 2x lane; at TP4 the
|
|
|
# FP8 tile fallbacks of the 2x regime apply (RoCEnante one-shot all-reduce
|
|
# FP8 tile fallbacks of the 2x regime apply (RoCEnante one-shot all-reduce
|
|
|
# is TP8-only, stock NCCL over the 200G fabric is used).
|
|
# is TP8-only, stock NCCL over the 200G fabric is used).
|
|
|
#
|
|
#
|
|
@@ -27,7 +27,7 @@
|
|
|
|
|
|
|
|
engine = "sglang"
|
|
engine = "sglang"
|
|
|
name = "MiMo-V2.6-Flash-MOPD (SGLang DFlash, 4x Spark TP4)"
|
|
name = "MiMo-V2.6-Flash-MOPD (SGLang DFlash, 4x Spark TP4)"
|
|
|
-docker_image = "mimo26-spark:a14"
|
|
|
|
|
|
|
+docker_image = "mimo26-spark:a15"
|
|
|
docker_image_dir = "mimo26-spark"
|
|
docker_image_dir = "mimo26-spark"
|
|
|
model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
|
|
model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
|
|
|
parent_model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
|
|
parent_model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
|
|
@@ -49,7 +49,7 @@ args = [
|
|
|
"--enable-multimodal",
|
|
"--enable-multimodal",
|
|
|
"--moe-runner-backend", "marlin",
|
|
"--moe-runner-backend", "marlin",
|
|
|
"--moe-a2a-backend", "none",
|
|
"--moe-a2a-backend", "none",
|
|
|
- "--mem-fraction-static", "0.94",
|
|
|
|
|
|
|
+ "--mem-fraction-static", "0.88",
|
|
|
"--context-length", "524288",
|
|
"--context-length", "524288",
|
|
|
"--max-running-requests", "8",
|
|
"--max-running-requests", "8",
|
|
|
"--chunked-prefill-size", "8192",
|
|
"--chunked-prefill-size", "8192",
|
|
@@ -87,8 +87,11 @@ host = "~/.cache/mimo26-jit/sglang"
|
|
|
container = "/root/.cache/sglang"
|
|
container = "/root/.cache/sglang"
|
|
|
|
|
|
|
|
[env]
|
|
[env]
|
|
|
-# RoCEnante overlay is TP8-only (needs the TP8 Gloo exchange group) — stock
|
|
|
|
|
-# NCCL on TP4.
|
|
|
|
|
|
|
+# RoCEnante (b12x one-shot RoCE all-reduce): the sglang port hard-gated it to
|
|
|
|
|
+# world=8 ("TP8 Gloo exchange group") while the b12x core supports 2..16 — the
|
|
|
|
|
+# vLLM lanes already ran it on 4 nodes. a15 relaxes the sglang guard to the
|
|
|
|
|
+# core range, so TP4 can try it; the two-active-HCA check is untouched.
|
|
|
|
|
+SGLANG8_ROCE_ALLREDUCE = "1"
|
|
|
# a13/a14 tuning switches
|
|
# a13/a14 tuning switches
|
|
|
MIMO26_FP8_OPROJ = "1"
|
|
MIMO26_FP8_OPROJ = "1"
|
|
|
MIMO26_FP8_DRAFT = "1"
|
|
MIMO26_FP8_DRAFT = "1"
|