launcher/recipes/mimo26-flash-mopd-4x-vllm/recipe.toml

# MiMo-V2.6-Flash-MOPD on 4x DGX Spark via vLLM + B12X — the engine A/B twin
# of mimo26-flash-mopd-4x (SGLang). No new image: r39 already carries the
# fork's MiMo-V2 integration (mimo_v2.py; SWA+DFlash PR #46104 merged into
# the lab fork main 2026-07-01, present at the 9e5d1793 tag), so this lane
# boots the SAME engine lineage our GLM-5.3 r39 lanes run — only the model
# and the launch policy differ.
#
# Launch policy mirrors local-inference-lab's lil-catalog MiMo manifests
# (MiMo-V2.6-Flash-RL TP4 qualified on SM120; Pro-MOPD TP8 with the DFlash
# `dflash/` subfolder draft, BF16 draft weights, b12x noncausal sliding
# attention). Deviations: GMU 0.88 -> 0.84 (operator call, GB10 UMA guard),
# TP8 -> TP4 (our 4-node lane).
#
# What we are testing:
#   * B12X attention long-context path on 192/128 GQA (the sglang-triton
#     lane decodes at 30% of d0 at 256K; the lab's vLLM stack has the
#     split-KV verify fix that fixed exactly this on their boxes).
#   * BF16 KV @ GMU 0.84 (their cache policy — fp8 KV skipped on both stacks).
#   * DFlash via vLLM speculative-config (method dflash, draft from the
#     checkpoint's dflash/ subfolder, 7 draft tokens) vs sglang's DFlash.
#
# Status: PARKED — dead end on r39 (2026-09-28). Four boot attempts, each one
# layer deeper: (1) EAGLE3 hook raised on the Omni wrapper — fixed by the
# image's inner-language-model patch; (2) B12X rejects the draft's non-causal
# attention — draft moved to TRITON_ATTN; (3) qwen3_dflash hard-imports
# ModelOptMxFp8LinearMethod that r39's modelopt doesn't ship — patched to be
# None-tolerant; (4) FATAL: KV-layout validator refuses the MiMo hybrid stack
# (SWA-128 + GA layers, mixed page sizes 131072/163840) because b12x declares
# LBNHC only and the check is backend-declared, not env-fixable. r39 was
# qualified for GLM-5.3, never for MiMo; lil-catalog itself marks their MiMo
# DFlash "runtime qualification unverified". Reopening recipe + external
# intel (madeby561 amd64-only rc4, eugr spark-vllm-b12x nightlies, lil-catalog
# flags): specs/bugs/BUG-2026-09-28-vllm-mimo-dflash-parked.md.
#
# REOPENED 2026-09-28 same evening: the eugr/spark-vllm-b12x nightly (source
# local-inference-lab/vllm@dev/karmic-kraken) landed all four missing pieces
# on 2026-09-22 — MiMo aux-state loading, block-outermost KV layouts for
# mixed page sizes, b12x non-causal attention (so the DFlash draft runs B12X
# too), and the eagle3 language-model unwrap. It also ships cached
# serve-launchers (serve-mimo26-flash.sh) that define the canonical MiMo
# flag set; we mirror them. No local image build: docker_image is the
# nightly tag, distributed to nodes by save/load.

engine = "vllm"
name = "MiMo-V2.6-Flash-MOPD (vLLM B12X, 4x Spark, DFlash-7, b12x-nightly)"
docker_image = "eugr/spark-vllm-b12x:latest"
entrypoint = ["vllm", "serve"]
model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
parent_model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
port = 30000
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    "--served-model-name", "mimo-v2.6-flash",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # Operator GMU (0.88 -> 0.84 for this first vLLM MiMo boot); their
    # manifest's 0.98 is RTX PRO 6000 territory, not GB10 UMA.
    "--gpu-memory-utilization", "0.85",
    "--block-size", "128",
    "--max-model-len", "524288",
    "--max-num-seqs", "16",
    "--max-num-batched-tokens", "8192",
    # SparkRing tp4 contract, concurrency-friendly scheduler bounds +
    # bounce-buffer loader. The 20 GiB KV pin is NOT adopted: on GB10's UMA
    # with GMU 0.85 the auto-sizing profiler already claims ~54 GiB free
    # (8.5M-token pool in boot logs), so pinning 20 GiB would strand ~34 GiB.
    "--model-loader-extra-config", """
'{"read_mode":"bounce","io_threads":8}'""",
    "--enable-chunked-prefill",
    "--async-scheduling",
    "--enable-prefix-caching",
    # B12X attention (lab-qualified on MiMo's 192/128 heads; non-causal now
    # supported by the nightly's b12x — the DFlash draft runs on B12X too).
    "--attention-backend", "B12X",
    # Mirror of the lab's serve-mimo26-flash.sh: b12x linear + MoE backends.
    "--linear-backend", "b12x",
    "--moe-backend", "b12x",
    # 2026-09-28 garble bisect, boot B1: mirror the lab launcher exactly —
    # the b12x loader (plugin env below), full Omni + --mm-encoder-tp-mode
    # weights instead of --language-model-only, explicit bf16 KV. instanttensor
    # was our deviation; MiMo's split-GDN weight mapping is what the b12x
    # loader path qualifies.
    "--load-format", "b12x",
    "--mm-encoder-tp-mode", "weights",
    "--kv-cache-dtype", "bfloat16",
    "--kernel-config", """
'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
    # DFlash draft from the checkpoint's dflash/ subfolder (7 tokens). The
    # karmic-kraken b12x supports non-causal attention, so the draft runs
    # B12X as well (the TRITON_ATTN workaround was only needed on r39).
    # DFlash-7 draft, B12X attention, kv_cache_dtype auto — the lab's
    # Pro-TP8 DFlash launcher contract verbatim. The 2026-09-28 garble was
    # NOT spec-related: instanttensor mis-mapped the split-GDN weights; the
    # b12x loader fixed it (see --load-format comment).
    "--speculative-config", """
'{"method":"dflash","model":"{{dflash}}","num_speculative_tokens":5,"kv_cache_dtype":"bfloat16","attention_backend":"B12X","draft_load_config":{"load_format":"safetensors","model_loader_extra_config":{}}}'""",
    # (the --language-model-only line removed for boot B1 — see above)
    "--reasoning-parser", "mimo",
    "--tool-call-parser", "mimo",
    "--enable-auto-tool-choice",
    "--generation-config", "vllm",
    # SparkRing compile contract (fuse_act_quant + explicit capture sizes
    # sized for 16 seqs x (5+1) draft slots).
    "--compilation-config", """
'{"pass_config":{"fuse_act_quant":true},"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":96,"cudagraph_capture_sizes":[1,2,4,6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96]}'""",
]

# DFlash draft weights: a subfolder of the main model repo, resolved per node
# (same mount as the sglang lane; {{dflash}} in the speculative config).
[[extra_mounts]]
repo = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
subfolder = "dflash"
container = "dflash"

# JIT caches: same per-node dirs the GLM r39 lane warms (b12x/flashinfer/
# torchinductor/vllm), so this lane boots JIT-warm off day zero.
[[extra_mounts]]
host = "~/sm121-cache/glm53-jit-121f/b12x"
container = "/root/.cache/b12x"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"

# Triton's own JIT cache (the vllm compile cache references cubins that live
# here + in TORCHINDUCTOR_CACHE_DIR; without these, every restart re-JITs and
# TritonBundler logs "cubin not found").
[[extra_mounts]]
host = "~/sm121-cache/jit-121f/triton"
container = "/root/.cache/triton"

[env]
HF_HUB_OFFLINE = "1"
VLLM_PLUGINS = "b12x_loader"
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
MAX_JOBS = "2"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
# Keep inductor + triton caches on the mounted volumes (defaults are the
# container /tmp and ~/.triton — both die every restart, leaving the vllm
# compile cache dangling at lost cubins).
TORCHINDUCTOR_CACHE_DIR = "/root/.cache/torchinductor"
TRITON_CACHE_DIR = "/root/.cache/triton"
# SparkRing tp4 serving-profile keys (quality flags from the DS4.1 garble
# line + b12x policy + v2 runner); NCCL transport stays on the switched
# cluster config — their switchless-ring keys are intentionally not ported.
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_MXFP8_LM_HEAD = "0"
VLLM_LM_HEAD_A16 = "1"
VLLM_KV_CACHE_LAYOUT = "BLHNC"
B12X_POLICY_MODE = "auto"
VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2097152"
VLLM_ROCE_ALLGATHER_MAX_SIZE = "16777216"
NCCL_IB_EXTENDED_IPV4_GIDS = "1"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_USE_AOT_COMPILE = "0"
VLLM_USE_STANDALONE_COMPILE = "1"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
TORCH_USE_RTLD_GLOBAL = "1"
VLLM_NO_USAGE_STATS = "1"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
# b12x CuTe kernels target sm_121a on GB10 (the lab launcher defaults to
# sm_120a = RTX PRO; eugr's own recipes override it the same way).
CUTE_DSL_ARCH = "sm_121a"
CMAKE_CUDA_ARCHITECTURES = "121"
OMP_NUM_THREADS = "2"