launcher/recipes/mimo26-pro-mopd-8x-sparkring/recipe.toml

# MiMo-V2.6-Pro-MOPD on 8x DGX Spark — SparkRing runtime stack applied to the
# Pro model (operator 2026-09-29: MOPD only, RL lanes retired — RL keeps
# training artifacts). There is NO upstream SparkRing profile for Pro, so this
# adapts our qualified tp4 contract: tp/nnodes/draft-tp = 8, per-model cache
# namespace. Served name stays the simple lane name `mimo-v2.6-pro`
# (operator 2026-09-29: no foreign TP/variant clutter in served names).
#
# Verified 2026-09-29:
#   * snapshot XiaomiMiMo/MiMo-V2.6-Pro-MOPD @ adea8e2c5373, 573.5 GB (NAS);
#   * config arch `MiMoV2ForCausalLM` (same as Flash → their qualified
#     Omni hf-overrides apply) and `processor_config` byte-equal to Flash's;
#   * dflash/ draft present (dflash_draft_model.safetensors + dflash.py);
#   * quant fp8 e4m3 + mxfp4_block_size → b12x MXFP4/MXFP8 backend, same env.
#
# Intended cluster: `8nodes-switch` (all 8; fleet coordination — prod must be
# down first; sparkring image sits only on dgx5-8 → `--sync-image` ships it).
# KV at TP8: weights ≈ 72 GiB/rank → auto pool ≈ 20-25 GiB ≈ 1M-1.2M tokens.
# Details: specs/tech-architecture/sparkring-mimo-lanes.md

engine = "vllm"
name = "MiMo-V2.6-Pro-MOPD (SparkRing installer image, 8x switched)"
docker_image = "ghcr.io/fujitsupolycom/sparkring@sha256:71d410571407fef3ce2959c6d392f5a2c3f44b757b856e853a71f6e3295620ad"
entrypoint = ["python3", "/opt/sparkring/toolchain/toolchain.py", "serve"]
# Floating model: the engine resolves the org/model id via the NAS HF cache
# (refs/main -> latest snapshot; the snapshot carries dflash/ for the draft
# loader). Our vendored NAS revision adea8e2c (no upstream Pro profile to
# cross-check) is what refs/main points at on 2026-10-02 — informational only.
model = "XiaomiMiMo/MiMo-V2.6-Pro-MOPD"
parent_model = "XiaomiMiMo/MiMo-V2.6-Pro-MOPD"
port = 30000
required_nodes = 8
tensor_parallel_size = 8
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
devices = ["/dev/infiniband"]

args = [
    "--served-model-name", "mimo-v2.6-pro",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    "--gpu-memory-utilization", "0.85",
    "--block-size", "128",
    # 524288 prod parity. (2026-09-29: the c2 "collapse" A/B at their 262144
    # was bit-identical — it later proved to be a BENCH artifact, not engine;
    # see specs/tech-architecture/sparkring-mimo-lanes.md.)
    "--max-model-len", "524288",
    "--max-num-seqs", "16",
    "--max-num-batched-tokens", "8192",
    "--model-loader-extra-config", """
'{"read_mode":"bounce","io_threads":8}'""",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    "--attention-backend", "B12X",
    "--linear-backend", "b12x",
    "--moe-backend", "b12x",
    "--no-enable-flashinfer-autotune",
    "--load-format", "b12x",
    "--kv-cache-dtype", "bfloat16",
    # NOTE: our nightly's `--kernel-config {enable_flashinfer_autotune:false}`
    # is mutually exclusive with their `--no-enable-flashinfer-autotune`
    # (ValueError at engine build) — their qualified shape wins, kernel-config
    # is dropped for this lane.
    # Their qualified multimodal shape: full Omni + data-parallel encoder
    # (this fork auto-falls back to weights for MiMo: "model does not support
    # --mm-encoder-tp-mode data"), per-prompt media limits, video kwargs,
    # pinned processor config.
    # Their qualified multimodal shape: full Omni + data-parallel encoder,
    # per-prompt media limits, video kwargs, pinned processor config.
    "--decode-context-parallel-size", "1",
    "--enable-prompt-tokens-details",
    "--limit-mm-per-prompt", """
'{"image":3,"video":1,"audio":1}'""",
    "--mm-processor-cache-gb", "0",
    "--mm-encoder-tp-mode", "data",
    "--media-io-kwargs", """
'{"video":{"num_frames":8}}'""",
    "--hf-overrides", """
'{"architectures":["MiMoV2OmniForCausalLM"],"processor_config":{"audio_avg_pooler":2,"audio_channels":20,"audio_end_token_id":151674,"audio_fmax":null,"audio_fmin":0,"audio_group_size":4,"audio_hop_length":240,"audio_input_id_per_second":25.0,"audio_kernel_size":3,"audio_n_mels":128,"audio_nfft":960,"audio_sampling_rate":24000,"audio_segment_size":6000,"audio_start_token_id":151673,"audio_stride_size":2,"audio_token_id":151669,"audio_window_size":960,"audio_zeroemb_idx":[1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024],"fps":1.0,"image_max_pixels":1048576,"image_min_pixels":8192,"image_token_id":151655,"max_frames":8,"merge_size":2,"min_frames":null,"num_frames":null,"pad_token_id":151643,"patch_size":16,"rope_type":"rope","temporal_compression_ratio":1,"temporal_patch_size":2,"use_per_grid_t_timestamps":false,"use_video_timestamps":true,"video_audio_interleave_length":0.0,"video_end_token_id":151671,"video_max_pixels":262144,"video_min_pixels":8192,"video_process_num_threads":16,"video_start_token_id":151670,"video_token_id":151656,"video_tokens_per_second":2,"video_total_max_pixels":268435456,"vision_end_token_id":151653,"vision_start_token_id":151652}}'""",
    # Draft: dflash/ inside the pinned snapshot (their /models/target/dflash).
    "--speculative-config", """
'{"model":"{{dflash}}","method":"dflash","num_speculative_tokens":5,"draft_tensor_parallel_size":8,"kv_cache_dtype":"bfloat16","attention_backend":"B12X","draft_load_config":{"load_format":"safetensors","model_loader_extra_config":{}}}'""",
    "--compilation-config", """
'{"pass_config":{"fuse_act_quant":true},"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":96,"cudagraph_capture_sizes":[1,2,4,6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96]}'""",
    "--reasoning-parser", "mimo",
    "--tool-call-parser", "mimo",
    "--enable-auto-tool-choice",
    "--generation-config", "vllm",
]

# DFlash drafter: the dflash/ subfolder of the main model repo, resolved per
# node via refs/main (floats with the model); substituted into the
# speculative-config as {{dflash}}. No bind-mount — the HF cache is already
# mounted at /root/.cache/huggingface.
[[extra_mounts]]
repo = "XiaomiMiMo/MiMo-V2.6-Pro-MOPD"
subfolder = "dflash"
container = "dflash"

# One cache root for their /cache layout (eight *_CACHE_DIR keys below point
# here; first boot warms JIT into it, the B12X selection-cache fix persists
# across restarts — the single-mount story our nightly lane never had).
[[extra_mounts]]
host = "~/sm121-cache/sparkring-pro-8x"
container = "/cache"

[env]
# --- SparkRing compose environment, verbatim except the noted drops ------
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
OMP_NUM_THREADS = "16"
MAX_JOBS = "16"
VLLM_ENABLE_PCIE_ALLREDUCE = "0"
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2097152"
VLLM_ROCE_ALLGATHER_MAX_SIZE = "16777216"
SPARKCACHE_ENABLED = "0"
SOURCE_IMAGE_PROFILE = ""
B12X_POLICY_MODE = "auto"
NCCL_NET = "IB"
NCCL_NET_PLUGIN = "none"
NCCL_IB_DISABLE = "0"
NCCL_IB_EXTENDED_IPV4_GIDS = "1"
NCCL_IB_MERGE_NICS = "0"
NCCL_IB_PRESERVE_PCI_DOMAIN = "1"
NCCL_IB_ROUTE_DIAGNOSTICS = "1"
NCCL_IB_SUBNET_AWARE_ROUTING = "1"
NCCL_IGNORE_CPU_AFFINITY = "1"
NCCL_CROSS_NIC = "1"
NCCL_ALGO = "Ring"
NCCL_CUMEM_ENABLE = "0"
NCCL_DEBUG = "WARN"
NCCL_DEBUG_SUBSYS = "NET,INIT,GRAPH"
NCCL_MAX_NCHANNELS = "4"
NCCL_MIN_NCHANNELS = "4"
NCCL_P2P_LEVEL = "SYS"
NCCL_PROTO = "LL,LL128,Simple"
NCCL_IB_GID_INDEX = "3"
# Their bundled NCCL 2.32.3 (loader search tree inside the image).
NCCL_ROOT = "/opt/sparkring/toolchain/nccl"
NCCL_LIB_DIR = "/opt/sparkring/toolchain/nccl/lib"
NCCL_INCLUDE_DIR = "/opt/sparkring/toolchain/nccl/include"
NCCL_LOCAL_INFERENCE_PATH = "/opt/sparkring/toolchain/nccl/lib/libnccl.so.2"
VLLM_NCCL_SO_PATH = "/opt/sparkring/toolchain/nccl/lib/libnccl.so.2"
VLLM_NCCL_INCLUDE_PATH = "/opt/sparkring/toolchain/nccl/include"
CUDA_HOME = "/usr/local/cuda-13.4"
CUDA_PATH = "/usr/local/cuda-13.4"
CUDA_VERSION = "13.4.2"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_MXFP8_LM_HEAD = "0"
VLLM_LM_HEAD_A16 = "1"
VLLM_B12X_MOE_FP4_FORCE_A16 = "0"
VLLM_KV_CACHE_LAYOUT = "BLHNC"
VLLM_PLUGINS = "b12x_loader,sparkring_status"
# DIAGNOSIS CLOSED 2026-09-29: the c2 depth "collapse" was the bench's
# own default (llama-benchy without --enable-prefix-caching serializes
# cold 67K prefills per pair — artifact, not engine). DEBUG removed.
B12X_ROCE_GID_INDEX = "3"
SPARKRING_FEATURES = ""
SPARKRING_TOOL_CHOICE_CONTRACT = "1"
SIRCL_ENABLED = "0"
VLLM_SPARK_TP4_MODE = ""
VLLM_SPARK_TP4_VOCAB_MODE = ""
# --- caches under /cache (compose names, verbatim) -----------------------
XDG_CACHE_HOME = "/cache/mimo-v26-pro-mopd-4b7049d1e00f-adea8e2c5373"
VLLM_CACHE_ROOT = "/cache/mimo-v26-pro-mopd-4b7049d1e00f-adea8e2c5373/vllm"
TORCHINDUCTOR_CACHE_DIR = "/cache/mimo-v26-pro-mopd-4b7049d1e00f-adea8e2c5373/inductor"
TRITON_CACHE_DIR = "/cache/mimo-v26-pro-mopd-4b7049d1e00f-adea8e2c5373/triton"
CUTE_DSL_CACHE_DIR = "/cache/mimo-v26-pro-mopd-4b7049d1e00f-adea8e2c5373/cute"
TILELANG_CACHE_DIR = "/cache/mimo-v26-pro-mopd-4b7049d1e00f-adea8e2c5373/tilelang"
TVM_FFI_CACHE_DIR = "/cache/mimo-v26-pro-mopd-4b7049d1e00f-adea8e2c5373/tvm-ffi"
FLASHINFER_WORKSPACE_BASE = "/cache/mimo-v26-pro-mopd-4b7049d1e00f-adea8e2c5373/flashinfer"
B12X_COMPILE_CACHE_DIR = "/cache/mimo-v26-pro-mopd-cuda13.4.2-adea8e2c5373/b12x"
# --- launcher-controlled bits -------------------------------------------
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_NO_USAGE_STATS = "1"
# DROPPED on purpose (see header): NCCL_SWITCHLESS_RING_ONLY,
# B12X_ROCE_OPPOSITE_PATHS, B12X_ROCE_PEER_HCA_MAP, B12X_ROCE_HCA,
# NCCL_IB_HCA (cluster block: f0 pair), NCCL_SOCKET_IFNAME/GLOO (cluster
# block: enp1s0f0np0), SPARKRING_TRANSPORT_* (ring pacing), mem_limit.