launcher/recipes/mimo26-flash-mopd-4x-sparkring/recipe.toml

# MiMo-V2.6-Flash-MOPD on 4x DGX Spark — the SparkRing *installer-image* lane.
# Their published per-rank compose file (no installer) transcribed into an
# sm121 recipe: image, entrypoint, flags and environment are the qualified
# sparkring tp4 profile; wiring/cluster facts stay ours.
#
# Image (anonymous digests pull):
#   ghcr.io/fujitsupolycom/sparkring@sha256:c977a2d2...af582
#   line dev-20260928-plainstatus-cuda1342-nccl2323-status033 =
#   eugr/spark-vllm-b12x:nightly-20260924 base + the SparkRing fixes we do
#   NOT have in our nightly:
#     * B12X selection-cache correction (restarted ranks reuse tuning;
#       their record: DS4.1 restart 200-225s vs 652-741s),
#     * MiMo vision attention sinks (in-softmax sinks as trained — without
#       them a red/blue test image reads black/white),
#     * Host-to-device staging fix (CpuGpuBuffer pinned-copy race: Qwen+MTP
#       +FULL graphs decoded token 8191 repetitions),
#     * tool-result contract (HTTP 400/500 instead of 200-without-tool-call),
#     * runtime-status dashboard at /v1/sparkring/status/view on the API port,
#     * CUDA 13.4.2 + NCCL 2.32.3 toolchain (their loader search tree).
#
# Checkpoint: their pin 2479e2d0029e = HF main = our NAS snapshot (verified
# against the profile's config/index sha256) — model points at the pinned
# container-side snapshot path (flat layout incl. the dflash/ draft inside).
# No installer, no local materialization: /models/target comes from the HF
# cache mount.
#
# Deviations from their compose (switched fabric / cluster conventions):
#   * RING-ONLY KEYS DROPPED: NCCL_SWITCHLESS_RING_ONLY, B12X_ROCE_OPPOSITE_
#     PATHS, B12X_ROCE_PEER_HCA_MAP — direct-cycle wiring assumptions (p0→p1
#     cabling), invalid on our 4nodes-switch fabric.
#   * 4-port HCA list → the cluster NCCL block's 2 ports: on the switch only
#     f0 pairs are ACTIVE (f1 DOWN on every dgx58 node).
#   * eth0 socket vars dropped — the cluster nccl block carries the correct
#     enp1s0f0np0 socket/GLOO ifaces.
#   * mem_limit/memswap omitted (GB10 UMA: our lanes run unbounded; a 108G
#     cgroup cap can OOM-kill mid-serve).
#   * port 30000 (cluster convention; their compose serves 8020).
#   * max-model-len 524288 (prod parity with the sglang lane; their profile
#     qualifies 262144), KV pin omitted (GB10 auto-sizing keeps ~54 GiB).
#   * RPC: --async-scheduling omitted — NOT in their qualified shape.
#     Re-add only as a measured A/B.
#   * --mm-encoder-tp-mode data (theirs; our nightly lane ran weights).

engine = "vllm"
name = "MiMo-V2.6-Flash-MOPD (SparkRing installer image, 4x switched)"
docker_image = "ghcr.io/fujitsupolycom/sparkring@sha256:c977a2d2efb7ecf9ea856cd0379fdd93a8913f0770f4627a3ae00a084cfaf582"
entrypoint = ["python3", "/opt/sparkring/toolchain/toolchain.py", "serve"]
# Pin = their profile revision (verified: config/index sha256 match the
# profile); flat snapshot dir contains dflash/ for the draft loader.
model = "/root/.cache/huggingface/hub/models--XiaomiMiMo--MiMo-V2.6-Flash-MOPD/snapshots/2479e2d0029eca9a34cc7e7f55a121925f81908e"
parent_model = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
port = 30000
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
devices = ["/dev/infiniband"]

args = [
    "--served-model-name", "mimo-v2.6-flash",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    "--gpu-memory-utilization", "0.85",
    "--block-size", "128",
    # EXPERIMENT 2026-09-29 (operator): A/B their qualified 262144 vs our
    # 524288 on the c2 long-context collapse — RESULT: NEGATIVE, collapse
    # is bit-identical at both (c2: 13.8 t/s @65K, 5.6 @131K; c1 healthy
    # 42-48 everywhere) → MML not the axis; back to 524288 for prod parity.
    # Pool stayed auto throughout (KV usage during the stall: 0.7% — memory
    # excluded, as the operator predicted).
    "--max-model-len", "524288",
    "--max-num-seqs", "16",
    "--max-num-batched-tokens", "8192",
    "--model-loader-extra-config", """
'{"read_mode":"bounce","io_threads":8}'""",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    "--attention-backend", "B12X",
    "--linear-backend", "b12x",
    "--moe-backend", "b12x",
    "--no-enable-flashinfer-autotune",
    "--load-format", "b12x",
    "--kv-cache-dtype", "bfloat16",
    # NOTE: our nightly's `--kernel-config {enable_flashinfer_autotune:false}`
    # is mutually exclusive with their `--no-enable-flashinfer-autotune`
    # (ValueError at engine build) — their qualified shape wins, kernel-config
    # is dropped for this lane.
    # Their qualified multimodal shape: full Omni + data-parallel encoder
    # (this fork auto-falls back to weights for MiMo: "model does not support
    # --mm-encoder-tp-mode data"), per-prompt media limits, video kwargs,
    # pinned processor config.
    # Their qualified multimodal shape: full Omni + data-parallel encoder,
    # per-prompt media limits, video kwargs, pinned processor config.
    "--decode-context-parallel-size", "1",
    "--enable-prompt-tokens-details",
    "--limit-mm-per-prompt", """
'{"image":3,"video":1,"audio":1}'""",
    "--mm-processor-cache-gb", "0",
    "--mm-encoder-tp-mode", "data",
    "--media-io-kwargs", """
'{"video":{"num_frames":8}}'""",
    "--hf-overrides", """
'{"architectures":["MiMoV2OmniForCausalLM"],"processor_config":{"audio_avg_pooler":2,"audio_channels":20,"audio_end_token_id":151674,"audio_fmax":null,"audio_fmin":0,"audio_group_size":4,"audio_hop_length":240,"audio_input_id_per_second":25.0,"audio_kernel_size":3,"audio_n_mels":128,"audio_nfft":960,"audio_sampling_rate":24000,"audio_segment_size":6000,"audio_start_token_id":151673,"audio_stride_size":2,"audio_token_id":151669,"audio_window_size":960,"audio_zeroemb_idx":[1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024,1024],"fps":1.0,"image_max_pixels":1048576,"image_min_pixels":8192,"image_token_id":151655,"max_frames":8,"merge_size":2,"min_frames":null,"num_frames":null,"pad_token_id":151643,"patch_size":16,"rope_type":"rope","temporal_compression_ratio":1,"temporal_patch_size":2,"use_per_grid_t_timestamps":false,"use_video_timestamps":true,"video_audio_interleave_length":0.0,"video_end_token_id":151671,"video_max_pixels":262144,"video_min_pixels":8192,"video_process_num_threads":16,"video_start_token_id":151670,"video_token_id":151656,"video_tokens_per_second":2,"video_total_max_pixels":268435456,"vision_end_token_id":151653,"vision_start_token_id":151652}}'""",
    # Draft: dflash/ inside the pinned snapshot (their /models/target/dflash).
    "--speculative-config", """
'{"model":"/root/.cache/huggingface/hub/models--XiaomiMiMo--MiMo-V2.6-Flash-MOPD/snapshots/2479e2d0029eca9a34cc7e7f55a121925f81908e/dflash","method":"dflash","num_speculative_tokens":5,"draft_tensor_parallel_size":4,"kv_cache_dtype":"bfloat16","attention_backend":"B12X","draft_load_config":{"load_format":"safetensors","model_loader_extra_config":{}}}'""",
    "--compilation-config", """
'{"pass_config":{"fuse_act_quant":true},"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":96,"cudagraph_capture_sizes":[1,2,4,6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96]}'""",
    "--reasoning-parser", "mimo",
    "--tool-call-parser", "mimo",
    "--enable-auto-tool-choice",
    "--generation-config", "vllm",
]

# One cache root for their /cache layout (eight *_CACHE_DIR keys below point
# here; first boot warms JIT into it, the B12X selection-cache fix persists
# across restarts — the single-mount story our nightly lane never had).
[[extra_mounts]]
host = "~/sm121-cache/sparkring-121f"
container = "/cache"

[env]
# --- SparkRing compose environment, verbatim except the noted drops ------
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
OMP_NUM_THREADS = "16"
MAX_JOBS = "16"
VLLM_ENABLE_PCIE_ALLREDUCE = "0"
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2097152"
VLLM_ROCE_ALLGATHER_MAX_SIZE = "16777216"
SPARKCACHE_ENABLED = "0"
SOURCE_IMAGE_PROFILE = ""
B12X_POLICY_MODE = "auto"
NCCL_NET = "IB"
NCCL_NET_PLUGIN = "none"
NCCL_IB_DISABLE = "0"
NCCL_IB_EXTENDED_IPV4_GIDS = "1"
NCCL_IB_MERGE_NICS = "0"
NCCL_IB_PRESERVE_PCI_DOMAIN = "1"
NCCL_IB_ROUTE_DIAGNOSTICS = "1"
NCCL_IB_SUBNET_AWARE_ROUTING = "1"
NCCL_IGNORE_CPU_AFFINITY = "1"
NCCL_CROSS_NIC = "1"
NCCL_ALGO = "Ring"
NCCL_CUMEM_ENABLE = "0"
NCCL_DEBUG = "WARN"
NCCL_DEBUG_SUBSYS = "NET,INIT,GRAPH"
NCCL_MAX_NCHANNELS = "4"
NCCL_MIN_NCHANNELS = "4"
NCCL_P2P_LEVEL = "SYS"
NCCL_PROTO = "LL,LL128,Simple"
NCCL_IB_GID_INDEX = "3"
# Their bundled NCCL 2.32.3 (loader search tree inside the image).
NCCL_ROOT = "/opt/sparkring/toolchain/nccl"
NCCL_LIB_DIR = "/opt/sparkring/toolchain/nccl/lib"
NCCL_INCLUDE_DIR = "/opt/sparkring/toolchain/nccl/include"
NCCL_LOCAL_INFERENCE_PATH = "/opt/sparkring/toolchain/nccl/lib/libnccl.so.2"
VLLM_NCCL_SO_PATH = "/opt/sparkring/toolchain/nccl/lib/libnccl.so.2"
VLLM_NCCL_INCLUDE_PATH = "/opt/sparkring/toolchain/nccl/include"
CUDA_HOME = "/usr/local/cuda-13.4"
CUDA_PATH = "/usr/local/cuda-13.4"
CUDA_VERSION = "13.4.2"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_MXFP8_LM_HEAD = "0"
VLLM_LM_HEAD_A16 = "1"
VLLM_B12X_MOE_FP4_FORCE_A16 = "0"
VLLM_KV_CACHE_LAYOUT = "BLHNC"
VLLM_PLUGINS = "b12x_loader,sparkring_status"
# DIAGNOSIS CLOSED 2026-09-29: the c2 depth "collapse" was the bench's
# own default (llama-benchy without --enable-prefix-caching serializes
# cold 67K prefills per pair — artifact, not engine). DEBUG removed.
B12X_ROCE_GID_INDEX = "3"
SPARKRING_FEATURES = ""
SPARKRING_TOOL_CHOICE_CONTRACT = "1"
SIRCL_ENABLED = "0"
VLLM_SPARK_TP4_MODE = ""
VLLM_SPARK_TP4_VOCAB_MODE = ""
# --- caches under /cache (compose names, verbatim) -----------------------
XDG_CACHE_HOME = "/cache/mimo-v26-flash-rl-4b7049d1e00f-2479e2d0029e"
VLLM_CACHE_ROOT = "/cache/mimo-v26-flash-rl-4b7049d1e00f-2479e2d0029e/vllm"
TORCHINDUCTOR_CACHE_DIR = "/cache/mimo-v26-flash-rl-4b7049d1e00f-2479e2d0029e/inductor"
TRITON_CACHE_DIR = "/cache/mimo-v26-flash-rl-4b7049d1e00f-2479e2d0029e/triton"
CUTE_DSL_CACHE_DIR = "/cache/mimo-v26-flash-rl-4b7049d1e00f-2479e2d0029e/cute"
TILELANG_CACHE_DIR = "/cache/mimo-v26-flash-rl-4b7049d1e00f-2479e2d0029e/tilelang"
TVM_FFI_CACHE_DIR = "/cache/mimo-v26-flash-rl-4b7049d1e00f-2479e2d0029e/tvm-ffi"
FLASHINFER_WORKSPACE_BASE = "/cache/mimo-v26-flash-rl-4b7049d1e00f-2479e2d0029e/flashinfer"
B12X_COMPILE_CACHE_DIR = "/cache/mimo-v26-flash-rl-cuda13.4.2-2479e2d0029e/b12x"
# --- launcher-controlled bits -------------------------------------------
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_NO_USAGE_STATS = "1"
# DROPPED on purpose (see header): NCCL_SWITCHLESS_RING_ONLY,
# B12X_ROCE_OPPOSITE_PATHS, B12X_ROCE_PEER_HCA_MAP, B12X_ROCE_HCA,
# NCCL_IB_HCA (cluster block: f0 pair), NCCL_SOCKET_IFNAME/GLOO (cluster
# block: enp1s0f0np0), SPARKRING_TRANSPORT_* (ring pacing), mem_limit.