launcher/recipes/mimo-v2.5-nvfp4/recipe.toml

engine = "vllm"
name = "MiMo-V2.5 Omni NVFP4 (vLLM Ray)"
docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
model = "lukealonso/MiMo-V2.5-NVFP4"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "ray"

# 1M context · NVFP4 4-bit KV · MTP1 · ~30 tok/s
# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
# REQUIRES patched vLLM container with mods (nvfp4-kv-diffkv, fix-mimo-v2-vllm, etc.)
# Stock vLLM will reject NVFP4 KV or OOM.

entrypoint = ["vllm", "serve"]

args = """
--served-model-name MiMo-V2.5-NVFP4
--trust-remote-code
--dtype auto
--load-format safetensors
--hf-overrides {"architectures":["MiMoV2OmniForCausalLM"]}
--limit-mm-per-prompt {"image":4,"video":1,"audio":1}
--mm-encoder-tp-mode data
--attention-backend triton_attn_diffkv
--kv-cache-dtype nvfp4
--gpu-memory-utilization 0.84
--max-model-len 1000000
--max-num-batched-tokens 4096
--max-num-seqs 4
--block-size 32
--enable-prefix-caching
--enable-chunked-prefill
--enable-auto-tool-choice
--tool-call-parser mimo
--reasoning-parser mimo
--speculative-config {"method":"mtp","num_speculative_tokens":1}
--enforce-eager
"""

# TP/PP/nnodes/node-rank/master-addr/distributed-executor-backend are
# auto-generated by the launcher from the recipe's top-level fields.

[env]
# Memory / Ray stability
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
RAY_memory_monitor_refresh_ms = "0"
VLLM_USE_RAY_V2_EXECUTOR_BACKEND = "0"
VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM = "0"
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"

# NVFP4 weights + KV + WMMA decode
VLLM_NVFP4_GEMM_BACKEND = "flashinfer-cutlass"
VLLM_USE_FLASHINFER_MOE_FP4 = "1"
VLLM_FLASHINFER_MOE_BACKEND = "throughput"
VLLM_NVFP4_INLINE = "1"
VLLM_WMMA_DECODE = "1"
VLLM_WMMA_INSPECT = "0"
VLLM_WMMA_COMPARE = "0"
VLLM_WMMA_AUTHCOMPARE = "0"