recipes: mimo-v2.5-nvfp4

recipes/mimo-v2.5-nvfp4/recipe.toml

@@ -0,0 +1,59 @@
+engine = "vllm"
+name = "MiMo-V2.5 Omni NVFP4 (vLLM)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
+model = "lukealonso/MiMo-V2.5-NVFP4"
+parent_model = "XiaomiMiMo/MiMo-V2.5"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+
+# 1M context · NVFP4 4-bit KV · MTP1 · ~30 tok/s
+# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
+# REQUIRES patched vLLM container with mods (nvfp4-kv-diffkv, fix-mimo-v2-vllm, etc.)
+# Stock vLLM will reject NVFP4 KV or OOM.
+
+entrypoint = ["vllm", "serve"]
+
+args = """
+--served-model-name MiMo-V2.5-NVFP4
+--trust-remote-code
+--dtype auto
+--load-format safetensors
+--hf-overrides {"architectures":["MiMoV2OmniForCausalLM"]}
+--limit-mm-per-prompt {"image":4,"video":1,"audio":1}
+--mm-encoder-tp-mode data
+--attention-backend triton_attn_diffkv
+--kv-cache-dtype nvfp4
+--gpu-memory-utilization 0.84
+--max-model-len 1000000
+--max-num-batched-tokens 4096
+--max-num-seqs 4
+--block-size 32
+--enable-prefix-caching
+--enable-chunked-prefill
+--enable-auto-tool-choice
+--tool-call-parser mimo
+--reasoning-parser mimo
+--speculative-config {"method":"mtp","num_speculative_tokens":1}
+--enforce-eager
+"""
+
+# TP/PP/nnodes/node-rank/master-addr are
+# auto-generated by the launcher from the recipe's top-level fields.
+# mp backend auto-detected by vLLM >0.20, no --distributed-executor-backend flag needed.
+
+[env]
+# Memory stability
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+
+# NVFP4 weights + KV + WMMA decode
+VLLM_NVFP4_GEMM_BACKEND = "flashinfer-cutlass"
+VLLM_USE_FLASHINFER_MOE_FP4 = "1"
+VLLM_FLASHINFER_MOE_BACKEND = "throughput"
+VLLM_NVFP4_INLINE = "1"
+VLLM_WMMA_DECODE = "1"
+VLLM_WMMA_INSPECT = "0"
+VLLM_WMMA_COMPARE = "0"
+VLLM_WMMA_AUTHCOMPARE = "0"