|
|
@@ -0,0 +1,59 @@
|
|
|
+engine = "vllm"
|
|
|
+name = "MiMo-V2.5 Omni NVFP4 (vLLM)"
|
|
|
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
|
|
|
+model = "lukealonso/MiMo-V2.5-NVFP4"
|
|
|
+parent_model = "XiaomiMiMo/MiMo-V2.5"
|
|
|
+required_nodes = 2
|
|
|
+tensor_parallel_size = 2
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+
|
|
|
+# 1M context · NVFP4 4-bit KV · MTP1 · ~30 tok/s
|
|
|
+# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
|
|
|
+# REQUIRES patched vLLM container with mods (nvfp4-kv-diffkv, fix-mimo-v2-vllm, etc.)
|
|
|
+# Stock vLLM will reject NVFP4 KV or OOM.
|
|
|
+
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+
|
|
|
+args = """
|
|
|
+--served-model-name MiMo-V2.5-NVFP4
|
|
|
+--trust-remote-code
|
|
|
+--dtype auto
|
|
|
+--load-format safetensors
|
|
|
+--hf-overrides {"architectures":["MiMoV2OmniForCausalLM"]}
|
|
|
+--limit-mm-per-prompt {"image":4,"video":1,"audio":1}
|
|
|
+--mm-encoder-tp-mode data
|
|
|
+--attention-backend triton_attn_diffkv
|
|
|
+--kv-cache-dtype nvfp4
|
|
|
+--gpu-memory-utilization 0.84
|
|
|
+--max-model-len 1000000
|
|
|
+--max-num-batched-tokens 4096
|
|
|
+--max-num-seqs 4
|
|
|
+--block-size 32
|
|
|
+--enable-prefix-caching
|
|
|
+--enable-chunked-prefill
|
|
|
+--enable-auto-tool-choice
|
|
|
+--tool-call-parser mimo
|
|
|
+--reasoning-parser mimo
|
|
|
+--speculative-config {"method":"mtp","num_speculative_tokens":1}
|
|
|
+--enforce-eager
|
|
|
+"""
|
|
|
+
|
|
|
+# TP/PP/nnodes/node-rank/master-addr are
|
|
|
+# auto-generated by the launcher from the recipe's top-level fields.
|
|
|
+# mp backend auto-detected by vLLM >0.20, no --distributed-executor-backend flag needed.
|
|
|
+
|
|
|
+[env]
|
|
|
+# Memory stability
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
|
|
|
+
|
|
|
+# NVFP4 weights + KV + WMMA decode
|
|
|
+VLLM_NVFP4_GEMM_BACKEND = "flashinfer-cutlass"
|
|
|
+VLLM_USE_FLASHINFER_MOE_FP4 = "1"
|
|
|
+VLLM_FLASHINFER_MOE_BACKEND = "throughput"
|
|
|
+VLLM_NVFP4_INLINE = "1"
|
|
|
+VLLM_WMMA_DECODE = "1"
|
|
|
+VLLM_WMMA_INSPECT = "0"
|
|
|
+VLLM_WMMA_COMPARE = "0"
|
|
|
+VLLM_WMMA_AUTHCOMPARE = "0"
|
