|
@@ -1,11 +1,11 @@
|
|
|
engine = "vllm"
|
|
engine = "vllm"
|
|
|
-name = "MiMo-V2.5 Omni NVFP4 (vLLM Ray)"
|
|
|
|
|
|
|
+name = "MiMo-V2.5 Omni NVFP4 (vLLM)"
|
|
|
docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
|
|
docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
|
|
|
model = "lukealonso/MiMo-V2.5-NVFP4"
|
|
model = "lukealonso/MiMo-V2.5-NVFP4"
|
|
|
required_nodes = 2
|
|
required_nodes = 2
|
|
|
tensor_parallel_size = 2
|
|
tensor_parallel_size = 2
|
|
|
pipeline_parallel_size = 1
|
|
pipeline_parallel_size = 1
|
|
|
-distributed_backend = "ray"
|
|
|
|
|
|
|
+distributed_backend = "mp"
|
|
|
|
|
|
|
|
# 1M context · NVFP4 4-bit KV · MTP1 · ~30 tok/s
|
|
# 1M context · NVFP4 4-bit KV · MTP1 · ~30 tok/s
|
|
|
# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
|
|
# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
|
|
@@ -38,15 +38,13 @@ args = """
|
|
|
--enforce-eager
|
|
--enforce-eager
|
|
|
"""
|
|
"""
|
|
|
|
|
|
|
|
-# TP/PP/nnodes/node-rank/master-addr/distributed-executor-backend are
|
|
|
|
|
|
|
+# TP/PP/nnodes/node-rank/master-addr are
|
|
|
# auto-generated by the launcher from the recipe's top-level fields.
|
|
# auto-generated by the launcher from the recipe's top-level fields.
|
|
|
|
|
+# mp backend auto-detected by vLLM >0.20, no --distributed-executor-backend flag needed.
|
|
|
|
|
|
|
|
[env]
|
|
[env]
|
|
|
-# Memory / Ray stability
|
|
|
|
|
|
|
+# Memory stability
|
|
|
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
-RAY_memory_monitor_refresh_ms = "0"
|
|
|
|
|
-VLLM_USE_RAY_V2_EXECUTOR_BACKEND = "0"
|
|
|
|
|
-VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM = "0"
|
|
|
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
|
|
|
|
|
|
|
|
# NVFP4 weights + KV + WMMA decode
|
|
# NVFP4 weights + KV + WMMA decode
|
