|
|
@@ -60,11 +60,16 @@ args = [
|
|
|
"--distributed-executor-backend", "mp",
|
|
|
# Operator GMU (0.88 -> 0.84 for this first vLLM MiMo boot); their
|
|
|
# manifest's 0.98 is RTX PRO 6000 territory, not GB10 UMA.
|
|
|
- "--gpu-memory-utilization", "0.84",
|
|
|
+ "--gpu-memory-utilization", "0.85",
|
|
|
"--block-size", "128",
|
|
|
"--max-model-len", "524288",
|
|
|
- "--max-num-seqs", "4",
|
|
|
- "--max-num-batched-tokens", "4096",
|
|
|
+ "--max-num-seqs", "16",
|
|
|
+ "--max-num-batched-tokens", "8192",
|
|
|
+ # SparkRing tp4 contract: deterministic 20 GiB KV pin (not auto-sizing),
|
|
|
+ # bounce-buffer loader, c1/c8/c16 concurrency-friendly scheduler bounds.
|
|
|
+ "--kv-cache-memory-bytes", "21474836480",
|
|
|
+ "--model-loader-extra-config", """
|
|
|
+'{"read_mode":"bounce","io_threads":8}'""",
|
|
|
"--enable-chunked-prefill",
|
|
|
"--async-scheduling",
|
|
|
"--enable-prefix-caching",
|
|
|
@@ -92,12 +97,16 @@ args = [
|
|
|
# NOT spec-related: instanttensor mis-mapped the split-GDN weights; the
|
|
|
# b12x loader fixed it (see --load-format comment).
|
|
|
"--speculative-config", """
|
|
|
-'{"method":"dflash","model":"{{dflash}}","num_speculative_tokens":7,"kv_cache_dtype":"auto","attention_backend":"B12X"}'""",
|
|
|
+'{"method":"dflash","model":"{{dflash}}","num_speculative_tokens":5,"kv_cache_dtype":"bfloat16","attention_backend":"B12X","draft_load_config":{"load_format":"safetensors","model_loader_extra_config":{}}}'""",
|
|
|
# (the --language-model-only line removed for boot B1 — see above)
|
|
|
"--reasoning-parser", "mimo",
|
|
|
"--tool-call-parser", "mimo",
|
|
|
"--enable-auto-tool-choice",
|
|
|
"--generation-config", "vllm",
|
|
|
+ # SparkRing compile contract (fuse_act_quant + explicit capture sizes
|
|
|
+ # sized for 16 seqs x (5+1) draft slots).
|
|
|
+ "--compilation-config", """
|
|
|
+'{"pass_config":{"fuse_act_quant":true},"cudagraph_mode":"FULL_AND_PIECEWISE","max_cudagraph_capture_size":96,"cudagraph_capture_sizes":[1,2,4,6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96]}'""",
|
|
|
]
|
|
|
|
|
|
# DFlash draft weights: a subfolder of the main model repo, resolved per node
|
|
|
@@ -131,6 +140,17 @@ VLLM_PLUGINS = "b12x_loader"
|
|
|
VLLM_ENABLE_ROCE_ALLREDUCE = "1"
|
|
|
MAX_JOBS = "2"
|
|
|
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
|
+# SparkRing tp4 serving-profile keys (quality flags from the DS4.1 garble
|
|
|
+# line + b12x policy + v2 runner); NCCL transport stays on the switched
|
|
|
+# cluster config — their switchless-ring keys are intentionally not ported.
|
|
|
+VLLM_USE_V2_MODEL_RUNNER = "1"
|
|
|
+VLLM_MXFP8_LM_HEAD = "0"
|
|
|
+VLLM_LM_HEAD_A16 = "1"
|
|
|
+VLLM_KV_CACHE_LAYOUT = "BLHNC"
|
|
|
+B12X_POLICY_MODE = "auto"
|
|
|
+VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2097152"
|
|
|
+VLLM_ROCE_ALLGATHER_MAX_SIZE = "16777216"
|
|
|
+NCCL_IB_EXTENDED_IPV4_GIDS = "1"
|
|
|
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
VLLM_USE_AOT_COMPILE = "0"
|
|
|
VLLM_USE_STANDALONE_COMPILE = "1"
|
