|
@@ -54,7 +54,8 @@ args = [
|
|
|
"--load-format", "instanttensor",
|
|
"--load-format", "instanttensor",
|
|
|
"--max-model-len", "524288",
|
|
"--max-model-len", "524288",
|
|
|
"--max-num-seqs", "8",
|
|
"--max-num-seqs", "8",
|
|
|
- "--max-num-batched-tokens", "8192",
|
|
|
|
|
|
|
+ "--max-num-batched-tokens", "4096",
|
|
|
|
|
+ "--long-prefill-token-threshold", "1024",
|
|
|
"--max-cudagraph-capture-size", "64",
|
|
"--max-cudagraph-capture-size", "64",
|
|
|
# 0.78, not higher: the SvangenStudios agent-serving benchmark (v0.1.1) on
|
|
# 0.78, not higher: the SvangenStudios agent-serving benchmark (v0.1.1) on
|
|
|
# this exact stack found "0.80 boots but dies under traffic". GMU is a KV
|
|
# this exact stack found "0.80 boots but dies under traffic". GMU is a KV
|
|
@@ -89,7 +90,7 @@ args = [
|
|
|
"--enable-auto-tool-choice",
|
|
"--enable-auto-tool-choice",
|
|
|
"--generation-config", "vllm",
|
|
"--generation-config", "vllm",
|
|
|
"--default-chat-template-kwargs.thinking=true",
|
|
"--default-chat-template-kwargs.thinking=true",
|
|
|
- "--default-chat-template-kwargs.reasoning_effort=high",
|
|
|
|
|
|
|
+ "--default-chat-template-kwargs.reasoning_effort=max",
|
|
|
"--disable-custom-all-reduce",
|
|
"--disable-custom-all-reduce",
|
|
|
"--trust-remote-code",
|
|
"--trust-remote-code",
|
|
|
]
|
|
]
|
|
@@ -118,6 +119,8 @@ VLLM_USE_V2_MODEL_RUNNER = "1"
|
|
|
B12X_MLA_SM120_UNIFIED = "1"
|
|
B12X_MLA_SM120_UNIFIED = "1"
|
|
|
B12X_MOE_FORCE_A8 = "1"
|
|
B12X_MOE_FORCE_A8 = "1"
|
|
|
|
|
|
|
|
|
|
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
|
|
|
|
|
+
|
|
|
# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
|
|
# --- arch / memory / GPU isolation (operational on the sm121 cluster) ---
|
|
|
TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
