|
@@ -48,6 +48,9 @@ args = [
|
|
|
"--dcp-comm-backend", "ag_rs",
|
|
"--dcp-comm-backend", "ag_rs",
|
|
|
# Engine backends
|
|
# Engine backends
|
|
|
"--attention-backend", "B12X_MLA_SPARSE",
|
|
"--attention-backend", "B12X_MLA_SPARSE",
|
|
|
|
|
+ # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
|
|
|
|
|
+ "--hf-overrides", """
|
|
|
|
|
+'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
|
|
|
# KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
|
|
# KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
|
|
|
"--kv-cache-dtype", "nvfp4_ds_mla",
|
|
"--kv-cache-dtype", "nvfp4_ds_mla",
|
|
|
"--kv-cache-memory-bytes", "5500000000",
|
|
"--kv-cache-memory-bytes", "5500000000",
|
|
@@ -61,6 +64,9 @@ args = [
|
|
|
# (refs/main → latest snapshot) so symlinks resolve correctly.
|
|
# (refs/main → latest snapshot) so symlinks resolve correctly.
|
|
|
"--speculative-config", """
|
|
"--speculative-config", """
|
|
|
'{"model":"{{mtp_draft}}","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
|
|
'{"model":"{{mtp_draft}}","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
|
|
|
|
|
+ # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
|
|
|
|
|
+ "--compilation-config", """
|
|
|
|
|
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
|
|
|
# Timeout safety net (cold boot: ranks finish minutes apart)
|
|
# Timeout safety net (cold boot: ranks finish minutes apart)
|
|
|
"--distributed-timeout-seconds", "3600",
|
|
"--distributed-timeout-seconds", "3600",
|
|
|
"--cpu-distributed-timeout-seconds", "3600",
|
|
"--cpu-distributed-timeout-seconds", "3600",
|
|
@@ -104,6 +110,9 @@ VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
|
|
|
VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
|
|
VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
|
|
|
VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
|
|
VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
|
|
|
|
|
|
|
|
|
|
+# ── Speculative decode ────────────────────────────────────────────────────
|
|
|
|
|
+VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
|
|
|
|
|
+
|
|
|
# ── NVFP4 KV cache ────────────────────────────────────────────────────────
|
|
# ── NVFP4 KV cache ────────────────────────────────────────────────────────
|
|
|
VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
|
|
VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
|
|
|
|
|
|