Forráskód Böngészése

feat(recipe): GLM-5.2 hf-overrides, full cudagraph compilation, adaptive spec

Add three GLM-5.2-MXPF4-Experts-GPTQ tuning knobs:
  --hf-overrides: 78-char index_topk_pattern for MLP/expert routing
  --compilation-config: FULL_AND_PIECEWISE cudagraph mode, custom_ops=all,
    capture sizes [1,2,3,4,5,6,8]
  VLLM_ADAPTIVE_SPEC_DEPTHS=2,4: adaptive speculative decode depths

Triple-quoted TOML strings render as single-line JSON in the docker
command (verified via -d serve dry-run across all 4 nodes).
Alexander Korolev 3 hónapja
szülő
commit
7ebb007f08
1 módosított fájl, 9 hozzáadás és 0 törlés
  1. 9 0
      recipes/glm52-mxfp4-experts-gptq/recipe.toml

+ 9 - 0
recipes/glm52-mxfp4-experts-gptq/recipe.toml

@@ -48,6 +48,9 @@ args = [
     "--dcp-comm-backend", "ag_rs",
     # Engine backends
     "--attention-backend", "B12X_MLA_SPARSE",
+    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
+    "--hf-overrides", """
+'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
     # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
     "--kv-cache-dtype", "nvfp4_ds_mla",
     "--kv-cache-memory-bytes", "5500000000",
@@ -61,6 +64,9 @@ args = [
     # (refs/main → latest snapshot) so symlinks resolve correctly.
     "--speculative-config", """
 '{"model":"{{mtp_draft}}","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
+    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
     # Timeout safety net (cold boot: ranks finish minutes apart)
     "--distributed-timeout-seconds", "3600",
     "--cpu-distributed-timeout-seconds", "3600",
@@ -104,6 +110,9 @@ VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
 VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
 VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
 
+# ── Speculative decode ────────────────────────────────────────────────────
+VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
+
 # ── NVFP4 KV cache ────────────────────────────────────────────────────────
 VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"