瀏覽代碼

fix(recipe): switch GLM-5.2 from SGLang to vLLM

SGLang variant OOM'd on 4x DGX Spark (128 GB shared memory).
Switch to vLLM with:

- ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1 image
- Mapika/GLM-5.2-NVFP4 model (410 GB, ~103 GB/GPU at TP=4)
- 5-token MTP speculative decoding
- glm47 tool-call / glm45 reasoning parsers
- Blackwell MoE kernels (VLLM_USE_B12X_MOE)
Alexander Korolev 4 月之前
父節點
當前提交
b5615f604d
共有 1 個文件被更改,包括 21 次插入 和 14 次删除
  1. 21 14
      recipes/glm52/recipe.toml

+ 21 - 14
recipes/glm52/recipe.toml

@@ -1,27 +1,34 @@
-engine = "sglang"
-name = "GLM-5.2 NVFP4"
-docker_image = "scitrera/dgx-spark-sglang-mm:v0"
+engine = "vllm"
+name = "GLM-5.2 NVFP4 (vLLM)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
 model = "Mapika/GLM-5.2-NVFP4"
 required_nodes = 4
 tensor_parallel_size = 4
+pipeline_parallel_size = 1
 distributed_backend = "mp"
 
-entrypoint = ["sglang", "serve"]
+# 410B NVFP4 model on 4× DGX Spark (GB10 Blackwell), 128 GB shared each.
+# TP=4 → ~103 GB/GPU for weights, ~25 GB for KV cache at 0.90 util.
 
 args = [
-    "--quantization", "modelopt_fp4",
-    "--mem-fraction-static", "0.85",
-    "--context-length", "32768",
-    "--fp4-gemm-backend", "cutlass",
-    "--kv-cache-dtype", "auto",
-    "--reasoning-parser", "glm45",
+    "--kv-cache-dtype", "fp8",
+    "--gpu-memory-utilization", "0.90",
+    "--load-format", "safetensors",
+    "--enable-prefix-caching",
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":5}'""",
     "--tool-call-parser", "glm47",
-    "--moe-runner-backend", "flashinfer_cutlass",
+    "--reasoning-parser", "glm45",
+    "--enable-auto-tool-choice",
+    "--enable-expert-parallel",
     "--trust-remote-code",
-    "--cuda-graph-max-bs", "16",
-    "--cuda-graph-bs", "1", "2", "4", "8", "12", "16",
     "--served-model-name", "glm52",
 ]
 
 [env]
-TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"
+# Fast safetensors loading (GPU‑side decompress)
+SAFETENSORS_FAST_GPU = "1"
+# Allow long context lengths
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+# Blackwell MoE kernels
+VLLM_USE_B12X_MOE = "1"