|
@@ -1,27 +1,34 @@
|
|
|
-engine = "sglang"
|
|
|
|
|
-name = "GLM-5.2 NVFP4"
|
|
|
|
|
-docker_image = "scitrera/dgx-spark-sglang-mm:v0"
|
|
|
|
|
|
|
+engine = "vllm"
|
|
|
|
|
+name = "GLM-5.2 NVFP4 (vLLM)"
|
|
|
|
|
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
|
|
|
model = "Mapika/GLM-5.2-NVFP4"
|
|
model = "Mapika/GLM-5.2-NVFP4"
|
|
|
required_nodes = 4
|
|
required_nodes = 4
|
|
|
tensor_parallel_size = 4
|
|
tensor_parallel_size = 4
|
|
|
|
|
+pipeline_parallel_size = 1
|
|
|
distributed_backend = "mp"
|
|
distributed_backend = "mp"
|
|
|
|
|
|
|
|
-entrypoint = ["sglang", "serve"]
|
|
|
|
|
|
|
+# 410B NVFP4 model on 4× DGX Spark (GB10 Blackwell), 128 GB shared each.
|
|
|
|
|
+# TP=4 → ~103 GB/GPU for weights, ~25 GB for KV cache at 0.90 util.
|
|
|
|
|
|
|
|
args = [
|
|
args = [
|
|
|
- "--quantization", "modelopt_fp4",
|
|
|
|
|
- "--mem-fraction-static", "0.85",
|
|
|
|
|
- "--context-length", "32768",
|
|
|
|
|
- "--fp4-gemm-backend", "cutlass",
|
|
|
|
|
- "--kv-cache-dtype", "auto",
|
|
|
|
|
- "--reasoning-parser", "glm45",
|
|
|
|
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
|
|
+ "--gpu-memory-utilization", "0.90",
|
|
|
|
|
+ "--load-format", "safetensors",
|
|
|
|
|
+ "--enable-prefix-caching",
|
|
|
|
|
+ "--speculative-config", """
|
|
|
|
|
+'{"method":"mtp","num_speculative_tokens":5}'""",
|
|
|
"--tool-call-parser", "glm47",
|
|
"--tool-call-parser", "glm47",
|
|
|
- "--moe-runner-backend", "flashinfer_cutlass",
|
|
|
|
|
|
|
+ "--reasoning-parser", "glm45",
|
|
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
|
|
+ "--enable-expert-parallel",
|
|
|
"--trust-remote-code",
|
|
"--trust-remote-code",
|
|
|
- "--cuda-graph-max-bs", "16",
|
|
|
|
|
- "--cuda-graph-bs", "1", "2", "4", "8", "12", "16",
|
|
|
|
|
"--served-model-name", "glm52",
|
|
"--served-model-name", "glm52",
|
|
|
]
|
|
]
|
|
|
|
|
|
|
|
[env]
|
|
[env]
|
|
|
-TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"
|
|
|
|
|
|
|
+# Fast safetensors loading (GPU‑side decompress)
|
|
|
|
|
+SAFETENSORS_FAST_GPU = "1"
|
|
|
|
|
+# Allow long context lengths
|
|
|
|
|
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
|
|
|
|
|
+# Blackwell MoE kernels
|
|
|
|
|
+VLLM_USE_B12X_MOE = "1"
|