|
|
@@ -26,19 +26,20 @@
|
|
|
|
|
|
engine = "vllm"
|
|
|
name = "GLM-5.2 Hybrid FP8-MXFP4 (vLLM)"
|
|
|
-docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.2"
|
|
|
+docker_image = "sparkrun-vllm-ds4-gb10:glm52"
|
|
|
model = "aidendle94/GLM-5.2-Hybrid-FP8-MXFP4"
|
|
|
required_nodes = 4
|
|
|
tensor_parallel_size = 4
|
|
|
pipeline_parallel_size = 1
|
|
|
distributed_backend = "mp"
|
|
|
shm_size = "10gb"
|
|
|
+docker_image_dir = "glm52-int4-int8"
|
|
|
|
|
|
entrypoint = ["vllm", "serve"]
|
|
|
|
|
|
args = [
|
|
|
# Model name
|
|
|
- "--served-model-name", "glm-5.2",
|
|
|
+ "--served-model-name", "glm52",
|
|
|
# General
|
|
|
"--trust-remote-code",
|
|
|
"--reasoning-parser", "glm45",
|
|
|
@@ -63,7 +64,7 @@ args = [
|
|
|
"--gpu-memory-utilization", "0.89",
|
|
|
# Speculative decode — MTP k=4 with adaptive depths
|
|
|
"--speculative-config", """
|
|
|
-'{"model":"aidendle94/GLM-5.2-Hybrid-FP8-MXFP4/mtp-draft","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
|
|
|
+'{"model":"/root/.cache/huggingface/links/mtp-draft","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
|
|
|
# Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
|
|
|
"--compilation-config", """
|
|
|
'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
|