|
|
@@ -0,0 +1,35 @@
|
|
|
+engine = "vllm"
|
|
|
+name = "GLM-5.1 NVFP8"
|
|
|
+docker_image = "vllm/vllm-openai:glm51-cu130"
|
|
|
+model = "lukealonso/GLM-5.1-NVFP4"
|
|
|
+parent_model = "zai-org/GLM-5.1"
|
|
|
+
|
|
|
+required_nodes = 4
|
|
|
+distributed_backend = "mp"
|
|
|
+pipeline_parallel_size = 0
|
|
|
+
|
|
|
+args = [
|
|
|
+ "--speculative-config.method", "mtp",
|
|
|
+ "--speculative-config.num_speculative_tokens", "3",
|
|
|
+ "--tool-call-parser", "glm47",
|
|
|
+ "--reasoning-parser", "glm45",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--chat-template-content-format", "string",
|
|
|
+ "--attention-backend", "TRITON_MLA",
|
|
|
+ "--disable-flashinfer",
|
|
|
+ "--disable-custom-all-reduce",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
+ "--max-model-len", "32768",
|
|
|
+ "--max-num-batched-tokens", "6144",
|
|
|
+ "--max-num-seqs", "48",
|
|
|
+ "--gpu-memory-utilization", "0.88",
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--served-model-name", "glm51", "default",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+NCCL_DEBUG = "WARN"
|
|
|
+NCCL_IB_MERGE_NICS = "1"
|
|
|
+NCCL_MAX_NCHANNELS = "12"
|
