| 1234567891011121314151617181920212223242526272829303132333435 |
- engine = "vllm"
- name = "GLM-5.1 NVFP8"
- docker_image = "vllm/vllm-openai:glm51-cu130"
- model = "lukealonso/GLM-5.1-NVFP4"
- parent_model = "zai-org/GLM-5.1"
- required_nodes = 4
- distributed_backend = "mp"
- pipeline_parallel_size = 0
- args = [
- "--speculative-config.method", "mtp",
- "--speculative-config.num_speculative_tokens", "3",
- "--tool-call-parser", "glm47",
- "--reasoning-parser", "glm45",
- "--enable-auto-tool-choice",
- "--chat-template-content-format", "string",
- "--attention-backend", "TRITON_MLA",
- "--disable-flashinfer",
- "--disable-custom-all-reduce",
- "--enable-prefix-caching",
- "--enable-chunked-prefill",
- "--kv-cache-dtype", "fp8",
- "--max-model-len", "32768",
- "--max-num-batched-tokens", "6144",
- "--max-num-seqs", "48",
- "--gpu-memory-utilization", "0.88",
- "--trust-remote-code",
- "--served-model-name", "glm51", "default",
- ]
- [env]
- NCCL_DEBUG = "WARN"
- NCCL_IB_MERGE_NICS = "1"
- NCCL_MAX_NCHANNELS = "12"
|
