| 12345678910111213141516171819202122232425262728293031323334 |
- engine = "vllm"
- name = "GLM-5.1 NVFP8"
- docker_image = "vllm/vllm-openai:glm51-cu130"
- model = "lukealonso/GLM-5.1-NVFP4"
- port = 30000
- required_nodes = 4 # 0 = любое, иначе строгое совпадение
- pipeline_parallel_size = 0 # 0 = авто (равно nnodes), иначе кастомное значение
- args = [
- "--speculative-config.method", "mtp",
- "--speculative-config.num_speculative_tokens", "3",
- "--tool-call-parser", "glm47",
- "--reasoning-parser", "glm45",
- "--enable-auto-tool-choice",
- "--chat-template-content-format", "string",
- "--attention-backend", "TRITON_MLA",
- "--disable-flashinfer",
- "--disable-custom-all-reduce",
- "--enable-prefix-caching",
- "--enable-chunked-prefill",
- "--kv-cache-dtype", "fp8",
- "--max-model-len", "32768",
- "--max-num-batched-tokens", "6144",
- "--max-num-seqs", "48",
- "--gpu-memory-utilization", "0.88",
- "--trust-remote-code",
- "--host", "0.0.0.0"
- ]
- [env]
- NCCL_DEBUG = "WARN"
- NCCL_IB_MERGE_NICS = "1"
- NCCL_MAX_NCHANNELS = "12"
|
