| 1234567891011121314151617181920212223242526272829303132333435 |
- engine = "vllm"
- name = "GLM-5.2 NVFP4 (vLLM)"
- docker_image = "vllm/vllm-openai:glm52"
- model = "Mapika/GLM-5.2-NVFP4"
- required_nodes = 4
- tensor_parallel_size = 4
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- # 410B NVFP4 model on 4× DGX Spark (GB10 Blackwell), 128 GB shared each.
- # TP=4 → ~103 GB/GPU for weights, ~25 GB for KV cache at 0.90 util.
- #entrypoint = ["vllm", "serve"]
- args = [
- # "--kv-cache-dtype", "fp8",
- "--gpu-memory-utilization", "0.85",
- "--load-format", "safetensors",
- "--enable-prefix-caching",
- # "--speculative-config", """
- #'{"method":"mtp","num_speculative_tokens":5}'""",
- "--tool-call-parser", "glm47",
- "--reasoning-parser", "glm45",
- "--enable-auto-tool-choice",
- # "--enable-expert-parallel",
- "--trust-remote-code",
- "--served-model-name", "glm52",
- ]
- [env]
- # Fast safetensors loading (GPU‑side decompress)
- SAFETENSORS_FAST_GPU = "1"
- # Allow long context lengths
- VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
- # Blackwell MoE kernels
- #VLLM_USE_B12X_MOE = "1"
|
