|
|
@@ -48,13 +48,14 @@ distributed_backend = "mp"
|
|
|
shm_size = "64gb"
|
|
|
docker_image_dir = "glm52-nf3-hybrid"
|
|
|
|
|
|
-# entrypoint = ["vllm", "serve"]
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
|
|
|
args = [
|
|
|
# Quantization & KV cache
|
|
|
"--quantization", "modelopt_fp4",
|
|
|
"--kv-cache-dtype", "fp8",
|
|
|
- "--load-format", "fastsafetensors",
|
|
|
+ "--load-format", "auto",
|
|
|
+# "--load-format", "fastsafetensors",
|
|
|
# Context length (800K max-len, 877K KV pool)
|
|
|
"--max-model-len", "800000",
|
|
|
# Batch & memory
|