|
|
@@ -0,0 +1,27 @@
|
|
|
+engine = "sglang"
|
|
|
+name = "GLM-5.2 NVFP4"
|
|
|
+docker_image = "scitrera/dgx-spark-sglang-mm:v0"
|
|
|
+model = "Mapika/GLM-5.2-NVFP4"
|
|
|
+required_nodes = 4
|
|
|
+tensor_parallel_size = 4
|
|
|
+distributed_backend = "mp"
|
|
|
+
|
|
|
+entrypoint = ["sglang", "serve"]
|
|
|
+
|
|
|
+args = [
|
|
|
+ "--quantization", "modelopt_fp4",
|
|
|
+ "--mem-fraction-static", "0.85",
|
|
|
+ "--context-length", "32768",
|
|
|
+ "--fp4-gemm-backend", "cutlass",
|
|
|
+ "--kv-cache-dtype", "auto",
|
|
|
+ "--reasoning-parser", "glm45",
|
|
|
+ "--tool-call-parser", "glm47",
|
|
|
+ "--moe-runner-backend", "flashinfer_cutlass",
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--cuda-graph-max-bs", "16",
|
|
|
+ "--cuda-graph-bs", "1", "2", "4", "8", "12", "16",
|
|
|
+ "--served-model-name", "glm52",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"
|