|
|
@@ -24,6 +24,15 @@ parent_model = "zai-org/GLM-5.3-Flash"
|
|
|
port = 30000
|
|
|
required_nodes = 2
|
|
|
tensor_parallel_size = 2
|
|
|
+# GB10 adaptation of the card's DGX-Spark-x2 block: the card ships
|
|
|
+# --max-model-len 500000 + --gpu-memory-utilization 0.87 + --max-num-seqs 4.
|
|
|
+# On GB10 (unified memory, not cgroup-contained — forum 386057) that first
|
|
|
+# attempt STALLED both nodes: 83 GB weights/rank + the VL tower + b12x + CUDA
|
|
|
+# graphs + a 500K-token KV reservation blew past the 119 GB host pool and
|
|
|
+# sshd could no longer fork (nodes ping but refuse SSH). Cut to GMU 0.75 +
|
|
|
+# 32768 ctx + 2 seqs — ample for tool-eval-bench scenarios, leaves host
|
|
|
+# headroom. Recovery from a stall is an HA plug cycle of switch.dgx3/dgx4
|
|
|
+# ONLY (never the shared fabric switch).
|
|
|
args = [
|
|
|
"--served-model-name", "glm53-fp4-local",
|
|
|
"--dtype", "bfloat16",
|
|
|
@@ -38,13 +47,13 @@ args = [
|
|
|
"--linear-backend", "b12x",
|
|
|
"--no-enable-flashinfer-autotune",
|
|
|
"--load-format", "b12x",
|
|
|
- "--max-model-len", "500000",
|
|
|
- "--max-num-seqs", "4",
|
|
|
+ "--max-model-len", "32768",
|
|
|
+ "--max-num-seqs", "2",
|
|
|
"--max-num-batched-tokens", "4096",
|
|
|
"--reasoning-parser", "glm45",
|
|
|
"--tool-call-parser", "glm47",
|
|
|
"--enable-auto-tool-choice",
|
|
|
- "--gpu-memory-utilization", "0.87",
|
|
|
+ "--gpu-memory-utilization", "0.75",
|
|
|
"--speculative-config", """
|
|
|
'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}'""",
|
|
|
]
|
