|
|
@@ -33,14 +33,14 @@ shm_size = "16g"
|
|
|
# full 166 GB checkpoint into a 121 GiB box — the root cause of the four
|
|
|
# "stalls" (b12x routing OOM) before this was caught.
|
|
|
# GB10 adaptation of the card's DGX-Spark-x2 block: the card ships
|
|
|
-# --max-model-len 500000 + --gpu-memory-utilization 0.87 + --max-num-seqs 4.
|
|
|
-# On GB10 (unified memory, not cgroup-contained — forum 386057) that first
|
|
|
-# attempt STALLED both nodes: 83 GB weights/rank + the VL tower + b12x + CUDA
|
|
|
-# graphs + a 500K-token KV reservation blew past the 119 GB host pool and
|
|
|
-# sshd could no longer fork (nodes ping but refuse SSH). Cut to GMU 0.75 +
|
|
|
-# 32768 ctx + 2 seqs — ample for tool-eval-bench scenarios, leaves host
|
|
|
-# headroom. Recovery from a stall is an HA plug cycle of switch.dgx3/dgx4
|
|
|
-# ONLY (never the shared fabric switch).
|
|
|
+# The card's DGX-Spark-x2 block: --max-model-len 500000 + GMU 0.87 +
|
|
|
+# --max-num-seqs 4. The earlier "GB10 adaptation" (GMU 0.75 / 32768 / 2 seqs)
|
|
|
+# was based on a misdiagnosis: the four stalls were the missing
|
|
|
+# distributed_backend (each node loading the FULL model), not memory pressure
|
|
|
+# from the card's values. With correct TP2 (~87 GiB/rank) the card config
|
|
|
+# boots fine (proven by the manual card-flow run: 42/42 routed, 92/100).
|
|
|
+# Recovery if a boot ever wedges: HA plug cycle of switch.dgx3/dgx4 ONLY
|
|
|
+# (never the shared fabric switch).
|
|
|
args = [
|
|
|
"--served-model-name", "glm53-fp4-local",
|
|
|
"--dtype", "bfloat16",
|
|
|
@@ -55,21 +55,21 @@ args = [
|
|
|
"--linear-backend", "b12x",
|
|
|
"--no-enable-flashinfer-autotune",
|
|
|
"--load-format", "b12x",
|
|
|
- "--max-model-len", "32768",
|
|
|
- "--max-num-seqs", "2",
|
|
|
+ "--max-model-len", "500000",
|
|
|
+ "--max-num-seqs", "4",
|
|
|
"--max-num-batched-tokens", "4096",
|
|
|
"--reasoning-parser", "glm45",
|
|
|
"--tool-call-parser", "glm47",
|
|
|
"--enable-auto-tool-choice",
|
|
|
- "--gpu-memory-utilization", "0.75",
|
|
|
+ "--gpu-memory-utilization", "0.87",
|
|
|
"--speculative-config", """
|
|
|
'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}'""",
|
|
|
]
|
|
|
|
|
|
# `serve --cold` overlays these over `args` (by flag name) for an extra-safe
|
|
|
-# first boot on a memory-tight node — lower GMU + smaller KV than the default.
|
|
|
-# The default `args` are already the GB10-safe cut of the card; `--cold` is the
|
|
|
-# cautious first launch, then drop the flag once the lane is proven to boot.
|
|
|
+# first boot on a memory-tight node — smaller KV than the card's default.
|
|
|
+# The default `args` are the card verbatim; `--cold` is the cautious first
|
|
|
+# launch, then drop the flag once the lane is proven to boot.
|
|
|
cold_args = [
|
|
|
"--max-model-len", "16384",
|
|
|
"--max-num-seqs", "1",
|
