|
|
@@ -58,6 +58,17 @@ args = [
|
|
|
'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}'""",
|
|
|
]
|
|
|
|
|
|
+# `serve --cold` overlays these over `args` (by flag name) for an extra-safe
|
|
|
+# first boot on a memory-tight node — lower GMU + smaller KV than the default.
|
|
|
+# The default `args` are already the GB10-safe cut of the card; `--cold` is the
|
|
|
+# cautious first launch, then drop the flag once the lane is proven to boot.
|
|
|
+cold_args = [
|
|
|
+ "--max-model-len", "16384",
|
|
|
+ "--max-num-seqs", "1",
|
|
|
+ "--max-num-batched-tokens", "2048",
|
|
|
+ "--gpu-memory-utilization", "0.60",
|
|
|
+]
|
|
|
+
|
|
|
[env]
|
|
|
# b12x MoE/FP4 path (card "both nodes" block)
|
|
|
CUTE_DSL_ARCH = "sm_121a"
|