|
|
@@ -50,10 +50,12 @@ args = [
|
|
|
# GMU is ONLY a startup guardrail: 0.88 = 107.0 GiB, below any
|
|
|
# realistic free-memory dip on GB10's 121.63 GiB (#48140).
|
|
|
"--gpu-memory-utilization", "0.88",
|
|
|
- # no KV pin: ~93 GiB weights/rank leaves ~10 GiB — let the engine
|
|
|
- # auto-size the pool (a fixed pin overshoots the tighter budget)
|
|
|
+ # no KV pin: ~93 GiB weights/rank leaves only ~2.4 GiB KV at GMU 0.88
|
|
|
+ # (the re-cut is heavier than the old 186 GB cut) — the engine needs
|
|
|
+ # 3.73 GiB for one 524288-token request, so cap the context at 256K
|
|
|
+ # (needs ~1.9 GiB; bench contexts are far below that)
|
|
|
"--block-size", "256",
|
|
|
- "--max-model-len", "524288",
|
|
|
+ "--max-model-len", "262144",
|
|
|
"--max-num-seqs", "16",
|
|
|
"--max-num-batched-tokens", "4096",
|
|
|
"--enable-chunked-prefill",
|
