|
|
@@ -40,8 +40,10 @@ args = [
|
|
|
"--linear-backend", "b12x",
|
|
|
"--kv-cache-dtype", "fp8",
|
|
|
"--block-size", "256",
|
|
|
- "--gpu-memory-utilization", "0.80",
|
|
|
- "--kv-cache-memory-bytes", "12884901888",
|
|
|
+ # his verified 2x (TP2) values: 0.75 fails KV alloc, 0.82 leaves one rank
|
|
|
+ # short, 0.85 works; KV pin left OFF (a pin caps the pool ~26 GB/rank and
|
|
|
+ # hides the layout headroom)
|
|
|
+ "--gpu-memory-utilization", "0.85",
|
|
|
"--max-model-len", "393216",
|
|
|
"--max-num-seqs", "8",
|
|
|
"--max-num-batched-tokens", "8192",
|