|
|
@@ -65,9 +65,10 @@ args = [
|
|
|
"--max-model-len", "524288",
|
|
|
"--max-num-seqs", "16",
|
|
|
"--max-num-batched-tokens", "8192",
|
|
|
- # SparkRing tp4 contract: deterministic 20 GiB KV pin (not auto-sizing),
|
|
|
- # bounce-buffer loader, c1/c8/c16 concurrency-friendly scheduler bounds.
|
|
|
- "--kv-cache-memory-bytes", "21474836480",
|
|
|
+ # SparkRing tp4 contract, concurrency-friendly scheduler bounds +
|
|
|
+ # bounce-buffer loader. The 20 GiB KV pin is NOT adopted: on GB10's UMA
|
|
|
+ # with GMU 0.85 the auto-sizing profiler already claims ~54 GiB free
|
|
|
+ # (8.5M-token pool in boot logs), so pinning 20 GiB would strand ~34 GiB.
|
|
|
"--model-loader-extra-config", """
|
|
|
'{"read_mode":"bounce","io_threads":8}'""",
|
|
|
"--enable-chunked-prefill",
|
