|
|
@@ -102,7 +102,7 @@ args = [
|
|
|
"--gpu-memory-utilization", "0.90",
|
|
|
# Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
|
|
|
# 92.72 GiB weights/node; pinning also keeps warmup off the swap path
|
|
|
- "--kv-cache-memory-bytes", "10737418240",
|
|
|
+ "--kv-cache-memory-bytes", "17179869184", # 16 GiB: 4x NVFP4 has the headroom (47.1 GiB weights/node vs 108 usable); 2x stays 10
|
|
|
"--decode-context-parallel-size", "4",
|
|
|
"--cp-kv-cache-interleave-size", "4",
|
|
|
# No --kv-cache-dtype: let the r12 fork pick its default (the explicit
|