|
@@ -49,14 +49,14 @@ args = [
|
|
|
"--load-format", "auto",
|
|
"--load-format", "auto",
|
|
|
"--safetensors-load-strategy", "eager",
|
|
"--safetensors-load-strategy", "eager",
|
|
|
"--gpu-memory-utilization", "0.86",
|
|
"--gpu-memory-utilization", "0.86",
|
|
|
- "--max-model-len", "163840",
|
|
|
|
|
|
|
+ "--max-model-len", "524288",
|
|
|
"--kv-cache-dtype", "fp8_e4m3",
|
|
"--kv-cache-dtype", "fp8_e4m3",
|
|
|
"--block-size", "2304",
|
|
"--block-size", "2304",
|
|
|
- "--kv-cache-memory", "8589934592",
|
|
|
|
|
|
|
+ "--kv-cache-memory", "10737418240",
|
|
|
"--trust-remote-code",
|
|
"--trust-remote-code",
|
|
|
"--enable-prefix-caching",
|
|
"--enable-prefix-caching",
|
|
|
- "--max-num-batched-tokens", "8192",
|
|
|
|
|
- "--max-num-seqs", "16",
|
|
|
|
|
|
|
+ "--max-num-batched-tokens", "6144",
|
|
|
|
|
+ "--max-num-seqs", "6",
|
|
|
"--long-prefill-token-threshold", "2304",
|
|
"--long-prefill-token-threshold", "2304",
|
|
|
# flashinfer_cutlass — native NVFP4 MoE on the checkpoint's per-projection
|
|
# flashinfer_cutlass — native NVFP4 MoE on the checkpoint's per-projection
|
|
|
# input scales (marlin branch of their entrypoint not used). NO explicit
|
|
# input scales (marlin branch of their entrypoint not used). NO explicit
|