|
|
@@ -48,7 +48,12 @@ args = [
|
|
|
"--moe-a2a-backend", "none",
|
|
|
# Memory / scheduling (128 GB unified per node)
|
|
|
"--mem-fraction-static", "0.85",
|
|
|
- "--context-length", "262144",
|
|
|
+ # KV pool at a14 defaults: 2,197,568 tokens cluster-wide (TP8, DFlash
|
|
|
+ # accounted) — a 512K request uses ~23% of it, so the context limit is a
|
|
|
+ # declaration, not a memory wall. The model is natively 1M
|
|
|
+ # (max_position_embeddings, no rope_scaling; 70 layers, mostly SWA-128).
|
|
|
+ # 1M is also pool-feasible (45% per request) if ever needed.
|
|
|
+ "--context-length", "524288",
|
|
|
"--max-running-requests", "8",
|
|
|
"--chunked-prefill-size", "8192",
|
|
|
"--page-size", "64",
|
