|
@@ -41,8 +41,13 @@ entrypoint = ["/opt/nvidia/nvidia_entrypoint.sh", "sglang", "serve"]
|
|
|
args = [
|
|
args = [
|
|
|
# Stable gateway alias (same as the vLLM lane).
|
|
# Stable gateway alias (same as the vLLM lane).
|
|
|
"--served-model-name", "mistral-large-4",
|
|
"--served-model-name", "mistral-large-4",
|
|
|
- "--mem-fraction-static", "0.85",
|
|
|
|
|
|
|
+ # Boot-7 KV pool was 288K tokens (needs 524K): after weights 70.1 GiB only
|
|
|
|
|
+ # 15.9 GB reached the pool — ~17 GB eaten by prefill cuda graphs (bs list
|
|
|
|
|
+ # up to 2048), DSpark graphs and workspaces. Disable the prefill graphs
|
|
|
|
|
+ # (the KV/token is fine, ~51 KB) and raise the fraction to 0.88.
|
|
|
|
|
+ "--mem-fraction-static", "0.88",
|
|
|
"--context-length", "524288",
|
|
"--context-length", "524288",
|
|
|
|
|
+ "--disable-prefill-cuda-graph",
|
|
|
"--chunked-prefill-size", "2048",
|
|
"--chunked-prefill-size", "2048",
|
|
|
# The embedded DSpark draft (do NOT pass --speculative-draft-model-path —
|
|
# The embedded DSpark draft (do NOT pass --speculative-draft-model-path —
|
|
|
# the checkpoint carries it).
|
|
# the checkpoint carries it).
|
