|
|
@@ -21,16 +21,20 @@ args = [
|
|
|
"--max-num-batched-tokens", "8192",
|
|
|
"--enable-chunked-prefill",
|
|
|
"--gpu-memory-utilization", "0.85",
|
|
|
- "--compilation-config", '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}',
|
|
|
+ "--compilation-config", """
|
|
|
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
|
|
|
"--enable-prefix-caching",
|
|
|
"--load-format", "safetensors",
|
|
|
- "--speculative-config", '{"method":"mtp","num_speculative_tokens":2}',
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"mtp","num_speculative_tokens":2}'""",
|
|
|
"--tokenizer-mode", "deepseek_v4",
|
|
|
"--tool-call-parser", "deepseek_v4",
|
|
|
"--enable-auto-tool-choice",
|
|
|
"--reasoning-parser", "deepseek_v4",
|
|
|
- "--reasoning-config", '{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}',
|
|
|
- "--default-chat-template-kwargs", '{"thinking":true,"preserve_thinking":true}',
|
|
|
+ "--reasoning-config", """
|
|
|
+'{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
|
|
|
+ "--default-chat-template-kwargs", """
|
|
|
+'{"thinking":true,"preserve_thinking":true}'""",
|
|
|
"--default-chat-template-kwargs.reasoning_effort", "high",
|
|
|
"--enable-flashinfer-autotune",
|
|
|
"--trust-remote-code",
|