|
@@ -8,9 +8,9 @@
|
|
|
# 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
|
|
# 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
|
|
|
# KV cache in FP8.
|
|
# KV cache in FP8.
|
|
|
#
|
|
#
|
|
|
-# Optional DFlash speculative decoding with draft model:
|
|
|
|
|
|
|
+# DFlash speculative decoding with quantization-matched draft model:
|
|
|
# poolside/Laguna-S-2.1-DFlash-NVFP4
|
|
# poolside/Laguna-S-2.1-DFlash-NVFP4
|
|
|
-# → 15 speculative tokens, requires --max-num-seqs 32.
|
|
|
|
|
|
|
+# → 15 speculative tokens, method dflash.
|
|
|
#
|
|
#
|
|
|
# REFERENCE:
|
|
# REFERENCE:
|
|
|
# https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
|
|
# https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
|
|
@@ -24,21 +24,25 @@ pipeline_parallel_size = 1
|
|
|
|
|
|
|
|
entrypoint = ["vllm", "serve"]
|
|
entrypoint = ["vllm", "serve"]
|
|
|
|
|
|
|
|
-args = """
|
|
|
|
|
---gpu-memory-utilization 0.85
|
|
|
|
|
---max-model-len 262144
|
|
|
|
|
---load-format instanttensor
|
|
|
|
|
---enable-prefix-caching
|
|
|
|
|
---tool-call-parser poolside_v1
|
|
|
|
|
---reasoning-parser poolside_v1
|
|
|
|
|
---enable-auto-tool-choice
|
|
|
|
|
---trust-remote-code
|
|
|
|
|
---kv-cache-dtype fp8
|
|
|
|
|
---enable-flashinfer-autotune
|
|
|
|
|
---max-num-seqs 32
|
|
|
|
|
---override-generation-config '{"temperature":0.7,"top_p":0.95}'
|
|
|
|
|
---served-model-name laguna-s-2.1
|
|
|
|
|
-"""
|
|
|
|
|
|
|
+args = [
|
|
|
|
|
+ "--gpu-memory-utilization", "0.85",
|
|
|
|
|
+ "--max-model-len", "262144",
|
|
|
|
|
+ "--load-format", "instanttensor",
|
|
|
|
|
+ "--enable-prefix-caching",
|
|
|
|
|
+ "--tool-call-parser", "poolside_v1",
|
|
|
|
|
+ "--reasoning-parser", "poolside_v1",
|
|
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
|
|
+ "--trust-remote-code",
|
|
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
|
|
+ "--enable-flashinfer-autotune",
|
|
|
|
|
+ "--max-num-seqs", "32",
|
|
|
|
|
+ "--override-generation-config", """
|
|
|
|
|
+'{"temperature":0.7,"top_p":0.95}'""",
|
|
|
|
|
+ # DFlash speculative decoding with quantization-matched draft
|
|
|
|
|
+ "--speculative-config", """
|
|
|
|
|
+'{"model":"poolside/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15,"method":"dflash"}'""",
|
|
|
|
|
+ "--served-model-name", "laguna-s-2.1", "laguna", "default",
|
|
|
|
|
+]
|
|
|
|
|
|
|
|
[env]
|
|
[env]
|
|
|
SAFETENSORS_FAST_GPU = "1"
|
|
SAFETENSORS_FAST_GPU = "1"
|