Browse Source

feat(recipe): enable DFlash speculative decoding for Laguna S 2.1

Add --speculative-config with draft model
poolside/Laguna-S-2.1-DFlash-NVFP4, 15 speculative tokens,
method dflash (quantization-matched NVFP4 draft).
Alexander Korolev 3 months ago
parent
commit
d14021070c
1 changed files with 21 additions and 17 deletions
  1. 21 17
      recipes/laguna-s2.1-nvfp4/recipe.toml

+ 21 - 17
recipes/laguna-s2.1-nvfp4/recipe.toml

@@ -8,9 +8,9 @@
 # 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
 # KV cache in FP8.
 #
-# Optional DFlash speculative decoding with draft model:
+# DFlash speculative decoding with quantization-matched draft model:
 #   poolside/Laguna-S-2.1-DFlash-NVFP4
-#   → 15 speculative tokens, requires --max-num-seqs 32.
+#   → 15 speculative tokens, method dflash.
 #
 # REFERENCE:
 #   https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
@@ -24,21 +24,25 @@ pipeline_parallel_size = 1
 
 entrypoint = ["vllm", "serve"]
 
-args = """
---gpu-memory-utilization 0.85
---max-model-len 262144
---load-format instanttensor
---enable-prefix-caching
---tool-call-parser poolside_v1
---reasoning-parser poolside_v1
---enable-auto-tool-choice
---trust-remote-code
---kv-cache-dtype fp8
---enable-flashinfer-autotune
---max-num-seqs 32
---override-generation-config '{"temperature":0.7,"top_p":0.95}'
---served-model-name laguna-s-2.1
-"""
+args = [
+    "--gpu-memory-utilization", "0.85",
+    "--max-model-len", "262144",
+    "--load-format", "instanttensor",
+    "--enable-prefix-caching",
+    "--tool-call-parser", "poolside_v1",
+    "--reasoning-parser", "poolside_v1",
+    "--enable-auto-tool-choice",
+    "--trust-remote-code",
+    "--kv-cache-dtype", "fp8",
+    "--enable-flashinfer-autotune",
+    "--max-num-seqs", "32",
+    "--override-generation-config", """
+'{"temperature":0.7,"top_p":0.95}'""",
+    # DFlash speculative decoding with quantization-matched draft
+    "--speculative-config", """
+'{"model":"poolside/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15,"method":"dflash"}'""",
+    "--served-model-name", "laguna-s-2.1", "laguna", "default",
+]
 
 [env]
 SAFETENSORS_FAST_GPU = "1"