recipes: laguna-s2.1-nvfp4

recipes/laguna-s2.1-nvfp4/recipe.toml

@@ -0,0 +1,87 @@
+# Laguna S 2.1 NVFP4 — 1× DGX Spark (GB10), TP=1.
+#
+# Model: poolside/Laguna-S-2.1-NVFP4
+#   → https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
+#
+# 117.6B total / 8.5B active MoE — NVFP4 quantized (~71 GB stored).
+# Fits on a single DGX Spark (128 GB unified memory).
+# 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
+# KV cache in FP8.
+#
+# DFlash speculative decoding with quantization-matched draft model:
+#   poolside/Laguna-S-2.1-DFlash-NVFP4
+#   → 15 speculative tokens, method dflash.
+#
+# REFERENCE:
+#   https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
+#
+# DEPLOYMENT: tmux (host venv). The pre-built Docker image
+# (ghcr.io/timothystewart6/vllm-gb10:v0.26.0-gb10.1) ships the wrong
+# flashinfer version, so Docker is non-functional. The sustainable path
+# is a host venv with the exact deps from the model card.
+#
+# ACTIVE (risky): vLLM 0.26.0 + flashinfer 0.6.16.dev20260729.
+#   0.6.16 adds SM121 (GB10/DGX Spark) NVFP4 attention support
+#   (flashinfer PR #3897) and Expert Parallelism, but introduces
+#   API changes that vLLM 0.26.0 (which requires flashinfer 0.6.14)
+#   may not fully track. Tested on dgx1.lan — monitor for MoE cubin
+#   validation regressions ("reject incompatible output-scale cubins").
+#
+# SAFE (commented out): vLLM 0.25.1 + flashinfer 0.6.15.dev20260728.
+#   Same minor as the original 0.6.15.dev20260712 — just 16 days of
+#   bug fixes. No API changes. Drop-in replacement.
+#
+#   uv venv ~/venvs/vllm025 -p 3.12
+#   uv pip install vllm==0.25.1 --torch-backend=cu130
+#   uv pip install flashinfer-python==0.6.15.dev20260728 flashinfer-cubin==0.6.15.dev20260728 flashinfer-jit-cache==0.6.15.dev20260728 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
+#   hf download poolside/Laguna-S-2.1-NVFP4
+#   hf download poolside/Laguna-S-2.1-DFlash-NVFP4
+#
+# The `venv` field below is used by tmux to activate the venv before
+# running the entrypoint.
+
+engine = "vllm"
+name = "Laguna S 2.1 NVFP4 (vLLM)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.26.0-gb10.1"
+model = "poolside/Laguna-S-2.1-NVFP4"
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+deploy = "tmux"
+venv = "~/venvs/vllm026"
+venv_setup = """
+    uv venv ~/venvs/vllm026 -p 3.12
+    uv pip install vllm==0.26.0 --torch-backend=cu130
+    uv pip install flashinfer-python==0.6.16.dev20260729 flashinfer-cubin==0.6.16.dev20260729 flashinfer-jit-cache==0.6.16.dev20260729 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
+    hf download poolside/Laguna-S-2.1-NVFP4
+    hf download poolside/Laguna-S-2.1-DFlash-NVFP4
+"""
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--gpu-memory-utilization", "0.85",
+    "--max-model-len", "262144",
+    "--load-format", "instanttensor",
+    "--enable-prefix-caching",
+    "--tool-call-parser", "poolside_v1",
+    "--reasoning-parser", "poolside_v1",
+    "--enable-auto-tool-choice",
+    "--trust-remote-code",
+    "--kv-cache-dtype", "fp8",
+    "--enable-flashinfer-autotune",
+    "--max-num-seqs", "32",
+    "--override-generation-config", "'{\"temperature\":0.7,\"top_p\":0.95}'",
+    # DFlash speculative decoding with quantization-matched draft
+    "--speculative-config", "'{\"model\":\"poolside/Laguna-S-2.1-DFlash-NVFP4\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}'",
+    "--served-model-name", "laguna-s-2.1", "laguna", "default",
+]
+
+[env]
+SAFETENSORS_FAST_GPU = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_FLOAT32_MATMUL_PRECISION = "high"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+MAX_JOBS = "4"