launcher/recipes/laguna-s2.1-nvfp4/recipe.toml

# Laguna S 2.1 NVFP4 — 1× DGX Spark (GB10), TP=1.
#
# Model: poolside/Laguna-S-2.1-NVFP4
#   → https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
#
# 117.6B total / 8.5B active MoE — NVFP4 quantized (~71 GB stored).
# Fits on a single DGX Spark (128 GB unified memory).
# 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
# KV cache in FP8.
#
# DFlash speculative decoding with quantization-matched draft model:
#   poolside/Laguna-S-2.1-DFlash-NVFP4
#   → 15 speculative tokens, method dflash.
#
# REFERENCE:
#   https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
#
# DEPLOYMENT: tmux (host venv). The pre-built Docker image
# (ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1) ships the wrong
# flashinfer version, so Docker is non-functional. The sustainable path
# is a host venv with the exact deps from the model card:
#
#   uv venv ~/venvs/vllm025 -p 3.12
#   uv pip install vllm==0.25.1 --torch-backend=cu130
#   uv pip install flashinfer-python==0.6.15.dev20260712 \
#     flashinfer-cubin==0.6.15.dev20260712 \
#     flashinfer-jit-cache==0.6.15.dev20260712 \
#     --extra-index-url https://flashinfer.ai/whl/nightly/ \
#     --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
#   hf download poolside/Laguna-S-2.1-NVFP4
#   hf download poolside/Laguna-S-2.1-DFlash-NVFP4
#
# The `venv` field below is used by tmux to activate the venv before
# running the entrypoint.

engine = "vllm"
name = "Laguna S 2.1 NVFP4 (vLLM)"
docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
model = "poolside/Laguna-S-2.1-NVFP4"
tensor_parallel_size = 1
pipeline_parallel_size = 1
deploy = "tmux"
venv = "~/venvs/vllm025"
venv_setup = """
    uv venv ~/venvs/vllm025 -p 3.12
    uv pip install vllm==0.25.1 --torch-backend=cu130
    uv pip install flashinfer-python==0.6.15.dev20260712 flashinfer-cubin==0.6.15.dev20260712 flashinfer-jit-cache==0.6.15.dev20260712 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
    hf download poolside/Laguna-S-2.1-NVFP4
    hf download poolside/Laguna-S-2.1-DFlash-NVFP4
"""

entrypoint = ["vllm", "serve"]

args = [
    "--gpu-memory-utilization", "0.85",
    "--max-model-len", "262144",
    "--load-format", "instanttensor",
    "--enable-prefix-caching",
    "--tool-call-parser", "poolside_v1",
    "--reasoning-parser", "poolside_v1",
    "--enable-auto-tool-choice",
    "--trust-remote-code",
    "--kv-cache-dtype", "fp8",
    "--enable-flashinfer-autotune",
    "--max-num-seqs", "32",
    "--override-generation-config", """
'{"temperature":0.7,"top_p":0.95}'""",
    # DFlash speculative decoding with quantization-matched draft
    "--speculative-config", """
'{"model":"poolside/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15,"method":"dflash"}'""",
    "--served-model-name", "laguna-s-2.1", "laguna", "default",
]

[env]
SAFETENSORS_FAST_GPU = "1"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
VLLM_FLOAT32_MATMUL_PRECISION = "high"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
MAX_JOBS = "4"