launcher/recipes/laguna-s2.1-nvfp4/recipe.toml

# Laguna S 2.1 NVFP4 — 1× DGX Spark (GB10), TP=1.
#
# Model: poolside/Laguna-S-2.1-NVFP4
#   → https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
#
# 117.6B total / 8.5B active MoE — NVFP4 quantized (~71 GB stored).
# Fits on a single DGX Spark (128 GB unified memory).
# 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
# KV cache in FP8.
#
# DFlash speculative decoding with quantization-matched draft model:
#   poolside/Laguna-S-2.1-DFlash-NVFP4
#   → 15 speculative tokens, method dflash.
#
# REFERENCE:
#   https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
#
# DEPLOYMENT: tmux (host venv). The pre-built Docker image
# (ghcr.io/timothystewart6/vllm-gb10:v0.26.0-gb10.1) ships the wrong
# flashinfer version, so Docker is non-functional. The sustainable path
# is a host venv with the exact deps from the model card.
#
# ACTIVE (risky): vLLM 0.26.0 + flashinfer 0.6.16.dev20260729.
#   0.6.16 adds SM121 (GB10/DGX Spark) NVFP4 attention support
#   (flashinfer PR #3897) and Expert Parallelism, but introduces
#   API changes that vLLM 0.26.0 (which requires flashinfer 0.6.14)
#   may not fully track. Tested on dgx1.lan — monitor for MoE cubin
#   validation regressions ("reject incompatible output-scale cubins").
#
# SAFE (commented out): vLLM 0.25.1 + flashinfer 0.6.15.dev20260728.
#   Same minor as the original 0.6.15.dev20260712 — just 16 days of
#   bug fixes. No API changes. Drop-in replacement.
#
#   uv venv ~/venvs/vllm025 -p 3.12
#   uv pip install vllm==0.25.1 --torch-backend=cu130
#   uv pip install flashinfer-python==0.6.15.dev20260728 flashinfer-cubin==0.6.15.dev20260728 flashinfer-jit-cache==0.6.15.dev20260728 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
#   hf download poolside/Laguna-S-2.1-NVFP4
#   hf download poolside/Laguna-S-2.1-DFlash-NVFP4
#
# The `venv` field below is used by tmux to activate the venv before
# running the entrypoint.

engine = "vllm"
name = "Laguna S 2.1 NVFP4 (vLLM)"
docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.26.0-gb10.1"
model = "poolside/Laguna-S-2.1-NVFP4"
tensor_parallel_size = 1
pipeline_parallel_size = 1
deploy = "tmux"
venv = "~/venvs/vllm026"
venv_setup = """
    uv venv ~/venvs/vllm026 -p 3.12
    uv pip install vllm==0.26.0 --torch-backend=cu130
    uv pip install flashinfer-python==0.6.16.dev20260729 flashinfer-cubin==0.6.16.dev20260729 flashinfer-jit-cache==0.6.16.dev20260729 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
    hf download poolside/Laguna-S-2.1-NVFP4
    hf download poolside/Laguna-S-2.1-DFlash-NVFP4
"""

entrypoint = ["vllm", "serve"]

args = [
    "--gpu-memory-utilization", "0.85",
    "--max-model-len", "262144",
    "--load-format", "instanttensor",
    "--enable-prefix-caching",
    "--tool-call-parser", "poolside_v1",
    "--reasoning-parser", "poolside_v1",
    "--enable-auto-tool-choice",
    "--trust-remote-code",
    "--kv-cache-dtype", "fp8",
    "--enable-flashinfer-autotune",
    "--max-num-seqs", "32",
    "--override-generation-config", """
'{"temperature":0.7,"top_p":0.95}'""",
    # DFlash speculative decoding with quantization-matched draft
    "--speculative-config", """
'{"model":"poolside/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15,"method":"dflash"}'""",
    "--served-model-name", "laguna-s-2.1", "laguna", "default",
]

[env]
SAFETENSORS_FAST_GPU = "1"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
VLLM_FLOAT32_MATMUL_PRECISION = "high"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
MAX_JOBS = "4"