| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475 |
- # Laguna S 2.1 NVFP4 — 1× DGX Spark (GB10), TP=1.
- #
- # Model: poolside/Laguna-S-2.1-NVFP4
- # → https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
- #
- # 117.6B total / 8.5B active MoE — NVFP4 quantized (~71 GB stored).
- # Fits on a single DGX Spark (128 GB unified memory).
- # 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
- # KV cache in FP8.
- #
- # DFlash speculative decoding with quantization-matched draft model:
- # poolside/Laguna-S-2.1-DFlash-NVFP4
- # → 15 speculative tokens, method dflash.
- #
- # REFERENCE:
- # https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
- #
- # DEPLOYMENT: tmux (host venv). The pre-built Docker image
- # (ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1) ships the wrong
- # flashinfer version, so Docker is non-functional. The sustainable path
- # is a host venv with the exact deps from the model card:
- #
- # uv venv ~/venvs/vllm025 -p 3.12
- # uv pip install vllm==0.25.1 --torch-backend=cu130
- # uv pip install flashinfer-python==0.6.15.dev20260712 \
- # flashinfer-cubin==0.6.15.dev20260712 \
- # flashinfer-jit-cache==0.6.15.dev20260712 \
- # --extra-index-url https://flashinfer.ai/whl/nightly/ \
- # --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
- # hf download poolside/Laguna-S-2.1-NVFP4
- # hf download poolside/Laguna-S-2.1-DFlash-NVFP4
- #
- # The `venv` field below is used by tmux to activate the venv before
- # running the entrypoint.
- engine = "vllm"
- name = "Laguna S 2.1 NVFP4 (vLLM)"
- docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
- model = "poolside/Laguna-S-2.1-NVFP4"
- tensor_parallel_size = 1
- pipeline_parallel_size = 1
- deploy = "tmux"
- venv = "~/venvs/vllm025"
- entrypoint = ["vllm", "serve"]
- args = [
- "--gpu-memory-utilization", "0.85",
- "--max-model-len", "262144",
- "--load-format", "instanttensor",
- "--enable-prefix-caching",
- "--tool-call-parser", "poolside_v1",
- "--reasoning-parser", "poolside_v1",
- "--enable-auto-tool-choice",
- "--trust-remote-code",
- "--kv-cache-dtype", "fp8",
- "--enable-flashinfer-autotune",
- "--max-num-seqs", "32",
- "--override-generation-config", """
- '{"temperature":0.7,"top_p":0.95}'""",
- # DFlash speculative decoding with quantization-matched draft
- "--speculative-config", """
- '{"model":"poolside/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15,"method":"dflash"}'""",
- "--served-model-name", "laguna-s-2.1", "laguna", "default",
- ]
- [env]
- SAFETENSORS_FAST_GPU = "1"
- TORCH_CUDA_ARCH_LIST = "12.1a"
- FLASHINFER_CUDA_ARCH_LIST = "12.1a"
- CUTE_DSL_ARCH = "sm_121a"
- VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
- VLLM_FLOAT32_MATMUL_PRECISION = "high"
- PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
- MAX_JOBS = "4"
|
