| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354 |
- # Laguna S 2.1 NVFP4 — 1× DGX Spark (GB10), TP=1.
- #
- # Model: poolside/Laguna-S-2.1-NVFP4
- # → https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
- #
- # 117.6B total / 8.5B active MoE — NVFP4 quantized (~71 GB stored).
- # Fits on a single DGX Spark (128 GB unified memory).
- # 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
- # KV cache in FP8.
- #
- # DFlash speculative decoding with quantization-matched draft model:
- # poolside/Laguna-S-2.1-DFlash-NVFP4
- # → 15 speculative tokens, method dflash.
- #
- # REFERENCE:
- # https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
- engine = "vllm"
- name = "Laguna S 2.1 NVFP4 (vLLM)"
- docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
- model = "poolside/Laguna-S-2.1-NVFP4"
- tensor_parallel_size = 1
- pipeline_parallel_size = 1
- entrypoint = ["vllm", "serve"]
- args = [
- "--gpu-memory-utilization", "0.85",
- "--max-model-len", "262144",
- "--load-format", "instanttensor",
- "--enable-prefix-caching",
- "--tool-call-parser", "poolside_v1",
- "--reasoning-parser", "poolside_v1",
- "--enable-auto-tool-choice",
- "--trust-remote-code",
- "--kv-cache-dtype", "fp8",
- "--enable-flashinfer-autotune",
- "--max-num-seqs", "32",
- "--override-generation-config", """
- '{"temperature":0.7,"top_p":0.95}'""",
- # DFlash speculative decoding with quantization-matched draft
- "--speculative-config", """
- '{"model":"poolside/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15,"method":"dflash"}'""",
- "--served-model-name", "laguna-s-2.1", "laguna", "default",
- ]
- [env]
- SAFETENSORS_FAST_GPU = "1"
- TORCH_CUDA_ARCH_LIST = "12.1a"
- FLASHINFER_CUDA_ARCH_LIST = "12.1a"
- VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
- VLLM_FLOAT32_MATMUL_PRECISION = "high"
- PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
- MAX_JOBS = "4"
|
