|
|
@@ -0,0 +1,50 @@
|
|
|
+# Laguna S 2.1 NVFP4 — 1× DGX Spark (GB10), TP=1.
|
|
|
+#
|
|
|
+# Model: poolside/Laguna-S-2.1-NVFP4
|
|
|
+# → https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
|
|
|
+#
|
|
|
+# 117.6B total / 8.5B active MoE — NVFP4 quantized (~71 GB stored).
|
|
|
+# Fits on a single DGX Spark (128 GB unified memory).
|
|
|
+# 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
|
|
|
+# KV cache in FP8.
|
|
|
+#
|
|
|
+# Optional DFlash speculative decoding with draft model:
|
|
|
+# poolside/Laguna-S-2.1-DFlash-NVFP4
|
|
|
+# → 15 speculative tokens, requires --max-num-seqs 32.
|
|
|
+#
|
|
|
+# REFERENCE:
|
|
|
+# https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "Laguna S 2.1 NVFP4 (vLLM)"
|
|
|
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
|
|
|
+model = "poolside/Laguna-S-2.1-NVFP4"
|
|
|
+tensor_parallel_size = 1
|
|
|
+pipeline_parallel_size = 1
|
|
|
+
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+
|
|
|
+args = """
|
|
|
+--gpu-memory-utilization 0.85
|
|
|
+--max-model-len 262144
|
|
|
+--load-format instanttensor
|
|
|
+--enable-prefix-caching
|
|
|
+--tool-call-parser poolside_v1
|
|
|
+--reasoning-parser poolside_v1
|
|
|
+--enable-auto-tool-choice
|
|
|
+--trust-remote-code
|
|
|
+--kv-cache-dtype fp8
|
|
|
+--enable-flashinfer-autotune
|
|
|
+--max-num-seqs 32
|
|
|
+--override-generation-config '{"temperature":0.7,"top_p":0.95}'
|
|
|
+--served-model-name laguna-s-2.1
|
|
|
+"""
|
|
|
+
|
|
|
+[env]
|
|
|
+SAFETENSORS_FAST_GPU = "1"
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
|
|
|
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
|
|
|
+VLLM_FLOAT32_MATMUL_PRECISION = "high"
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+MAX_JOBS = "4"
|