|
|
@@ -0,0 +1,37 @@
|
|
|
+engine = "vllm"
|
|
|
+name = "Laguna M.1 NVFP4 (vLLM)"
|
|
|
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
|
|
|
+model = "poolside/Laguna-M.1-NVFP4"
|
|
|
+required_nodes = 2
|
|
|
+tensor_parallel_size = 2
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+
|
|
|
+# 225B total / 23B active MoE — NVFP4 quantized (~131B stored).
|
|
|
+# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
|
|
|
+# TP=2 → ~66 GB/node weights, room for 262K KV cache.
|
|
|
+# Quantization auto-detected from model config.
|
|
|
+
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+
|
|
|
+args = """
|
|
|
+--gpu-memory-utilization 0.8
|
|
|
+--max-model-len 262144
|
|
|
+--load-format instanttensor
|
|
|
+--enable-prefix-caching
|
|
|
+--tool-call-parser poolside_v1
|
|
|
+--reasoning-parser poolside_v1
|
|
|
+--enable-auto-tool-choice
|
|
|
+--trust-remote-code
|
|
|
+--kv-cache-dtype fp8
|
|
|
+--enable-flashinfer-autotune
|
|
|
+--served-model-name laguna
|
|
|
+"""
|
|
|
+
|
|
|
+[env]
|
|
|
+SAFETENSORS_FAST_GPU = "1"
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
|
|
|
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
|
|
|
+VLLM_FLOAT32_MATMUL_PRECISION = "high"
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
