| 12345678910111213141516171819202122232425262728293031323334353637 |
- engine = "vllm"
- name = "Laguna M.1 NVFP4 (vLLM)"
- docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
- model = "poolside/Laguna-M.1-NVFP4"
- required_nodes = 2
- tensor_parallel_size = 2
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- # 225B total / 23B active MoE — NVFP4 quantized (~131B stored).
- # 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
- # TP=2 → ~66 GB/node weights, room for 262K KV cache.
- # Quantization auto-detected from model config.
- entrypoint = ["vllm", "serve"]
- args = """
- --gpu-memory-utilization 0.8
- --max-model-len 262144
- --load-format instanttensor
- --enable-prefix-caching
- --tool-call-parser poolside_v1
- --reasoning-parser poolside_v1
- --enable-auto-tool-choice
- --trust-remote-code
- --kv-cache-dtype fp8
- --enable-flashinfer-autotune
- --served-model-name laguna
- """
- [env]
- SAFETENSORS_FAST_GPU = "1"
- TORCH_CUDA_ARCH_LIST = "12.1a"
- FLASHINFER_CUDA_ARCH_LIST = "12.1a"
- VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
- VLLM_FLOAT32_MATMUL_PRECISION = "high"
- PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
