瀏覽代碼

feat(recipe): add Laguna S 2.1 NVFP4 (single-node DGX Spark)

117.6B total / 8.5B active MoE, NVFP4 quantized (~71 GB), fits on one
DGX Spark (128 GB). TP=1, 262K context, KV cache FP8.
- Model: poolside/Laguna-S-2.1-NVFP4
- Image: ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1
- DFlash-ready (draft model poolside/Laguna-S-2.1-DFlash-NVFP4)
- --max-num-seqs 32 (DFlash requirement)
- MAX_JOBS=4 for FlashInfer compilation
- Override generation config for NVFP4 quality
Alexander Korolev 3 月之前
父節點
當前提交
e844cb481a
共有 1 個文件被更改,包括 50 次插入 和 0 次删除
  1. 50 0
      recipes/laguna-s2.1-nvfp4/recipe.toml

+ 50 - 0
recipes/laguna-s2.1-nvfp4/recipe.toml

@@ -0,0 +1,50 @@
+# Laguna S 2.1 NVFP4 — 1× DGX Spark (GB10), TP=1.
+#
+# Model: poolside/Laguna-S-2.1-NVFP4
+#   → https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
+#
+# 117.6B total / 8.5B active MoE — NVFP4 quantized (~71 GB stored).
+# Fits on a single DGX Spark (128 GB unified memory).
+# 48 layers (36 SWA, 12 global), 256+1 experts, 262K context.
+# KV cache in FP8.
+#
+# Optional DFlash speculative decoding with draft model:
+#   poolside/Laguna-S-2.1-DFlash-NVFP4
+#   → 15 speculative tokens, requires --max-num-seqs 32.
+#
+# REFERENCE:
+#   https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
+
+engine = "vllm"
+name = "Laguna S 2.1 NVFP4 (vLLM)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
+model = "poolside/Laguna-S-2.1-NVFP4"
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+
+entrypoint = ["vllm", "serve"]
+
+args = """
+--gpu-memory-utilization 0.85
+--max-model-len 262144
+--load-format instanttensor
+--enable-prefix-caching
+--tool-call-parser poolside_v1
+--reasoning-parser poolside_v1
+--enable-auto-tool-choice
+--trust-remote-code
+--kv-cache-dtype fp8
+--enable-flashinfer-autotune
+--max-num-seqs 32
+--override-generation-config '{"temperature":0.7,"top_p":0.95}'
+--served-model-name laguna-s-2.1
+"""
+
+[env]
+SAFETENSORS_FAST_GPU = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_FLOAT32_MATMUL_PRECISION = "high"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+MAX_JOBS = "4"