|
|
@@ -14,6 +14,24 @@
|
|
|
#
|
|
|
# REFERENCE:
|
|
|
# https://huggingface.co/poolside/Laguna-S-2.1-NVFP4
|
|
|
+#
|
|
|
+# DEPLOYMENT: tmux (host venv). The pre-built Docker image
|
|
|
+# (ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1) ships the wrong
|
|
|
+# flashinfer version, so Docker is non-functional. The sustainable path
|
|
|
+# is a host venv with the exact deps from the model card:
|
|
|
+#
|
|
|
+# uv venv ~/venvs/vllm025 -p 3.12
|
|
|
+# uv pip install vllm==0.25.1 --torch-backend=cu130
|
|
|
+# uv pip install flashinfer-python==0.6.15.dev20260712 \
|
|
|
+# flashinfer-cubin==0.6.15.dev20260712 \
|
|
|
+# flashinfer-jit-cache==0.6.15.dev20260712 \
|
|
|
+# --extra-index-url https://flashinfer.ai/whl/nightly/ \
|
|
|
+# --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
|
|
|
+# hf download poolside/Laguna-S-2.1-NVFP4
|
|
|
+# hf download poolside/Laguna-S-2.1-DFlash-NVFP4
|
|
|
+#
|
|
|
+# Then ensure `vllm` is on PATH (activate the venv or add its bin/ to PATH).
|
|
|
+# The entrypoint below (`["vllm", "serve"]`) is used directly by tmux.
|
|
|
|
|
|
engine = "vllm"
|
|
|
name = "Laguna S 2.1 NVFP4 (vLLM)"
|
|
|
@@ -21,6 +39,7 @@ docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.25.1-gb10.1"
|
|
|
model = "poolside/Laguna-S-2.1-NVFP4"
|
|
|
tensor_parallel_size = 1
|
|
|
pipeline_parallel_size = 1
|
|
|
+deploy = "tmux"
|
|
|
|
|
|
entrypoint = ["vllm", "serve"]
|
|
|
|
|
|
@@ -48,6 +67,7 @@ args = [
|
|
|
SAFETENSORS_FAST_GPU = "1"
|
|
|
TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
|
|
|
VLLM_FLOAT32_MATMUL_PRECISION = "high"
|
|
|
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|