|
|
@@ -53,6 +53,7 @@ solo = false # optional; single-node on first free node, n
|
|
|
deploy = "docker" # optional; "docker" (default) or "tmux"
|
|
|
venv = "~/venvs/vllm025" # optional; Python venv for tmux (activates before entrypoint)
|
|
|
venv_setup = [...] # optional; commands to create venv if it doesn't exist (tmux only)
|
|
|
+venv_setup_script = "venv.sh" # optional; script file (relative to recipe dir) to create venv (tmux only)
|
|
|
tensor_parallel_size = 2
|
|
|
pipeline_parallel_size = 1 # vLLM only, 0 = auto
|
|
|
distributed_backend = "mp" # or "ray"
|
|
|
@@ -127,29 +128,49 @@ served via either `docker` or `tmux` without changes.
|
|
|
|
|
|
When `deploy = "tmux"`, the `venv` field specifies a Python virtualenv path that is
|
|
|
activated before the entrypoint (`source <venv>/bin/activate && ...`). The `venv_setup`
|
|
|
-field is an optional list of shell commands to create and initialize the venv **if it
|
|
|
+field is an optional set of shell commands to create and initialize the venv **if it
|
|
|
does not already exist** on a node. Before serve, the launcher checks
|
|
|
-`test -f <venv>/bin/activate` on each node; if absent, it runs the `venv_setup` commands
|
|
|
+`test -f <venv>/bin/activate` on each node; if absent, it runs the setup commands
|
|
|
in order, then re-checks. A failure aborts the serve.
|
|
|
|
|
|
+`venv_setup` accepts either an inline multi-line string (one command per line;
|
|
|
+blank lines and `#` comments are skipped) or a `Vec<String>` list:
|
|
|
+
|
|
|
```toml
|
|
|
deploy = "tmux"
|
|
|
venv = "~/venvs/vllm025"
|
|
|
-venv_setup = [
|
|
|
- "uv venv ~/venvs/vllm025 -p 3.12",
|
|
|
- "uv pip install vllm==0.25.1 --torch-backend=cu130",
|
|
|
- "uv pip install flashinfer-python==0.6.15.dev20260712 \
|
|
|
+venv_setup = """
|
|
|
+ uv venv ~/venvs/vllm025 -p 3.12
|
|
|
+ uv pip install vllm==0.25.1 --torch-backend=cu130
|
|
|
+ uv pip install flashinfer-python==0.6.15.dev20260712 \
|
|
|
flashinfer-cubin==0.6.15.dev20260712 \
|
|
|
flashinfer-jit-cache==0.6.15.dev20260712 \
|
|
|
--extra-index-url https://flashinfer.ai/whl/nightly/ \
|
|
|
- --extra-index-url https://flashinfer.ai/whl/nightly/cu130/",
|
|
|
- "hf download poolside/Laguna-S-2.1-NVFP4",
|
|
|
- "hf download poolside/Laguna-S-2.1-DFlash-NVFP4",
|
|
|
+ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/
|
|
|
+ hf download poolside/Laguna-S-2.1-NVFP4
|
|
|
+ hf download poolside/Laguna-S-2.1-DFlash-NVFP4
|
|
|
+"""
|
|
|
+```
|
|
|
+
|
|
|
+Or as a list:
|
|
|
+
|
|
|
+```toml
|
|
|
+venv_setup = [
|
|
|
+ "uv venv ~/venvs/vllm025 -p 3.12",
|
|
|
+ "uv pip install vllm==0.25.1 --torch-backend=cu130",
|
|
|
]
|
|
|
```
|
|
|
|
|
|
+Alternatively, `venv_setup_script` references a shell script file (relative to the
|
|
|
+recipe directory) that is read and run on each node if the venv doesn't exist.
|
|
|
+Script file takes precedence over `venv_setup` if both are set:
|
|
|
+
|
|
|
+```toml
|
|
|
+venv_setup_script = "venv_setup.sh"
|
|
|
+```
|
|
|
+
|
|
|
If the venv already exists on a node, the setup commands are skipped. `venv_setup`
|
|
|
-is ignored in Docker mode (the container has its own Python).
|
|
|
+and `venv_setup_script` are ignored in Docker mode (the container has its own Python).
|
|
|
|
|
|
## Env vars (auto-set)
|
|
|
|