Răsfoiți Sursa

feat(tmux): add venv_setup field for declarative venv creation

Add  field to Recipe — a list of shell commands that the
launcher runs on each tmux node to create and initialize the Python venv
if it doesn't already exist. Before serve, the launcher checks
 on each node; if absent, it runs the
venv_setup commands in order, then re-checks. A failure aborts the
serve. Nodes are prepared in parallel.

The laguna recipe now declares venv_setup with the exact uv/hf
commands from the model card. AGENTS.md documents the field. All
non-en locales updated with new i18n keys.
Alexander Korolev 2 luni în urmă
părinte
comite
5f646966b8

+ 29 - 0
AGENTS.md

@@ -52,6 +52,7 @@ required_nodes = 2                  # optional; also caps the launch set (N node
 solo = false                       # optional; single-node on first free node, no dist flags
 deploy = "docker"                   # optional; "docker" (default) or "tmux"
 venv = "~/venvs/vllm025"            # optional; Python venv for tmux (activates before entrypoint)
+venv_setup = [...]                 # optional; commands to create venv if it doesn't exist (tmux only)
 tensor_parallel_size = 2
 pipeline_parallel_size = 1          # vLLM only, 0 = auto
 distributed_backend = "mp"          # or "ray"
@@ -122,6 +123,34 @@ validate them, so the same `args` list works for both `vllm` and `sglang` recipe
 `entrypoint` field is shared: both deployment paths use it, so a single recipe can be
 served via either `docker` or `tmux` without changes.
 
+### Venv creation (`venv` + `venv_setup`)
+
+When `deploy = "tmux"`, the `venv` field specifies a Python virtualenv path that is
+activated before the entrypoint (`source <venv>/bin/activate && ...`). The `venv_setup`
+field is an optional list of shell commands to create and initialize the venv **if it
+does not already exist** on a node. Before serve, the launcher checks
+`test -f <venv>/bin/activate` on each node; if absent, it runs the `venv_setup` commands
+in order, then re-checks. A failure aborts the serve.
+
+```toml
+deploy = "tmux"
+venv = "~/venvs/vllm025"
+venv_setup = [
+    "uv venv ~/venvs/vllm025 -p 3.12",
+    "uv pip install vllm==0.25.1 --torch-backend=cu130",
+    "uv pip install flashinfer-python==0.6.15.dev20260712 \
+      flashinfer-cubin==0.6.15.dev20260712 \
+      flashinfer-jit-cache==0.6.15.dev20260712 \
+      --extra-index-url https://flashinfer.ai/whl/nightly/ \
+      --extra-index-url https://flashinfer.ai/whl/nightly/cu130/",
+    "hf download poolside/Laguna-S-2.1-NVFP4",
+    "hf download poolside/Laguna-S-2.1-DFlash-NVFP4",
+]
+```
+
+If the venv already exists on a node, the setup commands are skipped. `venv_setup`
+is ignored in Docker mode (the container has its own Python).
+
 ## Env vars (auto-set)
 
 Cluster-driven: `NCCL_IB_DISABLE/HCA/SOCKET_IFNAME/GID_INDEX`, `GLOO_/TP_SOCKET_IFNAME`, `MN_IF_NAME`, `UCX_NET_DEVICES`, `OMPI_MCA_btl_tcp_if_include`

+ 9 - 0
locales/de/main.ftl

@@ -163,3 +163,12 @@ composed-dir-built = Zusammengestelltes Modellverzeichnis '{ $host }' auf '{ $no
         [one] Datei
        *[other] Dateien
     })
+tmux-mods-unsupported = Recipe '{ $name }' verwendet [mods], was Docker-only ist. tmux-Bereitstellung unterstützt keine mods.
+tmux-deploy-ignored-opts = ⚠️ tmux-Bereitstellung: ignoriere Docker-only CLI-Optionen: { $opts }
+tmux-composed-model-unsupported = Recipe '{ $name }' verwendet [composed_model], was Docker-only ist. tmux-Bereitstellung unterstützt keine zusammengesetzten Modelldateien.
+venv-preparing = 🔧 Bereite venv vor: { $venv }
+venv-exists = ✓ venv existiert bereits auf { $name }
+venv-creating = venv nicht gefunden auf { $name } — erstelle...
+venv-created = ✓ venv erfolgreich erstellt auf { $name }
+venv-prepare-failed = Venv-Vorbereitung: Knoten { $name } fehlgeschlagen: { $error }
+venv-not-created = venv wurde nicht erfolgreich auf { $name } erstellt — Setup-Befehle wurden ausgeführt, aber das activate-Skript fehlt

+ 6 - 0
locales/en/main.ftl

@@ -152,6 +152,12 @@ tmux-following-logs = 📜 Following logs from '{ $sess }' on { $name }...
 tmux-deploy-ignored-opts = ⚠️ tmux deploy: ignoring Docker-only CLI opts: { $opts }
 tmux-mods-unsupported = Recipe '{ $name }' uses [mods], which is Docker-only. tmux deployment does not support mods.
 tmux-composed-model-unsupported = Recipe '{ $name }' uses [composed_model], which is Docker-only. tmux deployment does not support composed model directories.
+venv-preparing = 🔧 Preparing venv: { $venv }
+venv-exists = ✓ venv already exists on { $name }
+venv-creating = venv not found on { $name } — creating...
+venv-created = ✓ venv created successfully on { $name }
+venv-prepare-failed = venv prepare: node { $name } failed: { $error }
+venv-not-created = venv was not created successfully on { $name } — setup commands ran but activate script is missing
 composed-dir-dry-run = [dry-run] would build composed model dir '{ $host }' on '{ $node }' (mount: { $mount }, { $sources } { $sources ->
         [one] source
        *[other] sources

+ 9 - 0
locales/es/main.ftl

@@ -163,3 +163,12 @@ composed-dir-built = Creado directorio de modelo compuesto '{ $host }' en '{ $no
         [one] archivo
        *[other] archivos
     })
+tmux-mods-unsupported = Recipe '{ $name }' usa [mods], que es Docker-only. El despliegue tmux no admite mods.
+tmux-deploy-ignored-opts = ⚠️ tmux deploy: ignorando opciones CLI Docker-only: { $opts }
+tmux-composed-model-unsupported = Recipe '{ $name }' usa [composed_model], que es Docker-only. El despliegue tmux no admite directorios de modelos compuestos.
+venv-preparing = 🔧 Preparando venv: { $venv }
+venv-exists = ✓ venv ya existe en { $name }
+venv-creating = venv no encontrado en { $name } — creando...
+venv-created = ✓ venv creado correctamente en { $name }
+venv-prepare-failed = preparación venv: nodo { $name } falló: { $error }
+venv-not-created = venv no se creó correctamente en { $name } — los comandos de configuración se ejecutaron pero falta el script activate

+ 9 - 0
locales/fr/main.ftl

@@ -163,3 +163,12 @@ composed-dir-built = Répertoire de modèle composé créé : '{ $host }' sur '{
         [one] fichier
        *[other] fichiers
     })
+tmux-mods-unsupported = Recipe '{ $name }' utilise [mods], ce qui est Docker-only. Le déploiement tmux ne prend pas en charge les mods.
+tmux-deploy-ignored-opts = ⚠️ tmux deploy : ignore les options CLI Docker-only : { $opts }
+tmux-composed-model-unsupported = Recipe '{ $name }' utilise [composed_model], ce qui est Docker-only. Le déploiement tmux ne prend pas en charge les répertoires de modèles composés.
+venv-preparing = 🔧 Préparation du venv : { $venv }
+venv-exists = ✓ venv existe déjà sur { $name }
+venv-creating = venv non trouvé sur { $name } — création...
+venv-created = ✓ venv créé avec succès sur { $name }
+venv-prepare-failed = préparation venv : nœud { $name } échec : { $error }
+venv-not-created = venv n'a pas été créé avec succès sur { $name } — les commandes de configuration ont été exécutées mais le script activate est manquant

+ 9 - 0
locales/it/main.ftl

@@ -163,3 +163,12 @@ composed-dir-built = Creata directory del modello composto '{ $host }' su '{ $no
         [one] file
        *[other] file
     })
+tmux-mods-unsupported = Recipe '{ $name }' usa [mods], che è Docker-only. Il deployment tmux non supporta i mods.
+tmux-deploy-ignored-opts = ⚠️ tmux deploy: ignoro le opzioni CLI Docker-only: { $opts }
+tmux-composed-model-unsupported = Recipe '{ $name }' usa [composed_model], che è Docker-only. Il deployment tmux non supporta directory di modelli composti.
+venv-preparing = 🔧 Preparazione venv: { $venv }
+venv-exists = ✓ venv esiste già su { $name }
+venv-creating = venv non trovato su { $name } — creazione...
+venv-created = ✓ venv creato con successo su { $name }
+venv-prepare-failed = preparazione venv: nodo { $name } fallito: { $error }
+venv-not-created = venv non creato correttamente su { $name } — i comandi di configurazione sono stati eseguiti ma manca lo script activate

+ 9 - 0
locales/ru/main.ftl

@@ -172,3 +172,12 @@ composed-dir-built = Создан каталог составной модели
         [few] файла
        *[many] файлов
     })
+tmux-mods-unsupported = Recipe '{ $name }' использует [mods], который является Docker-only. Развертывание tmux не поддерживает mods.
+tmux-deploy-ignored-opts = ⚠️ tmux deploy: игнорируются Docker-only CLI опции: { $opts }
+tmux-composed-model-unsupported = Recipe '{ $name }' использует [composed_model], который является Docker-only. Развертывание tmux не поддерживает составные каталоги моделей.
+venv-preparing = 🔧 Подготовка venv: { $venv }
+venv-exists = ✓ venv уже существует на { $name }
+venv-creating = venv не найден на { $name } — создаю...
+venv-created = ✓ venv успешно создан на { $name }
+venv-prepare-failed = подготовка venv: узел { $name } завершился с ошибкой: { $error }
+venv-not-created = venv не был создан успешно на { $name } — команды настройки выполнились, но скрипт activate отсутствует

+ 7 - 0
recipes/laguna-s2.1-nvfp4/recipe.toml

@@ -41,6 +41,13 @@ tensor_parallel_size = 1
 pipeline_parallel_size = 1
 deploy = "tmux"
 venv = "~/venvs/vllm025"
+venv_setup = [
+    "uv venv ~/venvs/vllm025 -p 3.12",
+    "uv pip install vllm==0.25.1 --torch-backend=cu130",
+    "uv pip install flashinfer-python==0.6.15.dev20260712 flashinfer-cubin==0.6.15.dev20260712 flashinfer-jit-cache==0.6.15.dev20260712 --extra-index-url https://flashinfer.ai/whl/nightly/ --extra-index-url https://flashinfer.ai/whl/nightly/cu130/",
+    "hf download poolside/Laguna-S-2.1-NVFP4",
+    "hf download poolside/Laguna-S-2.1-DFlash-NVFP4",
+]
 
 entrypoint = ["vllm", "serve"]
 

+ 4 - 4
specs/state.yaml

@@ -23,9 +23,9 @@ session:
 
 git:
   branch: main
-  head: 240002c
-  clean: true                    # structured-output fix landed + pushed
+  head: d9b922e
+  clean: true                    # venv field + deploy field + structured-output fix landed + pushed
 
 handoff:
-  next_skill: survey-context    # fix shipped; survey to pick next task
-  note: "BUG-2026-07-27T120657 fixed and landed (82bbb91). Back to Sustain."
+  next_skill: survey-context    # venv_setup feature shipped; survey to pick next task
+  note: "venv_setup field added to Recipe; prepare_venv creates venvs on tmux nodes if absent. All 147 tests pass, make check green."

+ 1 - 0
src/after_stop.rs

@@ -200,6 +200,7 @@ mod tests {
             solo: None,
             deploy: crate::config::DeployTarget::Docker,
             venv: None,
+            venv_setup: vec![],
             mods: Default::default(),
             sync_image: true,
             docker_image_dir: None,

+ 1 - 0
src/before_serve.rs

@@ -310,6 +310,7 @@ mod tests {
             solo: None,
             deploy: crate::config::DeployTarget::Docker,
             venv: None,
+            venv_setup: vec![],
             mods: Default::default(),
             sync_image: true,
             docker_image_dir: None,

+ 29 - 0
src/config/tests.rs

@@ -226,6 +226,35 @@ fn test_venv_defaults_to_none() {
     assert_eq!(r.venv, None);
 }
 
+#[test]
+fn test_venv_setup_deserializes() {
+    let r: Recipe = toml::from_str(
+        r#"name = "x"
+model = "m"
+deploy = "tmux"
+venv = "~/venvs/vllm025"
+venv_setup = [
+    "uv venv ~/venvs/vllm025 -p 3.12",
+    "uv pip install vllm==0.25.1 --torch-backend=cu130",
+]
+"#,
+    )
+    .unwrap();
+    assert_eq!(r.venv.as_deref(), Some("~/venvs/vllm025"));
+    assert_eq!(r.venv_setup.len(), 2);
+    assert_eq!(r.venv_setup[0], "uv venv ~/venvs/vllm025 -p 3.12");
+    assert_eq!(
+        r.venv_setup[1],
+        "uv pip install vllm==0.25.1 --torch-backend=cu130"
+    );
+}
+
+#[test]
+fn test_venv_setup_defaults_to_empty() {
+    let r: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"\n").unwrap();
+    assert!(r.venv_setup.is_empty());
+}
+
 #[test]
 fn default_config_dir_appends_sm121() {
     let Some(d) = default_config_dir() else {

+ 18 - 0
src/config/types.rs

@@ -168,6 +168,24 @@ pub struct Recipe {
     /// Example: `venv = "~/venvs/vllm025"`
     #[serde(default)]
     pub venv: Option<String>,
+    /// Shell commands to create and initialize the venv at `venv` if it
+    /// does not already exist on a node. Each command runs as-is over SSH
+    /// (shell expands `~` and `$VAR`). Only used in tmux deployment mode.
+    ///
+    /// The launcher checks `test -f <venv>/bin/activate` on each node
+    /// before serve; if absent, it runs these commands in order, then
+    /// re-checks. A failure aborts the serve.
+    ///
+    /// Example:
+    /// ```toml
+    /// venv = "~/venvs/vllm025"
+    /// venv_setup = [
+    ///     "uv venv ~/venvs/vllm025 -p 3.12",
+    ///     "uv pip install vllm==0.25.1 --torch-backend=cu130",
+    /// ]
+    /// ```
+    #[serde(default)]
+    pub venv_setup: Vec<String>,
     #[serde(default)]
     pub mods: ModsTable,
     #[serde(default = "default_sync_image")]

+ 2 - 205
src/docker/command.rs

@@ -293,208 +293,5 @@ pub(crate) fn build_docker_command(
 }
 
 #[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::config::{
-        Cluster, DeployTarget, DistributedBackend, ExtraMount, NcclConfig, Node, Recipe, RecipeArgs,
-    };
-    use std::collections::HashMap;
-
-    fn tcls() -> Cluster {
-        Cluster {
-            name: "t".into(),
-            master_ib_ip: "1".into(),
-            nodes: vec![Node {
-                name: "n0".into(),
-                ssh_ip: "10.0.0.1".into(),
-                ib_ip: "1".into(),
-                rank: 0,
-                host_key: None,
-                entity: None,
-            }],
-            extra_entities: vec![],
-        }
-    }
-
-    fn tnccl() -> NcclConfig {
-        NcclConfig {
-            ib_hca: "r".into(),
-            socket_ifname: "e".into(),
-            ib_gid_index: 3,
-            ib_disable: 0,
-            gloo_socket_ifname: None,
-            ucx_net_devices: None,
-            nccl_cross_nic: None,
-            nccl_proto: None,
-            nccl_p2p_level: None,
-        }
-    }
-
-    fn minimal_config() -> Config {
-        Config {
-            ssh_user: "u".into(),
-            ssh_key_path: "/tmp/key".into(),
-            clusters: vec![],
-            nccl: HashMap::new(),
-            hf_cache_dir: None,
-            init_port: 25000,
-            before_serve: HashMap::new(),
-            exec_aliases: HashMap::new(),
-            home_assistant: None,
-            global_mods_dir: None,
-            drop_caches: true,
-            mount_aliases: HashMap::from([("models".to_string(), "/mnt/nas/models".to_string())]),
-        }
-    }
-
-    fn run_with(recipe: &Recipe) -> String {
-        let resolved = resolve_extra_mounts(&minimal_config(), recipe).unwrap();
-        let ctx = DockerCommandCtx {
-            cluster: tcls(),
-            nccl: tnccl(),
-            recipe: recipe.clone(),
-            recipe_name: "r".to_string(),
-            pp_size: 2,
-            tp_size: 1,
-            nnodes: 2,
-            cache_dir: "/home/user/.cache/huggingface".to_string(),
-            keep: false,
-            init_port: 25000,
-            mods_remote_path: String::new(),
-            extra_mounts: resolved.static_mounts,
-        };
-        build_docker_command(&ctx, &tcls().nodes[0], true).join(" ")
-    }
-
-    fn base_recipe() -> Recipe {
-        Recipe {
-            engine: "vllm".into(),
-            name: "T".into(),
-            docker_image: "img".into(),
-            model: "mdl".into(),
-            port: 8000,
-            required_nodes: Some(2),
-            pipeline_parallel_size: Some(2),
-            tensor_parallel_size: 1,
-            args: RecipeArgs::List(vec!["--x".into(), "4096".into()]),
-            env: HashMap::new(),
-            cluster: None,
-            entrypoint: vec![],
-            distributed_backend: Some(DistributedBackend::Mp),
-            before_serve: vec![],
-            after_stop: vec![],
-            solo: None,
-            deploy: DeployTarget::Docker,
-            venv: None,
-            mods: Default::default(),
-            shm_size: String::new(),
-            sync_image: true,
-            docker_image_dir: None,
-            extra_mounts: vec![],
-            build: Default::default(),
-            composed_model: None,
-        }
-    }
-
-    #[test]
-    fn extra_mounts_are_mounted() {
-        let mut r = base_recipe();
-        r.extra_mounts.push(ExtraMount {
-            alias: Some("models".into()),
-            host: Some("~/.cache/models".into()),
-            repo: None,
-            subfolder: None,
-            container: "/models".into(),
-            read_only: true,
-            env: Some("MODEL_DIR".into()),
-        });
-        let cmd = run_with(&r);
-        // alias present in minimal_config() → alias host wins
-        assert!(cmd.contains("-v /mnt/nas/models:/models:ro"));
-        assert!(cmd.contains("-v /home/user/.cache/huggingface:/root/.cache/huggingface"));
-        // env injected pointing at the container path, before recipe env
-        assert!(cmd.contains("-e MODEL_DIR=/models"));
-    }
-
-    #[test]
-    fn extra_mounts_fallback_to_host_when_alias_absent() {
-        let mut r = base_recipe();
-        r.extra_mounts.push(ExtraMount {
-            alias: Some("no_such_alias".into()),
-            host: Some("~/.cache/jit".into()),
-            repo: None,
-            subfolder: None,
-            container: "/root/.cache/jit".into(),
-            read_only: false,
-            env: None,
-        });
-        let cmd = run_with(&r);
-        // alias not in config → host default used with ~ expansion (ssh_user=u)
-        assert!(cmd.contains("-v /home/u/.cache/jit:/root/.cache/jit"));
-        assert!(!cmd.contains(":ro"));
-    }
-
-    #[test]
-    fn model_path_ref_is_partitioned_not_mounted() {
-        // repo-based mounts are path references, NOT bind-mounts. They are
-        // partitioned into path_refs and resolved per-node via substitution.
-        let mut r = base_recipe();
-        r.extra_mounts.push(ExtraMount {
-            alias: None,
-            host: None,
-            repo: Some("aidendle94/GLM-5.2-Hybrid-FP8-MXFP4".into()),
-            subfolder: Some("mtp-draft".into()),
-            container: "mtp_draft".into(),
-            read_only: true,
-            env: None,
-        });
-        let resolved = resolve_extra_mounts(&minimal_config(), &r).unwrap();
-        assert_eq!(resolved.path_refs.len(), 1);
-        assert_eq!(resolved.static_mounts.len(), 0);
-        let pr = &resolved.path_refs[0];
-        assert_eq!(pr.repo, "aidendle94/GLM-5.2-Hybrid-FP8-MXFP4");
-        assert_eq!(pr.subfolder.as_deref(), Some("mtp-draft"));
-        assert_eq!(pr.name, "mtp_draft");
-        // No -v flag for path refs in build_docker_command (substitution only)
-        let cmd = run_with(&r);
-        assert!(!cmd.contains("-v /models/mtp-draft"));
-    }
-
-    #[test]
-    fn substitute_path_refs_replaces_placeholders() {
-        let mut r = base_recipe();
-        r.args = RecipeArgs::List(vec![
-            "--speculative-config".into(),
-            "{\"model\":\"{{mtp_draft}}\"}".into(),
-        ]);
-        let path_refs = vec![(
-            "mtp_draft".to_string(),
-            "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft".to_string(),
-        )];
-        let substituted = substitute_path_refs(&r, &path_refs);
-        if let RecipeArgs::List(args) = &substituted.args {
-            assert_eq!(
-                args[1],
-                "{\"model\":\"/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft\"}"
-            );
-        } else {
-            panic!("expected list args");
-        }
-    }
-
-    #[test]
-    fn substitute_path_refs_in_env() {
-        let mut r = base_recipe();
-        r.env
-            .insert("MTP_PATH".to_string(), "{{mtp_draft}}".to_string());
-        let path_refs = vec![(
-            "mtp_draft".to_string(),
-            "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft".to_string(),
-        )];
-        let substituted = substitute_path_refs(&r, &path_refs);
-        assert_eq!(
-            substituted.env["MTP_PATH"],
-            "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft"
-        );
-    }
-}
+#[path = "command_tests.rs"]
+mod tests;

+ 204 - 0
src/docker/command_tests.rs

@@ -0,0 +1,204 @@
+use super::*;
+use crate::config::{
+    Cluster, DeployTarget, DistributedBackend, ExtraMount, NcclConfig, Node, Recipe, RecipeArgs,
+};
+use std::collections::HashMap;
+
+fn tcls() -> Cluster {
+    Cluster {
+        name: "t".into(),
+        master_ib_ip: "1".into(),
+        nodes: vec![Node {
+            name: "n0".into(),
+            ssh_ip: "10.0.0.1".into(),
+            ib_ip: "1".into(),
+            rank: 0,
+            host_key: None,
+            entity: None,
+        }],
+        extra_entities: vec![],
+    }
+}
+
+fn tnccl() -> NcclConfig {
+    NcclConfig {
+        ib_hca: "r".into(),
+        socket_ifname: "e".into(),
+        ib_gid_index: 3,
+        ib_disable: 0,
+        gloo_socket_ifname: None,
+        ucx_net_devices: None,
+        nccl_cross_nic: None,
+        nccl_proto: None,
+        nccl_p2p_level: None,
+    }
+}
+
+fn minimal_config() -> Config {
+    Config {
+        ssh_user: "u".into(),
+        ssh_key_path: "/tmp/key".into(),
+        clusters: vec![],
+        nccl: HashMap::new(),
+        hf_cache_dir: None,
+        init_port: 25000,
+        before_serve: HashMap::new(),
+        exec_aliases: HashMap::new(),
+        home_assistant: None,
+        global_mods_dir: None,
+        drop_caches: true,
+        mount_aliases: HashMap::from([("models".to_string(), "/mnt/nas/models".to_string())]),
+    }
+}
+
+fn run_with(recipe: &Recipe) -> String {
+    let resolved = resolve_extra_mounts(&minimal_config(), recipe).unwrap();
+    let ctx = DockerCommandCtx {
+        cluster: tcls(),
+        nccl: tnccl(),
+        recipe: recipe.clone(),
+        recipe_name: "r".to_string(),
+        pp_size: 2,
+        tp_size: 1,
+        nnodes: 2,
+        cache_dir: "/home/user/.cache/huggingface".to_string(),
+        keep: false,
+        init_port: 25000,
+        mods_remote_path: String::new(),
+        extra_mounts: resolved.static_mounts,
+    };
+    build_docker_command(&ctx, &tcls().nodes[0], true).join(" ")
+}
+
+fn base_recipe() -> Recipe {
+    Recipe {
+        engine: "vllm".into(),
+        name: "T".into(),
+        docker_image: "img".into(),
+        model: "mdl".into(),
+        port: 8000,
+        required_nodes: Some(2),
+        pipeline_parallel_size: Some(2),
+        tensor_parallel_size: 1,
+        args: RecipeArgs::List(vec!["--x".into(), "4096".into()]),
+        env: HashMap::new(),
+        cluster: None,
+        entrypoint: vec![],
+        distributed_backend: Some(DistributedBackend::Mp),
+        before_serve: vec![],
+        after_stop: vec![],
+        solo: None,
+        deploy: DeployTarget::Docker,
+        venv: None,
+        venv_setup: vec![],
+        mods: Default::default(),
+        shm_size: String::new(),
+        sync_image: true,
+        docker_image_dir: None,
+        extra_mounts: vec![],
+        build: Default::default(),
+        composed_model: None,
+    }
+}
+
+#[test]
+fn extra_mounts_are_mounted() {
+    let mut r = base_recipe();
+    r.extra_mounts.push(ExtraMount {
+        alias: Some("models".into()),
+        host: Some("~/.cache/models".into()),
+        repo: None,
+        subfolder: None,
+        container: "/models".into(),
+        read_only: true,
+        env: Some("MODEL_DIR".into()),
+    });
+    let cmd = run_with(&r);
+    // alias present in minimal_config() → alias host wins
+    assert!(cmd.contains("-v /mnt/nas/models:/models:ro"));
+    assert!(cmd.contains("-v /home/user/.cache/huggingface:/root/.cache/huggingface"));
+    // env injected pointing at the container path, before recipe env
+    assert!(cmd.contains("-e MODEL_DIR=/models"));
+}
+
+#[test]
+fn extra_mounts_fallback_to_host_when_alias_absent() {
+    let mut r = base_recipe();
+    r.extra_mounts.push(ExtraMount {
+        alias: Some("no_such_alias".into()),
+        host: Some("~/.cache/jit".into()),
+        repo: None,
+        subfolder: None,
+        container: "/root/.cache/jit".into(),
+        read_only: false,
+        env: None,
+    });
+    let cmd = run_with(&r);
+    // alias not in config → host default used with ~ expansion (ssh_user=u)
+    assert!(cmd.contains("-v /home/u/.cache/jit:/root/.cache/jit"));
+    assert!(!cmd.contains(":ro"));
+}
+
+#[test]
+fn model_path_ref_is_partitioned_not_mounted() {
+    // repo-based mounts are path references, NOT bind-mounts. They are
+    // partitioned into path_refs and resolved per-node via substitution.
+    let mut r = base_recipe();
+    r.extra_mounts.push(ExtraMount {
+        alias: None,
+        host: None,
+        repo: Some("aidendle94/GLM-5.2-Hybrid-FP8-MXPF4".into()),
+        subfolder: Some("mtp-draft".into()),
+        container: "mtp_draft".into(),
+        read_only: true,
+        env: None,
+    });
+    let resolved = resolve_extra_mounts(&minimal_config(), &r).unwrap();
+    assert_eq!(resolved.path_refs.len(), 1);
+    assert_eq!(resolved.static_mounts.len(), 0);
+    let pr = &resolved.path_refs[0];
+    assert_eq!(pr.repo, "aidendle94/GLM-5.2-Hybrid-FP8-MXPF4");
+    assert_eq!(pr.subfolder.as_deref(), Some("mtp-draft"));
+    assert_eq!(pr.name, "mtp_draft");
+    // No -v flag for path refs in build_docker_command (substitution only)
+    let cmd = run_with(&r);
+    assert!(!cmd.contains("-v /models/mtp-draft"));
+}
+
+#[test]
+fn substitute_path_refs_replaces_placeholders() {
+    let mut r = base_recipe();
+    r.args = RecipeArgs::List(vec![
+        "--speculative-config".into(),
+        "{\"model\":\"{{mtp_draft}}\"}".into(),
+    ]);
+    let path_refs = vec![(
+        "mtp_draft".to_string(),
+        "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft".to_string(),
+    )];
+    let substituted = substitute_path_refs(&r, &path_refs);
+    if let RecipeArgs::List(args) = &substituted.args {
+        assert_eq!(
+            args[1],
+            "{\"model\":\"/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft\"}"
+        );
+    } else {
+        panic!("expected list args");
+    }
+}
+
+#[test]
+fn substitute_path_refs_in_env() {
+    let mut r = base_recipe();
+    r.env
+        .insert("MTP_PATH".to_string(), "{{mtp_draft}}".to_string());
+    let path_refs = vec![(
+        "mtp_draft".to_string(),
+        "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft".to_string(),
+    )];
+    let substituted = substitute_path_refs(&r, &path_refs);
+    assert_eq!(
+        substituted.env["MTP_PATH"],
+        "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft"
+    );
+}

+ 2 - 0
src/docker/tests.rs

@@ -63,6 +63,7 @@ fn trec() -> Recipe {
         solo: None,
         deploy: DeployTarget::Docker,
         venv: None,
+        venv_setup: vec![],
         mods: Default::default(),
         shm_size: String::new(),
         sync_image: true,
@@ -254,6 +255,7 @@ fn sglang_recipe() -> Recipe {
         solo: None,
         deploy: DeployTarget::Docker,
         venv: None,
+        venv_setup: vec![],
         mods: Default::default(),
         shm_size: String::new(),
         sync_image: true,

+ 1 - 0
src/doctor.rs

@@ -373,6 +373,7 @@ mod tests {
             solo: None,
             deploy: crate::config::DeployTarget::Docker,
             venv: None,
+            venv_setup: vec![],
             mods: Default::default(),
             sync_image: true,
             docker_image_dir: None,

+ 1 - 0
src/engine_flags.rs

@@ -122,6 +122,7 @@ mod tests {
             solo: None,
             deploy: DeployTarget::Docker,
             venv: None,
+            venv_setup: vec![],
             mods: Default::default(),
             shm_size: String::new(),
             sync_image: true,

+ 1 - 0
src/tmux/commands.rs

@@ -155,6 +155,7 @@ mod tests {
             solo: None,
             deploy: DeployTarget::Docker,
             venv: None,
+            venv_setup: vec![],
             mods: Default::default(),
             shm_size: String::new(),
             sync_image: true,

+ 119 - 1
src/tmux/serve.rs

@@ -4,7 +4,7 @@ use std::time::Duration;
 use tokio::time::sleep;
 use tracing::{info, warn};
 
-use crate::config::{Config, Recipe};
+use crate::config::{Config, Node, Recipe};
 use crate::tmux::TmuxManager;
 
 /// Launches a cluster via tmux sessions.
@@ -68,6 +68,9 @@ pub async fn tmux_serve(
 
     crate::before_serve::run(config, &cluster, &recipe, skip_before_serve, dry_run).await?;
 
+    // Prepare the Python venv on each node if it doesn't exist yet.
+    prepare_venv(config, &recipe, &cluster, dry_run).await?;
+
     let manager = TmuxManager;
 
     let is_ray = recipe.distributed_backend == Some(crate::config::DistributedBackend::Ray);
@@ -223,3 +226,118 @@ async fn wait_for_head_ready(
     warn!("  Head node did not become ready within timeout");
     Ok(false)
 }
+
+/// Prepares the Python venv on each node if it doesn't exist.
+///
+/// Checks `test -f <venv>/bin/activate` on each node; if absent, runs
+/// `recipe.venv_setup` commands in order, then re-checks. A failure on
+/// any command aborts the serve. Nodes are prepared in parallel.
+async fn prepare_venv(
+    config: &Config,
+    recipe: &Recipe,
+    cluster: &crate::config::Cluster,
+    dry_run: bool,
+) -> Result<()> {
+    let venv = match &recipe.venv {
+        Some(v) if !recipe.venv_setup.is_empty() => v,
+        _ => return Ok(()),
+    };
+
+    info!("{}", fluent_i18n::t!("venv-preparing", { "venv" => venv }));
+
+    let mut handles = Vec::with_capacity(cluster.nodes.len());
+    for node in &cluster.nodes {
+        let config = config.clone();
+        let node = node.clone();
+        let venv = venv.clone();
+        let setup = recipe.venv_setup.clone();
+        let dry = dry_run;
+
+        handles.push(crate::locale::spawn_with_locale(async move {
+            (
+                node.name.clone(),
+                prepare_venv_for_node(&config, &node, &venv, &setup, dry).await,
+            )
+        }));
+    }
+
+    let mut any_failed = false;
+    for h in handles {
+        let (name, result) = h.await?;
+        if let Err(e) = result {
+            any_failed = true;
+            warn!(
+                "{}",
+                fluent_i18n::t!("venv-prepare-failed", { "name" => &name, "error" => e.to_string() })
+            );
+        }
+    }
+
+    if any_failed {
+        anyhow::bail!("Venv preparation failed on at least one node (see warnings above)");
+    }
+
+    Ok(())
+}
+
+/// Checks if the venv exists on a single node and creates it if not.
+async fn prepare_venv_for_node(
+    config: &Config,
+    node: &Node,
+    venv: &str,
+    setup: &[String],
+    dry_run: bool,
+) -> Result<()> {
+    let check_cmd = format!("test -f {}/bin/activate", venv);
+
+    if dry_run {
+        info!(
+            "  ssh {}@{}: {} (dry-run)",
+            config.ssh_user, node.ssh_ip, check_cmd
+        );
+        for cmd in setup {
+            info!("    {} (dry-run)", cmd);
+        }
+        return Ok(());
+    }
+
+    // Check if venv already exists.
+    match crate::ssh::run_ssh(config, node, check_cmd.clone(), false).await {
+        Ok(_) => {
+            info!(
+                "{}",
+                fluent_i18n::t!("venv-exists", { "name" => &node.name })
+            );
+            return Ok(());
+        }
+        Err(_) => {
+            info!(
+                "{}",
+                fluent_i18n::t!("venv-creating", { "name" => &node.name })
+            );
+        }
+    }
+
+    // Run setup commands in order.
+    for cmd in setup {
+        info!("  ssh {}@{}: {}", config.ssh_user, node.ssh_ip, cmd);
+        crate::ssh::run_ssh(config, node, cmd.clone(), false).await?;
+    }
+
+    // Re-check that the venv was created.
+    match crate::ssh::run_ssh(config, node, check_cmd, false).await {
+        Ok(_) => {
+            info!(
+                "{}",
+                fluent_i18n::t!("venv-created", { "name" => &node.name })
+            );
+            Ok(())
+        }
+        Err(_) => {
+            anyhow::bail!(
+                "{}",
+                fluent_i18n::t!("venv-not-created", { "name" => &node.name })
+            );
+        }
+    }
+}