feat: mods system — entrypoint wrapping + global/recipe mods with volume mount

- Add [mods] recipe table with commands (on_failure: abort/warn/continue)
- Mods run inside container via --entrypoint sh -c wrapping before engine exec
- Mods directory is tar'd → base64 → SSH → extract on remote nodes
- Global mods (config.global_mods_dir) and recipe mods merged into /mods/
- Named mod subdirectories: recipes/<name>/mods/<mod-name>/file
- Validation: entrypoint required when mods present
- New module: src/docker/mods.rs (build_mod_script, upload_mods_to_cluster)
- Extracted tail builder: src/docker/tail.rs (TailCtx)
- i18n messages for mods upload/validation
- Test recipe: recipes/test-mods/ (demonstrates named mods flow)
- Recipe: recipes/laguna-m1-nvfp4/ (Laguna M.1 NVFP4, 2×DGX Spark)

AGENTS.md

@@ -37,6 +37,13 @@ shm_size = "64gb"                   # optional, only set when non-empty
 entrypoint = ["vllm", "serve"]      # optional, after image
 before_serve = [{ command = "echo", on_failure = "abort" }]
 args = ["--kv-cache-dtype", "fp8"]
+
+[mods]
+commands = [                         # run *inside* container before engine
+    "pip install flash-attn",
+    { command = "./patch.sh", on_failure = "warn" },
+]
+
 [env]
 KEY = "val"                         # overrides launcher defaults
 ```
@@ -50,6 +57,16 @@ Head: `VLLM_LOGGING_COLOR=1`, `TERM=xterm-256color`, `-p <port>:<port>`
 Ray: `RAY_ADDRESS/PORT/NODE_IP_ADDRESS/OVERRIDE_NODE_IP_ADDRESS`
 Recipe `[env]` wins (last wins in `-e`).
 
+## Mods
+
+Mods run **inside** the container before the engine process via entrypoint
+wrapping.  When `[mods]` is present, the recipe **must** also set `entrypoint`
+(the known process to `exec` at the end).  Same `on_failure` semantics as
+`before_serve` (default: `abort`).  On failure the container exits; configure
+`docker run --restart` externally to restart on crash.
+
+Implementation: `--entrypoint sh image -c 'mods && exec <original entrypoint>'`
+
 ## Flags auto-generated
 
 - vLLM: `--tensor-parallel-size`, `--pipeline-parallel-size`, `--nnodes`, `--node-rank`, `--master-addr`, `--master-port`, `--host`, `--port`, `--headless` (workers)

locales/de/main.ftl

@@ -81,3 +81,6 @@ exec-cannot-use-alias = Cannot use --alias together with a trailing command
 exec-must-specify = Must specify either --alias <name> or a command after `--`
 exec-unknown-alias-empty = Unknown alias @{ $name } (no aliases configured)
 exec-unknown-alias-list = Unknown alias @{ $name }; known aliases: { $aliases }
+mods-uploading = 📤 Lade Mods auf { $node } hoch...
+mods-uploaded = ✅ Mods auf { $node } hochgeladen
+mods-entrypoint-required = Rezept '{ $name }' hat Mods aber keinen Entrypoint; Entrypoint ist bei Mods erforderlich

locales/en/main.ftl

@@ -81,3 +81,6 @@ exec-cannot-use-alias = Cannot use --alias together with a trailing command
 exec-must-specify = Must specify either --alias <name> or a command after `--`
 exec-unknown-alias-empty = Unknown alias @{ $name } (no aliases configured)
 exec-unknown-alias-list = Unknown alias @{ $name }; known aliases: { $aliases }
+mods-uploading = 📤 Uploading mods to { $node }...
+mods-uploaded = ✅ Mods uploaded to { $node }
+mods-entrypoint-required = Recipe '{ $name }' has mods but no entrypoint; entrypoint is required when mods are present

locales/es/main.ftl

@@ -81,3 +81,6 @@ exec-cannot-use-alias = Cannot use --alias together with a trailing command
 exec-must-specify = Must specify either --alias <name> or a command after `--`
 exec-unknown-alias-empty = Unknown alias @{ $name } (no aliases configured)
 exec-unknown-alias-list = Unknown alias @{ $name }; known aliases: { $aliases }
+mods-uploading = 📤 Subiendo mods a { $node }...
+mods-uploaded = ✅ Mods subidos a { $node }
+mods-entrypoint-required = La receta '{ $name }' tiene mods pero no tiene entrypoint; se requiere entrypoint cuando hay mods

locales/fr/main.ftl

@@ -81,3 +81,6 @@ exec-cannot-use-alias = Cannot use --alias together with a trailing command
 exec-must-specify = Must specify either --alias <name> or a command after `--`
 exec-unknown-alias-empty = Unknown alias @{ $name } (no aliases configured)
 exec-unknown-alias-list = Unknown alias @{ $name }; known aliases: { $aliases }
+mods-uploading = 📤 Téléchargement des mods vers { $node }...
+mods-uploaded = ✅ Mods téléchargés vers { $node }
+mods-entrypoint-required = La recette '{ $name }' a des mods mais pas d'entrypoint ; l'entrypoint est requis quand des mods sont présents

locales/it/main.ftl

@@ -81,3 +81,6 @@ exec-cannot-use-alias = Cannot use --alias together with a trailing command
 exec-must-specify = Must specify either --alias <name> or a command after `--`
 exec-unknown-alias-empty = Unknown alias @{ $name } (no aliases configured)
 exec-unknown-alias-list = Unknown alias @{ $name }; known aliases: { $aliases }
+mods-uploading = 📤 Caricamento mod su { $node }...
+mods-uploaded = ✅ Mod caricati su { $node }
+mods-entrypoint-required = La ricetta '{ $name }' ha mod ma nessun entrypoint; l'entrypoint è obbligatorio quando sono presenti mod

locales/ru/main.ftl

@@ -82,3 +82,6 @@ exec-cannot-use-alias = Cannot use --alias together with a trailing command
 exec-must-specify = Must specify either --alias <name> or a command after `--`
 exec-unknown-alias-empty = Unknown alias @{ $name } (no aliases configured)
 exec-unknown-alias-list = Unknown alias @{ $name }; known aliases: { $aliases }
+mods-uploading = 📤 Загрузка модов на { $node }...
+mods-uploaded = ✅ Моды загружены на { $node }
+mods-entrypoint-required = Рецепт '{ $name }' содержит моды, но не имеет entrypoint; entrypoint обязателен при наличии модов

recipes/laguna-m1-nvfp4/recipe.toml

@@ -0,0 +1,27 @@
+engine = "vllm"
+name = "Laguna M.1 NVFP4 (vLLM)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
+model = "poolside/Laguna-M.1-NVFP4"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+
+# 225B total / 23B active MoE — NVFP4 quantized (~131B stored).
+# 2× DGX Spark (GB10 Blackwell), 128 GB shared each.
+# TP=2 → ~66 GB/node weights, room for 262K KV cache.
+# Quantization auto-detected from model config.
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--gpu-memory-utilization", "0.90",
+    "--max-model-len", "262144",
+    "--load-format", "safetensors",
+    "--enable-prefix-caching",
+    "--tool-call-parser", "poolside_v1",
+    "--reasoning-parser", "poolside_v1",
+    "--enable-auto-tool-choice",
+    "--trust-remote-code",
+    "--served-model-name", "laguna",
+]

recipes/test-mods/mods/local-patch/patch.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+# Simulates a vLLM patch: write a marker that the engine can read
+echo "alpine:3.20" > /tmp/.vllm-patched
+echo "MODS_PATCHED=1" >> /tmp/.vllm-patched
+echo "[mods] Applied vLLM compatibility patch at $(date)" >&2

recipes/test-mods/recipe.toml

@@ -0,0 +1,17 @@
+engine = "vllm"
+name = "Test Mods"
+docker_image = "alpine:latest"
+model = "noop"
+port = 30000
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "none"
+
+# Entrypoint must be set when mods are present.
+entrypoint = ["sh", "-c", "i=0; while true; do echo \"[$(hostname)] heartbeat $i\"; i=$((i+1)); sleep 2; done"]
+
+[mods]
+commands = [
+    "sh /mods/local-patch/patch.sh",
+]

src/before_serve.rs

@@ -192,6 +192,7 @@ mod tests {
             before_serve,
             exec_aliases: HashMap::new(),
             home_assistant: None,
+            global_mods_dir: None,
         }
     }
 
@@ -246,6 +247,7 @@ mod tests {
             entrypoint: vec![],
             distributed_backend: "mp".to_string(),
             before_serve: vec![],
+            mods: Default::default(),
             shm_size: String::new(),
         }
     }

src/cli.rs

@@ -72,10 +72,7 @@ pub enum Commands {
             help = "Cluster name (e.g. 4nodes-switch). If omitted — checks all."
         )]
         cluster: Option<String>,
-        #[arg(
-            long,
-            help = "Show node host keys (for adding to config.toml)"
-        )]
+        #[arg(long, help = "Show node host keys (for adding to config.toml)")]
         fingerprint: bool,
     },
 
@@ -89,10 +86,7 @@ pub enum Commands {
             help = "Cluster (if not auto — uses recipe.cluster)"
         )]
         cluster: String,
-        #[arg(
-            long,
-            help = "Auto-fix issues (create Dockerfile + build image)"
-        )]
+        #[arg(long, help = "Auto-fix issues (create Dockerfile + build image)")]
         repair: bool,
     },
 
@@ -102,10 +96,7 @@ pub enum Commands {
 
     /// Auto-discover host keys and write them into config.toml
     InitHostKeys {
-        #[arg(
-            long,
-            help = "Cluster name (e.g. 4nodes-switch). If omitted — all."
-        )]
+        #[arg(long, help = "Cluster name (e.g. 4nodes-switch). If omitted — all.")]
         cluster: Option<String>,
     },
 

src/config.rs

@@ -1,4 +1,4 @@
-pub use crate::pre_serve::{BeforeServeTable, OnFailure, PreServeCommand};
+pub use crate::pre_serve::{BeforeServeTable, ModsTable, OnFailure, PreServeCommand};
 use anyhow::{Context, Result};
 use serde::Deserialize;
 use std::collections::HashMap;
@@ -50,6 +50,11 @@ pub struct Config {
     pub exec_aliases: HashMap<String, String>,
     #[serde(default)]
     pub home_assistant: Option<HomeAssistantConfig>,
+    /// Global mods directory — script files merged with recipe mods/ into
+    /// the container's /mods/.  Recipe-level [mods] commands explicitly
+    /// reference files from either source via /mods/<file>.
+    #[serde(default)]
+    pub global_mods_dir: Option<std::path::PathBuf>,
 }
 /// Home Assistant config for `power on`/`power off`.
 #[derive(Clone, Deserialize)]
@@ -115,6 +120,10 @@ pub struct Recipe {
     /// Recipe-level pre-serve commands (appended after cluster-level).
     #[serde(default)]
     pub before_serve: Vec<PreServeCommand>,
+    /// Mods: commands that run *inside* the container via entrypoint wrapping
+    /// before the engine process starts. Requires `entrypoint` to be set.
+    #[serde(default)]
+    pub mods: ModsTable,
 }
 /// Resolves cluster by exact name, then unique prefix. Errors if none or ambiguous.
 fn resolve_cluster_name<'a>(clusters: &'a [Cluster], name: &str) -> Result<&'a Cluster> {
@@ -205,6 +214,19 @@ pub fn resolve_cluster<'c>(
     };
     select_cluster(config, &choice, recipe.required_nodes)
 }
+/// Resolve the path to a recipe's `mods/` directory, if it exists.
+/// Checks cwd first, then the default config dir.
+pub async fn resolve_mods_dir(name: &str) -> Option<std::path::PathBuf> {
+    let cwd = std::path::PathBuf::from(format!("recipes/{name}/mods"));
+    if cwd.is_dir() {
+        return Some(cwd);
+    }
+    let cfg = default_config_dir()?
+        .join("recipes")
+        .join(name)
+        .join("mods");
+    cfg.is_dir().then_some(cfg)
+}
 /// Loads a recipe (cwd first, then `<default_config_dir>/recipes/<name>/recipe.toml`,
 /// falling back to cwd for the error).
 pub async fn load_recipe(recipe_name: &str) -> Result<Recipe> {
@@ -266,6 +288,7 @@ mod tests {
             before_serve: HashMap::new(),
             exec_aliases: HashMap::new(),
             home_assistant: None,
+            global_mods_dir: None,
         }
     }
     fn test_nccl() -> NcclConfig {
@@ -276,13 +299,9 @@ mod tests {
             ib_disable: 0,
         }
     }
-    // ── Deserialization tests ──────────────────────────────────
     #[test]
     fn test_recipe_deserialize_minimal() {
-        let toml_str = r#"
-name = "Minimal"
-model = "test/model"
-"#;
+        let toml_str = "\nname = \"Minimal\"\nmodel = \"test/model\"\n";
         let recipe: Recipe = toml::from_str(toml_str).unwrap();
         assert_eq!(recipe.name, "Minimal");
         assert_eq!(recipe.model, "test/model");
@@ -318,7 +337,6 @@ OTHER = "123"
         assert_eq!(recipe.port, 30000);
         assert_eq!(recipe.required_nodes, Some(4));
         assert_eq!(recipe.pipeline_parallel_size, Some(2));
-        assert_eq!(recipe.tensor_parallel_size, 1);
         assert_eq!(recipe.distributed_backend, "ray");
         assert_eq!(recipe.cluster, Some("my-cluster".to_string()));
         assert_eq!(recipe.args, vec!["--arg1", "val1", "--arg2"]);
@@ -358,25 +376,15 @@ ib_disable = 0
         assert_eq!(config.clusters.len(), 1);
         assert_eq!(config.clusters[0].nodes.len(), 2);
         assert_eq!(config.clusters[0].nodes[0].host_key, None);
-        assert!(config.nccl.contains_key("prod"));
-        assert_eq!(config.nccl["prod"].ib_hca, "mlx5_0");
+        assert!(config.nccl.contains_key("prod") && config.nccl["prod"].ib_hca == "mlx5_0");
     }
     #[test]
     fn test_config_deserialize_with_host_key() {
-        let toml_str = r#"
-ssh_user = "admin"
-ssh_key_path = "/path/to/key"
-[[clusters]]
-name = "secure"
-master_ib_ip = "1.1.1.1"
-nodes = [
-    { name = "n1", ssh_ip = "2.2.2.2", ib_ip = "1.1.1.1", rank = 0, host_key = "SHA256:abc123" },
-]
-"#;
+        let toml_str = "\nssh_user = \"admin\"\nssh_key_path = \"/path/to/key\"\n[[clusters]]\nname = \"secure\"\nmaster_ib_ip = \"1.1.1.1\"\nnodes = [\n    { name = \"n1\", ssh_ip = \"2.2.2.2\", ib_ip = \"1.1.1.1\", rank = 0, host_key = \"SHA256:abc123\" },\n]\n";
         let config: Config = toml::from_str(toml_str).unwrap();
         assert_eq!(
             config.clusters[0].nodes[0].host_key,
-            Some("SHA256:abc123".to_string())
+            Some("SHA256:abc123".into())
         );
     }
     #[test]
@@ -391,6 +399,7 @@ nodes = [
             before_serve: HashMap::new(),
             exec_aliases: HashMap::new(),
             home_assistant: None,
+            global_mods_dir: None,
         };
         config.nccl.insert("test-cluster".to_string(), test_nccl());
         let (cluster, nccl) = select_cluster(&config, "test-cluster", None).unwrap();
@@ -409,6 +418,7 @@ nodes = [
             before_serve: HashMap::new(),
             exec_aliases: HashMap::new(),
             home_assistant: None,
+            global_mods_dir: None,
         };
         config.nccl.insert("test-cluster".to_string(), test_nccl());
         let (cluster, nccl) = select_cluster(&config, "auto", None).unwrap();
@@ -475,14 +485,11 @@ nodes = [
         let recipe: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"").unwrap();
         assert_eq!(recipe.distributed_backend, "mp");
     }
-    // ── Default config dir ─────────────────────────────────────
     #[test]
     fn default_config_dir_appends_sm121_launcher() {
         let Some(dir) = default_config_dir() else {
-            // No config dir on this system; nothing to assert.
             return;
         };
-        assert!(dir.ends_with("sm121-launcher"));
-        assert!(dir.parent().is_some());
+        assert!(dir.ends_with("sm121-launcher") && dir.parent().is_some());
     }
 }

src/docker/mod.rs

@@ -1,16 +1,15 @@
 pub mod health;
-
-use std::borrow::Cow;
-
-use anyhow::{Context, Result};
-use tokio::time::{Duration, sleep};
-use tracing::info;
-
+pub mod mods;
+pub mod tail;
 use crate::{
     config::{Cluster, Config, NcclConfig, Node, Recipe, load_recipe, resolve_cluster},
     doctor::try_repaired_image,
     ssh::{run_ssh, run_ssh_follow, run_ssh_with_retry},
 };
+use anyhow::{Context, Result};
+use std::borrow::Cow;
+use tokio::time::{Duration, sleep};
+use tracing::info;
 
 pub async fn serve(
     config: &Config,
@@ -73,6 +72,11 @@ pub async fn serve(
         }
     }
     crate::before_serve::run(config, &cluster, &recipe, skip_before_serve, dry_run).await?;
+
+    let mods_remote_path =
+        mods::upload_mods_to_cluster(config, &cluster, &recipe, recipe_name, node_name, dry_run)
+            .await?;
+
     let image = try_repaired_image(config, &cluster, &recipe).await?;
     let mut recipe = recipe;
     if image != recipe.docker_image {
@@ -89,6 +93,7 @@ pub async fn serve(
             cache_dir,
             keep,
             node_name: node_name.map(String::from),
+            mods_remote_path,
             dry_run,
         },
     )
@@ -107,7 +112,7 @@ pub(crate) fn logs_follow_cmd(recipe_name: &str) -> String {
         recipe_name
     )
 }
-fn shq(s: &str) -> String {
+pub(crate) fn shq(s: &str) -> String {
     let mut out = String::with_capacity(s.len() + 2);
     out.push('\'');
     for c in s.chars() {
@@ -228,6 +233,7 @@ struct StartClusterOpts {
     cache_dir: std::path::PathBuf,
     keep: bool,
     node_name: Option<String>,
+    mods_remote_path: String,
     dry_run: bool,
 }
 
@@ -244,6 +250,7 @@ struct DockerCommandCtx {
     cache_dir: String,
     keep: bool,
     init_port: u16,
+    mods_remote_path: String,
 }
 
 async fn start_cluster(
@@ -299,6 +306,7 @@ async fn start_cluster(
             cache_dir: opts.cache_dir.to_string_lossy().into_owned(),
             keep: opts.keep,
             init_port: config.init_port,
+            mods_remote_path: opts.mods_remote_path.clone(),
         };
         handles.push(tokio::spawn(async move {
             if !dry {
@@ -347,11 +355,13 @@ fn build_docker_command(ctx: &DockerCommandCtx, node: &Node, is_head: bool) -> V
     if !ctx.keep {
         flags.push("--rm".to_string());
     }
-
     let mut cmd = vec![
         format!("docker run -d {}", flags.join(" ")),
         format!("-v {}:/root/.cache/huggingface", ctx.cache_dir),
     ];
+    if !ctx.mods_remote_path.is_empty() && !ctx.recipe.mods.commands.is_empty() {
+        cmd.push(format!("-v {}:/mods:ro", ctx.mods_remote_path));
+    }
 
     let mut env = vec![("HF_HOME", Cow::Borrowed("/root/.cache/huggingface"))];
     if ctx.nnodes > 1 {
@@ -405,69 +415,27 @@ fn build_docker_command(ctx: &DockerCommandCtx, node: &Node, is_head: bool) -> V
         cmd.push(format!("-e {k}={v}"));
     }
 
-    cmd.push(ctx.recipe.docker_image.clone());
-
-    if !ctx.recipe.entrypoint.is_empty() {
-        let mut ep = ctx.recipe.entrypoint.clone();
-        if ep.len() >= 3 && ep[ep.len() - 2] == "-c" {
-            let last = ep.len() - 1;
-            ep[last] = shq(&ep[last]);
-        }
-        cmd.extend(ep);
-    }
-
-    if is_sglang {
-        cmd.push("--model-path".to_string());
-    }
-    cmd.push(ctx.recipe.model.clone());
-
-    cmd.extend(ctx.recipe.args.clone());
-
-    let mut flags: Vec<String> = match ctx.recipe.distributed_backend.as_str() {
-        "ray" | "mp" if is_sglang => vec![
-            "--tp".into(),
-            ctx.tp_size.to_string(),
-            "--nnodes".into(),
-            ctx.nnodes.to_string(),
-            "--node-rank".into(),
-            node.rank.to_string(),
-            "--dist-init-addr".into(),
-            format!("{}:{}", ctx.cluster.master_ib_ip, ctx.init_port),
-        ],
-        "ray" | "mp" => {
-            let mut fp = vec![
-                "--tensor-parallel-size".into(),
-                ctx.tp_size.to_string(),
-                "--pipeline-parallel-size".into(),
-                ctx.pp_size.to_string(),
-                "--nnodes".into(),
-                ctx.nnodes.to_string(),
-                "--node-rank".into(),
-                node.rank.to_string(),
-                "--master-addr".into(),
-                ctx.cluster.master_ib_ip.clone(),
-                "--master-port".into(),
-                ctx.init_port.to_string(),
-            ];
-            if ctx.recipe.distributed_backend == "ray" {
-                fp.push("--distributed-executor-backend".into());
-                fp.push(ctx.recipe.distributed_backend.clone());
-            }
-            fp
-        }
-        _ => vec![],
+    let tctx = tail::TailCtx {
+        recipe: &ctx.recipe,
+        tp_size: ctx.tp_size,
+        pp_size: ctx.pp_size,
+        nnodes: ctx.nnodes,
+        init_port: ctx.init_port,
+        master_ib_ip: &ctx.cluster.master_ib_ip,
     };
-    if !is_sglang
-        && node.rank > 0
-        && matches!(ctx.recipe.distributed_backend.as_str(), "mp" | "ray")
-    {
-        flags.push("--headless".into());
+    let tail = tail::build_tail(&tctx, node);
+    if ctx.recipe.mods.commands.is_empty() {
+        cmd.push(ctx.recipe.docker_image.clone());
+        cmd.extend(tail);
+    } else {
+        let script = mods::build_mod_script(&ctx.recipe.mods.commands, &tail);
+        cmd.extend([
+            "--entrypoint sh".into(),
+            ctx.recipe.docker_image.clone(),
+            "-c".into(),
+            script,
+        ]);
     }
-    cmd.extend(flags);
-    cmd.push("--host".to_string());
-    cmd.push("0.0.0.0".to_string());
-    cmd.push("--port".to_string());
-    cmd.push(ctx.recipe.port.to_string());
     cmd
 }
 

src/docker/mods.rs

@@ -0,0 +1,263 @@
+//! Mods: commands that run *inside* the container via entrypoint wrapping,
+//! before the engine process starts.  Each command runs inside `sh -c` on
+//! container start; the original entrypoint is `exec`'d at the end so the
+//! engine becomes PID 1 and signals pass through correctly.
+
+use anyhow::{Context, Result};
+use tracing::info;
+
+use crate::config::{Cluster, Config, Node, OnFailure, PreServeCommand, Recipe, resolve_mods_dir};
+use crate::ssh::upload_dir;
+
+use super::shq;
+
+/// Build a shell script that runs `mods` then `exec`s the original tail
+/// (entrypoint + model + args + auto-generated flags).
+///
+/// Mod commands are shell code (piped directly into the script) — only
+/// embedded single quotes are escaped so the outer [`shq`] wrapper doesn't
+/// break.  Each tail element is individually [`shq`]'d so multi-word or
+/// special-containing parts survive word-splitting inside `sh -c`.
+///
+/// The returned string is already wrapped in single quotes (via `shq`),
+/// ready to use as the `-c` argument to `sh -c`.
+pub fn build_mod_script(mods: &[PreServeCommand], tail: &[String]) -> String {
+    let mut script = String::new();
+
+    for m in mods {
+        // Mod commands are shell code — do NOT wrap in shq (which would
+        // prevent word splitting).  Embed directly; the outer shq wrapper
+        // handles any embedded single quotes.
+        let cmd = m.command();
+        match m.on_failure() {
+            OnFailure::Abort => {
+                script.push_str(&format!("{cmd} || exit 1; "));
+            }
+            OnFailure::Warn | OnFailure::Continue => {
+                script.push_str(&format!("{cmd} || true; "));
+            }
+        }
+    }
+
+    script.push_str("exec ");
+    for part in tail {
+        script.push_str(&shq(part));
+        script.push(' ');
+    }
+
+    let trimmed = script.trim_end().to_string();
+    shq(&trimmed)
+}
+
+/// Upload the mods directory to every node and return the remote path, or
+/// empty string when no mods directory exists.  Also validates that
+/// `entrypoint` is set when mods are present.
+pub async fn upload_mods_to_cluster(
+    config: &Config,
+    cluster: &Cluster,
+    recipe: &Recipe,
+    recipe_name: &str,
+    node_name: Option<&str>,
+    dry_run: bool,
+) -> Result<String> {
+    if recipe.mods.commands.is_empty() {
+        return Ok(String::new());
+    }
+    if recipe.entrypoint.is_empty() {
+        anyhow::bail!(
+            "{}",
+            fluent_i18n::t!("mods-entrypoint-required", { "name" => &recipe.name })
+        );
+    }
+    let nodes: Vec<&Node> = if let Some(ref nn) = node_name {
+        vec![
+            cluster
+                .nodes
+                .iter()
+                .find(|n| n.name == *nn)
+                .context(format!(
+                    "Node '{}' not found in cluster '{}'",
+                    nn, cluster.name
+                ))?,
+        ]
+    } else {
+        cluster.nodes.iter().collect()
+    };
+    let remote = format!("/tmp/sm121-mods-{}", recipe_name);
+    // Upload global mods first (shared scripts), then recipe mods on top
+    // (recipe-specific overrides). Both are merged into the same /mods/ dir.
+    if let Some(ref global_dir) = config.global_mods_dir
+        && global_dir.is_dir()
+    {
+        for node in &nodes {
+            upload_dir(config, node, global_dir, &remote, dry_run).await?;
+        }
+    }
+    if let Some(local_mods_dir) = resolve_mods_dir(recipe_name).await {
+        for node in &nodes {
+            info!(
+                "{}",
+                fluent_i18n::t!("mods-uploading", { "node" => &node.name })
+            );
+            upload_dir(config, node, &local_mods_dir, &remote, dry_run).await?;
+            info!(
+                "{}",
+                fluent_i18n::t!("mods-uploaded", { "node" => &node.name })
+            );
+        }
+    }
+    Ok(remote)
+}
+
+#[cfg(test)]
+fn unshq(s: &str) -> String {
+    let inner = s
+        .strip_prefix('\'')
+        .and_then(|s| s.strip_suffix('\''))
+        .unwrap_or(s);
+    inner.replace(r"'\''", "'")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::PreServeCommand;
+
+    #[test]
+    fn simple_mods_wraps_tail() {
+        let mods = vec![
+            PreServeCommand::Bare("pip install flash-attn".into()),
+            PreServeCommand::Bare("python patch.py".into()),
+        ];
+        let tail = vec![
+            "vllm".to_string(),
+            "serve".to_string(),
+            "org/model".to_string(),
+            "--host".to_string(),
+            "0.0.0.0".to_string(),
+        ];
+
+        let script = build_mod_script(&mods, &tail);
+        assert!(script.starts_with('\''), "shq-wrapped: {script}");
+        assert!(script.ends_with('\''), "shq-wrapped: {script}");
+
+        let inner = unshq(&script);
+        // Mod commands are NOT shq-wrapped — they're raw shell code
+        assert!(inner.starts_with("pip install flash-attn || exit 1; "));
+        assert!(inner.contains("python patch.py || exit 1; "));
+        assert!(
+            inner.contains("exec 'vllm' 'serve' 'org/model' '--host' '0.0.0.0'"),
+            "inner: {inner}"
+        );
+    }
+
+    #[test]
+    fn on_failure_abort_exits_on_error() {
+        let mods = vec![PreServeCommand::Bare("false".into())];
+        let tail = vec!["echo".to_string(), "ok".to_string()];
+        let script = build_mod_script(&mods, &tail);
+        let inner = unshq(&script);
+        assert!(inner.contains("|| exit 1"));
+    }
+
+    #[test]
+    fn on_failure_warn_continues() {
+        let mods = vec![PreServeCommand::Configured {
+            command: "false".into(),
+            on_failure: OnFailure::Warn,
+        }];
+        let tail = vec!["echo".to_string(), "ok".to_string()];
+        let script = build_mod_script(&mods, &tail);
+        let inner = unshq(&script);
+        assert!(inner.contains("|| true"), "warn continues: {inner}");
+        assert!(inner.contains("exec"));
+    }
+
+    #[test]
+    fn on_failure_continue_is_silent() {
+        let mods = vec![PreServeCommand::Configured {
+            command: "false".into(),
+            on_failure: OnFailure::Continue,
+        }];
+        let tail = vec!["true".to_string()];
+        let script = build_mod_script(&mods, &tail);
+        let inner = unshq(&script);
+        assert!(inner.contains("|| true"));
+        assert!(inner.contains("exec 'true'"));
+    }
+
+    #[test]
+    fn empty_mods_preserves_tail() {
+        let mods = vec![];
+        let tail = vec!["vllm".to_string(), "serve".to_string(), "noop".to_string()];
+        let script = build_mod_script(&mods, &tail);
+        let inner = unshq(&script);
+        assert_eq!(inner, "exec 'vllm' 'serve' 'noop'");
+    }
+
+    #[test]
+    fn escaping_special_chars_in_commands() {
+        let mods = vec![PreServeCommand::Bare("echo 'hello world'".into())];
+        let tail = vec!["true".to_string()];
+        let script = build_mod_script(&mods, &tail);
+        // The inner single quote in `echo 'hello world'` gets escaped as
+        // '\'' so the outer shq doesn't break.  After unshq, the inner
+        // script should have `echo '\''hello world'\''` (the outer shq
+        // wrapper's quotes are stripped, but the escaped inner quotes
+        // are restored to literal single quotes).
+        let inner = unshq(&script);
+        assert!(
+            inner.contains("echo 'hello world'"),
+            "inner single quotes should survive: {inner}"
+        );
+    }
+
+    #[test]
+    fn mods_with_entrypoint_sh_c() {
+        let mods = vec![PreServeCommand::Bare("echo mod".into())];
+        let tail = vec![
+            "sh".to_string(),
+            "-c".to_string(),
+            "i=0; while true; do echo $i; i=$((i+1)); done".to_string(),
+            "noop".to_string(),
+        ];
+        let script = build_mod_script(&mods, &tail);
+        let inner = unshq(&script);
+        assert!(inner.starts_with("echo mod || exit 1;"));
+        assert!(inner.contains("exec 'sh' '-c' '"));
+        assert!(inner.contains("'noop'"));
+    }
+
+    #[test]
+    fn mods_wrapping_uses_entrypoint_sh() {
+        let mods = vec![PreServeCommand::Bare("pip list".into())];
+        let tail = vec!["vllm".to_string(), "serve".to_string()];
+        let script = build_mod_script(&mods, &tail);
+        let inner = unshq(&script);
+        assert!(
+            inner.contains("exec 'vllm' 'serve'"),
+            "tail should be exec'd: {inner}"
+        );
+    }
+
+    #[test]
+    fn deserialize_mods_toml() {
+        let toml_str = r#"
+name = "Modded"
+model = "test/model"
+entrypoint = ["vllm", "serve"]
+
+[mods]
+commands = [
+    "pip install flash-attn",
+    { command = "./patch.sh", on_failure = "warn" },
+]
+"#;
+        let recipe: crate::config::Recipe = toml::from_str(toml_str).unwrap();
+        assert_eq!(recipe.mods.commands.len(), 2);
+        assert_eq!(recipe.mods.commands[0].command(), "pip install flash-attn");
+        assert_eq!(recipe.mods.commands[0].on_failure(), OnFailure::Abort);
+        assert_eq!(recipe.mods.commands[1].command(), "./patch.sh");
+        assert_eq!(recipe.mods.commands[1].on_failure(), OnFailure::Warn);
+    }
+}

src/docker/tail.rs

@@ -0,0 +1,84 @@
+//! Builds the "tail" of the docker command — everything after the image
+//! name (entrypoint, model, args, auto-generated flags).
+
+use crate::config::{Node, Recipe};
+
+use super::shq;
+
+/// The subset of [`DockerCommandCtx`] that `build_tail` needs.
+pub(crate) struct TailCtx<'a> {
+    pub recipe: &'a Recipe,
+    pub tp_size: u32,
+    pub pp_size: u32,
+    pub nnodes: u32,
+    pub init_port: u16,
+    pub master_ib_ip: &'a str,
+}
+
+/// Build the docker command tail — everything after the image name.
+pub fn build_tail(ctx: &TailCtx<'_>, node: &Node) -> Vec<String> {
+    let is_sglang = ctx.recipe.engine == "sglang";
+    let mut tail = Vec::new();
+
+    if !ctx.recipe.entrypoint.is_empty() {
+        let mut ep = ctx.recipe.entrypoint.clone();
+        // Non-mods path: inner shq for sh -c entrypoints
+        if ctx.recipe.mods.commands.is_empty() && ep.len() >= 3 && ep[ep.len() - 2] == "-c" {
+            let last = ep.len() - 1;
+            ep[last] = shq(&ep[last]);
+        }
+        tail.extend(ep);
+    }
+
+    if is_sglang {
+        tail.push("--model-path".to_string());
+    }
+    tail.push(ctx.recipe.model.clone());
+    tail.extend(ctx.recipe.args.clone());
+
+    let mut flags: Vec<String> = match ctx.recipe.distributed_backend.as_str() {
+        "ray" | "mp" if is_sglang => vec![
+            "--tp".into(),
+            ctx.tp_size.to_string(),
+            "--nnodes".into(),
+            ctx.nnodes.to_string(),
+            "--node-rank".into(),
+            node.rank.to_string(),
+            "--dist-init-addr".into(),
+            format!("{}:{}", ctx.master_ib_ip, ctx.init_port),
+        ],
+        "ray" | "mp" => {
+            let mut fp = vec![
+                "--tensor-parallel-size".into(),
+                ctx.tp_size.to_string(),
+                "--pipeline-parallel-size".into(),
+                ctx.pp_size.to_string(),
+                "--nnodes".into(),
+                ctx.nnodes.to_string(),
+                "--node-rank".into(),
+                node.rank.to_string(),
+                "--master-addr".into(),
+                ctx.master_ib_ip.to_string(),
+                "--master-port".into(),
+                ctx.init_port.to_string(),
+            ];
+            if ctx.recipe.distributed_backend == "ray" {
+                fp.push("--distributed-executor-backend".into());
+                fp.push(ctx.recipe.distributed_backend.clone());
+            }
+            fp
+        }
+        _ => vec![],
+    };
+    if !is_sglang
+        && node.rank > 0
+        && matches!(ctx.recipe.distributed_backend.as_str(), "mp" | "ray")
+    {
+        flags.push("--headless".into());
+    }
+    tail.extend(flags);
+    tail.push("--host".to_string());
+    tail.push("0.0.0.0".to_string());
+    tail.extend(["--port".into(), ctx.recipe.port.to_string()]);
+    tail
+}

src/docker/tests.rs

@@ -52,6 +52,7 @@ fn trec() -> Recipe {
         entrypoint: vec![],
         distributed_backend: "mp".into(),
         before_serve: vec![],
+        mods: Default::default(),
         shm_size: String::new(),
     }
 }
@@ -68,6 +69,7 @@ fn make_dm_ctx(recipe: &Recipe, nnodes: u32, pp: u32, tp: u32, keep: bool) -> Do
         cache_dir: "/home/user/.cache/huggingface".to_string(),
         keep,
         init_port: 25000,
+        mods_remote_path: String::new(),
     }
 }
 
@@ -231,6 +233,7 @@ fn sglang_recipe() -> Recipe {
         entrypoint: vec!["sglang".into(), "serve".into()],
         distributed_backend: "mp".into(),
         before_serve: vec![],
+        mods: Default::default(),
         shm_size: String::new(),
     }
 }

src/doctor.rs

@@ -341,6 +341,7 @@ mod tests {
             entrypoint: vec![],
             distributed_backend: "mp".to_string(),
             before_serve: vec![],
+            mods: Default::default(),
             shm_size: String::new(),
         }
     }

src/exec.rs

@@ -166,6 +166,7 @@ mod tests {
             before_serve: HashMap::new(),
             exec_aliases,
             home_assistant: None,
+            global_mods_dir: None,
         }
     }
 

src/pre_serve.rs

@@ -54,3 +54,11 @@ pub struct BeforeServeTable {
     #[serde(default)]
     pub commands: Vec<PreServeCommand>,
 }
+
+/// TOML sub-table under `[mods]` in a recipe.  Mirrors [`BeforeServeTable`].
+#[derive(Clone, Debug, Default, Deserialize)]
+pub struct ModsTable {
+    /// Commands to run inside the container before the engine starts.
+    #[serde(default)]
+    pub commands: Vec<PreServeCommand>,
+}

src/ssh.rs

@@ -5,6 +5,7 @@ use russh::client;
 use russh::keys::{PrivateKeyWithHashAlg, load_secret_key as russh_load_secret_key};
 use russh::{ChannelMsg, Error as SshError};
 use sha2::{Digest, Sha256};
+use std::path::Path;
 use std::sync::Arc;
 use tokio::time::{Duration, sleep};
 use tracing::warn;
@@ -219,8 +220,56 @@ pub async fn fetch_host_key(config: &Config, node: &Node) -> Result<String> {
     Ok(fp)
 }
 
+/// Upload a local directory to a remote node by tarring, base64-encoding,
+/// and piping the result through SSH for decoding and extraction.
+///
+/// The remote directory is created if it doesn't exist.
+pub async fn upload_dir(
+    config: &Config,
+    node: &Node,
+    local_dir: &Path,
+    remote_dir: &str,
+    dry_run: bool,
+) -> Result<()> {
+    if dry_run {
+        println!(
+            "  ssh {}@{}: tar -> base64 -> decode -> extract -> {} (dry-run)",
+            config.ssh_user, node.ssh_ip, remote_dir
+        );
+        return Ok(());
+    }
+
+    // Tar only the *contents* of the directory (not the directory itself)
+    // so that files extract directly into remote_dir.
+    let output = std::process::Command::new("tar")
+        .args(["czf", "-", "--dereference", "-C"])
+        .arg(local_dir)
+        .arg(".")
+        .output()
+        .context(format!("Failed to tar directory '{}'", local_dir.display()))?;
+    if !output.status.success() {
+        anyhow::bail!(
+            "tar failed on '{}': {}",
+            local_dir.display(),
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+    let b64 = STANDARD.encode(&output.stdout);
+
+    // Base64 is safe inside single quotes (charset: A-Za-z0-9+/=).
+    // `printf '%s'` is portable and doesn't interpret escape sequences.
+    let cmd = format!(
+        "mkdir -p {dir} && cd {dir} && printf '%s' '{b64}' | base64 -d | tar xzf -",
+        dir = remote_dir,
+        b64 = b64
+    );
+
+    run_ssh(config, node, cmd, false).await?;
+    Ok(())
+}
+
 /// Loads an SSH private key from the given path.
-pub fn load_secret_key(path: &std::path::Path) -> Result<russh::keys::PrivateKey> {
+pub fn load_secret_key(path: &Path) -> Result<russh::keys::PrivateKey> {
     russh_load_secret_key(path, None).context("Failed to load SSH key")
 }