serve: --clear flag to wipe host-side extra_mount folders (sudo)

Caches in extra_mount host dirs (JIT/kernel caches, etc.) are usually
root-owned because the containers run as root, so `sm121 stop`/operator
can't easily reset them. `serve --clear` runs `sudo rm -rf` on each
alias/host extra_mount dir on the launch nodes before containers start.

- Repo-based path refs (HF cache) are never touched.
- Only the launch set is cleared (--node/required_nodes trimmed), so a
  cache shared with another running container on a non-launch node is
  left alone.
- Failure on any node aborts the serve (no launching against a stale
  cache). `-d` prints the exact sudo rm -rf command per node.
- Dedup + sort via clear_paths (tested).

AGENTS.md documents the flag under Commands.

AGENTS.md

@@ -54,6 +54,13 @@ from the shared cache (`cp -a`, one-time per-node cost); a repo absent from
 BOTH locations aborts with a `huggingface-cli download` hint. Later serves read
 the model locally — no streaming weights over slow shared storage at load time.
 
+`serve --clear` deletes the **host-side `extra_mounts` folders** (alias/host
+— repo-based path refs are never touched) on the launch nodes with
+`sudo rm -rf` before containers start. The containers run as root, so the
+cache files there are usually root-owned. Use to reset JIT/kernel caches
+between serves. `-d --clear` prints the exact `sudo rm -rf '...'` command
+per node instead of running it.
+
 > ⚠️ **Cluster may be serving live traffic.** A model (often the one serving this
 > very agent) may be running on the cluster at any time. Before running `stop`,
 > `power off`, or `power switch`, ALWAYS first run `health` and confirm with the

src/cli.rs

@@ -87,6 +87,14 @@ pub enum Commands {
         /// left untouched.
         #[arg(long, num_args = 0..=1, default_missing_value = "")]
         fit: Option<String>,
+
+        /// Clear the host-side `extra_mounts` folders (alias/host) on the
+        /// launch nodes with `sudo rm -rf` before containers start — the
+        /// containers run as root, so caches there are usually root-owned.
+        /// Repo-based mounts (HF cache path refs) are never cleared. `-d`
+        /// prints the exact command instead of running it.
+        #[arg(long)]
+        clear: bool,
     },
 
     /// Show container logs

src/docker/command.rs

@@ -6,8 +6,10 @@
 
 use super::{mods, shq, tail};
 use crate::config::{Cluster, Config, NcclConfig, Node, Recipe};
+use crate::ssh::run_ssh;
 use anyhow::Result;
 use std::borrow::Cow;
+use tracing::info;
 
 /// Shared context for [`build_docker_command`] — constructed once, cloned per node.
 #[derive(Clone)]
@@ -114,6 +116,49 @@ pub(crate) struct ResolvedExtraMounts {
     pub path_refs: Vec<PathRef>,
 }
 
+/// `--clear`: delete the host-side static mount dirs on the launch nodes
+/// before containers start. Containers run as root, so caches there are
+/// usually root-owned → `sudo rm -rf`. Dry run prints the exact command
+/// (via [`run_ssh`]) and runs nothing. A failure on any node aborts the
+/// serve — never start against a stale cache. Repo-based path refs are
+/// never cleared (they point into the HF cache).
+pub(crate) async fn clear_extra_mounts(
+    config: &Config,
+    nodes: &[&Node],
+    resolved: &ResolvedExtraMounts,
+    clear: bool,
+    dry_run: bool,
+) -> Result<()> {
+    if !clear {
+        return Ok(());
+    }
+    let paths = clear_paths(&resolved.static_mounts);
+    if paths.is_empty() {
+        info!("--clear: no host-side extra_mounts to clear");
+        return Ok(());
+    }
+    info!(
+        "--clear: clearing {} host extra_mount dir(s) on {} launch node(s)",
+        paths.len(),
+        nodes.len()
+    );
+    for node in nodes {
+        for p in &paths {
+            let cmd = format!("sudo rm -rf {}", shq(p));
+            run_ssh(config, node, cmd, dry_run).await?;
+        }
+    }
+    Ok(())
+}
+
+/// Dedup + sort the static mount host paths (the set `--clear` deletes).
+pub(crate) fn clear_paths(mounts: &[ResolvedMount]) -> Vec<String> {
+    let mut paths: Vec<String> = mounts.iter().map(|m| m.host.clone()).collect();
+    paths.sort();
+    paths.dedup();
+    paths
+}
+
 /// Substitute `{{<name>}}` placeholders in `args` and `env` with resolved paths.
 /// `path_refs` maps placeholder name → resolved path (per-node). Done before
 /// building the docker command so vLLM args reference the correct snapshot.

src/docker/command_tests.rs

@@ -238,3 +238,16 @@ fn substitute_path_refs_in_env() {
         "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft"
     );
 }
+
+#[test]
+fn clear_paths_dedups_and_sorts() {
+    let m = |host: &str| ResolvedMount {
+        host: host.into(),
+        container: "/c".into(),
+        read_only: false,
+        env: None,
+    };
+    let mounts = vec![m("/b/.cache/jit"), m("/a/.cache/jit"), m("/b/.cache/jit")];
+    assert_eq!(clear_paths(&mounts), vec!["/a/.cache/jit", "/b/.cache/jit"]);
+    assert!(clear_paths(&[]).is_empty());
+}

src/docker/mod.rs

@@ -44,6 +44,7 @@ pub struct ServeOpts {
     /// the whole selected cluster; `Some("N")` = scale to N nodes;
     /// `None` = unset. See `docker::serve`.
     pub fit: Option<String>,
+    pub clear: bool,
 }
 /// Shell-quote a string for safe interpolation into a remote `sh -c`
 /// command (single-quote wrapping, embedded quotes escaped). Re-exported
@@ -96,11 +97,10 @@ pub async fn serve(
     use crate::ends::{SshNodePinger, SshProber, ha_controller_from_config};
     use crate::orchestrate::{BootTiming, PrepareRequest, prepare_cluster};
 
-    let power_ctrl = if opts.auto_power_on {
-        ha_controller_from_config(config)
-    } else {
-        None
-    };
+    let power_ctrl = opts
+        .auto_power_on
+        .then(|| ha_controller_from_config(config))
+        .flatten();
     // Effective solo: CLI flag overrides recipe field. When solo and no
     // explicit --node, launch on exactly one node (the first free node in
     // the smallest cluster that has one) with no distributed flags. When
@@ -177,13 +177,10 @@ pub async fn serve(
     )
     .await?;
 
-    use self::sync::sync_image_to_nodes;
-
-    // Build image on specified node if needed.
-    // `built_on` records the node that actually (re)built the image. After a
-    // fresh build that image is authoritative — the sync step below must pull
-    // FROM it, never let a peer's older image (by creation timestamp) overwrite
-    // the new build.
+    // `built_on` records the node that (re)built the image this serve: after
+    // a fresh build that image is authoritative — the sync step below must
+    // pull FROM it, never let a peer's older image (by creation timestamp)
+    // overwrite the new build.
     let mut built_on: Option<String> = None;
     if let Some(ref build_node_name) = opts.build_node {
         if !opts.dry_run && !cluster.nodes.iter().any(|n| n.name == *build_node_name) {
@@ -233,8 +230,8 @@ pub async fn serve(
         // the new build. The "most recently created" heuristic only applies
         // when nothing was built here and no explicit --sync-image-source was
         // given — i.e. the nodes merely disagree and we pick the newest.
-        let sync_source = built_on.as_deref().or(opts.sync_image_source.as_deref());
-        sync_image_to_nodes(config, &cluster, &recipe.docker_image, sync_source).await?;
+        let source = built_on.as_deref().or(opts.sync_image_source.as_deref());
+        self::sync::sync_image_to_nodes(config, &cluster, &recipe.docker_image, source).await?;
     }
 
     // Drop page cache on all nodes right before starting containers.
@@ -272,6 +269,7 @@ pub async fn serve(
             node_name: node_name.map(String::from),
             mods_remote_path,
             dry_run: opts.dry_run,
+            clear: opts.clear,
         },
     )
     .await?;
@@ -363,6 +361,7 @@ struct StartClusterOpts {
     node_name: Option<String>,
     mods_remote_path: String,
     dry_run: bool,
+    clear: bool,
 }
 async fn start_cluster(
     config: &Config,
@@ -422,6 +421,7 @@ async fn start_cluster(
     // (`repo`+`subfolder`) are kept as specs and resolved per node inside the
     // spawn loop, since the HF snapshot hash can differ across nodes.
     let resolved = command::resolve_extra_mounts(config, recipe)?;
+    command::clear_extra_mounts(config, &nodes, &resolved, opts.clear, opts.dry_run).await?;
     let mut handles = Vec::new();
     for node in &nodes {
         let node = (*node).clone();

src/main.rs

@@ -106,6 +106,7 @@ async fn main() -> Result<()> {
             solo,
             mirror,
             fit,
+            clear,
         } => {
             handle_serve(
                 &config,
@@ -125,6 +126,7 @@ async fn main() -> Result<()> {
                     solo,
                     mirror,
                     fit,
+                    clear,
                 },
                 follow,
                 dry_run,
@@ -295,6 +297,7 @@ async fn handle_build_cmd(
         solo: None,
         mirror: None,
         fit: None,
+        clear: false,
     };
     docker::build::handle_build(config, &cluster, recipe, &recipe_obj, &opts, build_node).await?;
     Ok(())