Browse Source

fix: make NCCL config optional for single-node, capture stderr, support --on for tmux

- select_cluster returns Option<&NcclConfig>: NCCL config only required
  when required_nodes > 1 (multi-node). Single-node recipes no longer
  need NCCL config or IB addresses.
- run_ssh/run_ssh_long_running now capture stderr (ChannelMsg::ExtendedData)
  so error output from venv setup and other commands is visible.
- tmux_serve uses prepare_cluster instead of bare select_cluster, enabling
  --on (auto-power-on) and --force for tmux deploy.
- build_nccl_setup/build_session_commands accept Option<&NcclConfig>,
  returning empty NCCL exports when None (single-node).
Alexander Korolev 2 tháng trước cách đây
mục cha
commit
1b4d07309a

+ 98 - 14
src/cluster_state.rs

@@ -5,7 +5,7 @@
 //! State collection (the "control" side) lives in [`ClusterState::probe`].
 //! Selection logic (the "state" side) lives on [`ClusterState`] methods.
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 use std::collections::{HashMap, HashSet};
 use tracing::{info, warn};
 
@@ -104,29 +104,48 @@ impl ClusterState {
 }
 
 /// Select a cluster given the current cluster state.
+///
+/// NCCL config is only required for multi-node recipes
+/// (`required_nodes > 1`). For single-node recipes, clusters without
+/// NCCL config are allowed — the config is returned as `None`.
 pub fn select_cluster<'c>(
     config: &'c Config,
     cluster_name: &str,
     required_nodes: Option<u32>,
     state: &ClusterState,
-) -> Result<(Cluster, &'c NcclConfig)> {
+) -> Result<(Cluster, Option<&'c NcclConfig>)> {
     if cluster_name != "auto" {
         let cluster = crate::config::resolve_cluster_name(&config.clusters, cluster_name)?;
-        let nccl = config
-            .nccl
-            .get(&cluster.name)
-            .context("no NCCL config for cluster")?;
+        let nccl = config.nccl.get(&cluster.name);
+        if required_nodes.is_some_and(|n| n > 1) && nccl.is_none() {
+            anyhow::bail!("no NCCL config for cluster '{}'", cluster.name);
+        }
         return Ok((cluster.clone(), nccl));
     }
-    let mut candidates: Vec<(&Cluster, &NcclConfig)> = config
-        .clusters
-        .iter()
-        .filter_map(|c| Some((c, config.nccl.get(&c.name)?)))
-        .filter(|(c, _)| required_nodes.is_none_or(|r| state.free_count(c) >= r))
-        .collect();
-    candidates.sort_by_key(|(c, _)| c.nodes.len());
-    if let Some((cluster, nccl)) = candidates.into_iter().next() {
+    // For single-node recipes, NCCL config is not required — allow clusters
+    // without it. For multi-node, require it.
+    let candidates: Vec<&Cluster> = if required_nodes.is_some_and(|n| n > 1) {
+        config
+            .clusters
+            .iter()
+            .filter_map(|c| {
+                let _nccl = config.nccl.get(&c.name)?;
+                Some(c)
+            })
+            .filter(|c| required_nodes.is_none_or(|r| state.free_count(c) >= r))
+            .collect()
+    } else {
+        config
+            .clusters
+            .iter()
+            .filter(|c| required_nodes.is_none_or(|r| state.free_count(c) >= r))
+            .collect()
+    };
+    let mut candidates = candidates;
+    candidates.sort_by_key(|c| c.nodes.len());
+    if let Some(cluster) = candidates.into_iter().next() {
         let avail = state.free_count(cluster);
+        let nccl = config.nccl.get(&cluster.name);
         info!(
             "{}",
             fluent_i18n::t!("cluster-selected", {
@@ -386,4 +405,69 @@ mod tests {
         };
         assert!(select_cluster(&cfg, "auto", Some(1), &state).is_err());
     }
+
+    #[test]
+    fn single_node_allows_cluster_without_nccl() {
+        let mut state = ClusterState::default();
+        state.nodes.insert(
+            "n1".into(),
+            NodeState {
+                reachable: true,
+                occupied: false,
+            },
+        );
+        let cfg = Config {
+            ssh_user: "u".to_string(),
+            ssh_key_path: "/k".into(),
+            clusters: vec![test_cluster("c", &["n1"])],
+            nccl: HashMap::new(), // no NCCL config
+            hf_cache_dir: None,
+            init_port: 25000,
+            before_serve: HashMap::new(),
+            exec_aliases: HashMap::new(),
+            home_assistant: None,
+            drop_caches: true,
+            mount_aliases: HashMap::new(),
+            global_mods_dir: None,
+        };
+        // Single-node recipe on a cluster without NCCL config → OK
+        let (c, nccl) = select_cluster(&cfg, "auto", Some(1), &state).unwrap();
+        assert_eq!(c.name, "c");
+        assert!(nccl.is_none());
+    }
+
+    #[test]
+    fn multi_node_requires_nccl() {
+        let mut state = ClusterState::default();
+        state.nodes.insert(
+            "n1".into(),
+            NodeState {
+                reachable: true,
+                occupied: false,
+            },
+        );
+        state.nodes.insert(
+            "n2".into(),
+            NodeState {
+                reachable: true,
+                occupied: false,
+            },
+        );
+        let cfg = Config {
+            ssh_user: "u".to_string(),
+            ssh_key_path: "/k".into(),
+            clusters: vec![test_cluster("c", &["n1", "n2"])],
+            nccl: HashMap::new(), // no NCCL config
+            hf_cache_dir: None,
+            init_port: 25000,
+            before_serve: HashMap::new(),
+            exec_aliases: HashMap::new(),
+            home_assistant: None,
+            drop_caches: true,
+            mount_aliases: HashMap::new(),
+            global_mods_dir: None,
+        };
+        // Multi-node recipe on a cluster without NCCL config → error
+        assert!(select_cluster(&cfg, "auto", Some(2), &state).is_err());
+    }
 }

+ 40 - 15
src/config.rs

@@ -1,4 +1,4 @@
-use anyhow::{Context, Result};
+use anyhow::Result;
 use std::collections::HashSet;
 use tracing::info;
 
@@ -29,18 +29,27 @@ pub fn resolve_cluster_name<'a>(clusters: &'a [Cluster], name: &str) -> Result<&
     }
 }
 
+/// Whether NCCL configuration is required for cluster selection.
+///
+/// NCCL config is only needed for multi-node distributed runs — for
+/// single-node recipes (`required_nodes == Some(1)`), no NCCL env vars
+/// are emitted, so the config is not required.
+fn nccl_required(required_nodes: Option<u32>) -> bool {
+    required_nodes.is_some_and(|n| n > 1)
+}
+
 pub fn select_cluster<'c>(
     config: &'c Config,
     cluster_name: &str,
     required_nodes: Option<u32>,
     occupied: &HashSet<String>,
-) -> Result<(Cluster, &'c NcclConfig)> {
+) -> Result<(Cluster, Option<&'c NcclConfig>)> {
     if cluster_name != "auto" {
         let cluster = resolve_cluster_name(&config.clusters, cluster_name)?;
-        let nccl = config
-            .nccl
-            .get(&cluster.name)
-            .context(fluent_i18n::t!("nccl-config-missing"))?;
+        let nccl = config.nccl.get(&cluster.name);
+        if nccl_required(required_nodes) && nccl.is_none() {
+            anyhow::bail!("{}", fluent_i18n::t!("nccl-config-missing"));
+        }
         return Ok((cluster.clone(), nccl));
     }
     let free = |c: &Cluster| -> u32 {
@@ -49,15 +58,31 @@ pub fn select_cluster<'c>(
             .filter(|n| !occupied.contains(&n.name))
             .count() as u32
     };
-    let mut candidates: Vec<(&Cluster, &NcclConfig)> = config
-        .clusters
-        .iter()
-        .filter_map(|c| Some((c, config.nccl.get(&c.name)?)))
-        .filter(|(c, _)| required_nodes.is_none_or(|r| free(c) >= r))
-        .collect();
-    candidates.sort_by_key(|(c, _)| c.nodes.len());
-    if let Some((cluster, nccl)) = candidates.into_iter().next() {
+    // For single-node recipes, NCCL config is not required — allow clusters
+    // without it. For multi-node, require it.
+    let candidates: Vec<&Cluster> = if nccl_required(required_nodes) {
+        config
+            .clusters
+            .iter()
+            .filter_map(|c| {
+                let nccl = config.nccl.get(&c.name)?;
+                Some((c, nccl))
+            })
+            .filter(|(c, _)| required_nodes.is_none_or(|r| free(c) >= r))
+            .map(|(c, _)| c)
+            .collect()
+    } else {
+        config
+            .clusters
+            .iter()
+            .filter(|c| required_nodes.is_none_or(|r| free(c) >= r))
+            .collect()
+    };
+    let mut candidates = candidates;
+    candidates.sort_by_key(|c| c.nodes.len());
+    if let Some(cluster) = candidates.into_iter().next() {
         let avail = free(cluster);
+        let nccl = config.nccl.get(&cluster.name);
         info!(
             "{}",
             fluent_i18n::t!("cluster-selected", { "name" => cluster.name.to_string(), "nodes" => avail })
@@ -76,7 +101,7 @@ pub fn resolve_cluster<'c>(
     recipe: &Recipe,
     cluster_name: &str,
     occupied: &HashSet<String>,
-) -> Result<(Cluster, &'c NcclConfig)> {
+) -> Result<(Cluster, Option<&'c NcclConfig>)> {
     let choice = if cluster_name != "auto" {
         cluster_name.to_string()
     } else if let Some(ref rc) = recipe.cluster {

+ 15 - 3
src/config/tests.rs

@@ -133,7 +133,7 @@ fn test_select_cluster_by_name() {
     cfg.nccl.insert("test-cluster".into(), tnccl());
     let (c, n) = select_cluster(&cfg, "test-cluster", None, &HashSet::new()).unwrap();
     assert_eq!(c.name, "test-cluster");
-    assert_eq!(n.ib_hca, "rocep1s0f0");
+    assert_eq!(n.unwrap().ib_hca, "rocep1s0f0");
 }
 
 #[test]
@@ -145,8 +145,20 @@ fn test_select_cluster_auto() {
 }
 
 #[test]
-fn test_select_cluster_auto_no_match() {
-    assert!(select_cluster(&tcfg(), "auto", None, &HashSet::new()).is_err());
+fn test_select_cluster_auto_no_nccl_allows_single_node() {
+    // With required_nodes = None, NCCL config is not required — the cluster
+    // is selected even without NCCL config (the node count is unknown at
+    // selection time, so we can't require it).
+    let cfg = tcfg();
+    let (c, n) = select_cluster(&cfg, "auto", None, &HashSet::new()).unwrap();
+    assert_eq!(c.name, "test-cluster");
+    assert!(n.is_none());
+}
+
+#[test]
+fn test_select_cluster_multi_node_requires_nccl() {
+    // Multi-node (required_nodes > 1) without NCCL config → error.
+    assert!(select_cluster(&tcfg(), "auto", Some(2), &HashSet::new()).is_err());
 }
 
 #[test]

+ 40 - 30
src/docker/command.rs

@@ -13,7 +13,9 @@ use std::borrow::Cow;
 #[derive(Clone)]
 pub(crate) struct DockerCommandCtx {
     pub cluster: Cluster,
-    pub nccl: NcclConfig,
+    /// NCCL config — `None` for single-node recipes (no NCCL env vars
+    /// are emitted). For multi-node, always `Some`.
+    pub nccl: Option<NcclConfig>,
     pub recipe: Recipe,
     pub recipe_name: String,
     pub pp_size: u32,
@@ -140,38 +142,46 @@ pub(crate) fn path_ref_placeholder(spec: &PathRef) -> String {
 }
 
 /// Collect NCCL cluster env vars for multi-node deployments.
+///
+/// `ctx.nccl` is `None` for single-node recipes — but this function is
+/// only called when `ctx.nnodes > 1`, so `None` indicates a configuration
+/// error (multi-node without NCCL config). In that case, only the worker
+/// IP env var is emitted (NCCL env vars are skipped), which will cause
+/// NCCL to fail at runtime with a visible error.
 fn collect_nccl_env<'a>(ctx: &'a DockerCommandCtx, node: &'a Node) -> Vec<(&'a str, Cow<'a, str>)> {
     let is_sglang = ctx.recipe.engine == "sglang";
-    let sock = ctx.nccl.socket_ifname.as_str();
-    let gloo_if = ctx.nccl.gloo_socket_ifname.as_deref().unwrap_or(sock);
-    let ucx_dev = ctx.nccl.ucx_net_devices.as_deref().unwrap_or(sock);
-    let mut env: Vec<(&str, Cow<str>)> = vec![
-        ("NCCL_IB_DISABLE", format!("{}", ctx.nccl.ib_disable).into()),
-        ("NCCL_IB_HCA", ctx.nccl.ib_hca.as_str().into()),
-        ("NCCL_SOCKET_IFNAME", sock.into()),
-        ("GLOO_SOCKET_IFNAME", gloo_if.into()),
-        ("TP_SOCKET_IFNAME", gloo_if.into()),
-        (
-            "NCCL_IB_GID_INDEX",
-            ctx.nccl.ib_gid_index.to_string().into(),
-        ),
-        ("MN_IF_NAME", gloo_if.into()),
-        ("UCX_NET_DEVICES", ucx_dev.into()),
-        ("OMPI_MCA_btl_tcp_if_include", gloo_if.into()),
-        ("OMP_NUM_THREADS", "4".into()),
-        ("NCCL_IGNORE_CPU_AFFINITY", "1".into()),
-        ("NCCL_NET", "IB".into()),
-        ("NCCL_DEBUG", "WARN".into()),
-    ];
-    if let Some(v) = ctx.nccl.nccl_cross_nic {
-        env.push(("NCCL_CROSS_NIC", v.to_string().into()));
-    }
-    if let Some(ref v) = ctx.nccl.nccl_proto {
-        env.push(("NCCL_PROTO", v.as_str().into()));
-    }
-    if let Some(ref v) = ctx.nccl.nccl_p2p_level {
-        env.push(("NCCL_P2P_LEVEL", v.as_str().into()));
+    let mut env: Vec<(&str, Cow<str>)> = Vec::new();
+    if let Some(nccl) = ctx.nccl.as_ref() {
+        let sock = nccl.socket_ifname.as_str();
+        let gloo_if = nccl.gloo_socket_ifname.as_deref().unwrap_or(sock);
+        let ucx_dev = nccl.ucx_net_devices.as_deref().unwrap_or(sock);
+        env.extend(vec![
+            ("NCCL_IB_DISABLE", format!("{}", nccl.ib_disable).into()),
+            ("NCCL_IB_HCA", nccl.ib_hca.as_str().into()),
+            ("NCCL_SOCKET_IFNAME", sock.into()),
+            ("GLOO_SOCKET_IFNAME", gloo_if.into()),
+            ("TP_SOCKET_IFNAME", gloo_if.into()),
+            ("NCCL_IB_GID_INDEX", nccl.ib_gid_index.to_string().into()),
+            ("MN_IF_NAME", gloo_if.into()),
+            ("UCX_NET_DEVICES", ucx_dev.into()),
+            ("OMPI_MCA_btl_tcp_if_include", gloo_if.into()),
+            ("OMP_NUM_THREADS", "4".into()),
+            ("NCCL_IGNORE_CPU_AFFINITY", "1".into()),
+            ("NCCL_NET", "IB".into()),
+            ("NCCL_DEBUG", "WARN".into()),
+        ]);
+        if let Some(v) = nccl.nccl_cross_nic {
+            env.push(("NCCL_CROSS_NIC", v.to_string().into()));
+        }
+        if let Some(ref v) = nccl.nccl_proto {
+            env.push(("NCCL_PROTO", v.as_str().into()));
+        }
+        if let Some(ref v) = nccl.nccl_p2p_level {
+            env.push(("NCCL_P2P_LEVEL", v.as_str().into()));
+        }
     }
+    // Worker IP is always set for multi-node (collect_nccl_env is only
+    // called when nnodes > 1).
     let worker_ip = Cow::Borrowed(node.ib_ip.as_str());
     if is_sglang {
         env.push(("SGLANG_WORKER_IP", worker_ip));

+ 1 - 1
src/docker/command_tests.rs

@@ -56,7 +56,7 @@ fn run_with(recipe: &Recipe) -> String {
     let resolved = resolve_extra_mounts(&minimal_config(), recipe).unwrap();
     let ctx = DockerCommandCtx {
         cluster: tcls(),
-        nccl: tnccl(),
+        nccl: Some(tnccl()),
         recipe: recipe.clone(),
         recipe_name: "r".to_string(),
         pp_size: 2,

+ 2 - 2
src/docker/mod.rs

@@ -193,7 +193,7 @@ pub async fn serve(
     start_cluster(
         config,
         &cluster,
-        &nccl,
+        nccl,
         recipe_name,
         &recipe,
         StartClusterOpts {
@@ -294,7 +294,7 @@ struct StartClusterOpts {
 async fn start_cluster(
     config: &Config,
     cluster: &Cluster,
-    nccl: &NcclConfig,
+    nccl: Option<NcclConfig>,
     recipe_name: &str,
     recipe: &Recipe,
     opts: StartClusterOpts,

+ 1 - 1
src/docker/tests.rs

@@ -78,7 +78,7 @@ fn trec() -> Recipe {
 fn make_dm_ctx(recipe: &Recipe, nnodes: u32, pp: u32, tp: u32, keep: bool) -> DockerCommandCtx {
     DockerCommandCtx {
         cluster: tcls(),
-        nccl: tnccl(),
+        nccl: Some(tnccl()),
         recipe: recipe.clone(),
         recipe_name: "r".to_string(),
         pp_size: pp,

+ 2 - 2
src/main.rs

@@ -235,8 +235,6 @@ async fn handle_serve(
                 (opts.build_node.is_some(), "--build-node"),
                 (opts.force_build, "--force-build"),
                 (opts.keep, "--keep"),
-                (opts.auto_power_on, "--on"),
-                (opts.force, "--force"),
                 (opts.solo.is_some(), "--solo/--no-solo"),
             ]
             .iter()
@@ -254,6 +252,8 @@ async fn handle_serve(
                 cluster,
                 opts.skip_before_serve,
                 dry_run,
+                opts.auto_power_on,
+                opts.force,
             )
             .await?;
             if follow && !dry_run {

+ 14 - 23
src/orchestrate/mod.rs

@@ -18,7 +18,8 @@ use crate::ends::{NodePinger, NodeProber, PowerController, cluster_entities};
 /// select a cluster with its free nodes.
 ///
 /// Returns a `Cluster` whose `nodes` field contains only free (reachable
-/// and not occupied) nodes, plus the matching NCCL config.
+/// and not occupied) nodes, plus the matching NCCL config (or `None` for
+/// single-node recipes, where NCCL config is not required).
 ///
 /// Timing for [`wait_for_all_nodes`] after a power-on: how long to wait
 /// for the machines to boot before the first poll, the poll interval, and
@@ -62,7 +63,7 @@ pub async fn prepare_cluster<P, C, N>(
     cluster_name: &str,
     node_name: Option<&str>,
     req: &PrepareRequest<'_, P, C, N>,
-) -> Result<(Cluster, NcclConfig)>
+) -> Result<(Cluster, Option<NcclConfig>)>
 where
     P: NodeProber,
     C: PowerController,
@@ -110,7 +111,7 @@ where
             })
             .collect();
         let c = Cluster { nodes: avail, ..c };
-        (c, n.clone())
+        (c, n.cloned())
     };
 
     // Validate node count.
@@ -131,7 +132,11 @@ where
 }
 
 /// Collect a single node from any cluster.
-fn resolve_solo_node(config: &Config, node_name: &str) -> Result<(Cluster, NcclConfig)> {
+///
+/// Returns `None` for the NCCL config when the cluster has no NCCL entry
+/// — this is valid for single-node deployments where NCCL env vars are
+/// never emitted.
+fn resolve_solo_node(config: &Config, node_name: &str) -> Result<(Cluster, Option<NcclConfig>)> {
     let (solo_c, solo_n) = config
         .clusters
         .iter()
@@ -143,21 +148,7 @@ fn resolve_solo_node(config: &Config, node_name: &str) -> Result<(Cluster, NcclC
                     nodes: vec![node.clone()],
                     extra_entities: vec![],
                 };
-                let nc = config
-                    .nccl
-                    .get(&c.name)
-                    .cloned()
-                    .unwrap_or_else(|| NcclConfig {
-                        ib_hca: String::new(),
-                        socket_ifname: "lo".to_string(),
-                        ib_gid_index: 3,
-                        ib_disable: 1,
-                        gloo_socket_ifname: None,
-                        ucx_net_devices: None,
-                        nccl_cross_nic: None,
-                        nccl_proto: None,
-                        nccl_p2p_level: None,
-                    });
+                let nc = config.nccl.get(&c.name).cloned();
                 (solo, nc)
             })
         })
@@ -173,7 +164,7 @@ fn resolve_cluster_simple<'c>(
     config: &'c Config,
     recipe: &Recipe,
     cluster_name: &str,
-) -> Result<(Cluster, &'c NcclConfig)> {
+) -> Result<(Cluster, Option<&'c NcclConfig>)> {
     use crate::cluster_state::NodeState;
     let mut state = ClusterState::default();
     for cluster in &config.clusters {
@@ -199,13 +190,13 @@ pub fn prepare_cluster_dry_run(
     recipe: &Recipe,
     cluster_name: &str,
     node_name: Option<&str>,
-) -> Result<(Cluster, NcclConfig)> {
+) -> Result<(Cluster, Option<NcclConfig>)> {
     if let Some(nn) = node_name {
         let (c, n) = resolve_solo_node(config, nn)?;
-        return Ok((c, n.clone()));
+        return Ok((c, n));
     }
     let (c, n) = resolve_cluster_simple(config, recipe, cluster_name)?;
-    Ok((c.clone(), n.clone()))
+    Ok((c, n.cloned()))
 }
 
 /// Does the state contain any unreachable nodes?

+ 6 - 0
src/ssh.rs

@@ -107,6 +107,9 @@ pub async fn run_ssh(
             ChannelMsg::Data { data } => {
                 output.push_str(&String::from_utf8_lossy(&data));
             }
+            ChannelMsg::ExtendedData { data, .. } => {
+                output.push_str(&String::from_utf8_lossy(&data));
+            }
             ChannelMsg::ExitStatus { exit_status: s } => {
                 exit_status = Some(s);
             }
@@ -205,6 +208,9 @@ pub async fn run_ssh_long_running(
     while let Some(msg) = channel.wait().await {
         match msg {
             ChannelMsg::Data { data } => output.push_str(&String::from_utf8_lossy(&data)),
+            ChannelMsg::ExtendedData { data, .. } => {
+                output.push_str(&String::from_utf8_lossy(&data))
+            }
             ChannelMsg::ExitStatus { exit_status: s } => exit_status = Some(s),
             _ => {}
         }

+ 1 - 1
src/tmux/commands.rs

@@ -70,7 +70,7 @@ impl TmuxManager {
         &self,
         node: &Node,
         recipe: &Recipe,
-        nccl: &crate::config::NcclConfig,
+        nccl: Option<&crate::config::NcclConfig>,
         cluster: &Cluster,
     ) -> Vec<String> {
         let sess = self.session_name(recipe, node.rank);

+ 20 - 8
src/tmux/mod.rs

@@ -22,13 +22,16 @@ impl TmuxManager {
     }
 
     /// Builds the list of `export` commands for NCCL environment variables.
-    pub fn build_nccl_setup(&self, nccl: &NcclConfig) -> Vec<String> {
-        vec![
-            format!("export NCCL_IB_HCA=\"{}\"", nccl.ib_hca),
-            format!("export NCCL_SOCKET_IFNAME=\"{}\"", nccl.socket_ifname),
-            format!("export NCCL_IB_GID_INDEX={}", nccl.ib_gid_index),
-            format!("export NCCL_IB_DISABLE={}", nccl.ib_disable),
-        ]
+    pub fn build_nccl_setup(&self, nccl: Option<&NcclConfig>) -> Vec<String> {
+        match nccl {
+            Some(nccl) => vec![
+                format!("export NCCL_IB_HCA=\"{}\"", nccl.ib_hca),
+                format!("export NCCL_SOCKET_IFNAME=\"{}\"", nccl.socket_ifname),
+                format!("export NCCL_IB_GID_INDEX={}", nccl.ib_gid_index),
+                format!("export NCCL_IB_DISABLE={}", nccl.ib_disable),
+            ],
+            None => Vec::new(),
+        }
     }
 
     /// Builds engine-specific host IP environment variable.
@@ -71,7 +74,16 @@ pub async fn handle_tmux(config: &Config, cmd: TmuxCommands, dry_run: bool) -> R
             cluster,
             skip_before_serve,
         } => {
-            serve::tmux_serve(config, &recipe, &cluster, skip_before_serve, dry_run).await?;
+            serve::tmux_serve(
+                config,
+                &recipe,
+                &cluster,
+                skip_before_serve,
+                dry_run,
+                false,
+                false,
+            )
+            .await?;
         }
         TmuxCommands::Stop { recipe, cluster } => {
             ops::tmux_stop(config, &recipe, &cluster, dry_run).await?;

+ 36 - 6
src/tmux/serve.rs

@@ -1,5 +1,4 @@
 use anyhow::{Context, Result};
-use std::collections::HashSet;
 use std::time::Duration;
 use tokio::time::sleep;
 use tracing::{info, warn};
@@ -15,6 +14,8 @@ pub async fn tmux_serve(
     cluster_name: &str,
     skip_before_serve: bool,
     dry_run: bool,
+    auto_power_on: bool,
+    force: bool,
 ) -> Result<()> {
     let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
     let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
@@ -50,8 +51,36 @@ pub async fn tmux_serve(
     } else {
         "auto".to_string()
     };
-    let (cluster, nccl) =
-        crate::config::select_cluster(config, &choice, recipe.required_nodes, &HashSet::new())?;
+    // Use prepare_cluster (not select_cluster) so --on (auto_power_on) and
+    // --force (force_occupied) are supported for tmux deploy — they have
+    // nothing to do with Docker.
+    let recipe_for_select = recipe.clone();
+    let (cluster, nccl) = if dry_run {
+        crate::orchestrate::prepare_cluster_dry_run(config, &recipe_for_select, &choice, None)?
+    } else {
+        use crate::ends::{SshNodePinger, SshProber, ha_controller_from_config};
+        use crate::orchestrate::{BootTiming, PrepareRequest, prepare_cluster};
+        let power_ctrl = if auto_power_on {
+            ha_controller_from_config(config)
+        } else {
+            None
+        };
+        prepare_cluster(
+            config,
+            &recipe_for_select,
+            &choice,
+            None,
+            &PrepareRequest {
+                prober: &SshProber,
+                power: power_ctrl.as_ref(),
+                pinger: &SshNodePinger,
+                auto_power_on,
+                boot_timing: BootTiming::default(),
+                force_occupied: force,
+            },
+        )
+        .await?
+    };
 
     if let Some(required) = recipe.required_nodes {
         let actual = cluster.nodes.len() as u32;
@@ -84,7 +113,7 @@ pub async fn tmux_serve(
             .find(|n| n.rank == 0)
             .context("No head node (rank 0) in cluster")?;
 
-        let head_cmds = manager.build_session_commands(head_node, &recipe, nccl, &cluster);
+        let head_cmds = manager.build_session_commands(head_node, &recipe, nccl.as_ref(), &cluster);
         manager
             .create_session(config, head_node, &head_cmds, dry_run)
             .await?;
@@ -104,7 +133,8 @@ pub async fn tmux_serve(
             if node.rank == 0 {
                 continue;
             }
-            let worker_cmds = manager.build_session_commands(node, &recipe, nccl, &cluster);
+            let worker_cmds =
+                manager.build_session_commands(node, &recipe, nccl.as_ref(), &cluster);
             manager
                 .create_session(config, node, &worker_cmds, dry_run)
                 .await?;
@@ -123,7 +153,7 @@ pub async fn tmux_serve(
             let dry = dry_run;
 
             handles.push(crate::locale::spawn_with_locale(async move {
-                let cmds = manager.build_session_commands(&node, &recipe, &nccl, &cluster);
+                let cmds = manager.build_session_commands(&node, &recipe, nccl.as_ref(), &cluster);
                 manager
                     .create_session(&config, &node, &cmds, dry)
                     .await