power on: ensure-powered — nodes believed down with an already-on plug get a 30 s plug power cycle (2-5 s cuts left the PSU rail energized and the machine in soft-off); per-node outcome report becomes the job message; CLI uses a 1.5 s TCP reachability probe, web uses the last snapshot

locales/de/main.ftl

@@ -229,3 +229,11 @@ push-failed = Push von { $ref} auf { $node} ist nach { $elapsed} fehlgeschlagen.
 pushed-image = Image { $ref} gepusht, { $latest} aktualisiert ({ $elapsed })
 push-running = Push läuft seit { $elapsed }...
 push-already-published = { $ref} bereits veröffentlicht ({ $label}): unveränderter Image-Digest — übersprungen, zum erneuten Push --force nutzen
+
+power-on-on = { $name }: eingeschaltet
+power-on-cycled = { $name }: Steckdose war on, Knoten war aus — Stromzyklus
+power-on-was-on = { $name }: Steckdose bereits on (Knoten aktiv?) — unverändert
+power-on-skip = { $name }: keine HA-Entität — übersprungen
+power-on-failed = { $name }: HA-Fehler: { $error }
+power-on-cycling = ⟳ { $name }: Steckdose ist on, aber der Knoten antwortet nicht — Stromzyklus { $entity }
+power-on-off-wait-timeout = { $name }: Steckdose meldete nicht off binnen 10 s (Relay klemmt evtl.)

locales/en/main.ftl

@@ -248,3 +248,11 @@ push-failed = Push of { $ref } on { $node } failed after { $elapsed }.\n{ $log }
 pushed-image = Pushed { $ref } and refreshed { $latest } ({ $elapsed })
 push-running = Push running for { $elapsed }...
 push-already-published = { $ref } already published ({ $label}): unchanged image digest — skipping, use --force to re-publish
+
+power-on-on = { $name }: turned on
+power-on-cycled = { $name }: plug was on, node down — power cycled
+power-on-was-on = { $name }: plug already on (node up?) — left as is
+power-on-skip = { $name }: no HA entity — skipped
+power-on-failed = { $name }: HA call failed: { $error }
+power-on-cycling = ⟳ { $name }: plug reads on but the node is down — cycling { $entity }
+power-on-off-wait-timeout = { $name }: plug did not report off within 10 s (relay may have stuck)

locales/es/main.ftl

@@ -229,3 +229,11 @@ push-failed = El push de { $ref} en { $node} falló tras { $elapsed}.\n{ $log }\
 pushed-image = Imagen { $ref} con push hecho, { $latest} actualizada ({ $elapsed })
 push-running = Push en curso desde hace { $elapsed }...
 push-already-published = { $ref} ya publicado ({ $label}): digest de imagen sin cambios — omitido, use --force para volver a publicar
+
+power-on-on = { $name }: encendido
+power-on-cycled = { $name }: el enchufe estaba on con el nodo apagado — ciclo de energía
+power-on-was-on = { $name }: el enchufe ya está on (¿nodo activo?) — sin cambios
+power-on-skip = { $name }: sin entidad de HA — omitido
+power-on-failed = { $name }: fallo de HA: { $error }
+power-on-cycling = ⟳ { $name }: el enchufe está on pero el nodo no responde — ciclando { $entity }
+power-on-off-wait-timeout = { $name }: el enchufe no reportó off en 10 s (relé posiblemente atascado)

locales/fr/main.ftl

@@ -229,3 +229,11 @@ push-failed = Le push de { $ref} sur { $node} a échoué après { $elapsed}.\n{
 pushed-image = Image { $ref} poussée, { $latest} mise à jour ({ $elapsed })
 push-running = Push en cours depuis { $elapsed }...
 push-already-published = { $ref} déjà publié ({ $label}) : digest d'image inchangé — ignoré, utiliser --force pour repousser
+
+power-on-on = { $name }: allumé
+power-on-cycled = { $name }: la prise était on mais le nœud est éteint — alimentation recycler
+power-on-was-on = { $name }: prise déjà on (nœud actif ?) — laissée telle quelle
+power-on-skip = { $name }: pas d'entité HA — ignoré
+power-on-failed = { $name }: échec HA : { $error }
+power-on-cycling = ⟳ { $name }: la prise est on mais le nœud ne répond pas — cycle d'alimentation { $entity }
+power-on-off-wait-timeout = { $name }: la prise n'a pas signalé off en 10 s (relais possiblement bloqué)

locales/it/main.ftl

@@ -229,3 +229,11 @@ push-failed = Il push di { $ref} su { $node} è fallito dopo { $elapsed}.\n{ $lo
 pushed-image = Immagine { $ref} pushata, { $latest} aggiornata ({ $elapsed })
 push-running = Push in corso da { $elapsed }...
 push-already-published = { $ref} già pubblicato ({ $label}): digest immagine invariato — saltato, usare --force per ripubblicare
+
+power-on-on = { $name }: acceso
+power-on-cycled = { $name }: la presa era on con il nodo spento — alimentazione ri-ciclata
+power-on-was-on = { $name }: la presa è già on (nodo attivo?) — lasciata com'è
+power-on-skip = { $name }: nessuna entità HA — saltato
+power-on-failed = { $name }: errore HA: { $error }
+power-on-cycling = ⟳ { $name }: la presa è on ma il nodo non risponde — ciclo di alimentazione { $entity }
+power-on-off-wait-timeout = { $name }: la presa non ha riportato off entro 10 s (relè forse bloccato)

locales/ru/main.ftl

@@ -258,3 +258,11 @@ push-failed = Push образа { $ref} на { $node} не удался за { $
 pushed-image = Образ { $ref} запушен, { $latest} обновлён ({ $elapsed })
 push-running = Push идёт { $elapsed }...
 push-already-published = { $ref} уже опубликован ({ $label}): digest изображения не изменился — пропускаю, для повторного push — --force
+
+power-on-on = { $name }: включено
+power-on-cycled = { $name }: вилка была on при выключенном узле — питание перециклено
+power-on-was-on = { $name }: вилка уже on (узел жив?) — оставлено как есть
+power-on-skip = { $name }: нет HA-сущности — пропущено
+power-on-failed = { $name }: ошибка вызова HA: { $error }
+power-on-cycling = ⟳ { $name }: вилка on, но узел недоступен — перецикливаю { $entity }
+power-on-off-wait-timeout = { $name }: вилка не ушла в off за 10 с (возможно, реле залипло)

src/cli.rs

@@ -1,5 +1,9 @@
 use crate::runlock::RunlockCommands;
-use clap::{Args, Parser, Subcommand, ValueEnum};
+use clap::{Parser, Subcommand, ValueEnum};
+
+pub use crate::cli_sub::{
+    ConfigCommands, PowerAction, PublishArgs, RegistryCommands, SyncImageArgs, TmuxCommands,
+};
 
 /// Output format for human-readable reports (`--format`).
 #[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)]
@@ -333,165 +337,7 @@ pub enum Commands {
         action: PowerAction,
     },
 }
-#[derive(Subcommand)]
-pub enum PowerAction {
-    /// Power off: SSH `sudo shutdown now` + wait + HA `turn_off`.
-    Off {
-        #[arg(long)]
-        cluster: String,
-
-        /// Target a specific node within the cluster (default: all).
-        #[arg(long)]
-        node: Option<String>,
-    },
-    /// Power on: HA `turn_on` only (no SSH; the smart plug wakes the machine).
-    On {
-        #[arg(long)]
-        cluster: String,
 
-        /// Target a specific node within the cluster (default: all).
-        #[arg(long)]
-        node: Option<String>,
-    },
-    /// Switch clusters: disable nodes from source, enable nodes in target.
-    Switch {
-        /// Current cluster (nodes + entities to shut down).
-        #[arg(long)]
-        from: String,
-        /// Target cluster (nodes + entities to wake up).
-        #[arg(long)]
-        to: String,
-    },
-}
-/// `sm121 config` subcommands — manage the global config file.
-#[derive(Subcommand)]
-pub enum ConfigCommands {
-    /// Show the active config path and where the global config lives
-    Path,
-    /// Create the global config file (template) if it does not exist
-    InitGlobal,
-}
-
-#[derive(Args)]
-pub struct PublishArgs {
-    #[arg(long, required = true)]
-    pub recipe: String,
-
-    /// Cluster whose fabric env + node count fill the generated run script
-    #[arg(long)]
-    pub cluster: Option<String>,
-
-    /// Output dir (default: ./publish/<recipe>)
-    #[arg(long)]
-    pub out: Option<std::path::PathBuf>,
-
-    /// Also create a public GitHub gist (requires `gh` CLI, authenticated)
-    #[arg(long)]
-    pub gist: bool,
-
-    /// After bundling, push the recipe's image to the image registry
-    /// (`[registry]` config) under its version tag + `latest`.
-    #[arg(long)]
-    pub push: bool,
-
-    /// Node that has the image for `--push` (default: probe all cluster
-    /// nodes for the first match).
-    #[arg(long)]
-    pub node: Option<String>,
-
-    /// Re-push even if the publication ledger already records this image
-    /// digest (with `--push`).
-    #[arg(long)]
-    pub force: bool,
-
-    /// Upload the bundle to a recipe registry (see `sm121 registry --help`)
-    #[arg(long, value_name = "URL")]
-    pub registry: Option<String>,
-
-    /// Registry publish token (or $SM121RECIPES_TOKEN)
-    #[arg(long)]
-    pub token: Option<String>,
-}
-#[derive(Subcommand)]
-pub enum RegistryCommands {
-    /// List recipes on the registry
-    List {
-        #[arg(long, value_name = "URL")]
-        registry: Option<String>,
-    },
-    /// Print a registry recipe (recipe.toml)
-    Show {
-        name: String,
-        #[arg(long, value_name = "URL")]
-        registry: Option<String>,
-    },
-    /// Download a recipe into ./recipes/<name>/ (+ ./docker-images/<name>/ if it has an image)
-    Install {
-        name: String,
-        #[arg(long, value_name = "URL")]
-        registry: Option<String>,
-        /// Overwrite an existing ./recipes/<name>/
-        #[arg(long)]
-        force: bool,
-    },
-}
-
-#[derive(Subcommand)]
-pub enum TmuxCommands {
-    /// Launch cluster via tmux
-    Serve {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-        /// Skip pre-serve hooks (cluster-level + recipe-level).
-        #[arg(long)]
-        skip_before_serve: bool,
-    },
-    /// Stop tmux sessions
-    Stop {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-    /// Detailed tmux session status
-    Status {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-    /// Show tmux session logs
-    Logs {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, action = clap::ArgAction::SetTrue)]
-        follow: bool,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-    /// Attach to head node tmux session
-    Attach {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-}
 #[cfg(test)]
 #[path = "cli_tests.rs"]
 mod tests;
-
-#[derive(Args)]
-pub struct SyncImageArgs {
-    #[arg(long, required = true)]
-    pub recipe: String,
-
-    #[arg(long, default_value = "auto")]
-    pub cluster: String,
-
-    /// Explicit source node (default: newest image present in the set).
-    #[arg(long)]
-    pub source: Option<String>,
-}

src/cli_sub.rs

@@ -0,0 +1,165 @@
+//! Subcommand enums split out of cli.rs (the 500-line cap).
+//! Clap-derive types only — `crate::cli` re-exports them so callers and
+//! dispatch keep their import paths.
+
+use clap::{Args, Subcommand};
+
+#[derive(Subcommand)]
+pub enum PowerAction {
+    /// Power off: SSH `sudo shutdown now` + wait + HA `turn_off`.
+    Off {
+        #[arg(long)]
+        cluster: String,
+
+        /// Target a specific node within the cluster (default: all).
+        #[arg(long)]
+        node: Option<String>,
+    },
+    /// Power on: HA `turn_on` only (no SSH; the smart plug wakes the machine).
+    On {
+        #[arg(long)]
+        cluster: String,
+
+        /// Target a specific node within the cluster (default: all).
+        #[arg(long)]
+        node: Option<String>,
+    },
+    /// Switch clusters: disable nodes from source, enable nodes in target.
+    Switch {
+        /// Current cluster (nodes + entities to shut down).
+        #[arg(long)]
+        from: String,
+        /// Target cluster (nodes + entities to wake up).
+        #[arg(long)]
+        to: String,
+    },
+}
+/// `sm121 config` subcommands — manage the global config file.
+#[derive(Subcommand)]
+pub enum ConfigCommands {
+    /// Show the active config path and where the global config lives
+    Path,
+    /// Create the global config file (template) if it does not exist
+    InitGlobal,
+}
+
+#[derive(Args)]
+pub struct PublishArgs {
+    #[arg(long, required = true)]
+    pub recipe: String,
+
+    /// Cluster whose fabric env + node count fill the generated run script
+    #[arg(long)]
+    pub cluster: Option<String>,
+
+    /// Output dir (default: ./publish/<recipe>)
+    #[arg(long)]
+    pub out: Option<std::path::PathBuf>,
+
+    /// Also create a public GitHub gist (requires `gh` CLI, authenticated)
+    #[arg(long)]
+    pub gist: bool,
+
+    /// After bundling, push the recipe's image to the image registry
+    /// (`[registry]` config) under its version tag + `latest`.
+    #[arg(long)]
+    pub push: bool,
+
+    /// Node that has the image for `--push` (default: probe all cluster
+    /// nodes for the first match).
+    #[arg(long)]
+    pub node: Option<String>,
+
+    /// Re-push even if the publication ledger already records this image
+    /// digest (with `--push`).
+    #[arg(long)]
+    pub force: bool,
+
+    /// Upload the bundle to a recipe registry (see `sm121 registry --help`)
+    #[arg(long, value_name = "URL")]
+    pub registry: Option<String>,
+
+    /// Registry publish token (or $SM121RECIPES_TOKEN)
+    #[arg(long)]
+    pub token: Option<String>,
+}
+#[derive(Subcommand)]
+pub enum RegistryCommands {
+    /// List recipes on the registry
+    List {
+        #[arg(long, value_name = "URL")]
+        registry: Option<String>,
+    },
+    /// Print a registry recipe (recipe.toml)
+    Show {
+        name: String,
+        #[arg(long, value_name = "URL")]
+        registry: Option<String>,
+    },
+    /// Download a recipe into ./recipes/<name>/ (+ ./docker-images/<name>/ if it has an image)
+    Install {
+        name: String,
+        #[arg(long, value_name = "URL")]
+        registry: Option<String>,
+        /// Overwrite an existing ./recipes/<name>/
+        #[arg(long)]
+        force: bool,
+    },
+}
+
+#[derive(Subcommand)]
+pub enum TmuxCommands {
+    /// Launch cluster via tmux
+    Serve {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+        /// Skip pre-serve hooks (cluster-level + recipe-level).
+        #[arg(long)]
+        skip_before_serve: bool,
+    },
+    /// Stop tmux sessions
+    Stop {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+    /// Detailed tmux session status
+    Status {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+    /// Show tmux session logs
+    Logs {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, action = clap::ArgAction::SetTrue)]
+        follow: bool,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+    /// Attach to head node tmux session
+    Attach {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+}
+
+#[derive(Args)]
+pub struct SyncImageArgs {
+    #[arg(long, required = true)]
+    pub recipe: String,
+
+    #[arg(long, default_value = "auto")]
+    pub cluster: String,
+
+    /// Explicit source node (default: newest image present in the set).
+    #[arg(long)]
+    pub source: Option<String>,
+}

src/docker/mod.rs

@@ -12,16 +12,17 @@ pub mod push;
 pub use logs::show_logs;
 pub mod build_dir;
 pub mod stale;
+mod start;
 pub mod stop;
 pub mod sync;
 pub mod tail;
 use crate::{
-    config::{Cluster, Config, NcclConfig, Node, Recipe, load_recipe},
+    config::{Cluster, Config, NcclConfig, Recipe, load_recipe},
     doctor::try_repaired_image,
     ssh::run_ssh,
 };
 use anyhow::{Context, Result};
-use tokio::time::{Duration, sleep};
+
 use tracing::{info, warn};
 /// Options for the serve operation.
 #[derive(Clone, Debug)]
@@ -284,7 +285,7 @@ pub async fn serve(
         opts.dry_run,
     )
     .await?;
-    start_cluster(
+    start::start_cluster(
         config,
         &cluster,
         nccl,
@@ -322,6 +323,7 @@ pub async fn serve(
     }
     Ok(cluster)
 }
+
 pub async fn resolve_cluster_by_container_label<'c>(
     config: &'c Config,
     recipe_name: &str,
@@ -351,7 +353,7 @@ pub async fn resolve_cluster_by_container_label<'c>(
     anyhow::bail!("No running containers found for recipe '{}'", recipe_name);
 }
 /// Options for [start_cluster].
-struct StartClusterOpts {
+pub(crate) struct StartClusterOpts {
     cache_dir: std::path::PathBuf,
     /// When set, the per-node mirror dir mounted at /root/.cache/huggingface
     /// (repos are copied into it from the shared cache before launch).
@@ -362,138 +364,3 @@ struct StartClusterOpts {
     dry_run: bool,
     clear: bool,
 }
-async fn start_cluster(
-    config: &Config,
-    cluster: &Cluster,
-    nccl: Option<NcclConfig>,
-    recipe_name: &str,
-    recipe: &Recipe,
-    opts: StartClusterOpts,
-) -> Result<()> {
-    let nodes: Vec<&Node> = if let Some(ref nn) = opts.node_name {
-        let v: Vec<&Node> = cluster.nodes.iter().filter(|n| n.name == *nn).collect();
-        if v.is_empty() {
-            anyhow::bail!("Node '{}' not found in cluster '{}'", nn, cluster.name);
-        }
-        v
-    } else {
-        // Trim to `required_nodes` when set: a recipe that needs N nodes
-        // must launch on exactly N nodes, not on every free node in the
-        // cluster. Previously a `required_nodes = 1` recipe on a 4-node
-        // cluster launched 4 containers, each told `--nnodes 4 --tp 4` —
-        // a broken rendezvous that never coordinated. `prepare_cluster`
-        // guarantees >= required_nodes are free, so taking the first N is
-        // safe and deterministic.
-        let all = cluster.nodes.iter().collect::<Vec<&Node>>();
-        match recipe.required_nodes {
-            Some(n) if (n as usize) < all.len() => all.into_iter().take(n as usize).collect(),
-            _ => all,
-        }
-    };
-    let nnodes = nodes.len() as u32;
-    // `--mirror`: ensure every repo the recipe needs exists in the per-node
-    // mirror before launch (copies from the shared HF cache when missing).
-    if let Some(ref mirror) = opts.mirror {
-        mirror::ensure_mirrors(
-            config,
-            &nodes,
-            recipe,
-            &config.resolved_hf_cache_dir(),
-            mirror,
-            opts.dry_run,
-        )
-        .await?;
-    }
-    let pp_size = if recipe.pipeline_parallel_size.unwrap_or(0) == 0
-        || recipe.required_nodes == recipe.pipeline_parallel_size
-    {
-        nnodes
-    } else {
-        recipe.pipeline_parallel_size.unwrap()
-    };
-    let tp_size = if recipe.required_nodes == Some(recipe.tensor_parallel_size) {
-        nnodes
-    } else {
-        recipe.tensor_parallel_size
-    };
-    // Resolve static mounts once (alias/host + tilde). Model path references
-    // (`repo`+`subfolder`) are kept as specs and resolved per node inside the
-    // spawn loop, since the HF snapshot hash can differ across nodes.
-    let resolved = command::resolve_extra_mounts(config, recipe)?;
-    command::clear_extra_mounts(config, &nodes, &resolved, opts.clear, opts.dry_run).await?;
-    let mut handles = Vec::new();
-    for node in &nodes {
-        let node = (*node).clone();
-        let config = config.clone();
-        let rn = recipe_name.to_string();
-        let dry = opts.dry_run;
-        let engine = recipe.engine.clone();
-        let is_head = node.rank == 0;
-        let cleanup = format!(
-            "docker rm -f {} 2>/dev/null; true",
-            shq(&format!("sm121_{}_{}", recipe_name, node.rank))
-        );
-        let dm_ctx = command::DockerCommandCtx {
-            cluster: cluster.clone(),
-            nccl: nccl.clone(),
-            recipe: recipe.clone(),
-            recipe_name: rn.clone(),
-            pp_size,
-            tp_size,
-            nnodes,
-            cache_dir: opts.cache_dir.to_string_lossy().into_owned(),
-            keep: opts.keep,
-            init_port: config.init_port,
-            mods_remote_path: opts.mods_remote_path.clone(),
-            extra_mounts: resolved.static_mounts.clone(),
-        };
-        let hf_cache = opts.cache_dir.clone();
-        let path_refs = resolved.path_refs.clone();
-        handles.push(crate::locale::spawn_with_locale(async move {
-            launch_node::launch_node(
-                &config,
-                launch_node::LaunchNodeCtx {
-                    node: &node,
-                    rn: &rn,
-                    dry,
-                    engine: &engine,
-                    is_head,
-                    cleanup,
-                    dm_ctx: &dm_ctx,
-                    hf_cache: &hf_cache,
-                    path_refs: &path_refs,
-                },
-            )
-            .await
-        }));
-    }
-    // NOTE: all node launches are awaited (join_all) before reporting — a
-    // failure on one node does not cancel the others, so a partial cluster
-    // may be left running. Intentional: reporting every node's result is
-    // more useful than aborting mid-launch and leaving the operator to
-    // guess which nodes actually started.
-    for result in futures::future::join_all(handles).await {
-        match result {
-            Ok((name, rank, engine, Ok(_))) => info!(
-                "{}",
-                fluent_i18n::t!("engine-started", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank })
-            ),
-            Ok((name, rank, engine, Err(e))) => anyhow::bail!(
-                "{}",
-                fluent_i18n::t!("engine-start-failed", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank, "error" => e.to_string() })
-            ),
-            Err(e) => anyhow::bail!("Task join error: {}", e),
-        }
-    }
-    if !opts.dry_run {
-        sleep(Duration::from_secs(10)).await;
-        info!(
-            "{}",
-            fluent_i18n::t!("cluster-started", { "ip" => cluster.effective_master_ip(), "port" => recipe.port })
-        );
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests;

src/docker/start.rs

@@ -0,0 +1,147 @@
+//! Cluster launch (the `serve` inner half): build/sync the image, compose
+//! the docker run per node, start containers and wait for readiness.
+//! Split from mod.rs for the 500-line cap.
+
+use super::command;
+use super::launch_node;
+use super::mirror;
+use super::shq;
+use super::{Config, NcclConfig, Recipe, StartClusterOpts};
+use crate::config::{Cluster, Node};
+use anyhow::Result;
+
+use tokio::time::{Duration, sleep};
+use tracing::info;
+
+pub(crate) async fn start_cluster(
+    config: &Config,
+    cluster: &Cluster,
+    nccl: Option<NcclConfig>,
+    recipe_name: &str,
+    recipe: &Recipe,
+    opts: StartClusterOpts,
+) -> Result<()> {
+    let nodes: Vec<&Node> = if let Some(ref nn) = opts.node_name {
+        let v: Vec<&Node> = cluster.nodes.iter().filter(|n| n.name == *nn).collect();
+        if v.is_empty() {
+            anyhow::bail!("Node '{}' not found in cluster '{}'", nn, cluster.name);
+        }
+        v
+    } else {
+        // Trim to `required_nodes` when set: a recipe that needs N nodes
+        // must launch on exactly N nodes, not on every free node in the
+        // cluster. Previously a `required_nodes = 1` recipe on a 4-node
+        // cluster launched 4 containers, each told `--nnodes 4 --tp 4` —
+        // a broken rendezvous that never coordinated. `prepare_cluster`
+        // guarantees >= required_nodes are free, so taking the first N is
+        // safe and deterministic.
+        let all = cluster.nodes.iter().collect::<Vec<&Node>>();
+        match recipe.required_nodes {
+            Some(n) if (n as usize) < all.len() => all.into_iter().take(n as usize).collect(),
+            _ => all,
+        }
+    };
+    let nnodes = nodes.len() as u32;
+    // `--mirror`: ensure every repo the recipe needs exists in the per-node
+    // mirror before launch (copies from the shared HF cache when missing).
+    if let Some(ref mirror) = opts.mirror {
+        mirror::ensure_mirrors(
+            config,
+            &nodes,
+            recipe,
+            &config.resolved_hf_cache_dir(),
+            mirror,
+            opts.dry_run,
+        )
+        .await?;
+    }
+    let pp_size = if recipe.pipeline_parallel_size.unwrap_or(0) == 0
+        || recipe.required_nodes == recipe.pipeline_parallel_size
+    {
+        nnodes
+    } else {
+        recipe.pipeline_parallel_size.unwrap()
+    };
+    let tp_size = if recipe.required_nodes == Some(recipe.tensor_parallel_size) {
+        nnodes
+    } else {
+        recipe.tensor_parallel_size
+    };
+    // Resolve static mounts once (alias/host + tilde). Model path references
+    // (`repo`+`subfolder`) are kept as specs and resolved per node inside the
+    // spawn loop, since the HF snapshot hash can differ across nodes.
+    let resolved = command::resolve_extra_mounts(config, recipe)?;
+    command::clear_extra_mounts(config, &nodes, &resolved, opts.clear, opts.dry_run).await?;
+    let mut handles = Vec::new();
+    for node in &nodes {
+        let node = (*node).clone();
+        let config = config.clone();
+        let rn = recipe_name.to_string();
+        let dry = opts.dry_run;
+        let engine = recipe.engine.clone();
+        let is_head = node.rank == 0;
+        let cleanup = format!(
+            "docker rm -f {} 2>/dev/null; true",
+            shq(&format!("sm121_{}_{}", recipe_name, node.rank))
+        );
+        let dm_ctx = command::DockerCommandCtx {
+            cluster: cluster.clone(),
+            nccl: nccl.clone(),
+            recipe: recipe.clone(),
+            recipe_name: rn.clone(),
+            pp_size,
+            tp_size,
+            nnodes,
+            cache_dir: opts.cache_dir.to_string_lossy().into_owned(),
+            keep: opts.keep,
+            init_port: config.init_port,
+            mods_remote_path: opts.mods_remote_path.clone(),
+            extra_mounts: resolved.static_mounts.clone(),
+        };
+        let hf_cache = opts.cache_dir.clone();
+        let path_refs = resolved.path_refs.clone();
+        handles.push(crate::locale::spawn_with_locale(async move {
+            launch_node::launch_node(
+                &config,
+                launch_node::LaunchNodeCtx {
+                    node: &node,
+                    rn: &rn,
+                    dry,
+                    engine: &engine,
+                    is_head,
+                    cleanup,
+                    dm_ctx: &dm_ctx,
+                    hf_cache: &hf_cache,
+                    path_refs: &path_refs,
+                },
+            )
+            .await
+        }));
+    }
+    // NOTE: all node launches are awaited (join_all) before reporting — a
+    // failure on one node does not cancel the others, so a partial cluster
+    // may be left running. Intentional: reporting every node's result is
+    // more useful than aborting mid-launch and leaving the operator to
+    // guess which nodes actually started.
+    for result in futures::future::join_all(handles).await {
+        match result {
+            Ok((name, rank, engine, Ok(_))) => info!(
+                "{}",
+                fluent_i18n::t!("engine-started", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank })
+            ),
+            Ok((name, rank, engine, Err(e))) => anyhow::bail!(
+                "{}",
+                fluent_i18n::t!("engine-start-failed", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank, "error" => e.to_string() })
+            ),
+            Err(e) => anyhow::bail!("Task join error: {}", e),
+        }
+    }
+    if !opts.dry_run {
+        sleep(Duration::from_secs(10)).await;
+        info!(
+            "{}",
+            fluent_i18n::t!("cluster-started", { "ip" => cluster.effective_master_ip(), "port" => recipe.port })
+        );
+    }
+    Ok(())
+}

src/handlers.rs

@@ -0,0 +1,156 @@
+//! Per-command handlers extracted from main.rs (the 500-line cap).
+//! `main.rs` keeps the clap dispatch; each `handle_*` does the command work.
+
+use crate::cli::{OutputFormat, PowerAction};
+use crate::config::Config;
+use crate::config::load_recipe;
+use crate::config::resolve_cluster;
+use crate::config::{Cluster, select_cluster_by_name};
+use crate::docker;
+use crate::exec;
+use crate::power;
+use crate::ssh::fetch_host_key;
+use anyhow::Result;
+use std::collections::HashSet;
+use tracing::info;
+
+/// Handle `logs` command: show container logs.
+pub(crate) async fn handle_logs(
+    config: &Config,
+    recipe_name: &str,
+    cluster: &str,
+    follow: bool,
+    node: Option<String>,
+    dry_run: bool,
+) -> Result<()> {
+    let cluster = resolve_cluster_for_op(config, recipe_name, cluster).await?;
+    docker::show_logs(
+        config,
+        &cluster,
+        recipe_name,
+        follow,
+        node.as_deref(),
+        dry_run,
+    )
+    .await?;
+    Ok(())
+}
+/// Handle `health` command: check cluster health or show host key fingerprints.
+pub(crate) async fn handle_health(
+    config: &Config,
+    cluster_name: Option<String>,
+    fingerprint: bool,
+    format: OutputFormat,
+) -> Result<()> {
+    if fingerprint {
+        let clusters_to_check: Vec<Cluster> = match &cluster_name {
+            Some(name) => vec![select_cluster_by_name(config, name)?],
+            None => config.clusters.clone(),
+        };
+        for cluster in &clusters_to_check {
+            println!(
+                "\n{}",
+                fluent_i18n::t!("cluster-section", { "name" => &cluster.name })
+            );
+            for node in &cluster.nodes {
+                match fetch_host_key(config, node).await {
+                    Ok(fp) => println!(
+                        "{}",
+                        fluent_i18n::t!("host-key-line", { "name" => &node.name, "ip" => &node.ssh_ip, "key" => &fp })
+                    ),
+                    Err(e) => println!(
+                        "{}",
+                        fluent_i18n::t!("host-key-error", { "name" => &node.name, "ip" => &node.ssh_ip, "error" => &e.to_string() })
+                    ),
+                }
+            }
+        }
+    } else {
+        match cluster_name {
+            Some(ref name) => {
+                let cluster = select_cluster_by_name(config, name)?;
+                docker::health::health_check(config, &cluster, false, format).await?;
+            }
+            None => {
+                let fleet = docker::health::collect(config, None, false).await?;
+                docker::health::print_fleet(&fleet, format);
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Handle `exec` command: run a command on cluster nodes via SSH.
+pub(crate) async fn handle_exec(
+    config: &Config,
+    cluster: Option<String>,
+    recipe: Option<String>,
+    node: Option<String>,
+    alias: Option<String>,
+    command: Vec<String>,
+    dry_run: bool,
+) -> Result<()> {
+    let (cluster, _recipe) =
+        exec::resolve_target(config, cluster.as_deref(), recipe.as_deref()).await?;
+    let cmd_str = exec::resolve_command(config, alias.as_deref(), &command)?;
+    exec::run(config, &cluster, node.as_deref(), &cmd_str, dry_run).await?;
+    Ok(())
+}
+
+/// Handle `power` command: power on/off/switch nodes.
+pub(crate) async fn handle_power(
+    config: &Config,
+    action: PowerAction,
+    dry_run: bool,
+) -> Result<()> {
+    match action {
+        PowerAction::Off { cluster, node } => {
+            let cluster = select_cluster_by_name(config, &cluster)?;
+            let nodes = node.into_iter().collect::<Vec<_>>();
+            power::power_off(config, &cluster, &nodes, dry_run).await?;
+        }
+        PowerAction::On { cluster, node } => {
+            let cluster = select_cluster_by_name(config, &cluster)?;
+            let nodes = node.into_iter().collect::<Vec<_>>();
+            let targets = power::select_targets(&cluster, &nodes, "power on")?;
+            let stale = if dry_run {
+                Vec::new()
+            } else {
+                power::stale_nodes(&targets).await
+            };
+            let report = power::power_on(config, &cluster, &nodes, dry_run, &stale).await?;
+            for line in report {
+                info!("{}", line);
+            }
+        }
+        PowerAction::Switch { from, to } => {
+            let from = select_cluster_by_name(config, &from)?;
+            let to = select_cluster_by_name(config, &to)?;
+            power::power_switch(config, &from, &to, dry_run).await?;
+        }
+    }
+    Ok(())
+}
+
+/// Resolve a cluster for container operations (stop/logs).
+/// When `auto`, tries live container label lookup first, then falls back
+/// to config-based auto-selection (for dry-run / offline scenarios).
+pub(crate) async fn resolve_cluster_for_op(
+    config: &Config,
+    recipe_name: &str,
+    cluster_name: &str,
+) -> Result<Cluster> {
+    if cluster_name != "auto" {
+        let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
+        let (cluster, _) = resolve_cluster(config, &recipe, cluster_name, &HashSet::new())?;
+        return Ok(cluster);
+    }
+    // Try live container label lookup first
+    if let Ok(c) = docker::resolve_cluster_by_container_label(config, recipe_name).await {
+        return Ok(c.clone());
+    }
+    // Fall back to config-based auto (dry-run / containers not running)
+    let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
+    let (cluster, _) = resolve_cluster(config, &recipe, "auto", &HashSet::new())?;
+    Ok(cluster)
+}

src/main.rs

@@ -15,6 +15,7 @@ mod bench_remote;
 mod bench_report;
 mod builtin_scripts;
 mod cli;
+mod cli_sub;
 mod cluster_state;
 mod config;
 mod detect;
@@ -24,6 +25,7 @@ mod ends;
 mod engine_flags;
 mod exec;
 mod gconfig;
+mod handlers;
 mod health;
 mod home_assistant;
 mod host_keys;
@@ -43,12 +45,9 @@ mod staging;
 mod stamp;
 mod tmux;
 mod web;
-use crate::cli::{Cli, Commands, OutputFormat, PowerAction};
-use crate::config::{
-    Cluster, Config, DeployTarget, load_recipe, resolve_cluster, select_cluster_by_name,
-};
-use crate::ssh::fetch_host_key;
-use std::collections::HashSet;
+use crate::cli::{Cli, Commands};
+use crate::config::{Config, DeployTarget, load_recipe};
+use crate::handlers::{handle_exec, handle_health, handle_logs, handle_power};
 
 #[tokio::main]
 async fn main() -> Result<()> {
@@ -370,131 +369,3 @@ pub(crate) async fn handle_serve(
     }
     Ok(())
 }
-
-/// Handle `logs` command: show container logs.
-async fn handle_logs(
-    config: &Config,
-    recipe_name: &str,
-    cluster: &str,
-    follow: bool,
-    node: Option<String>,
-    dry_run: bool,
-) -> Result<()> {
-    let cluster = resolve_cluster_for_op(config, recipe_name, cluster).await?;
-    docker::show_logs(
-        config,
-        &cluster,
-        recipe_name,
-        follow,
-        node.as_deref(),
-        dry_run,
-    )
-    .await?;
-    Ok(())
-}
-/// Handle `health` command: check cluster health or show host key fingerprints.
-async fn handle_health(
-    config: &Config,
-    cluster_name: Option<String>,
-    fingerprint: bool,
-    format: OutputFormat,
-) -> Result<()> {
-    if fingerprint {
-        let clusters_to_check: Vec<Cluster> = match &cluster_name {
-            Some(name) => vec![select_cluster_by_name(config, name)?],
-            None => config.clusters.clone(),
-        };
-        for cluster in &clusters_to_check {
-            println!(
-                "\n{}",
-                fluent_i18n::t!("cluster-section", { "name" => &cluster.name })
-            );
-            for node in &cluster.nodes {
-                match fetch_host_key(config, node).await {
-                    Ok(fp) => println!(
-                        "{}",
-                        fluent_i18n::t!("host-key-line", { "name" => &node.name, "ip" => &node.ssh_ip, "key" => &fp })
-                    ),
-                    Err(e) => println!(
-                        "{}",
-                        fluent_i18n::t!("host-key-error", { "name" => &node.name, "ip" => &node.ssh_ip, "error" => &e.to_string() })
-                    ),
-                }
-            }
-        }
-    } else {
-        match cluster_name {
-            Some(ref name) => {
-                let cluster = select_cluster_by_name(config, name)?;
-                docker::health::health_check(config, &cluster, false, format).await?;
-            }
-            None => {
-                let fleet = docker::health::collect(config, None, false).await?;
-                docker::health::print_fleet(&fleet, format);
-            }
-        }
-    }
-    Ok(())
-}
-
-/// Handle `exec` command: run a command on cluster nodes via SSH.
-async fn handle_exec(
-    config: &Config,
-    cluster: Option<String>,
-    recipe: Option<String>,
-    node: Option<String>,
-    alias: Option<String>,
-    command: Vec<String>,
-    dry_run: bool,
-) -> Result<()> {
-    let (cluster, _recipe) =
-        exec::resolve_target(config, cluster.as_deref(), recipe.as_deref()).await?;
-    let cmd_str = exec::resolve_command(config, alias.as_deref(), &command)?;
-    exec::run(config, &cluster, node.as_deref(), &cmd_str, dry_run).await?;
-    Ok(())
-}
-
-/// Handle `power` command: power on/off/switch nodes.
-async fn handle_power(config: &Config, action: PowerAction, dry_run: bool) -> Result<()> {
-    match action {
-        PowerAction::Off { cluster, node } => {
-            let cluster = select_cluster_by_name(config, &cluster)?;
-            let nodes = node.into_iter().collect::<Vec<_>>();
-            power::power_off(config, &cluster, &nodes, dry_run).await?;
-        }
-        PowerAction::On { cluster, node } => {
-            let cluster = select_cluster_by_name(config, &cluster)?;
-            let nodes = node.into_iter().collect::<Vec<_>>();
-            power::power_on(config, &cluster, &nodes, dry_run).await?;
-        }
-        PowerAction::Switch { from, to } => {
-            let from = select_cluster_by_name(config, &from)?;
-            let to = select_cluster_by_name(config, &to)?;
-            power::power_switch(config, &from, &to, dry_run).await?;
-        }
-    }
-    Ok(())
-}
-
-/// Resolve a cluster for container operations (stop/logs).
-/// When `auto`, tries live container label lookup first, then falls back
-/// to config-based auto-selection (for dry-run / offline scenarios).
-pub(crate) async fn resolve_cluster_for_op(
-    config: &Config,
-    recipe_name: &str,
-    cluster_name: &str,
-) -> Result<Cluster> {
-    if cluster_name != "auto" {
-        let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
-        let (cluster, _) = resolve_cluster(config, &recipe, cluster_name, &HashSet::new())?;
-        return Ok(cluster);
-    }
-    // Try live container label lookup first
-    if let Ok(c) = docker::resolve_cluster_by_container_label(config, recipe_name).await {
-        return Ok(c.clone());
-    }
-    // Fall back to config-based auto (dry-run / containers not running)
-    let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
-    let (cluster, _) = resolve_cluster(config, &recipe, "auto", &HashSet::new())?;
-    Ok(cluster)
-}

src/power.rs

@@ -112,12 +112,60 @@ pub async fn power_off(
 /// Flow for `power on`: HA `turn_on` for each node with an `entity`
 /// and each cluster-level `extra_entities`.  No SSH needed — the smart
 /// plug handles waking the machine.
+/// What `power on` should do with one node's plug.
+#[derive(Debug, PartialEq, Eq)]
+pub(crate) enum OnAction {
+    /// Plug off (or state unknown) — plain `turn_on`.
+    TurnOn,
+    /// Plug reads on but the node is believed down — cut and re-apply
+    /// (a graceful shutdown leaves the plug on while the machine sits in
+    /// soft-off, so a plain `turn_on` would be a silent no-op).
+    Cycle,
+    /// Plug on and the node is believed up — leave it alone.
+    AlreadyOn,
+}
+
+/// SGR-free decision: pure so the semantics are testable.
+pub(crate) fn decide_on(state: Option<&str>, node_is_down: bool) -> OnAction {
+    match state {
+        Some("on") if node_is_down => OnAction::Cycle,
+        Some("on") => OnAction::AlreadyOn,
+        _ => OnAction::TurnOn,
+    }
+}
+
+/// TCP connect to the node's SSH port (1.5 s) — a soft-off machine refuses
+/// or times out, a live one accepts. Not an SSH handshake: no auth, cheap.
+pub(crate) async fn stale_nodes(targets: &[Node]) -> Vec<String> {
+    let mut out = Vec::new();
+    for node in targets {
+        let (host, port) = match node.ssh_ip.rsplit_once(':') {
+            Some((h, p)) if p.parse::<u16>().is_ok() => (h.to_string(), p.parse().unwrap()),
+            _ => (node.ssh_ip.clone(), 22),
+        };
+        let attempt = tokio::time::timeout(
+            Duration::from_millis(1500),
+            tokio::net::TcpStream::connect((host.as_str(), port)),
+        )
+        .await;
+        if !matches!(attempt, Ok(Ok(_))) {
+            out.push(node.name.clone());
+        }
+    }
+    out
+}
+
+/// Flow for `power on`: per-node plug decisions (`decide_on` + the caller's
+/// `stale` list), then cluster-level `extra_entities`.  Returns one report
+/// line per node — the job surface (web job message, CLI log) tells what
+/// actually happened instead of a blanket "ok".
 pub async fn power_on(
     config: &Config,
     cluster: &Cluster,
     node_names: &[String],
     dry_run: bool,
-) -> Result<()> {
+    stale: &[String],
+) -> Result<Vec<String>> {
     let targets = select_targets(cluster, node_names, "power on")?;
 
     let Some(ha_cfg) = &config.home_assistant else {
@@ -132,23 +180,113 @@ pub async fn power_on(
     if dry_run {
         warn!("{}", fluent_i18n::t!("power-on-dryrun"));
         mikro::dry_on(config, &targets, &extra_ids(cluster));
-        return Ok(());
+        return Ok(Vec::new());
     }
 
     info!(
         "{}",
         fluent_i18n::t!("power-on-head", { "nodes" => targets.len() })
     );
+    let mut report: Vec<String> = Vec::new();
+    for node in &targets {
+        let Some(entity) = &node.entity else {
+            warn!(
+                "{}",
+                fluent_i18n::t!("ha-skip-no-entity", { "name" => &node.name })
+            );
+            report.push(fluent_i18n::t!("power-on-skip", { "name" => &node.name }));
+            continue;
+        };
+        if mikro::device(config.mikrotik.as_ref(), entity).is_some() {
+            // RouterOS devices are not HA plugs; wake via the relay as before
+            toggle_one(
+                &ha,
+                "turn_on",
+                config.mikrotik.as_ref(),
+                entity,
+                &node.name,
+                &mut Vec::new(),
+            )
+            .await;
+            report.push(fluent_i18n::t!("power-on-on", { "name" => &node.name }));
+            continue;
+        }
+        let action = decide_on(
+            ha.get_state(entity).await.as_deref(),
+            stale.contains(&node.name),
+        );
+        match action {
+            OnAction::TurnOn => {
+                if let Err(e) = call(&ha, entity, "turn_on").await {
+                    warn!(
+                        "{}",
+                        fluent_i18n::t!("ha-call-failed", { "service" => "turn_on", "entity" => entity.clone(), "error" => e.to_string() })
+                    );
+                    report.push(fluent_i18n::t!("power-on-failed", { "name" => &node.name, "error" => e.to_string() }));
+                } else {
+                    report.push(fluent_i18n::t!("power-on-on", { "name" => &node.name }));
+                }
+            }
+            OnAction::Cycle => {
+                warn!(
+                    "{}",
+                    fluent_i18n::t!("power-on-cycling", { "name" => &node.name, "entity" => entity.clone() })
+                );
+                let r1 = call(&ha, entity, "turn_off").await;
+                // the relay opens asynchronously — wait until HA actually
+                // reports off (≤10 s), then a settle dwell, then re-apply
+                let mut saw_off = false;
+                for _ in 0..20 {
+                    tokio::time::sleep(Duration::from_millis(500)).await;
+                    if ha.get_state(entity).await.as_deref() != Some("on") {
+                        saw_off = true;
+                        break;
+                    }
+                }
+                if !saw_off {
+                    warn!(
+                        "{}",
+                        fluent_i18n::t!("power-on-off-wait-timeout", { "name" => &node.name, "entity" => entity.clone() })
+                    );
+                }
+                // the PDU needs a real outage: a 2-3 s cut left the PSU
+                // rail energized and the machine stayed in soft-off — 30 s
+                // reliably re-triggers the power-on
+                tokio::time::sleep(Duration::from_secs(30)).await;
+                let r2 = call(&ha, entity, "turn_on").await;
+                match (r1, r2) {
+                    (Ok(()), Ok(())) => {
+                        report.push(fluent_i18n::t!("power-on-cycled", { "name" => &node.name }))
+                    }
+                    (e1, e2) => {
+                        let e = e1
+                            .err()
+                            .or(e2.err())
+                            .unwrap_or_else(|| anyhow::anyhow!("cycle"));
+                        report.push(fluent_i18n::t!("power-on-failed", { "name" => &node.name, "error" => e.to_string() }));
+                    }
+                }
+            }
+            OnAction::AlreadyOn => {
+                warn!(
+                    "{}",
+                    fluent_i18n::t!("power-on-was-on", { "name" => &node.name })
+                );
+                report.push(fluent_i18n::t!("power-on-was-on", { "name" => &node.name }));
+            }
+        }
+    }
+    // cluster-level extras (fan / relay / fabric switch) as before
     let _ = toggle_entities(
         &ha,
         "turn_on",
         config.mikrotik.as_ref(),
-        &targets,
+        &[],
         &extra_ids(cluster),
         "cluster",
     )
     .await;
-    Ok(())
+    Ok(report)
 }
 
 /// Entity ids of a cluster's `extra_entities` (both forms).
@@ -385,7 +523,11 @@ pub async fn power_switch(
     Ok(())
 }
 
-fn select_targets(cluster: &Cluster, node_names: &[String], op: &str) -> Result<Vec<Node>> {
+pub(crate) fn select_targets(
+    cluster: &Cluster,
+    node_names: &[String],
+    op: &str,
+) -> Result<Vec<Node>> {
     let targets: Vec<Node> = if node_names.is_empty() {
         cluster.nodes.clone()
     } else {

src/power/tests.rs

@@ -174,3 +174,13 @@ password = "p"
     let dev = super::mikro::device(cfg.mikrotik.as_ref(), "router.lab");
     assert!(dev.is_some());
 }
+
+#[test]
+fn decide_on_cycles_only_when_plug_on_and_node_down() {
+    use crate::power::OnAction;
+    use crate::power::decide_on;
+    assert_eq!(decide_on(Some("on"), true), OnAction::Cycle);
+    assert_eq!(decide_on(Some("on"), false), OnAction::AlreadyOn);
+    assert_eq!(decide_on(Some("off"), true), OnAction::TurnOn);
+    assert_eq!(decide_on(None, true), OnAction::TurnOn);
+}

src/web/api.rs

@@ -139,15 +139,20 @@ pub(crate) async fn power(
     let action = req.action.clone();
     let cluster = req.cluster.clone();
     let to = req.to.clone();
-    super::jobs::spawn(app2, id.clone(), async move {
+    let stale = stale_from_snapshot(&app, &req.cluster);
+    super::jobs::spawn_report(app2, id.clone(), async move {
         match action.as_str() {
             "off" => {
                 let c = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
-                crate::power::power_off(&config, c, &[], false).await
+                crate::power::power_off(&config, c, &[], false).await?;
+                Ok(Vec::new())
             }
             "on" => {
                 let c = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
-                crate::power::power_on(&config, c, &[], false).await
+                // nodes the last probe saw down: their plugs may be stale-on
+                // (a graceful shutdown leaves the plug on), so power on
+                // cycles those instead of issuing a no-op turn_on
+                crate::power::power_on(&config, c, &[], false, &stale).await
             }
             "switch" => {
                 let to = to
@@ -155,7 +160,8 @@ pub(crate) async fn power(
                     .ok_or_else(|| anyhow::anyhow!("`to` cluster is required for power switch"))?;
                 let from = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
                 let to = crate::config::resolve_cluster_name(&config.clusters, to)?;
-                crate::power::power_switch(&config, from, to, false).await
+                crate::power::power_switch(&config, from, to, false).await?;
+                Ok(Vec::new())
             }
             other => Err(anyhow::anyhow!(
                 "unknown power action '{other}' (off|on|switch)"
@@ -165,6 +171,30 @@ pub(crate) async fn power(
     Ok(Json(serde_json::json!({ "job": id })))
 }
 
+/// Nodes of `cluster` that the last fleet probe saw unreachable — the
+/// `power on` cycle list (a plug left on by a graceful shutdown needs a
+/// cycle, not a no-op `turn_on`).
+pub(crate) fn stale_from_snapshot(app: &AppState, cluster: &str) -> Vec<String> {
+    let snap = app
+        .last
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner)
+        .clone();
+    let Some(snap) = snap else { return Vec::new() };
+    let mut out = Vec::new();
+    for c in &snap.clusters {
+        if c.name != cluster {
+            continue;
+        }
+        for n in &c.nodes {
+            if !n.reachable {
+                out.push(n.name.clone());
+            }
+        }
+    }
+    out
+}
+
 /// `GET /api/recipes` — all discoverable recipes (cwd + `recipe_sources` +
 /// the global config dir), with deploy target + node requirements.
 pub(crate) async fn recipes(app: State<AppState>) -> Json<serde_json::Value> {

src/web/jobs.rs

@@ -162,3 +162,29 @@ pub fn spawn(
         finish_job(&app, &id, ok, message);
     });
 }
+
+/// Like [`spawn`], but the future reports per-item lines (e.g. one per node
+/// for power actions) — they become the job's message instead of "ok".
+pub fn spawn_report(
+    app: AppState,
+    id: String,
+    fut: impl std::future::Future<Output = anyhow::Result<Vec<String>>> + Send + 'static,
+) {
+    tokio::spawn(async move {
+        let (ok, message) = match fut.await {
+            Ok(lines) => {
+                let joined = lines.join("; ");
+                (
+                    true,
+                    if joined.is_empty() {
+                        String::from("ok")
+                    } else {
+                        joined
+                    },
+                )
+            }
+            Err(e) => (false, e.to_string()),
+        };
+        finish_job(&app, &id, ok, message);
+    });
+}

src/web/tests.rs

@@ -161,6 +161,38 @@ async fn token_gate_enforces_when_set() {
     );
 }
 
+#[test]
+fn stale_from_snapshot_picks_unreachable_nodes() {
+    let app = app_state(None);
+    use crate::web::api::stale_from_snapshot;
+    let node = |name: &str, ip: &str, reachable: bool| crate::web::events::NodeSnap {
+        name: name.into(),
+        ssh_ip: ip.into(),
+        reachable,
+        occupied: false,
+        containers: Vec::new(),
+        rank: 0,
+        ha: None,
+    };
+    let snap = crate::web::events::Snapshot {
+        ts: 1,
+        clusters: vec![crate::web::events::ClusterSnap {
+            name: "c".into(),
+            up: false,
+            switch_off: false,
+            soft_off: Vec::new(),
+            free: 0,
+            nodes: vec![node("a", "1.1.1.1", true), node("b", "1.1.1.2", false)],
+        }],
+        jobs: Vec::new(),
+        topo: Default::default(),
+        attaches: Vec::new(),
+    };
+    *app.last.lock().unwrap() = Some(snap);
+    assert_eq!(stale_from_snapshot(&app, "c"), vec!["b".to_string()]);
+    assert!(stale_from_snapshot(&app, "other").is_empty());
+}
+
 #[tokio::test]
 async fn token_gate_covers_recipes_panel() {
     let app = app_state(Some("s3cret"));

src/web/ui.html

@@ -2035,7 +2035,7 @@ function doPower(action, cluster) {
     });
     return;
   }
-  twoStep(`power on ${cluster}`, `Turns on <code>${esc(cluster)}</code> via Home Assistant (SSH is not used).`, cluster, () => {
+  twoStep(`power on ${cluster}`, `Turns on <code>${esc(cluster)}</code> via Home Assistant (SSH is not used). Nodes that are down but whose plug reads on get a power cycle — the job reports each node.`, cluster, () => {
     api("/api/power", { action: "on", cluster }).catch((e) => openDialog("power failed", esc(e.message), () => {}, true));
   });
 }