docs: real hash for web P2

specs/tech-architecture/web-dashboard.md

@@ -204,7 +204,17 @@ toggle is one click away:
    against the real fleet: dgx1 up+occupied, dgx2-4 offline; SSE + token
    gate + non-loopback refusal verified).
 2. **P2 — actions.** jobs registry, serve/stop/power POSTs, confirm flows,
-   jobs window, job LEDs, dry-run preview.
+   jobs window, job LEDs, dry-run preview.  ✅ done (`aa2ccd5`) — jobs are
+   tracked tokio tasks calling the CLI handlers (`handle_serve` /
+   `handle_stop` / power fns), single-flight 409, `job` SSE events (started /
+   ok / failed, with kind), state pushed on job finish. UI: per-cluster
+   action bar (recipe select, dry-run), stop confirm lists running
+   containers (new web probe fetches per-node sm121 containers), power
+   off/switch type-the-name confirm + occupied-block. `GET /api/recipes`
+   lists discoverable recipes (deploy + required_nodes). Deviations: job
+   events are started/ok/failed (progress lines deferred to P3's log
+   streams); serve/stop jobs run the *whole* CLI path (including
+   occupancy / NCCL / venv checks) — a web serve is exactly a CLI serve.
 3. **P3 — logs & windows.** log attaches + ring buffers + SSE log events,
    window manager polish (resize/minimize/dock), "new window" flows from
    the LED grid.

src/config.rs

@@ -234,6 +234,19 @@ pub async fn resolve_mods_dir(name: &str) -> Option<std::path::PathBuf> {
     cfg.is_dir().then_some(cfg)
 }
 
+/// All recipe dirs in search order: `./recipes`, each `recipe_sources`
+/// entry (`~` expanded), then the global config dir's `recipes/`.
+pub fn recipe_dirs(config: &Config) -> Vec<std::path::PathBuf> {
+    let mut v = vec![std::path::PathBuf::from("recipes")];
+    for s in &config.recipe_sources {
+        v.push(local_path(s));
+    }
+    if let Some(dir) = default_config_dir() {
+        v.push(dir.join("recipes"));
+    }
+    v
+}
+
 /// Load a recipe by name. Search order: `./recipes/<name>/recipe.toml` (cwd),
 /// then each `sources` entry (a dir containing `<name>/recipe.toml`, `~`
 /// expanded), then `<default_config_dir>/recipes/<name>/recipe.toml`.

src/main.rs

@@ -196,7 +196,7 @@ async fn main() -> Result<()> {
 /// `deploy` field. Docker-only CLI opts (sync_image, build_node,
 /// force_build, keep, auto_power_on) are ignored with a warning when
 /// `deploy = "tmux"`.
-async fn handle_serve(
+pub(crate) async fn handle_serve(
     config: &Config,
     recipe: &str,
     cluster: &str,
@@ -310,7 +310,7 @@ async fn handle_logs(
 }
 
 /// Handle `stop` command: stop containers and optionally power off.
-async fn handle_stop(
+pub(crate) async fn handle_stop(
     config: &Config,
     recipe_name: &str,
     cluster: &str,

src/web/api.rs

@@ -1,11 +1,11 @@
 //! REST endpoints. P1: `GET /api/state` (latest snapshot; probes once if
 //! the supervisor hasn't ticked yet).
 
-use crate::cluster_state::ClusterState;
-use axum::{extract::State, response::Json};
+use axum::{extract::State, http::StatusCode, response::Json};
+use serde::Deserialize;
 
 use super::AppState;
-use super::events::{Event, Snapshot, build_snapshot};
+use super::events::Snapshot;
 
 pub(crate) async fn state(app: State<AppState>) -> Json<Snapshot> {
     if app.last.lock().unwrap().is_none() {
@@ -13,13 +13,7 @@ pub(crate) async fn state(app: State<AppState>) -> Json<Snapshot> {
         // UI gets data immediately (probe is read-only `docker ps` per node
         // and degrades to `reachable: false` on failure). Two racing first
         // requests may both probe — harmless (idempotent, last write wins).
-        let st = ClusterState::probe(&app.config).await;
-        let snap = build_snapshot(&app.config, &st);
-        let _ = app.bus.send(Event::State {
-            ts: snap.ts,
-            snapshot: snap.clone(),
-        });
-        *app.last.lock().unwrap() = Some(snap);
+        super::push_state(&app).await;
     }
     app.last
         .lock()
@@ -28,3 +22,180 @@ pub(crate) async fn state(app: State<AppState>) -> Json<Snapshot> {
         .map(|s| Json(s.clone()))
         .unwrap()
 }
+
+// ---------------------------------------------------------------------------
+// P2: actions (serve/stop/power run as tracked jobs via the same handlers
+// the CLI uses) + recipe list.
+
+type ApiError = (StatusCode, String);
+
+#[derive(Deserialize)]
+pub(crate) struct ServeReq {
+    pub cluster: String,
+    pub recipe: String,
+    #[serde(default)]
+    pub node: Option<String>,
+    #[serde(default)]
+    pub solo: Option<bool>,
+    #[serde(default)]
+    pub force: bool,
+    #[serde(default)]
+    pub fit: Option<String>,
+    #[serde(default)]
+    pub clear: bool,
+    #[serde(default)]
+    pub mirror: Option<String>,
+    #[serde(default)]
+    pub keep: bool,
+    #[serde(default)]
+    pub dry_run: bool,
+}
+
+/// `POST /api/serve` — same dispatch as the CLI (docker vs tmux by
+/// `recipe.deploy`), run as a tracked job.
+pub(crate) async fn serve(
+    app: State<AppState>,
+    Json(req): Json<ServeReq>,
+) -> Result<Json<serde_json::Value>, ApiError> {
+    let id = super::jobs::new_job(&app, "serve", &req.cluster, Some(&req.recipe))?;
+    let config = app.config.clone();
+    let app2 = (*app).clone();
+    let opts = crate::docker::ServeOpts {
+        keep: req.keep,
+        skip_before_serve: false,
+        dry_run: req.dry_run,
+        auto_power_on: false,
+        sync_image: true,
+        sync_image_source: None,
+        build_node: None,
+        force_build: false,
+        force: req.force,
+        solo: req.solo,
+        mirror: req.mirror,
+        fit: req.fit,
+        clear: req.clear,
+    };
+    let recipe = req.recipe.clone();
+    let cluster = req.cluster.clone();
+    let node = req.node.clone();
+    super::jobs::spawn(app2, id.clone(), async move {
+        crate::handle_serve(&config, &recipe, &cluster, node, opts, false, false).await
+    });
+    Ok(Json(serde_json::json!({ "job": id })))
+}
+
+#[derive(Deserialize)]
+pub(crate) struct StopReq {
+    pub cluster: String,
+    pub recipe: String,
+    #[serde(default)]
+    pub node: Option<String>,
+}
+
+/// `POST /api/stop` — stops the recipe's containers (auto-discovers the
+/// cluster when the recipe is running exactly once), as a tracked job.
+pub(crate) async fn stop(
+    app: State<AppState>,
+    Json(req): Json<StopReq>,
+) -> Result<Json<serde_json::Value>, ApiError> {
+    let id = super::jobs::new_job(&app, "stop", &req.cluster, Some(&req.recipe))?;
+    let config = app.config.clone();
+    let app2 = (*app).clone();
+    let recipe = req.recipe.clone();
+    let cluster = req.cluster.clone();
+    let node = req.node.clone();
+    super::jobs::spawn(app2, id.clone(), async move {
+        let nodes = vec![node.unwrap_or_default()];
+        let nodes = if nodes[0].is_empty() { vec![] } else { nodes };
+        crate::handle_stop(&config, &recipe, &cluster, &nodes, false, false).await
+    });
+    Ok(Json(serde_json::json!({ "job": id })))
+}
+
+#[derive(Deserialize)]
+pub(crate) struct PowerReq {
+    /// "off" | "on" | "switch"
+    pub action: String,
+    pub cluster: String,
+    /// switch target (required only for `switch`)
+    #[serde(default)]
+    pub to: Option<String>,
+}
+
+/// `POST /api/power` — HA power actions as a tracked job. The UI enforces
+/// the type-the-name confirm; this endpoint trusts the caller (it is
+/// token-gated / loopback-bound).
+pub(crate) async fn power(
+    app: State<AppState>,
+    Json(req): Json<PowerReq>,
+) -> Result<Json<serde_json::Value>, ApiError> {
+    let id = super::jobs::new_job(&app, "power", &req.cluster, None)?;
+    let config = app.config.clone();
+    let app2 = (*app).clone();
+    let action = req.action.clone();
+    let cluster = req.cluster.clone();
+    let to = req.to.clone();
+    super::jobs::spawn(app2, id.clone(), async move {
+        match action.as_str() {
+            "off" => {
+                let c = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
+                crate::power::power_off(&config, c, &[], false).await
+            }
+            "on" => {
+                let c = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
+                crate::power::power_on(&config, c, &[], false).await
+            }
+            "switch" => {
+                let to = to
+                    .as_deref()
+                    .ok_or_else(|| anyhow::anyhow!("`to` cluster is required for power switch"))?;
+                let from = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
+                let to = crate::config::resolve_cluster_name(&config.clusters, to)?;
+                crate::power::power_switch(&config, from, to, false).await
+            }
+            other => Err(anyhow::anyhow!(
+                "unknown power action '{other}' (off|on|switch)"
+            )),
+        }
+    });
+    Ok(Json(serde_json::json!({ "job": id })))
+}
+
+/// `GET /api/recipes` — all discoverable recipes (cwd + `recipe_sources` +
+/// the global config dir), with deploy target + node requirements.
+pub(crate) async fn recipes(app: State<AppState>) -> Json<serde_json::Value> {
+    let dirs = crate::config::recipe_dirs(&app.config);
+    let mut out = Vec::new();
+    for dir in dirs {
+        let mut entries = match tokio::fs::read_dir(&dir).await {
+            Ok(e) => e,
+            Err(_) => continue,
+        };
+        loop {
+            let Ok(Some(entry)) = entries.next_entry().await else {
+                break;
+            };
+            let path = entry.path();
+            if !path.is_dir() {
+                continue;
+            }
+            let Some(name) = entry.file_name().to_str().map(String::from) else {
+                continue;
+            };
+            let file = path.join("recipe.toml");
+            let Ok(toml_str) = tokio::fs::read_to_string(&file).await else {
+                continue;
+            };
+            match toml::from_str::<crate::config::Recipe>(&toml_str) {
+                Ok(r) => out.push(serde_json::json!({
+                    "name": name,
+                    "engine": r.engine,
+                    "deploy": format!("{:?}", r.deploy).to_lowercase(),
+                    "required_nodes": r.required_nodes,
+                })),
+                Err(e) => tracing::debug!("skipping recipe {}: {e}", name),
+            }
+        }
+    }
+    Json(serde_json::json!({ "recipes": out }))
+}

src/web/events.rs

@@ -1,18 +1,32 @@
 //! SSE event bus + state snapshot for the dashboard.
 //!
 //! One multiplexed `broadcast` channel; `/events` forwards every event and
-//! the UI routes by `type`. P1 emits `state` only — `job` and `log`
-//! variants are added in P2/P3 when their producers exist.
+//! the UI routes by `type`. P2 emits `state` (supervisor + job finishes)
+//! and `job` (registry transitions).
 
-use crate::cluster_state::ClusterState;
 use crate::config::Config;
+use crate::web::probe::ContainerSnap;
 use serde::Serialize;
+use std::collections::HashMap;
+use std::time::{SystemTime, UNIX_EPOCH};
 
-/// One SSE event (serialized as `{"type":"state", ...}` on the wire).
+/// One SSE event (serialized as `{"type":"state"|"job", ...}` on the wire).
 #[derive(Serialize, Clone, Debug)]
 #[serde(tag = "type", rename_all = "lowercase")]
 pub enum Event {
-    State { ts: u64, snapshot: Snapshot },
+    State {
+        ts: u64,
+        snapshot: Snapshot,
+    },
+    Job {
+        id: String,
+        /// "serve" | "stop" | "power"
+        kind: &'static str,
+        event: String,
+        cluster: String,
+        recipe: Option<String>,
+        message: String,
+    },
 }
 
 /// Per-node view for the LED grid.
@@ -22,6 +36,7 @@ pub struct NodeSnap {
     pub ssh_ip: String,
     pub reachable: bool,
     pub occupied: bool,
+    pub containers: Vec<ContainerSnap>,
 }
 
 /// Per-cluster view.
@@ -37,117 +52,168 @@ pub struct ClusterSnap {
 pub struct Snapshot {
     pub ts: u64,
     pub clusters: Vec<ClusterSnap>,
+    pub jobs: Vec<crate::web::jobs::Job>,
+}
+
+fn now_ts() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(0)
 }
 
 /// Pure snapshot builder — unit-testable without SSH.
-pub fn build_snapshot(config: &Config, st: &ClusterState) -> Snapshot {
+pub fn build_snapshot(
+    config: &Config,
+    nodes: &HashMap<String, crate::web::probe::WebNodeState>,
+    jobs: &[crate::web::jobs::Job],
+) -> Snapshot {
     let mut clusters = Vec::with_capacity(config.clusters.len());
     for c in &config.clusters {
-        let mut nodes = Vec::with_capacity(c.nodes.len());
+        let mut node_snaps = Vec::with_capacity(c.nodes.len());
         let mut free = 0u32;
         for n in &c.nodes {
-            let s = st.nodes.get(&n.name);
+            let s = nodes.get(&n.name);
             let reachable = s.is_some_and(|x| x.reachable);
             let occupied = s.is_some_and(|x| x.occupied);
+            let containers = s.map(|x| x.containers.clone()).unwrap_or_default();
             if reachable && !occupied {
                 free += 1;
             }
-            nodes.push(NodeSnap {
+            node_snaps.push(NodeSnap {
                 name: n.name.clone(),
                 ssh_ip: n.ssh_ip.clone(),
                 reachable,
                 occupied,
+                containers,
             });
         }
         clusters.push(ClusterSnap {
             name: c.name.clone(),
-            nodes,
+            nodes: node_snaps,
             free,
         });
     }
     Snapshot {
-        ts: std::time::SystemTime::now()
-            .duration_since(std::time::UNIX_EPOCH)
-            .map(|d| d.as_secs())
-            .unwrap_or(0),
+        ts: now_ts(),
         clusters,
+        jobs: jobs.to_vec(),
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::config::{Cluster, Node};
-    use std::collections::HashMap;
-
-    fn node(name: &str, ip: &str, rank: u32) -> Node {
-        Node {
-            name: name.into(),
-            ssh_ip: ip.into(),
-            ib_ip: None,
-            rank,
-            host_key: None,
-            entity: None,
-        }
-    }
 
-    #[test]
-    fn snapshot_reflects_probe_state() {
-        let config = Config {
+    fn config() -> Config {
+        Config {
             ssh_user: "u".into(),
             ssh_key_path: "/k".into(),
             clusters: vec![
-                Cluster {
+                crate::config::Cluster {
                     name: "a".into(),
                     master_ib_ip: None,
-                    nodes: vec![node("n0", "10.0.0.1", 0), node("n1", "10.0.0.2", 1)],
+                    nodes: vec![
+                        crate::config::Node {
+                            name: "n0".into(),
+                            ssh_ip: "10.0.0.1".into(),
+                            ib_ip: None,
+                            rank: 0,
+                            host_key: None,
+                            entity: None,
+                        },
+                        crate::config::Node {
+                            name: "n1".into(),
+                            ssh_ip: "10.0.0.2".into(),
+                            ib_ip: None,
+                            rank: 1,
+                            host_key: None,
+                            entity: None,
+                        },
+                    ],
                     extra_entities: vec![],
                 },
-                Cluster {
+                crate::config::Cluster {
                     name: "b".into(),
                     master_ib_ip: None,
-                    nodes: vec![node("n2", "10.0.0.3", 0)],
+                    nodes: vec![crate::config::Node {
+                        name: "n2".into(),
+                        ssh_ip: "10.0.0.3".into(),
+                        ib_ip: None,
+                        rank: 0,
+                        host_key: None,
+                        entity: None,
+                    }],
                     extra_entities: vec![],
                 },
             ],
-            nccl: HashMap::new(),
+            nccl: std::collections::HashMap::new(),
             hf_cache_dir: None,
             init_port: 25000,
-            before_serve: HashMap::new(),
-            exec_aliases: HashMap::new(),
+            before_serve: std::collections::HashMap::new(),
+            exec_aliases: std::collections::HashMap::new(),
             home_assistant: None,
             global_mods_dir: None,
             drop_caches: true,
-            mount_aliases: HashMap::new(),
+            mount_aliases: std::collections::HashMap::new(),
             recipe_sources: Vec::new(),
             cluster_sources: Vec::new(),
-        };
-        let mut st = ClusterState::default();
-        st.nodes.insert(
+        }
+    }
+
+    #[test]
+    fn snapshot_reflects_probe_state() {
+        let config = config();
+        let mut nodes = HashMap::new();
+        nodes.insert(
             "n0".into(),
-            crate::cluster_state::NodeState {
+            crate::web::probe::WebNodeState {
                 reachable: true,
                 occupied: true,
+                containers: vec![ContainerSnap {
+                    name: "sm121_r_0".into(),
+                    status: "Up 3 hours".into(),
+                    cluster: "a".into(),
+                    recipe: "r".into(),
+                }],
             },
         );
-        st.nodes.insert(
+        nodes.insert(
             "n1".into(),
-            crate::cluster_state::NodeState {
+            crate::web::probe::WebNodeState {
                 reachable: true,
                 occupied: false,
+                containers: vec![],
             },
         );
-        // n2 absent from the probe → treated as unreachable.
-        let snap = build_snapshot(&config, &st);
+        // n2 absent → unreachable.
+        let snap = build_snapshot(&config, &nodes, &[]);
         assert_eq!(snap.clusters.len(), 2);
         let a = &snap.clusters[0];
-        assert_eq!(a.name, "a");
-        assert!(a.nodes[0].reachable);
-        assert!(a.nodes[0].occupied);
+        assert!(a.nodes[0].reachable && a.nodes[0].occupied);
+        assert_eq!(a.nodes[0].containers[0].recipe, "r");
         assert!(a.nodes[1].reachable && !a.nodes[1].occupied);
         assert_eq!(a.free, 1);
         let b = &snap.clusters[1];
         assert!(!b.nodes[0].reachable);
         assert_eq!(b.free, 0);
+        assert!(snap.jobs.is_empty());
+    }
+
+    #[test]
+    fn job_event_serializes_with_type_tag() {
+        let ev = Event::Job {
+            id: "j1".into(),
+            kind: "serve",
+            event: "started".into(),
+            cluster: "c".into(),
+            recipe: Some("r".into()),
+            message: "ok".into(),
+        };
+        let json = serde_json::to_value(&ev).unwrap();
+        assert_eq!(json["type"], "job");
+        assert_eq!(json["kind"], "serve");
+        assert_eq!(json["id"], "j1");
+        assert_eq!(json["event"], "started");
     }
 }

src/web/jobs.rs

@@ -0,0 +1,164 @@
+//! Job registry — serve/stop/power run as tracked tokio tasks that call the
+//! same handler functions the CLI uses (`crate::handle_serve`,
+//! `crate::handle_stop`, power fns). One running job per (kind, cluster,
+//! recipe); a second is rejected with 409.
+
+use super::AppState;
+use super::events::Event;
+use serde::Serialize;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+#[derive(Serialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum JobStatus {
+    Running,
+    Ok,
+    Failed,
+}
+
+#[derive(Serialize, Clone, Debug)]
+pub struct Job {
+    pub id: String,
+    /// "serve" | "stop" | "power"
+    pub kind: &'static str,
+    pub cluster: String,
+    pub recipe: Option<String>,
+    pub status: JobStatus,
+    pub message: String,
+    pub started: u64,
+}
+
+static COUNTER: AtomicU64 = AtomicU64::new(0);
+const MAX_JOBS: usize = 20;
+
+fn now_ts() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(0)
+}
+
+fn next_id() -> String {
+    format!("j{:x}", COUNTER.fetch_add(1, Ordering::Relaxed) + 1)
+}
+
+fn emit(
+    app: &AppState,
+    id: &str,
+    kind: &'static str,
+    event: &str,
+    cluster: &str,
+    recipe: Option<&str>,
+    message: &str,
+) {
+    let _ = app.bus.send(Event::Job {
+        id: id.to_string(),
+        kind,
+        event: event.to_string(),
+        cluster: cluster.to_string(),
+        recipe: recipe.map(String::from),
+        message: message.to_string(),
+    });
+}
+
+/// Register a running job. Returns the job id, or 409 when a job of the
+/// same kind for the same cluster/recipe is already running.
+pub fn new_job(
+    app: &AppState,
+    kind: &'static str,
+    cluster: &str,
+    recipe: Option<&str>,
+) -> Result<String, (axum::http::StatusCode, String)> {
+    let id = next_id();
+    {
+        let mut jobs = app.jobs.lock().unwrap();
+        let dup = jobs.iter().any(|j| {
+            j.status == JobStatus::Running
+                && j.kind == kind
+                && j.cluster == cluster
+                && j.recipe.as_deref() == recipe
+        });
+        if dup {
+            let what = recipe.map_or(String::new(), |r| format!("{cluster}:{r}"));
+            return Err((
+                axum::http::StatusCode::CONFLICT,
+                format!("a {kind} job for {what} is already running"),
+            ));
+        }
+        jobs.push(Job {
+            id: id.clone(),
+            kind,
+            cluster: cluster.to_string(),
+            recipe: recipe.map(String::from),
+            status: JobStatus::Running,
+            message: "started".into(),
+            started: now_ts(),
+        });
+        trim_finished(&mut jobs);
+    }
+    emit(app, &id, kind, "started", cluster, recipe, "started");
+    Ok(id)
+}
+
+/// Mark a job finished, emit the job event, and push a fresh state
+/// snapshot so the LED board flips without waiting for the next poll.
+pub fn finish_job(app: &AppState, id: &str, ok: bool, message: String) {
+    let target = {
+        let mut jobs = app.jobs.lock().unwrap();
+        if let Some(job) = jobs.iter_mut().find(|j| j.id == id) {
+            job.status = if ok { JobStatus::Ok } else { JobStatus::Failed };
+            job.message = message.clone();
+        }
+        trim_finished(&mut jobs);
+        jobs.iter()
+            .find(|j| j.id == id)
+            .map(|j| (j.kind, j.cluster.clone(), j.recipe.clone()))
+    };
+    if let Some((kind, cluster, recipe)) = target {
+        emit(
+            app,
+            id,
+            kind,
+            if ok { "ok" } else { "failed" },
+            &cluster,
+            recipe.as_deref(),
+            &message,
+        );
+    }
+    let app = app.clone();
+    tokio::spawn(async move {
+        super::push_state(&app).await;
+    });
+}
+
+fn trim_finished(jobs: &mut Vec<Job>) {
+    if jobs.len() <= MAX_JOBS {
+        return;
+    }
+    // Keep all running; drop oldest finished first.
+    let excess = jobs.len() - MAX_JOBS;
+    let mut removed = 0;
+    jobs.retain(|j| {
+        if j.status == JobStatus::Running || removed >= excess {
+            return true;
+        }
+        removed += 1;
+        false
+    });
+}
+
+/// Run `fut` as job `id`, recording the outcome.
+pub fn spawn(
+    app: AppState,
+    id: String,
+    fut: impl std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
+) {
+    tokio::spawn(async move {
+        let (ok, message) = match fut.await {
+            Ok(()) => (true, String::from("ok")),
+            Err(e) => (false, e.to_string()),
+        };
+        finish_job(&app, &id, ok, message);
+    });
+}

src/web/mod.rs

@@ -1,16 +1,17 @@
 //! `sm121 web` — axum + SSE dashboard.
 //!
-//! P1 (read-only): `/` serves the embedded UI, `/api/state` the snapshot,
-//! `/events` a multiplexed SSE stream fed by a supervisor task that polls
-//! `cluster_state::probe` every 30 s. Data endpoints are token-gated when
-//! `--token` is set (required when binding off-loopback).
+//! P2: read-only LED board (state snapshots every 30 s + on job finish)
+//! plus actions — serve/stop/power run as tracked jobs calling the same
+//! handlers the CLI uses. Data endpoints are token-gated when `--token`
+//! is set (required when binding off-loopback).
 //!
 //! Design: `specs/tech-architecture/web-dashboard.md`.
 
 mod api;
 pub mod events;
+pub mod jobs;
+pub mod probe;
 
-use crate::cluster_state::ClusterState;
 use crate::config::Config;
 use crate::web::events::{Event, build_snapshot};
 use axum::{
@@ -22,7 +23,7 @@ use axum::{
         Html, Response,
         sse::{Event as SseEvent, Sse},
     },
-    routing::get,
+    routing::{get, post},
 };
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
@@ -40,6 +41,21 @@ pub struct AppState {
     pub token: Option<String>,
     /// Latest snapshot (written by the supervisor; `/api/state` serves it).
     pub last: Arc<Mutex<Option<events::Snapshot>>>,
+    /// Tracked action jobs (serve/stop/power); capped at `jobs::MAX_JOBS`.
+    pub jobs: Arc<Mutex<Vec<jobs::Job>>>,
+}
+
+/// Probe the fleet and broadcast a fresh state snapshot (supervisor tick
+/// and job-finish both route through here).
+pub(crate) async fn push_state(app: &AppState) {
+    let st = probe::probe(&app.config).await;
+    let jobs = app.jobs.lock().unwrap().clone();
+    let snap = build_snapshot(&app.config, &st, &jobs);
+    let _ = app.bus.send(Event::State {
+        ts: snap.ts,
+        snapshot: snap.clone(),
+    });
+    *app.last.lock().unwrap() = Some(snap);
 }
 
 /// `GET /events` — forward every bus event as an SSE message.
@@ -97,13 +113,7 @@ async fn token_guard(
 /// Supervisor: poll the fleet and broadcast a state snapshot every tick.
 async fn supervisor(app: AppState) {
     loop {
-        let st = ClusterState::probe(&app.config).await;
-        let snap = build_snapshot(&app.config, &st);
-        let _ = app.bus.send(Event::State {
-            ts: snap.ts,
-            snapshot: snap.clone(),
-        });
-        *app.last.lock().unwrap() = Some(snap);
+        push_state(&app).await;
         tokio::time::sleep(Duration::from_secs(POLL_SECS)).await;
     }
 }
@@ -117,6 +127,10 @@ fn is_loopback(addr: SocketAddr) -> bool {
 pub fn router(app: AppState) -> Router {
     let protected = Router::new()
         .route("/api/state", get(api::state))
+        .route("/api/serve", post(api::serve))
+        .route("/api/stop", post(api::stop))
+        .route("/api/power", post(api::power))
+        .route("/api/recipes", get(api::recipes))
         .route("/events", get(events_stream))
         .layer(middleware::from_fn_with_state(app.clone(), token_guard));
     Router::new()
@@ -145,6 +159,7 @@ pub async fn run(
         bus: tx,
         token,
         last: Arc::new(Mutex::new(None)),
+        jobs: Arc::new(Mutex::new(Vec::new())),
     };
     tokio::spawn(supervisor(app.clone()));
     let listener = tokio::net::TcpListener::bind(addr).await?;

src/web/probe.rs

@@ -0,0 +1,81 @@
+//! Web fleet probe: like `cluster_state::probe` but also returns each node's
+//! running sm121 containers (the stop-confirm dialog names them).
+
+use crate::config::Config;
+use crate::ssh::run_ssh;
+use std::collections::HashMap;
+
+/// A running sm121-labeled container on a node.
+#[derive(Clone, Debug, serde::Serialize)]
+pub struct ContainerSnap {
+    pub name: String,
+    pub status: String,
+    pub cluster: String,
+    pub recipe: String,
+}
+
+/// Per-node state with containers.
+#[derive(Clone, Debug, Default)]
+pub struct WebNodeState {
+    pub reachable: bool,
+    pub occupied: bool,
+    pub containers: Vec<ContainerSnap>,
+}
+
+/// One `docker ps` over SSH per unique node, parallel. Unreachable nodes
+/// come back as the default state (reachable: false) — probe never fails.
+pub async fn probe(config: &Config) -> HashMap<String, WebNodeState> {
+    const CMD: &str = r#"docker ps --filter label=sm121.cluster --format '{{.Names}}\t{{.Status}}\t{{.Label "sm121.cluster"}}\t{{.Label "sm121.recipe"}}' 2>/dev/null"#;
+    let mut seen = std::collections::HashSet::new();
+    let mut handles = Vec::new();
+    for cluster in &config.clusters {
+        for node in &cluster.nodes {
+            if !seen.insert(node.ssh_ip.clone()) {
+                continue;
+            }
+            let config = config.clone();
+            let node = node.clone();
+            let name = node.name.clone();
+            let cmd = CMD.to_string();
+            handles.push(crate::locale::spawn_with_locale(async move {
+                (name, run_ssh(&config, &node, cmd, false).await)
+            }));
+        }
+    }
+    let mut out: HashMap<String, WebNodeState> = HashMap::new();
+    for h in futures::future::join_all(handles).await {
+        match h {
+            Ok((name, Ok(output))) => {
+                let mut containers = Vec::new();
+                for line in output.lines() {
+                    let mut f = line.split('\t');
+                    if let (Some(n), Some(s), Some(c), Some(r)) =
+                        (f.next(), f.next(), f.next(), f.next())
+                    {
+                        containers.push(ContainerSnap {
+                            name: n.to_string(),
+                            status: s.to_string(),
+                            cluster: c.to_string(),
+                            recipe: r.to_string(),
+                        });
+                    }
+                }
+                out.insert(
+                    name,
+                    WebNodeState {
+                        reachable: true,
+                        occupied: !containers.is_empty(),
+                        containers,
+                    },
+                );
+            }
+            Ok((name, Err(_))) => {
+                out.insert(name, WebNodeState::default());
+            }
+            Err(e) => {
+                tracing::debug!("web probe task panicked: {}", e);
+            }
+        }
+    }
+    out
+}

src/web/tests.rs

@@ -4,9 +4,18 @@
 use crate::config::{Cluster, Config, Node};
 use crate::web::events::{Event, build_snapshot};
 use crate::web::{AppState, router};
+use axum::http::StatusCode;
 use std::collections::HashMap;
 use std::net::SocketAddr;
 
+async fn call(addr: SocketAddr, path: &str) -> reqwest::Response {
+    reqwest::Client::new()
+        .get(format!("http://{addr}{path}"))
+        .send()
+        .await
+        .unwrap()
+}
+
 /// Config with one node at 127.0.0.1:2222 — connection refused instantly,
 /// machine-independent (no real SSH, no hardware).
 fn fake_config() -> Config {
@@ -47,6 +56,7 @@ fn app_state(token: Option<&str>) -> AppState {
         bus: tx,
         token: token.map(String::from),
         last: std::sync::Arc::new(std::sync::Mutex::new(None)),
+        jobs: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())),
     }
 }
 
@@ -111,6 +121,7 @@ async fn bus_delivers_state_events() {
     let snap = crate::web::events::Snapshot {
         ts: 1,
         clusters: Vec::new(),
+        jobs: Vec::new(),
     };
     tx.send(Event::State {
         ts: snap.ts,
@@ -123,6 +134,7 @@ async fn bus_delivers_state_events() {
             assert_eq!(ts, 1);
             assert!(snapshot.clusters.is_empty());
         }
+        Event::Job { .. } => panic!("unexpected job event"),
     }
 }
 
@@ -130,13 +142,75 @@ async fn bus_delivers_state_events() {
 async fn snapshot_serializes_with_type_tag() {
     let ev = Event::State {
         ts: 7,
-        snapshot: build_snapshot(
-            &fake_config(),
-            &crate::cluster_state::ClusterState::default(),
-        ),
+        snapshot: build_snapshot(&fake_config(), &std::collections::HashMap::new(), &[]),
     };
     let json = serde_json::to_value(&ev).unwrap();
     assert_eq!(json["type"], "state");
     assert_eq!(json["ts"], 7);
     assert!(json["snapshot"].is_object());
 }
+
+// --- P2 -----------------------------------------------------------------------
+
+#[tokio::test]
+async fn single_flight_rejects_duplicate_then_allows_after_finish() {
+    let app = app_state(None);
+    let id = crate::web::jobs::new_job(&app, "serve", "c1", Some("r1")).unwrap();
+    let dup = crate::web::jobs::new_job(&app, "serve", "c1", Some("r1"));
+    assert!(matches!(dup, Err((StatusCode::CONFLICT, _))));
+    // a different recipe on the same cluster is fine
+    let id2 = crate::web::jobs::new_job(&app, "serve", "c1", Some("r2")).unwrap();
+    assert_ne!(id, id2);
+    // stop is a different kind: allowed even while serve r1 runs
+    let _id3 = crate::web::jobs::new_job(&app, "stop", "c1", Some("r1")).unwrap();
+    crate::web::jobs::finish_job(&app, &id, true, "done".into());
+    // after r1's serve finished, a fresh serve r1 is allowed
+    let id4 = crate::web::jobs::new_job(&app, "serve", "c1", Some("r1")).unwrap();
+    assert_ne!(id, id4);
+}
+
+#[tokio::test]
+async fn recipes_endpoint_lists_repo_recipes() {
+    let (addr, _h) = boot().await;
+    let res = call(addr, "/api/recipes").await;
+    assert_eq!(res.status(), StatusCode::OK);
+    let v: serde_json::Value = res.json().await.unwrap();
+    let arr = v["recipes"].as_array().unwrap();
+    assert!(!arr.is_empty(), "repo recipes/ should be discoverable");
+    let names: Vec<&str> = arr.iter().map(|r| r["name"].as_str().unwrap()).collect();
+    assert!(
+        names.contains(&"qwen38-27b-nvfp4-unsloth"),
+        "got: {names:?}"
+    );
+}
+
+#[tokio::test]
+async fn serve_endpoint_registers_job_and_rejects_concurrent() {
+    let (addr, _h) = boot().await;
+    let body = serde_json::json!({ "cluster": "c1", "recipe": "r1", "dry_run": true });
+    let res = reqwest::Client::new()
+        .post(format!("http://{addr}/api/serve"))
+        .json(&body)
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+    let v: serde_json::Value = res.json().await.unwrap();
+    let id = v["job"].as_str().unwrap().to_string();
+    // the dry-run job targets the fake config's clusters? No — "c1" doesn't
+    // exist in the real config; the job will fail asynchronously (cluster
+    // resolution error). Single-flight still holds while it runs.
+    let res2 = reqwest::Client::new()
+        .post(format!("http://{addr}/api/serve"))
+        .json(&body)
+        .send()
+        .await
+        .unwrap();
+    // Either 409 (still running) or 200 (already finished and released).
+    assert!(
+        matches!(res2.status(), StatusCode::CONFLICT | StatusCode::OK),
+        "unexpected status {}",
+        res2.status()
+    );
+    let _ = id;
+}

src/web/ui.html

@@ -10,12 +10,14 @@
     --border: #30363d; --accent: #58a6ff;
     --led-on: #3fb950; --led-off: #f85149; --led-busy: #d29922; --led-free: #30363d;
     --glow-on: 0 0 8px 2px rgba(63,185,80,.55); --glow-busy: 0 0 8px 2px rgba(210,153,34,.55);
+    --btn: #21262d; --btn-hover: #30363d;
   }
   :root[data-theme="light"] {
     --bg: #f6f8fa; --panel: #ffffff; --fg: #1f2328; --muted: #656d76;
     --border: #d0d7de; --accent: #0969da;
     --led-on: #1a7f37; --led-off: #cf222e; --led-busy: #9a6700; --led-free: #d0d7de;
     --glow-on: 0 0 8px 2px rgba(26,127,55,.35); --glow-busy: 0 0 8px 2px rgba(154,103,0,.35);
+    --btn: #f6f8fa; --btn-hover: #eaeef2;
   }
   * { box-sizing: border-box; }
   body {
@@ -26,139 +28,432 @@
   header {
     display: flex; align-items: center; gap: 12px;
     padding: 10px 16px; border-bottom: 1px solid var(--border);
-    background: var(--panel);
-  }
-  header h1 { font-size: 15px; font-weight: 600; margin: 0; letter-spacing: .3px; }
-  #sse {
-    display: inline-flex; align-items: center; gap: 6px;
-    font-size: 12px; color: var(--muted);
-  }
-  #sse .dot { width: 8px; height: 8px; border-radius: 50%; background: var(--led-free); transition: background .3s; }
-  #sse.ok .dot { background: var(--led-on); box-shadow: var(--glow-on); }
-  #sse.err .dot { background: var(--led-off); }
-  #theme {
-    margin-left: auto; cursor: pointer; border: 1px solid var(--border);
-    background: var(--panel); color: var(--fg); border-radius: 6px;
-    padding: 4px 10px; font-size: 12px; transition: transform .12s;
-  }
-  #theme:active { transform: scale(.95); }
-
-  .stage { padding: 20px; }
-  .win {
+    background: var(--panel); position: sticky; top: 0; z-index: 5;
+  }
+  header h1 { font-size: 15px; margin: 0; font-weight: 650; letter-spacing: .3px; }
+  header .spacer { flex: 1; }
+  .ssec { font-size: 12px; color: var(--muted); display: flex; align-items: center; gap: 6px; }
+  .ssec .dot { width: 8px; height: 8px; border-radius: 50%; background: var(--led-off); }
+  .ssec.ok .dot { background: var(--led-on); }
+  .ssec.re .dot { animation: pulse 1s infinite; }
+  button.tbtn {
+    background: var(--btn); color: var(--fg); border: 1px solid var(--border);
+    border-radius: 6px; padding: 4px 10px; cursor: pointer; font: inherit; font-size: 13px;
+  }
+  button.tbtn:hover { background: var(--btn-hover); }
+  main { display: grid; grid-template-columns: 1fr 1fr; gap: 14px; padding: 14px; max-width: 1200px; margin: 0 auto; }
+  @media (max-width: 860px) { main { grid-template-columns: 1fr; } }
+  .card {
     background: var(--panel); border: 1px solid var(--border); border-radius: 10px;
-    box-shadow: 0 8px 28px rgba(0,0,0,.25); overflow: hidden;
-    animation: win-in .28s cubic-bezier(.2,.9,.3,1.2);
-  }
-  @keyframes win-in { from { opacity: 0; transform: scale(.94) translateY(6px); } to { opacity: 1; transform: none; } }
-  .titlebar {
-    padding: 7px 12px; font-size: 12px; color: var(--muted);
-    border-bottom: 1px solid var(--border); display: flex; gap: 8px; align-items: center;
-    user-select: none; cursor: grab;
-  }
-  .titlebar b { color: var(--fg); font-weight: 600; }
-  .grid { padding: 10px 12px; }
-  .cluster { margin-bottom: 14px; }
-  .cluster-head {
-    display: flex; gap: 10px; align-items: baseline; font-size: 13px;
-    font-weight: 600; margin: 0 0 6px 2px;
-  }
-  .cluster-head .free { font-weight: 400; font-size: 12px; color: var(--muted); }
-  .node {
-    display: grid; grid-template-columns: 14px 14px 190px 1fr; gap: 10px; align-items: center;
-    padding: 6px 8px; border-radius: 8px; margin-bottom: 2px;
-    transition: background .15s;
+    padding: 14px; min-height: 120px;
   }
-  .node:hover { background: color-mix(in srgb, var(--fg) 6%, transparent); }
-  .led { width: 10px; height: 10px; border-radius: 50%; background: var(--led-free); transition: background .35s, box-shadow .35s; }
-  .led.on  { background: var(--led-on);  box-shadow: var(--glow-on); }
+  .card h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .8px; color: var(--muted); margin: 0 0 10px; }
+  .cluster { margin-bottom: 14px; animation: winIn .35s ease both; }
+  .cluster-head { display: flex; align-items: baseline; gap: 8px; margin-bottom: 6px; }
+  .cluster-head b { font-size: 13px; }
+  .cluster-head .free { color: var(--muted); font-size: 12px; }
+  .node { display: flex; align-items: center; gap: 10px; padding: 5px 0; border-bottom: 1px solid var(--border); }
+  .node:last-child { border-bottom: none; }
+  .led {
+    width: 11px; height: 11px; border-radius: 50%; background: var(--led-off);
+    transition: background .4s, box-shadow .4s; flex-shrink: 0;
+  }
+  .led.on { background: var(--led-on); box-shadow: var(--glow-on); }
+  .led.b { background: var(--led-busy); box-shadow: var(--glow-busy); }
   .led.off { background: var(--led-off); }
-  .led.busy{ background: var(--led-busy); box-shadow: var(--glow-busy); }
-  .led.pulse { animation: pulse 1.1s ease-in-out infinite; }
-  @keyframes pulse { 50% { opacity: .35; } }
-  .nname { font-weight: 500; font-size: 13px; }
-  .nmeta { font-size: 12px; color: var(--muted); font-family: ui-monospace, monospace; }
-  .nstate { font-size: 12px; }
-  .nstate .up { color: var(--led-on); } .nstate .down { color: var(--led-off); }
-  .nstate .occupied { color: var(--led-busy); }
-  .empty { color: var(--muted); padding: 18px; text-align: center; }
+  .led.flip { animation: flip .45s ease; }
+  .nname { width: 64px; font-family: ui-monospace, monospace; font-size: 12.5px; }
+  .nmeta { color: var(--muted); font-size: 12px; flex: 1; overflow: hidden; text-overflow: ellipsis; }
+  .containers { width: 100%; margin-top: 2px; }
+  .crow { display: flex; align-items: center; gap: 8px; padding: 2px 0 2px 21px; font-size: 12px; color: var(--muted); }
+  .cled { width: 7px; height: 7px; border-radius: 50%; background: var(--led-busy); box-shadow: var(--glow-busy); flex-shrink: 0; }
+  .cname { font-family: ui-monospace, monospace; color: var(--fg); }
+  .actions { display: flex; gap: 8px; align-items: center; margin: 6px 0 0 21px; flex-wrap: wrap; }
+  .actions select, .actions input[type=checkbox] { font: inherit; }
+  .actions select {
+    background: var(--btn); color: var(--fg); border: 1px solid var(--border);
+    border-radius: 6px; padding: 3px 6px; font-size: 12.5px; max-width: 260px;
+  }
+  .actions label { font-size: 12px; color: var(--muted); display: flex; align-items: center; gap: 4px; }
+  .abtn {
+    background: var(--btn); color: var(--fg); border: 1px solid var(--border);
+    border-radius: 6px; padding: 3px 12px; cursor: pointer; font: inherit; font-size: 12.5px;
+    transition: background .15s;
+  }
+  .abtn:hover:not(:disabled) { background: var(--btn-hover); }
+  .abtn:disabled { opacity: .45; cursor: default; }
+  .abtn.danger { color: var(--led-off); }
+  .jobs { font-family: ui-monospace, monospace; font-size: 12px; }
+  .job { display: flex; gap: 8px; align-items: baseline; padding: 4px 0; border-bottom: 1px solid var(--border); animation: winIn .3s ease both; }
+  .job:last-child { border-bottom: none; }
+  .jdot { width: 8px; height: 8px; border-radius: 50%; background: var(--accent); flex-shrink: 0; align-self: center; }
+  .job.ok .jdot { background: var(--led-on); }
+  .job.failed .jdot { background: var(--led-off); }
+  .jkind { width: 48px; color: var(--muted); }
+  .jmsg { color: var(--muted); flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+  .jjob .jmsg { color: var(--fg); }
+  .jobs-empty { color: var(--muted); font-size: 12.5px; padding: 6px 0; }
+  /* confirm dialog */
+  .dlg-back {
+    position: fixed; inset: 0; background: rgba(0,0,0,.5); z-index: 10;
+    display: flex; align-items: center; justify-content: center; animation: fadeIn .15s ease;
+  }
+  .dlg {
+    background: var(--panel); border: 1px solid var(--border); border-radius: 10px;
+    padding: 18px; width: min(460px, 92vw); max-height: 80vh; overflow: auto;
+    box-shadow: 0 12px 40px rgba(0,0,0,.5); animation: winIn .2s ease both;
+  }
+  .dlg h3 { margin: 0 0 10px; font-size: 14px; }
+  .dlg .body { font-size: 13px; color: var(--fg); margin-bottom: 12px; }
+  .dlg .body code { font-family: ui-monospace, monospace; font-size: 12px; background: var(--btn); padding: 1px 5px; border-radius: 4px; }
+  .dlg .cbox { font-family: ui-monospace, monospace; font-size: 12px; color: var(--muted); margin: 8px 0 0; padding: 8px; background: var(--btn); border-radius: 6px; }
+  .dlg .cbox li { margin: 2px 0; }
+  .dlg input {
+    width: 100%; background: var(--btn); color: var(--fg); border: 1px solid var(--border);
+    border-radius: 6px; padding: 6px 8px; font: inherit; margin: 8px 0;
+  }
+  .dlg .row { display: flex; justify-content: flex-end; gap: 8px; }
+  @keyframes flip { 0% { transform: scale(1); } 40% { transform: scale(1.5); } 100% { transform: scale(1); } }
+  @keyframes pulse { 0%,100% { opacity: 1; } 50% { opacity: .35; } }
+  @keyframes fadeIn { from { opacity: 0; } to { opacity: 1; } }
+  @keyframes winIn { from { opacity: 0; transform: translateY(-4px) scale(.985); } to { opacity: 1; transform: none; } }
+  @media (prefers-reduced-motion: reduce) {
+    *, .led, .led.flip, .ssec.re .dot, .cluster, .job, .dlg { animation: none !important; }
+  }
 </style>
 </head>
 <body>
 <header>
   <h1>sm121</h1>
-  <span id="sse"><span class="dot"></span><span id="sse-text">connecting…</span></span>
-  <button id="theme">🌙 dark</button>
+  <div class="spacer"></div>
+  <div class="ssec" id="sse"><span class="dot"></span><span id="sse-t">connecting…</span></div>
+  <button class="tbtn" id="theme">◐</button>
 </header>
-<div class="stage">
-  <div class="win" style="max-width: 760px">
-    <div class="titlebar"><b>state</b><span id="updated"></span></div>
-    <div class="grid" id="grid"><div class="empty">waiting for state…</div></div>
+<main>
+  <section class="card" id="state-card"><h2>Cluster state</h2><div id="clusters"></div></section>
+  <section class="card"><h2>Jobs</h2><div class="jobs" id="jobs"><div class="jobs-empty">no jobs yet</div></div></section>
+</main>
+<div class="dlg-back" id="dlg" hidden>
+  <div class="dlg">
+    <h3 id="dlg-title"></h3>
+    <div class="body" id="dlg-body"></div>
+    <div class="row">
+      <button class="tbtn" id="dlg-cancel">Cancel</button>
+      <button class="abtn" id="dlg-ok">Confirm</button>
+    </div>
   </div>
 </div>
 <script>
 "use strict";
-// ---- theme -------------------------------------------------------------
+const $ = (s) => document.querySelector(s);
+const esc = (s) => String(s ?? "").replace(/[&<>"']/g, (c) => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
+const state = { snap: null, jobs: new Map(), recipes: [] };
+
+// --- theme ---------------------------------------------------------------
 const root = document.documentElement;
-const saved = localStorage.getItem("theme");
-const prefersLight = matchMedia("(prefers-color-scheme: light)").matches;
-setTheme(saved || (prefersLight ? "light" : "dark"));
-document.getElementById("theme").onclick = () => setTheme(root.dataset.theme === "dark" ? "light" : "dark");
-function setTheme(t) {
-  root.dataset.theme = t;
-  localStorage.setItem("theme", t);
-  document.getElementById("theme").textContent = t === "dark" ? "🌙 dark" : "☀️ light";
-}
-// ---- state grid ----------------------------------------------------------
-const grid = document.getElementById("grid");
-const updated = document.getElementById("updated");
-function esc(s) { const d = document.createElement("i"); d.textContent = s; return d.innerHTML; }
-function renderState(snap) {
-  if (!snap.clusters || !snap.clusters.length) {
-    grid.innerHTML = '<div class="empty">no clusters in config</div>';
+root.dataset.theme = localStorage.getItem("sm121-theme") || (matchMedia("(prefers-color-scheme: light)").matches ? "light" : "dark");
+$("#theme").onclick = () => {
+  root.dataset.theme = root.dataset.theme === "dark" ? "light" : "dark";
+  localStorage.setItem("sm121-theme", root.dataset.theme);
+};
+
+// --- SSE -------------------------------------------------------------------
+function sse(cls, txt) { $("#sse").className = "ssec " + cls; $("#sse-t").textContent = txt; }
+function connectSSE() {
+  const es = new EventSource("/events");
+  es.onmessage = (e) => {
+    sse("ok", "live");
+    let ev;
+    try { ev = JSON.parse(e.data); } catch { return; }
+    if (ev.type === "state") onState(ev.snapshot);
+    else if (ev.type === "job") onJob(ev);
+  };
+  es.onerror = () => { sse("re", "reconnecting…"); };
+}
+
+// --- state render ------------------------------------------------------------
+function ledState(n) { return n.reachable ? (n.occupied ? "b" : "on") : "off"; }
+function meta(n, c) {
+  if (!n.reachable) return "offline";
+  if (n.occupied) return n.containers.map((x) => x.status).join(", ");
+  return c.free ? "free" : "no free nodes in cluster";
+}
+function renderCluster(c, prev) {
+  const el = document.createElement("div");
+  el.className = "cluster";
+  const head = document.createElement("div");
+  head.className = "cluster-head";
+  head.innerHTML = `<b>${esc(c.name)}</b><span class="free">${c.free} free</span>`;
+  el.appendChild(head);
+  const prevNodes = prev ? Object.fromEntries(prev.nodes.map((n) => [n.name, n])) : {};
+  for (const n of c.nodes) {
+    const row = document.createElement("div");
+    row.className = "node";
+    const cls = ledState(n);
+    const flip = prevNodes[n.name] && ledState(prevNodes[n.name]) !== cls ? " flip" : "";
+    row.innerHTML = `<span class="led ${cls}${flip}"></span>
+      <span class="nname">${esc(n.name)}</span>
+      <span class="nmeta" title="${esc(n.ssh_ip)}">${esc(meta(n, c))}</span>`;
+    el.appendChild(row);
+    if (n.occupied && n.containers.length) {
+      const box = document.createElement("div");
+      box.className = "containers";
+      for (const x of n.containers) {
+        const row = document.createElement("div");
+        row.className = "crow";
+        row.innerHTML = `<span class="cled"></span><span class="cname">${esc(x.name)}</span><span>${esc(x.recipe)} · ${esc(x.status)}</span>`;
+        box.appendChild(row);
+      }
+      el.appendChild(box);
+    }
+  }
+  el.appendChild(actionBar(c, prevNodes));
+  return el;
+}
+function actionBar(c, prevNodes) {
+  const bar = document.createElement("div");
+  bar.className = "actions";
+  const sel = document.createElement("select");
+  sel.id = "recipe-" + c.name;
+  if (!state.recipes.length) {
+    sel.innerHTML = "<option value=''></option>";
+  } else {
+    for (const r of state.recipes) {
+      const o = document.createElement("option");
+      o.value = r.name;
+      o.textContent = r.name + (r.deploy === "tmux" ? " (tmux)" : "");
+      sel.appendChild(o);
+    }
+  }
+  bar.appendChild(sel);
+  const dry = document.createElement("label");
+  dry.innerHTML = `<input type="checkbox" id="dry-${esc(c.name)}"> dry-run`;
+  bar.appendChild(dry);
+  bar.appendChild(abtn("serve-" + c.name, "serve", () => doServe(c.name)));
+  bar.appendChild(abtn("stop-" + c.name, "stop", () => doStop(c.name), "danger"));
+  bar.appendChild(abtn("power-" + c.name, "power off", () => doPower("off", c.name), "danger"));
+  bar.appendChild(abtn("pwswitch-" + c.name, "switch", () => doPower("switch", c.name), ""));
+  return bar;
+}
+function abtn(id, label, fn, cls) {
+  const b = document.createElement("button");
+  b.className = "abtn " + (cls || "");
+  b.id = id;
+  b.textContent = label;
+  b.onclick = fn;
+  return b;
+}
+function onState(snap) {
+  const prev = state.snap;
+  state.snap = snap;
+  state.jobs = new Map(snap.jobs.map((j) => [j.id, j]));
+  const box = $("#clusters");
+  box.innerHTML = "";
+  for (const c of snap.clusters) box.appendChild(renderCluster(c, prev ? prev.clusters.find((p) => p.name === c.name) : null));
+  renderJobs();
+}
+
+// --- jobs ---------------------------------------------------------------------
+function renderJobs() {
+  const box = $("#jobs");
+  box.innerHTML = "";
+  const js = [...state.jobs.values()].sort((a, b) => b.started - a.started);
+  if (!js.length) { box.innerHTML = '<div class="jobs-empty">no jobs yet</div>'; return; }
+  for (const j of js) {
+    const el = document.createElement("div");
+    el.className = "job " + (j.status === "ok" ? "ok" : j.status === "failed" ? "failed" : "jjob");
+    const when = new Date(j.started * 1000).toLocaleTimeString();
+    el.innerHTML = `<span class="jdot"></span><span class="jkind">${esc(j.kind)}</span>
+      <span class="jmsg">${esc(j.cluster)}${j.recipe ? ":" + esc(j.recipe) : ""} — ${esc(j.message)} <span>(${when})</span></span>`;
+    box.appendChild(el);
+    setBtnBusy(j, j.status === "running");
+  }
+}
+function setBtnBusy(j, busy) {
+  // power jobs disable both the off and switch buttons (the job event
+  // doesn't carry the action)
+  const keys = j.kind === "power" ? ["power", "pwswitch"] : [j.kind];
+  for (const k of keys) {
+    const el = document.getElementById(k + "-" + j.cluster);
+    if (el) el.disabled = busy;
+  }
+}
+
+// --- job events ------------------------------------------------------------------
+function onJob(ev) {
+  if (ev.event === "started") {
+    state.jobs.set(ev.id, { id: ev.id, kind: ev.kind, cluster: ev.cluster, recipe: ev.recipe, status: "running", message: ev.message, started: Date.now() / 1000 });
+  }
+  const j = state.jobs.get(ev.id);
+  if (j && ev.event !== "started") {
+    j.message = ev.message;
+    j.status = ev.event === "ok" ? "ok" : "failed";
+  }
+  renderJobs();
+}
+
+// --- actions -----------------------------------------------------------------------
+const cur = () => state.snap;
+function recipeOf(cluster) {
+  const sel = document.getElementById("recipe-" + cluster);
+  return sel ? sel.value : "";
+}
+function dryOf(cluster) {
+  const d = document.getElementById("dry-" + cluster);
+  return d ? d.checked : false;
+}
+async function api(path, body) {
+  const res = await fetch(path, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
+  if (!res.ok) {
+    let m = res.status;
+    try { m = (await res.json()).error || m; } catch {}
+    throw new Error(m);
+  }
+  return res.json();
+}
+function doServe(cluster) {
+  const recipe = recipeOf(cluster);
+  if (!recipe) { alert("pick a recipe first"); return; }
+  const dry = dryOf(cluster);
+  const body = { cluster, recipe, dry_run: dry };
+  if (dry) {
+    api("/api/serve", body).then((r) => openDialog("dry-run: " + r.job, "started (watch the jobs window)", () => {}));
     return;
   }
-  grid.innerHTML = snap.clusters.map(c => `
-    <div class="cluster">
-      <div class="cluster-head">${esc(c.name)}<span class="free">${c.free}/${c.nodes.length} free</span></div>
-      ${c.nodes.map(n => `
-        <div class="node">
-          <span class="led ${n.reachable ? "on" : "off"} ${!n.reachable ? "pulse" : ""}" title="reachability"></span>
-          <span class="led ${n.occupied ? "busy" : "free"}" title="occupancy"></span>
-          <span class="nname">${esc(n.name)}</span>
-          <span class="nmeta">${esc(n.ssh_ip)}</span>
-        </div>
-        <div class="node" style="padding-top:0;margin-left:38px">
-          <span></span><span></span>
-          <span class="nstate">
-            ${n.reachable
-              ? (n.occupied ? '<span class="occupied">● occupied</span>' : '<span class="up">○ free</span>')
-              : '<span class="down">● unreachable</span>'}
-          </span><span></span>
-        </div>`).join("")}
-    </div>`).join("");
-  updated.textContent = "updated " + new Date(snap.ts * 1000).toLocaleTimeString();
-}
-// ---- SSE -----------------------------------------------------------------
-const TOKEN = new URLSearchParams(location.search).get("token") || "";
-const sseEl = document.getElementById("sse"), sseText = document.getElementById("sse-text");
-function connect() {
-  const url = "/events" + (TOKEN ? `?token=${encodeURIComponent(TOKEN)}` : "");
-  const es = new EventSource(url);
-  es.onopen = () => { sseEl.classList.add("ok"); sseEl.classList.remove("err"); sseText.textContent = "live"; };
-  es.onerror = () => { sseEl.classList.add("err"); sseEl.classList.remove("ok"); sseText.textContent = "reconnecting…"; };
-  es.onmessage = (e) => {
+  openDialog(`serve ${recipe} on ${cluster}`, `Launches <code>${esc(recipe)}</code> on cluster <code>${esc(cluster)}</code>.`, () => {
+    api("/api/serve", body).catch((e) => openDialog("serve failed", esc(e.message), () => {}, true));
+  });
+}
+function doStop(cluster) {
+  const snap = cur();
+  const c = snap ? snap.clusters.find((x) => x.name === cluster) : null;
+  const containers = c ? c.nodes.flatMap((n) => n.containers) : [];
+  const recipe = recipeOf(cluster);
+  if (!recipe) { alert("pick the recipe to stop first"); return; }
+  const list = containers.length
+    ? `<ul class="cbox">${containers.map((x) => `<li><b>${esc(x.name)}</b> — ${esc(x.status)}</li>`).join("")}</ul>`
+    : `<div class="cbox">no containers detected on ${esc(cluster)} (recipe may not be running here)</div>`;
+  openDialog(`stop ${recipe}`, `Will run <code>sm121 stop</code> for <code>${esc(recipe)}</code> on <code>${esc(cluster)}</code>. Currently running:${list}`, () => {
+    api("/api/stop", { cluster, recipe }).catch((e) => openDialog("stop failed", esc(e.message), () => {}, true));
+  });
+}
+function doPower(action, cluster) {
+  if (action === "off") {
+    const snap = cur();
+    const c = snap ? snap.clusters.find((x) => x.name === cluster) : null;
+    const busy = c ? c.nodes.filter((n) => n.occupied) : [];
+    if (busy.length) {
+      openDialog(`power off ${cluster} blocked`,
+        `Cluster has ${busy.length} occupied node(s): <code>${esc(busy.map((n) => n.name).join(", "))}</code>. Stop its workloads first.`,
+        () => {}, true);
+      return;
+    }
+    twoStep(`power off ${cluster}`,
+      `Shuts down <b>all nodes</b> of <code>${esc(cluster)}</code> (SSH then Home Assistant). This can kill live traffic.`,
+      cluster, () => {
+        api("/api/power", { action: "off", cluster }).catch((e) => openDialog("power failed", esc(e.message), () => {}, true));
+      });
+    return;
+  }
+  if (action === "switch") {
+    const snap = cur();
+    const names = snap ? snap.clusters.map((c) => c.name) : [];
+    const body = document.createElement("div");
+    body.className = "body";
+    let inner = `Power <code>${esc(cluster)}</code> off and bring up another cluster instead.`;
+    const sel = document.createElement("select");
+    sel.style.cssText = "width:100%;margin:8px 0;padding:6px;background:var(--btn);color:var(--fg);border:1px solid var(--border);border-radius:6px;";
+    for (const n of names.filter((n) => n !== cluster)) {
+      const o = document.createElement("option");
+      o.value = n; o.textContent = n;
+      sel.appendChild(o);
+    }
+    body.innerHTML = inner;
+    body.appendChild(sel);
+    openDialogHtml(`power switch from ${cluster}`, body, () => {
+      const to = sel.value;
+      api("/api/power", { action: "switch", cluster, to }).catch((e) => openDialog("power failed", esc(e.message), () => {}, true));
+    });
+    return;
+  }
+  twoStep(`power on ${cluster}`, `Turns on <code>${esc(cluster)}</code> via Home Assistant (SSH is not used).`, cluster, () => {
+    api("/api/power", { action: "on", cluster }).catch((e) => openDialog("power failed", esc(e.message), () => {}, true));
+  });
+}
+function twoStep(title, bodyHtml, nameToType, go) {
+  openDialogHtml(title, (() => { const d = document.createElement("div"); d.className = "body"; d.innerHTML = bodyHtml; return d; })(),
+    () => {}, { twoStep: nameToType, go });
+}
+
+// --- dialog -----------------------------------------------------------------------
+let dlg = { ok: null, twoStep: null };
+function openDialog(title, bodyHtml, go, danger) {
+  $("#dlg-title").textContent = title;
+  $("#dlg-body").innerHTML = bodyHtml;
+  const ok = $("#dlg-ok");
+  ok.style.display = "";
+  ok.textContent = "Confirm";
+  ok.className = "abtn" + (danger ? " danger" : "");
+  dlg = { ok: go, twoStep: null };
+  $("#dlg").hidden = false;
+}
+function openDialogHtml(title, bodyEl, go, opts) {
+  $("#dlg-title").textContent = title;
+  $("#dlg-body").innerHTML = "";
+  $("#dlg-body").appendChild(bodyEl);
+  const ok = $("#dlg-ok");
+  ok.style.display = "";
+  if (opts && opts.twoStep) {
+    ok.style.display = "none";
+    const wrap = document.createElement("div");
+    wrap.innerHTML = `<div style="font-size:12.5px;color:var(--muted)">Type <b>${esc(opts.twoStep)}</b> to confirm.</div>
+      <input id="dlg-type" placeholder="${esc(opts.twoStep)}" autocomplete="off">`;    $("#dlg-body").appendChild(wrap);
+    const inp = wrap.querySelector("#dlg-type");
+    inp.oninput = () => {
+      if (inp.value === opts.twoStep) {
+        ok.style.display = "";
+        ok.textContent = "⚡ Confirm power action";
+        ok.className = "abtn danger";
+      } else { ok.style.display = "none"; }
+    };
+    dlg = { ok: null, twoStep: opts.twoStep, go };
+  } else {
+    dlg = { ok: go, twoStep: null };
+  }
+  $("#dlg").hidden = false;
+}
+$("#dlg-ok").onclick = () => {
+  const go = dlg.twoStep ? dlg.go : dlg.ok;
+  $("#dlg").hidden = true;
+  if (go) go();
+};
+$("#dlg-cancel").onclick = () => { $("#dlg").hidden = true; };
+document.addEventListener("keydown", (e) => { if (e.key === "Escape") $("#dlg").hidden = true; });
+
+// --- boot ---------------------------------------------------------------------------
+async function boot() {
+  try {
+    const res = await fetch("/api/state");
+    if (res.ok) onState(await res.json());
+  } catch {}
+  try {
+    const res = await fetch("/api/recipes");
+    if (res.ok) {
+      const d = await res.json();
+      state.recipes = d.recipes || [];
+      if (state.snap) onState(state.snap); // re-render with recipes
+    }
+  } catch {}
+  connectSSE();
+  // 30 s heartbeat refresh (mirrors the server poll; harmless if state is identical)
+  setInterval(async () => {
     try {
-      const ev = JSON.parse(e.data);
-      if (ev.type === "state") renderState(ev.snapshot);
-    } catch (_) {}
-  };
+      const res = await fetch("/api/state");
+      if (res.ok) onState(await res.json());
+    } catch {}
+  }, 30000);
 }
-// Initial snapshot, then live updates.
-fetch("/api/state" + (TOKEN ? `?token=${encodeURIComponent(TOKEN)}` : ""))
-  .then(r => r.ok ? r.json() : null)
-  .then(snap => snap && renderState(snap))
-  .catch(() => {});
-connect();
+boot();
 </script>
 </body>
 </html>