report: stale cross-check of run ledger vs live probe (orange + runlock hint); new 'sm121 runlock list|close' offline ledger command

AGENTS.md

@@ -17,7 +17,7 @@ See [`DEVELOPMENT.md`](DEVELOPMENT.md) for architecture, module layout, and dev
 ## Commands
 
 ```
-serve/stop/bench/logs/health/doctor/exec/power/tmux/init-host-keys/config/web/report
+serve/stop/bench/logs/health/doctor/exec/power/tmux/init-host-keys/config/web/report/runlock
 ```
 
 `serve` records the launch in the recipe's run ledger (`recipes/<name>/.lock`,
@@ -42,6 +42,17 @@ curation: `--archive <name>` / `--unarchive <name>` flip the recipe's
 actually stopped and removed (polls up 60 s per node) — an immediate `serve`
 on the same nodes would otherwise race the dying containers' GPU release.
 
+`runlock` — offline run-ledger management (the per-recipe `.lock`):
+`list` shows every recipe with an open (running) entry, `close <recipe>`
+closes the open run without any node verification. `stop` closes the ledger
+only after verifying every node of the cluster, so a node that died
+mid-run (or a killed serve) leaves the run open — `sm121 report --serve`
+cross-checks the ledger against a live fleet probe and renders such rows as
+orange "stale" (tooltip: `sm121 runlock close <recipe>`) instead of green
+"up" when all nodes of the run's cluster are reachable but no running
+container of the recipe exists; an unreachable node is inconclusive and
+keeps the green (the fix for a confirmed stale row is `runlock close`).
+
 `bench` runs `tool-eval-bench` (from PATH) against a live serve and lands
 `<UTC-date>_<label>.{md,json,log}` in `recipes/<lane>/reports/` — it does the
 scratch-cwd/absolute-json/log juggling, polls the API for readiness first,

history/night-shifts/2026-09-13.md

@@ -516,3 +516,19 @@ closed (sm121 stop can't verify with dgx3/4 offline). Closed the record per
 mark_stop semantics + docker rm'd the exited r37 containers on dgx1/2.
 Systemic options on the table: (1) report cross-checks ledger against the
 live probe, (2) `sm121 runlock close <recipe>` offline command.
+
+## Report stale-status + sm121 runlock (2026-09-14, ~11:40 UTC)
+
+Operator asked for both fixes after the r37 false-"up":
+1. `sm121 report --serve` now cross-checks the run ledger against a live
+   fleet probe every refresh tick (30 s, only when an open run exists): an
+   open run whose recipe has no running container on any reachable node is
+   rendered orange "stale" with a `sm121 runlock close <recipe>` tooltip
+   instead of green "up". Inconclusive (unknown cluster, unreachable node)
+   stays green. tmux deploys are never probed. `src/report/stale.rs`
+   (mark_stale + 4 unit tests), serve.rs refresh hook, page.html state cell.
+2. `sm121 runlock list|close <recipe>` — offline ledger management (the
+   `mark_stop` file op without node verification). RunlockCommands lives in
+   runlock.rs (cli.rs at the 500-line cap).
+All gates green (310 tests). Live-verified: eugr 2x shows up (probe sees the
+containers), r37 shows stopped.

src/cli.rs

@@ -1,3 +1,4 @@
+use crate::runlock::RunlockCommands;
 use clap::{Args, Parser, Subcommand, ValueEnum};
 
 /// Output format for human-readable reports (`--format`).
@@ -223,6 +224,10 @@ pub enum Commands {
     #[command(subcommand)]
     Config(ConfigCommands),
 
+    /// Run-ledger management (per-recipe .lock; offline)
+    #[command(subcommand)]
+    Runlock(RunlockCommands),
+
     /// Launch the web dashboard (axum + SSE) on a local port
     Web {
         #[arg(long, default_value_t = 3100)]
@@ -281,7 +286,6 @@ pub enum Commands {
         action: PowerAction,
     },
 }
-
 #[derive(Subcommand)]
 pub enum PowerAction {
     /// Power off: SSH `sudo shutdown now` + wait + HA `turn_off`.
@@ -312,7 +316,6 @@ pub enum PowerAction {
         to: String,
     },
 }
-
 /// `sm121 config` subcommands — manage the global config file.
 #[derive(Subcommand)]
 pub enum ConfigCommands {
@@ -347,7 +350,6 @@ pub struct PublishArgs {
     #[arg(long)]
     pub token: Option<String>,
 }
-
 #[derive(Subcommand)]
 pub enum RegistryCommands {
     /// List recipes on the registry

src/main.rs

@@ -53,9 +53,7 @@ async fn main() -> Result<()> {
         .ok()
         .map(|l| l.split('.').next().unwrap_or(&l).replace('_', "-"));
     let _ = fluent_i18n::set_locale(locale.as_deref());
-    // Capture for spawned tasks — set_locale is thread-local, so tokio
-    // workers miss it. spawn_with_locale re-applies from this global.
-    locale::set_global_locale(locale.clone());
+    locale::set_global_locale(locale.clone()); // tokio workers miss the thread-local; spawn_with_locale re-applies
     tracing_subscriber::fmt()
         .with_env_filter(
             tracing_subscriber::EnvFilter::try_from_default_env()
@@ -256,6 +254,7 @@ async fn main() -> Result<()> {
             )
             .await?;
         }
+        Commands::Runlock(cmd) => crate::runlock::run(&config, &cmd)?,
     }
 
     Ok(())

src/report/mod.rs

@@ -64,6 +64,11 @@ pub(crate) struct ReportEntry {
     provenance: Provenance,
 }
 
+/// serde helper: omit `false` flags from the rows JSON.
+fn is_false(b: &bool) -> bool {
+    !b
+}
+
 #[derive(Serialize, Deserialize)]
 pub(crate) struct ReportRow {
     name: String,
@@ -87,6 +92,15 @@ pub(crate) struct ReportRow {
     /// recipe's .lock ledger; None = not running.
     #[serde(default)]
     up: Option<i64>,
+    /// The cluster of that open run (None = tmux deploy). Server-side only:
+    /// feeds the live stale cross-check, never displayed.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    up_cluster: Option<String>,
+    /// The ledger says running but the live fleet probe finds no running
+    /// container for this recipe (see `stale.rs`). Rendered orange, with a
+    /// `sm121 runlock close` hint, instead of the green "up".
+    #[serde(default, skip_serializing_if = "is_false")]
+    stale: bool,
     /// Lifetime launch count from the .lock ledger.
     #[serde(default)]
     runs: u64,
@@ -366,6 +380,8 @@ pub(crate) fn collect(config: &Config, show_archived: bool) -> Vec<ReportRow> {
                 port: r.port,
                 incomplete,
                 up: lock.running.as_ref().map(|r| r.start),
+                up_cluster: lock.running.as_ref().and_then(|r| r.cluster.clone()),
+                stale: false,
                 runs: lock.total,
                 medals: quality::Medals::default(),
                 reports,
@@ -432,6 +448,8 @@ pub async fn run(
     Ok(())
 }
 
+mod stale;
+
 #[cfg(test)]
 mod tests;
 

src/report/page.html

@@ -447,7 +447,9 @@ function render() {
           : `<span title="${si.pct}/100${pe && pe.points ? " (" + pe.points + ")" : ""}">${starGlyphs(si)}</span>`;
       } else if (c[0] === "state") {
         body = r.up != null
-          ? `<span title="up since ${fmtSince(r.up)} UTC"><span style="color:green">●</span><span class="dim"> up</span></span>`
+          ? (r.stale
+              ? `<span title="ledger says running but no live container found (node down? killed serve?) — verify and close: sm121 runlock close ${esc(r.name)}"><span style="color:#d2822a">●</span><span class="dim"> stale</span></span>`
+              : `<span title="up since ${fmtSince(r.up)} UTC"><span style="color:green">●</span><span class="dim"> up</span></span>`)
           : '<span class="dim" title="not running">○</span>';
       } else if (c[0] === "uptime") {
         body = r.up != null ? fmtUp(v) : '<span class="dim">—</span>';

src/report/serve.rs

@@ -26,6 +26,7 @@ use axum::{
 };
 use futures::{Stream, StreamExt};
 use notify::Watcher;
+use std::collections::HashMap;
 use std::convert::Infallible;
 use std::sync::Arc;
 use std::time::Duration;
@@ -97,7 +98,34 @@ pub(crate) async fn refresh_loop(
         // debounce: let the burst settle, then drain the remaining events
         tokio::time::sleep(DEBOUNCE).await;
         while ev_rx.try_recv().is_ok() {}
-        let json = serde_json::to_string(&collect(&cfg, archived)).unwrap_or_default();
+        let mut rows = collect(&cfg, archived);
+        // Live cross-check: an open ledger run with no running container
+        // anywhere (all its nodes reachable) is marked stale — otherwise a
+        // killed serve or a node that died before `stop` could verify would
+        // show as "up" forever. No open run with a cluster → no probe.
+        if rows
+            .iter()
+            .any(|r| r.up.is_some() && r.up_cluster.is_some())
+        {
+            let fleet = crate::web::probe::probe(&cfg).await;
+            let mut cluster_nodes: HashMap<String, Vec<String>> = HashMap::new();
+            for c in &cfg.clusters {
+                cluster_nodes.insert(
+                    c.name.clone(),
+                    c.nodes.iter().map(|n| n.name.clone()).collect(),
+                );
+            }
+            let mut reachable: HashMap<String, bool> = HashMap::new();
+            let mut live: std::collections::HashSet<String> = std::collections::HashSet::new();
+            for (node, st) in &fleet.nodes {
+                reachable.insert(node.clone(), st.reachable);
+                for c in &st.containers {
+                    live.insert(c.recipe.clone());
+                }
+            }
+            crate::report::stale::mark_stale(&mut rows, &cluster_nodes, &reachable, &live);
+        }
+        let json = serde_json::to_string(&rows).unwrap_or_default();
         if json != last {
             last = json;
             let _ = tx.send(last.clone());

src/report/stale.rs

@@ -0,0 +1,119 @@
+//! Live cross-check of the run ledger: the `.lock` file is written by
+//! `sm121 serve`/`stop` and survives process death — a killed serve (or a
+//! node that died before `stop` could verify) leaves the run open forever,
+//! and the report shows the recipe as "up" against reality. `mark_stale`
+//! reconciles the ledger against the live fleet probe: an open run whose
+//! recipe has no running container anywhere is flagged `stale` (orange in
+//! the UI) instead of green.
+//!
+//! Inconclusive probes are left alone: an unknown cluster (removed from the
+//! config) or any unreachable node of the run's cluster means we cannot
+//! tell — a wrong "stale" is the same class of bug this fixes. The fix
+//! for a confirmed stale row is `sm121 runlock close <recipe>`.
+
+use super::ReportRow;
+use std::collections::{HashMap, HashSet};
+
+/// Flag every open ledger run the live probe contradicts.
+///
+/// `cluster_nodes`: cluster name → its node names (from the config).
+/// `reachable`: node name → probe reachability. `live_recipes`: recipe
+/// names with a running sm121 container on any reachable node (fleet-wide —
+/// a run is live as long as its containers exist anywhere).
+pub(crate) fn mark_stale(
+    rows: &mut [ReportRow],
+    cluster_nodes: &HashMap<String, Vec<String>>,
+    reachable: &HashMap<String, bool>,
+    live_recipes: &HashSet<String>,
+) {
+    for row in rows {
+        if row.up.is_none() {
+            continue;
+        }
+        // tmux deploys have no containers to probe; the ledger stays as-is.
+        let Some(cluster) = &row.up_cluster else {
+            continue;
+        };
+        // cluster no longer in the config: the probe cannot know.
+        let Some(nodes) = cluster_nodes.get(cluster) else {
+            continue;
+        };
+        let inconclusive = nodes
+            .iter()
+            .any(|n| !reachable.get(n).copied().unwrap_or(false));
+        if inconclusive || live_recipes.contains(&row.name) {
+            continue;
+        }
+        row.stale = true;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::report::quality::Medals;
+
+    fn row(name: &str, up: Option<i64>, up_cluster: Option<&str>) -> ReportRow {
+        ReportRow {
+            name: name.into(),
+            title: name.into(),
+            model: "m".into(),
+            parent: "m".into(),
+            quant: "nvfp4".into(),
+            engine: "vllm".into(),
+            image: "img".into(),
+            nodes: None,
+            tp: 1,
+            deploy: "docker".into(),
+            port: 30000,
+            incomplete: false,
+            up,
+            up_cluster: up_cluster.map(String::from),
+            runs: 1,
+            medals: Medals::default(),
+            stale: false,
+            reports: vec![],
+        }
+    }
+
+    #[test]
+    fn open_run_without_live_container_is_stale() {
+        let mut rows = vec![row("r-a", Some(1000), Some("c1"))];
+        let cn = HashMap::from([("c1".to_string(), vec!["n1".to_string(), "n2".to_string()])]);
+        let reach = HashMap::from([("n1".to_string(), true), ("n2".to_string(), true)]);
+        mark_stale(&mut rows, &cn, &reach, &HashSet::new());
+        assert!(rows[0].stale, "no live container anywhere -> stale");
+    }
+
+    #[test]
+    fn live_container_anywhere_keeps_it_running() {
+        let mut rows = vec![row("r-a", Some(1000), Some("c1"))];
+        let cn = HashMap::from([("c1".to_string(), vec!["n1".to_string()])]);
+        let reach = HashMap::from([("n1".to_string(), true)]);
+        let live = HashSet::from(["r-a".to_string()]);
+        mark_stale(&mut rows, &cn, &reach, &live);
+        assert!(!rows[0].stale, "the lane is up -> not stale");
+    }
+
+    #[test]
+    fn unreachable_node_is_inconclusive() {
+        let mut rows = vec![row("r-a", Some(1000), Some("c1"))];
+        let cn = HashMap::from([("c1".to_string(), vec!["n1".to_string(), "n2".to_string()])]);
+        let reach = HashMap::from([("n1".to_string(), true), ("n2".to_string(), false)]);
+        mark_stale(&mut rows, &cn, &reach, &HashSet::new());
+        assert!(!rows[0].stale, "cannot verify while a node is down");
+    }
+
+    #[test]
+    fn unknown_cluster_and_tmux_rows_are_left_alone() {
+        let mut rows = vec![
+            row("r-a", Some(1000), Some("gone")), // cluster removed from config
+            row("r-b", Some(1000), None),         // tmux deploy
+            row("r-c", None, Some("c1")),         // not running
+        ];
+        let cn: HashMap<String, Vec<String>> = HashMap::new();
+        let reach: HashMap<String, bool> = HashMap::new();
+        mark_stale(&mut rows, &cn, &reach, &HashSet::new());
+        assert!(!rows[0].stale && !rows[1].stale && !rows[2].stale);
+    }
+}

src/report/ui_tests.rs

@@ -22,6 +22,16 @@ fn page_has_bell_and_notif_panel() {
     }
 }
 
+/// The stale branch of the state cell: an open ledger run the live probe
+/// contradicts renders orange with the `sm121 runlock close` hint.
+#[test]
+fn page_renders_stale_state() {
+    let page = html_page(&[]);
+    for needle in ["r.stale", "sm121 runlock close", "stale"] {
+        assert!(page.contains(needle), "page missing {needle}");
+    }
+}
+
 /// The `notifDiff` JS function (extracted between markers from the live
 /// page) must classify every update kind and order newest first — probed
 /// under node, like the web dashboard's UI probes.

src/runlock.rs

@@ -14,6 +14,21 @@ use anyhow::Result;
 use serde::{Deserialize, Serialize};
 use std::path::{Path, PathBuf};
 
+/// `sm121 runlock` subcommands — offline .lock ledger management. `sm121
+/// stop` only closes the ledger after verifying every node of the cluster,
+/// so a dead node (or a killed serve) leaves the run open forever; these
+/// fix that without touching the nodes.
+#[derive(clap::Subcommand)]
+pub enum RunlockCommands {
+    /// Close the open run of a recipe's .lock ledger (offline)
+    Close {
+        /// Recipe name (its directory under the recipe sources)
+        recipe: String,
+    },
+    /// List every recipe with an open (running) ledger entry
+    List,
+}
+
 /// One launch: unix-epoch seconds (UTC), open-ended while running.
 #[derive(Serialize, Deserialize, Clone)]
 pub(crate) struct RunEntry {
@@ -110,6 +125,73 @@ pub(crate) fn mark_stop(recipe: &str, sources: &[String]) -> Result<()> {
     l.save(&dir)
 }
 
+/// Every recipe with an open (running) ledger entry: (name, start, cluster).
+/// Same dir precedence as collect: first dir wins on name collisions.
+pub(crate) fn open_runs(config: &crate::config::Config) -> Vec<(String, i64, Option<String>)> {
+    let mut out = Vec::new();
+    let mut seen = std::collections::HashSet::new();
+    for dir in crate::config::recipe_dirs(config) {
+        let Ok(entries) = std::fs::read_dir(&dir) else {
+            continue;
+        };
+        for entry in entries.flatten() {
+            let path = entry.path();
+            let Some(name) = entry.file_name().to_str().map(String::from) else {
+                continue;
+            };
+            if !seen.insert(name.clone()) || !path.join("recipe.toml").is_file() {
+                continue;
+            }
+            if let Some(run) = Lock::load(&path).running {
+                out.push((name, run.start, run.cluster));
+            }
+        }
+    }
+    out.sort_by(|a, b| a.0.cmp(&b.0));
+    out
+}
+
+/// `sm121 runlock` — offline ledger management: no node verification
+/// (unlike `sm121 stop`, which refuses to close the ledger when it cannot
+/// reach every node of the cluster). Used after a node died mid-run or the
+/// serve process was killed.
+pub(crate) fn run(config: &crate::config::Config, cmd: &RunlockCommands) -> Result<()> {
+    match cmd {
+        RunlockCommands::Close { recipe } => {
+            let open = open_runs(config)
+                .into_iter()
+                .find(|(n, _, _)| n == recipe)
+                .map(|(_, start, cluster)| (start, cluster));
+            match open {
+                None => println!("no open run for '{recipe}' (nothing to close)"),
+                Some((start, cluster)) => {
+                    mark_stop(recipe, &config.recipe_sources)?;
+                    println!(
+                        "closed open run of '{recipe}' (started {} UTC, cluster {})",
+                        crate::stamp::fmt_ts(start),
+                        cluster.as_deref().unwrap_or("tmux"),
+                    );
+                }
+            }
+        }
+        RunlockCommands::List => {
+            let open = open_runs(config);
+            if open.is_empty() {
+                println!("no open runs (all ledgers closed)");
+                return Ok(());
+            }
+            for (name, start, cluster) in open {
+                println!(
+                    "{name}\tstarted {} UTC\tcluster {}",
+                    crate::stamp::fmt_ts(start),
+                    cluster.as_deref().unwrap_or("tmux"),
+                );
+            }
+        }
+    }
+    Ok(())
+}
+
 /// Flip the recipe's `archived` flag in its own toml (comment-preserving).
 pub(crate) fn set_archived(recipe: &str, sources: &[String], archived: bool) -> Result<()> {
     let file = crate::config::recipe_file(recipe, sources)
@@ -198,6 +280,32 @@ mod tests {
         std::fs::remove_dir_all(&d).ok();
     }
 
+    #[test]
+    fn open_runs_lists_only_open_ledgers() {
+        let d = dir("open");
+        let src = sources(&d);
+        mark_start("r-lock", &src, Some("c1")).unwrap();
+        std::fs::create_dir_all(d.join("other")).unwrap();
+        std::fs::write(
+            d.join("other/recipe.toml"),
+            "engine = \"vllm\"\nname = \"O\"\n",
+        )
+        .unwrap();
+        mark_start("other", &src, Some("c1")).unwrap();
+        mark_stop("other", &src).unwrap();
+        let cfg: crate::config::Config =
+            toml::from_str(&format!("recipe_sources = [\"{}\"]\n", d.display())).unwrap();
+        let open = open_runs(&cfg);
+        let mine = open.iter().find(|(n, _, _)| n == "r-lock");
+        assert!(mine.is_some(), "open run not listed: {open:?}");
+        assert_eq!(mine.unwrap().2.as_deref(), Some("c1"));
+        assert!(
+            open.iter().all(|(n, _, _)| n != "other"),
+            "closed run must not be listed: {open:?}",
+        );
+        std::fs::remove_dir_all(&d).ok();
+    }
+
     #[test]
     fn archived_flag_flips_round_trip() {
         let d = dir("arch");