Przeglądaj źródła

serve/stop reliability + run ledger + archive curation

- serve: records the launch in recipes/<name>/.lock (gitignored ledger: lifetime launch count, open run, capped history) and then WAITS for engine readiness (/v1/models on the head node, up to 30 min, --no-wait skips); a run that never reaches readiness is closed with its uptime and fails loudly
- follow: engine-liveness watch (30 s probes, 2 misses = died) via tokio::select! against the log stream; an ended stream is verified once — a dead engine fails loudly and closes the run (docker/logs.rs follow_with_liveness)
- stop: all nodes run the verifying command in parallel, every node is attempted even when a sibling fails, failures aggregate and name the stragglers (stop-not-verified); the verified stop closes the run
- report: State (● up since), Uptime (live, 30 s tick), Launches columns + '● running' filter; readiness/waitReady extracted to readiness.rs (bench reuses), stamp.rs (utc_stamp/fmt_ts)
- archive: --archive/--unarchive flip the recipe's archived flag (toml_edit, comments preserved), --archived shows them instead; archived recipes leave the default output
Alexander Korolev 1 miesiąc temu
rodzic
commit
1d73fd0945

+ 3 - 0
.gitignore

@@ -13,3 +13,6 @@ docker-images/*/deps-src.tar.gz
 data/benchmarks.sqlite
 docker-images/*/vllm-src.tar.gz
 docker-images/*/b12x-src.tar.gz
+
+# per-recipe run ledger (machine state, not source)
+recipes/*/.lock

+ 23 - 4
AGENTS.md

@@ -17,12 +17,30 @@ See [`DEVELOPMENT.md`](DEVELOPMENT.md) for architecture, module layout, and dev
 ## Commands
 
 ```
-serve/stop/bench/logs/health/doctor/exec/power/tmux/init-host-keys/config/web
+serve/stop/bench/logs/health/doctor/exec/power/tmux/init-host-keys/config/web/report
 ```
 
-`stop` waits until the containers are actually stopped and removed (polls up
-60 s per node) — an immediate `serve` on the same nodes would otherwise race
-the dying containers' GPU release. `--no-wait` opts out.
+`serve` records the launch in the recipe's run ledger (`recipes/<name>/.lock`,
+gitignored) and then **waits for the engine to actually come up** — polls
+`GET /v1/models` on the head node up to 30 min (`--no-wait` skips); a run that
+never reaches readiness is closed with its uptime and the serve fails.
+Follow mode watches liveness (30 s probes; 2 misses = died): a dead engine —
+or an ended log stream with a dead engine — fails loudly and closes the run.
+
+`stop` stops ALL nodes in parallel, every node runs the verifying command
+(polls up to 60 s that no labeled container remains) and failures are
+aggregated: one straggler cannot leave the other nodes un-stopped, and the
+error names the nodes that did not verify. The verified stop closes the run
+in the ledger. `--no-wait` opts out of the per-node verification.
+
+`report` — the table of tested recipes: markdown (default) or an interactive
+HTML table via `--serve`/`--open`. It shows each recipe's run-ledger state
+(● running + uptime — live in the HTML, launch count) and supports archive
+curation: `--archive <name>` / `--unarchive <name>` flip the recipe's
+`archived` flag (comment-preserving), archived recipes leave the output,
+`--archived` shows them instead. `stop` waits until the containers are
+actually stopped and removed (polls up 60 s per node) — an immediate `serve`
+on the same nodes would otherwise race the dying containers' GPU release.
 
 `bench` runs `tool-eval-bench` (from PATH) against a live serve and lands
 `<UTC-date>_<label>.{md,json,log}` in `recipes/<lane>/reports/` — it does the
@@ -119,6 +137,7 @@ engine = "vllm"                     # or "sglang"
 docker_image = "img:tag"
 model = "org/model"
 parent_model = "org/base-model"     # optional; report category (medals/tree) ranks under the base model
+archived = false                    # optional; true hides the recipe from `sm121 report` (set via `--archive`)
 port = 30000                        # default
 required_nodes = 2                  # optional; also caps the launch set (N nodes, not all)
 solo = false                       # optional; single-node on first free node, no dist flags

+ 3 - 0
locales/en/main.ftl

@@ -220,3 +220,6 @@ report-col-points = Points
 report-col-score = Score /100
 report-col-tps1 = Tok/s c1
 report-col-tps2 = Tok/s c2
+stop-not-verified = Containers not verified stopped on: { $nodes }
+report-col-up = Up
+report-col-launches = Launches

+ 1 - 0
recipes/test-mods/recipe.toml

@@ -9,6 +9,7 @@ pipeline_parallel_size = 1
 
 # Entrypoint must be set when mods are present.
 entrypoint = ["sh", "-c", "i=0; while true; do echo \"[$(hostname)] heartbeat $i\"; i=$((i+1)); sleep 2; done"]
+archived = false
 
 [mods]
 commands = [

+ 1 - 0
src/after_stop.rs

@@ -188,6 +188,7 @@ mod tests {
         Recipe {
             build_profile: None,
             parent_model: None,
+            archived: false,
             engine: "vllm".to_string(),
             name: "T".to_string(),
             docker_image: String::new(),

+ 1 - 0
src/before_serve.rs

@@ -356,6 +356,7 @@ mod tests {
         Recipe {
             build_profile: None,
             parent_model: None,
+            archived: false,
             engine: "vllm".to_string(),
             name: "Test".to_string(),
             docker_image: String::new(),

+ 5 - 102
src/bench.rs

@@ -14,14 +14,12 @@ use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 use std::process::Stdio;
 use std::sync::{Arc, Mutex};
-use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+use std::time::SystemTime;
 
 use anyhow::Result;
 
 use crate::ansi::strip_ansi_step;
-use crate::config::{
-    Config, Recipe, default_config_dir, load_recipe, local_path, select_cluster_by_name,
-};
+use crate::config::{Config, Recipe, load_recipe, select_cluster_by_name};
 
 /// Fields of the `sm121 bench` subcommand (mirrored by the clap variant in
 /// `cli.rs`). Served model name and port come from the recipe
@@ -35,27 +33,6 @@ pub struct Args {
     pub seed: u64,
 }
 
-/// How long to poll the API for readiness before giving up.
-const READY_TIMEOUT_SECS: u64 = 900;
-/// Readiness poll interval.
-const READY_POLL_SECS: u64 = 5;
-
-/// Locate the recipe file (same search order as [`load_recipe`]) so reports
-/// land next to it.
-fn recipe_file(recipe_name: &str, sources: &[String]) -> Option<PathBuf> {
-    let mut paths: Vec<PathBuf> = vec![
-        PathBuf::from(format!("recipes/{}/recipe.toml", recipe_name)),
-        PathBuf::from(format!("{}/recipe.toml", recipe_name)),
-    ];
-    for src in sources {
-        paths.push(local_path(src).join(recipe_name).join("recipe.toml"));
-    }
-    if let Some(dir) = default_config_dir() {
-        paths.push(dir.join("recipes").join(recipe_name).join("recipe.toml"));
-    }
-    paths.into_iter().find(|p| p.is_file())
-}
-
 /// Served model name from the recipe's `--served-model-name` arg.
 fn served_model(recipe: &Recipe) -> Option<String> {
     let args = recipe.args.to_vec();
@@ -65,37 +42,6 @@ fn served_model(recipe: &Recipe) -> Option<String> {
         .cloned()
 }
 
-/// `YYYY-MM-DD_HHMMSS` in UTC (no chrono dependency).
-fn utc_stamp() -> String {
-    let secs: i64 = SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .map(|d| d.as_secs() as i64)
-        .unwrap_or(0);
-    let (h, m, s) = ((secs % 86400) / 3600, (secs % 3600) / 60, secs % 60);
-    let (y, mo, d) = civil_date(secs / 86400);
-    format!(
-        "{y:04}-{mo:02}-{d:02}_{h:02}{m:02}{s:02}",
-        y = y,
-        mo = mo,
-        d = d
-    )
-}
-
-/// Days since 1970-01-01 → (year, month, day); Howard Hinnant's civil_from_days.
-fn civil_date(z: i64) -> (i64, i64, i64) {
-    let z = z + 719_468;
-    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
-    let doe = z - era * 146_097;
-    let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
-    let y = yoe + era * 400;
-    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
-    let mp = (5 * doy + 2) / 153;
-    let d = doy - (153 * mp + 2) / 5 + 1;
-    let mo = if mp < 10 { mp + 3 } else { mp - 9 };
-    let y = if mo <= 2 { y + 1 } else { y };
-    (y, mo, d)
-}
-
 /// Keep the label filesystem-safe.
 fn sanitize(label: &str) -> String {
     label
@@ -110,33 +56,6 @@ fn sanitize(label: &str) -> String {
         .collect()
 }
 
-/// Poll `GET <base>/models` until it answers (engine up).
-async fn wait_ready(base_url: &str) -> Result<()> {
-    let client = reqwest::Client::builder()
-        .timeout(Duration::from_secs(10))
-        .build()?;
-    let deadline = Instant::now() + Duration::from_secs(READY_TIMEOUT_SECS);
-    let mut since_last_note = Instant::now();
-    loop {
-        match client.get(format!("{base_url}/models")).send().await {
-            Ok(r) if r.status().is_success() => return Ok(()),
-            Ok(_) => {}
-            Err(_) => {}
-        }
-        if since_last_note.elapsed() >= Duration::from_secs(30) {
-            since_last_note = Instant::now();
-            eprintln!(
-                "bench: waiting for API at {base_url} (up to {} s)...",
-                READY_TIMEOUT_SECS
-            );
-        }
-        if Instant::now() > deadline {
-            anyhow::bail!("bench: API at {base_url} not ready after {READY_TIMEOUT_SECS} s");
-        }
-        tokio::time::sleep(Duration::from_secs(READY_POLL_SECS)).await;
-    }
-}
-
 /// Tee one output pipe to the console and the log file.
 ///
 /// The console copy keeps the child's ANSI colors; the file copy is
@@ -268,7 +187,7 @@ fn latest_md(scratch: &Path) -> Option<PathBuf> {
 /// Run the bench and land `recipes/<lane>/reports/<stamp>_<label>.{md,json,log}`.
 pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
     let recipe = load_recipe(&a.recipe, &config.recipe_sources).await?;
-    let file = recipe_file(&a.recipe, &config.recipe_sources)
+    let file = crate::config::recipe_file(&a.recipe, &config.recipe_sources)
         .ok_or_else(|| anyhow::anyhow!("recipe file not found for '{}'", a.recipe))?
         .canonicalize()?;
     let cluster = select_cluster_by_name(config, &a.cluster)?;
@@ -287,7 +206,7 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
         )
     })?;
     let base_url = format!("http://{}:{}/v1", head.ssh_ip, port);
-    let stamp = utc_stamp();
+    let stamp = crate::stamp::utc_stamp();
     let label = sanitize(&a.label);
     let stem = format!("{stamp}_{label}");
     let reports_dir = file
@@ -334,7 +253,7 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
     fs::create_dir_all(&reports_dir)?;
 
     eprintln!("bench: waiting for {base_url} ...");
-    wait_ready(&base_url).await?;
+    crate::readiness::wait_ready(&base_url, crate::readiness::BENCH_WAIT_SECS).await?;
     eprintln!("bench: API ready — running {} (seed {})", bin, a.seed);
 
     let code = run_tool(&bin, &args, &scratch, &log_path)?;
@@ -386,22 +305,6 @@ mod tests {
     use super::*;
     use std::process::Command;
 
-    #[test]
-    fn stamp_format() {
-        let s = utc_stamp();
-        assert_eq!(s.len(), 17, "{s}");
-        assert_eq!(&s[4..5], "-");
-        assert_eq!(&s[7..8], "-");
-        assert_eq!(&s[10..11], "_");
-    }
-
-    #[test]
-    fn civil_date_known_values() {
-        assert_eq!(civil_date(0), (1970, 1, 1));
-        assert_eq!(civil_date(19_723), (2024, 1, 1));
-        assert_eq!(civil_date(20_818), (2026, 12, 31));
-    }
-
     /// Regression: the pump must not treat a not-yet-written pipe as EOF.
     /// `read()` into a zero-length buffer returns 0, which the old pump
     /// treated as EOF — closing the pipe and killing the child on its first

+ 31 - 28
src/cli.rs

@@ -43,10 +43,14 @@ pub enum Commands {
         #[arg(long)]
         skip_before_serve: bool,
 
-        /// Don't follow logs after serving (following is the default).
+        /// Don't follow logs after serving (default: follow).
         #[arg(long, action = clap::ArgAction::SetTrue)]
         no_follow: bool,
 
+        /// Skip the engine-readiness wait after launch (no /v1/models poll).
+        #[arg(long)]
+        no_wait: bool,
+
         /// Power on cluster if switch is off (requires HA config).
         #[arg(short = 'O', long = "on")]
         auto_power_on: bool,
@@ -62,45 +66,32 @@ pub enum Commands {
         /// Build image on this node if not found (requires docker-images/<recipe>/).
         #[arg(long)]
         build_node: Option<String>,
-
         /// Force rebuild even if checksum matches.
         #[arg(long)]
         force_build: bool,
 
-        /// Run even if nodes are occupied (another container running).
-        /// Skips the occupancy check during cluster selection — use to run
-        /// a lightweight test recipe alongside a live serve, or to force
-        /// onto a busy node. Dangerous for GPU recipes (OOM, port
-        /// conflicts); the operator is responsible for coexistence.
+        /// Run even on occupied nodes (dangerous for GPU recipes: OOM,
+        /// port conflicts). Skips the occupancy check in cluster selection.
         #[arg(long)]
         force: bool,
 
-        /// Serve single-node on the first free node found (overrides recipe
-        /// `solo`). Use `--no-solo` to force multi-node even if the recipe
-        /// sets `solo = true`.
+        /// Serve single-node on the first free node (overrides recipe `solo`);
+        /// `--no-solo` forces multi-node.
         #[arg(long)]
         solo: Option<bool>,
 
-        /// Mirror the model into a per-node local folder and mount that
-        /// instead of the shared HF cache (first serve copies the model from
-        /// the shared cache; later serves load it locally). Without a value
-        /// uses ~/.cache/hf-models; `--mirror <path>` uses that exact path.
+        /// Mount a per-node local HF mirror (bare = ~/.cache/hf-models;
+        /// `--mirror <path>` = that path; first serve copies per node).
         #[arg(long, num_args = 0..=1, default_missing_value = "")]
         mirror: Option<String>,
 
-        /// Scale the recipe to a node count and let tensor-parallel track it
-        /// (overrides `required_nodes` and `tensor_parallel_size`).
-        /// `--fit=N` scales to N nodes; bare `--fit` uses the whole selected
-        /// cluster. Incompatible with `--solo`. `pipeline_parallel_size` is
-        /// left untouched.
+        /// Scale the recipe to N nodes (`--fit=N`; bare = whole cluster, TP
+        /// tracks it; incompatible with --solo).
         #[arg(long, num_args = 0..=1, default_missing_value = "")]
         fit: Option<String>,
 
-        /// Clear the host-side `extra_mounts` folders (alias/host) on the
-        /// launch nodes with `sudo rm -rf` before containers start — the
-        /// containers run as root, so caches there are usually root-owned.
-        /// Repo-based mounts (HF cache path refs) are never cleared. `-d`
-        /// prints the exact command instead of running it.
+        /// `sudo rm -rf` the host-side extra_mounts folders on the launch
+        /// nodes before containers start (repo-based mounts untouched).
         #[arg(long)]
         clear: bool,
     },
@@ -251,17 +242,21 @@ pub enum Commands {
 
     /// Table of all tested recipes and their benchmark history
     Report {
-        /// Serve the interactive HTML table (default: print Markdown)
+        /// Interactive HTML table via --serve (default: print Markdown)
         #[arg(long)]
         serve: bool,
-
         /// Port for --serve
         #[arg(long, default_value_t = 3101)]
         port: u16,
-
         /// Open the browser (implies --serve)
         #[arg(long)]
         open: bool,
+        #[arg(long, help = "Archive a recipe (hidden from report output)")]
+        archive: Option<String>,
+        #[arg(long, help = "Un-archive a recipe")]
+        unarchive: Option<String>,
+        #[arg(long, help = "Show the archived recipes instead of the active ones")]
+        archived: bool,
     },
 
     /// Build Docker image for a recipe on a remote node
@@ -480,7 +475,15 @@ mod tests {
     #[test]
     fn report_flag_defaults_and_overrides() {
         let cli = Cli::try_parse_from(["sm121", "report"]).unwrap();
-        let Commands::Report { serve, port, open } = cli.command else {
+        let Commands::Report {
+            serve,
+            port,
+            open,
+            archive: _,
+            unarchive: _,
+            archived: _,
+        } = cli.command
+        else {
             unreachable!()
         };
         assert!(!serve && !open);

+ 17 - 0
src/config.rs

@@ -1,6 +1,7 @@
 use anyhow::{Context, Result};
 use serde::Deserialize;
 use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
 use tracing::info;
 
 /// A cluster-level HA entity (smart plug, switch, fan, …).
@@ -306,6 +307,22 @@ pub fn recipe_dirs(config: &Config) -> Vec<std::path::PathBuf> {
     v
 }
 
+/// Locate the recipe file (same search order as [`load_recipe`]) so reports
+/// land next to it.
+pub(crate) fn recipe_file(recipe_name: &str, sources: &[String]) -> Option<PathBuf> {
+    let mut paths: Vec<PathBuf> = vec![
+        PathBuf::from(format!("recipes/{}/recipe.toml", recipe_name)),
+        PathBuf::from(format!("{}/recipe.toml", recipe_name)),
+    ];
+    for src in sources {
+        paths.push(local_path(src).join(recipe_name).join("recipe.toml"));
+    }
+    if let Some(dir) = default_config_dir() {
+        paths.push(dir.join("recipes").join(recipe_name).join("recipe.toml"));
+    }
+    paths.into_iter().find(|p| p.is_file())
+}
+
 /// Load a recipe by name. Search order: `./recipes/<name>/recipe.toml` (cwd),
 /// then each `sources` entry (a dir containing `<name>/recipe.toml`, `~`
 /// expanded), then `<default_config_dir>/recipes/<name>/recipe.toml`.

+ 4 - 0
src/config/recipe.rs

@@ -67,6 +67,10 @@ pub struct Recipe {
     /// (`zai-org/GLM-5.3-Flash`).
     #[serde(default)]
     pub parent_model: Option<String>,
+    /// Archived recipes stay on disk but leave the report output
+    /// (`sm121 report --archive <name>` flips this).
+    #[serde(default)]
+    pub archived: bool,
     #[serde(default = "default_port")]
     pub port: u16,
     #[serde(default)]

+ 1 - 0
src/docker/command_tests.rs

@@ -80,6 +80,7 @@ fn base_recipe() -> Recipe {
     Recipe {
         build_profile: None,
         parent_model: None,
+        archived: false,
         engine: "vllm".into(),
         name: "T".into(),
         docker_image: "img".into(),

+ 30 - 0
src/docker/logs.rs

@@ -90,6 +90,36 @@ pub async fn show_logs(
     Ok(())
 }
 
+/// Follow the head node's logs with an engine-liveness watch: a dead engine
+/// (the API stops answering for two consecutive probes, or the log stream
+/// ends while the engine is down) fails loudly and closes the recipe's run
+/// in the ledger. Returns cleanly only when the follow ends with the engine
+/// still up (ssh drop).
+pub async fn follow_with_liveness(
+    config: &Config,
+    cluster: &Cluster,
+    recipe_name: &str,
+    port: u16,
+    node_name: Option<&str>,
+    dry_run: bool,
+) -> Result<()> {
+    let watch = crate::readiness::watch(cluster, port);
+    tokio::select! {
+        r = show_logs(config, cluster, recipe_name, true, node_name, dry_run) => {
+            r?;
+            if !crate::readiness::alive(cluster, port).await {
+                let _ = crate::runlock::mark_stop(recipe_name, &config.recipe_sources);
+                anyhow::bail!("log stream ended and the engine is down — containers died mid-follow");
+            }
+            tracing::warn!("log stream ended but the engine is still up (ssh drop?)");
+            Ok(())
+        }
+        r = watch => {
+            let _ = crate::runlock::mark_stop(recipe_name, &config.recipe_sources);
+            r
+        }
+    }
+}
 #[cfg(test)]
 mod tests {
     use super::{logs_follow_cmd, logs_tail_cmd};

+ 1 - 0
src/docker/mirror.rs

@@ -172,6 +172,7 @@ mod tests {
         Recipe {
             build_profile: None,
             parent_model: None,
+            archived: false,
             engine: "vllm".into(),
             name: "T".into(),
             docker_image: "img".into(),

+ 20 - 43
src/docker/mod.rs

@@ -28,6 +28,8 @@ pub struct ServeOpts {
     pub keep: bool,
     pub skip_before_serve: bool,
     pub dry_run: bool,
+    /// Skip the engine-readiness wait after launch (`--no-wait`).
+    pub no_wait_ready: bool,
     pub auto_power_on: bool,
     pub sync_image: bool,
     pub sync_image_source: Option<String>,
@@ -274,51 +276,26 @@ pub async fn serve(
         },
     )
     .await?;
-    Ok(cluster)
-}
-pub async fn stop_cluster(
-    config: &Config,
-    cluster: &Cluster,
-    recipe_name: &str,
-    nodes: &[String],
-    wait: bool,
-    dry_run: bool,
-) -> Result<()> {
-    info!(
-        "{}",
-        fluent_i18n::t!("cluster-stopping", { "name" => &cluster.name })
-    );
-    let nodes: Vec<&Node> = if nodes.is_empty() {
-        cluster.nodes.iter().collect()
-    } else {
-        let mut found_nodes: Vec<&Node> = Vec::new();
-        let mut not_found: Vec<String> = Vec::new();
-        for nn in nodes {
-            if let Some(node) = cluster.nodes.iter().find(|n| n.name == *nn) {
-                found_nodes.push(node);
-            } else {
-                not_found.push(nn.clone());
-            }
-        }
-        if !not_found.is_empty() {
-            let names = not_found.join(", ");
-            anyhow::bail!(
-                fluent_i18n::t!("exec-node-not-found", { "name" => &names, "cluster" => &cluster.name })
-            );
+
+    // Launch ledger + readiness: the run is recorded at launch time; then
+    // the engine must actually come up (bench-style /v1/models probe). A
+    // run that never reaches readiness is closed with its uptime and the
+    // serve fails loudly instead of returning a half-started serve.
+    if !opts.dry_run {
+        crate::runlock::mark_start(recipe_name, &config.recipe_sources, Some(&cluster.name))?;
+        if !opts.no_wait_ready
+            && let Err(e) = crate::readiness::wait_engine(
+                &cluster,
+                recipe.port,
+                crate::readiness::SERVE_WAIT_SECS,
+            )
+            .await
+        {
+            let _ = crate::runlock::mark_stop(recipe_name, &config.recipe_sources);
+            return Err(e);
         }
-        found_nodes
-    };
-    for node in &nodes {
-        run_ssh(
-            config,
-            node,
-            stop::remote_command(recipe_name, &cluster.name, node.rank, wait),
-            dry_run,
-        )
-        .await?;
     }
-    info!("{}", fluent_i18n::t!("cluster-all-stopped"));
-    Ok(())
+    Ok(cluster)
 }
 pub async fn resolve_cluster_by_container_label<'c>(
     config: &'c Config,

+ 1 - 0
src/docker/sglang_tests.rs

@@ -8,6 +8,7 @@ fn sglang_recipe() -> Recipe {
     Recipe {
         build_profile: None,
         parent_model: None,
+        archived: false,
         engine: "sglang".into(),
         name: "S".into(),
         docker_image: "sglang-img".into(),

+ 73 - 1
src/docker/stop.rs

@@ -8,6 +8,8 @@
 //! reporting success; `--no-wait` restores the legacy fire-and-forget behavior.
 
 use super::shq;
+use crate::config::{Cluster, Config, Node};
+use tracing::info;
 
 /// Builds the per-node remote stop command.
 ///
@@ -43,6 +45,71 @@ pub fn remote_command(recipe_name: &str, cluster_name: &str, rank: u32, wait: bo
     )
 }
 
+/// Stop the recipe's containers across `nodes` (all cluster nodes when the
+/// list is empty) and verify they are really gone on EVERY node: each node
+/// runs the verifying remote command, all nodes are attempted even when a
+/// sibling fails, and any failure names the straggler nodes.
+pub async fn stop_cluster(
+    config: &Config,
+    cluster: &Cluster,
+    recipe_name: &str,
+    nodes: &[String],
+    wait: bool,
+    dry_run: bool,
+) -> Result<(), anyhow::Error> {
+    info!(
+        "{}",
+        fluent_i18n::t!("cluster-stopping", { "name" => &cluster.name })
+    );
+    let nodes: Vec<&Node> = if nodes.is_empty() {
+        cluster.nodes.iter().collect()
+    } else {
+        let mut found: Vec<&Node> = Vec::new();
+        let mut not_found: Vec<String> = Vec::new();
+        for nn in nodes {
+            if let Some(node) = cluster.nodes.iter().find(|n| n.name == *nn) {
+                found.push(node);
+            } else {
+                not_found.push(nn.clone());
+            }
+        }
+        if !not_found.is_empty() {
+            let names = not_found.join(", ");
+            anyhow::bail!(
+                fluent_i18n::t!("exec-node-not-found", { "name" => &names, "cluster" => &cluster.name })
+            );
+        }
+        found
+    };
+    let mut handles = Vec::new();
+    for node in &nodes {
+        let config = config.clone();
+        let node = (*node).clone();
+        let cmd = remote_command(recipe_name, &cluster.name, node.rank, wait);
+        let dry = dry_run;
+        let name = node.name.clone();
+        handles.push(crate::locale::spawn_with_locale(async move {
+            (name, super::run_ssh(&config, &node, cmd, dry).await)
+        }));
+    }
+    let mut failed = Vec::new();
+    for r in futures::future::join_all(handles).await {
+        match r {
+            Ok((_, Ok(_))) => {}
+            Ok((name, Err(e))) => failed.push(format!("{name}: {e}")),
+            // a panicked node task is an unverified node
+            Err(e) => failed.push(format!("task: {e}")),
+        }
+    }
+    if failed.is_empty() {
+        info!("{}", fluent_i18n::t!("cluster-all-stopped"));
+        Ok(())
+    } else {
+        let nodes = failed.join(", ");
+        anyhow::bail!(fluent_i18n::t!("stop-not-verified", { "nodes" => &nodes }))
+    }
+}
+
 /// Handle the `stop` command: stop containers and optionally power off.
 ///
 /// Waits for the containers to actually stop (see [`remote_command`]), then
@@ -76,7 +143,12 @@ pub async fn handle(
             ));
         }
     };
-    crate::docker::stop_cluster(config, &cluster, recipe_name, &nodes, wait, dry_run).await?;
+    stop_cluster(config, &cluster, recipe_name, &nodes, wait, dry_run).await?;
+    // the stop is verified (every node ran the verifying command) — close
+    // the recipe's open run in the ledger
+    if !dry_run {
+        crate::runlock::mark_stop(recipe_name, &config.recipe_sources)?;
+    }
     // Run recipe `after_stop` hooks (e.g. restore_dgx_services) on the stopped nodes; best-effort.
     if let Ok(recipe) = crate::config::load_recipe(recipe_name, &config.recipe_sources).await {
         crate::after_stop::run(config, &cluster, &recipe, dry_run).await?;

+ 1 - 0
src/docker/tests.rs

@@ -48,6 +48,7 @@ fn trec() -> Recipe {
     Recipe {
         build_profile: None,
         parent_model: None,
+        archived: false,
         engine: "vllm".into(),
         name: "T".into(),
         docker_image: "img".into(),

+ 1 - 0
src/doctor.rs

@@ -355,6 +355,7 @@ mod tests {
         Recipe {
             build_profile: None,
             parent_model: None,
+            archived: false,
             engine: "vllm".to_string(),
             name: "Test Model".to_string(),
             docker_image: "test/image:latest".to_string(),

+ 1 - 0
src/engine_flags.rs

@@ -112,6 +112,7 @@ mod tests {
         Recipe {
             build_profile: None,
             parent_model: None,
+            archived: false,
             engine: "vllm".into(),
             name: "T".into(),
             docker_image: "img".into(),

+ 37 - 3
src/main.rs

@@ -29,9 +29,12 @@ mod orchestrate;
 mod power;
 mod pre_serve;
 mod publish;
+mod readiness;
 mod registry;
 mod report;
+mod runlock;
 mod ssh;
+mod stamp;
 mod tmux;
 mod web;
 
@@ -79,6 +82,7 @@ async fn main() -> Result<()> {
             keep,
             skip_before_serve,
             no_follow,
+            no_wait,
             auto_power_on,
             sync_image,
             sync_image_source,
@@ -98,6 +102,7 @@ async fn main() -> Result<()> {
                 docker::ServeOpts {
                     keep,
                     skip_before_serve,
+                    no_wait_ready: no_wait,
                     dry_run,
                     auto_power_on,
                     sync_image,
@@ -232,8 +237,24 @@ async fn main() -> Result<()> {
         Commands::Power { action } => {
             handle_power(&config, action, dry_run).await?;
         }
-        Commands::Report { serve, port, open } => {
-            crate::report::run(&config, serve || open, port, open).await?;
+        Commands::Report {
+            serve,
+            port,
+            open,
+            archive,
+            unarchive,
+            archived,
+        } => {
+            crate::report::run(
+                &config,
+                serve || open,
+                port,
+                open,
+                archive,
+                unarchive,
+                archived,
+            )
+            .await?;
         }
     }
 
@@ -261,7 +282,15 @@ pub(crate) async fn handle_serve(
             let cluster = docker::serve(config, recipe, cluster, node.as_deref(), opts).await?;
             if follow && !dry_run {
                 tracing::info!("{}", fluent_i18n::t!("container-log-header"));
-                docker::show_logs(config, &cluster, recipe, true, node.as_deref(), dry_run).await?;
+                docker::logs::follow_with_liveness(
+                    config,
+                    &cluster,
+                    recipe,
+                    recipe_obj.port,
+                    node.as_deref(),
+                    dry_run,
+                )
+                .await?;
             }
         }
         DeployTarget::Tmux => {
@@ -295,6 +324,10 @@ pub(crate) async fn handle_serve(
                 opts.force,
             )
             .await?;
+            if !dry_run {
+                // no readiness probe for tmux; cluster stays None (unresolved here)
+                crate::runlock::mark_start(recipe, &config.recipe_sources, None)?;
+            }
             if follow && !dry_run {
                 tracing::info!(
                     "{}",
@@ -321,6 +354,7 @@ async fn handle_build_cmd(
     let opts = docker::ServeOpts {
         keep: false,
         skip_before_serve: true,
+        no_wait_ready: false,
         dry_run,
         auto_power_on: false,
         sync_image: false,

+ 96 - 0
src/readiness.rs

@@ -0,0 +1,96 @@
+//! Engine readiness probes: poll `GET /v1/models` on the head node until
+//! the inference API answers — after `sm121 serve` launches containers
+//! (the model load can take many minutes) and before `sm121 bench` runs —
+//! plus a liveness watch for follow mode (the serve must fail loudly when
+//! the containers die mid-stream instead of tailing a dead engine).
+
+use crate::config::Cluster;
+use anyhow::Result;
+use std::time::{Duration, Instant};
+
+/// `sm121 serve`: big checkpoints load for many minutes.
+pub(crate) const SERVE_WAIT_SECS: u64 = 1800;
+/// `sm121 bench`: the serve is expected to already be up.
+pub(crate) const BENCH_WAIT_SECS: u64 = 900;
+/// Liveness watch: probe interval / consecutive misses that mean "died".
+const WATCH_POLL_SECS: u64 = 30;
+const WATCH_MISSES: u32 = 2;
+
+fn head_url(cluster: &Cluster, port: u16) -> String {
+    let head = cluster
+        .nodes
+        .iter()
+        .find(|n| n.rank == 0)
+        .or_else(|| cluster.nodes.first());
+    format!(
+        "http://{}:{}/v1",
+        head.map_or("", |n| n.ssh_ip.as_str()),
+        port
+    )
+}
+
+/// Poll `GET <base>/models` until it answers (engine up) or the timeout
+/// hits; progress note every 30 s.
+pub(crate) async fn wait_ready(base_url: &str, timeout_secs: u64) -> Result<()> {
+    let client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(10))
+        .build()?;
+    let deadline = Instant::now() + Duration::from_secs(timeout_secs);
+    let mut since_note = Instant::now();
+    loop {
+        if let Ok(r) = client.get(format!("{base_url}/models")).send().await
+            && r.status().is_success()
+        {
+            return Ok(());
+        }
+        if since_note.elapsed() >= Duration::from_secs(30) {
+            since_note = Instant::now();
+            eprintln!("waiting for API at {base_url} (up to {timeout_secs} s)...");
+        }
+        if Instant::now() > deadline {
+            anyhow::bail!("API at {base_url} not ready after {timeout_secs} s");
+        }
+        tokio::time::sleep(Duration::from_secs(5)).await;
+    }
+}
+
+/// Serve-side readiness: the head node's engine API must answer.
+pub(crate) async fn wait_engine(cluster: &Cluster, port: u16, timeout_secs: u64) -> Result<()> {
+    wait_ready(&head_url(cluster, port), timeout_secs).await
+}
+
+/// One-shot liveness: is the engine answering right now?
+pub(crate) async fn alive(cluster: &Cluster, port: u16) -> bool {
+    let client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(10))
+        .build()
+        .ok();
+    let Some(client) = client else {
+        return false;
+    };
+    client
+        .get(format!("{}/models", head_url(cluster, port)))
+        .send()
+        .await
+        .is_ok_and(|r| r.status().is_success())
+}
+
+/// Follow-mode liveness watch: resolves `Err` once the engine misses
+/// `WATCH_MISSES` consecutive probes — the containers died mid-follow.
+pub(crate) async fn watch(cluster: &Cluster, port: u16) -> Result<()> {
+    let mut misses = 0;
+    loop {
+        tokio::time::sleep(Duration::from_secs(WATCH_POLL_SECS)).await;
+        if alive(cluster, port).await {
+            misses = 0;
+        } else {
+            misses += 1;
+            if misses >= WATCH_MISSES {
+                anyhow::bail!(
+                    "engine stopped answering at {} — containers died mid-follow",
+                    head_url(cluster, port)
+                );
+            }
+        }
+    }
+}

+ 41 - 100
src/report/mod.rs

@@ -21,6 +21,8 @@ use std::path::Path;
 pub(crate) mod parse;
 pub(crate) mod provenance;
 pub(crate) mod quality;
+pub(crate) mod render;
+pub(crate) use render::{html_page, markdown};
 pub(crate) mod serve;
 pub(crate) use parse::{PerfCell, bench_complete, parse_bench_json, perf_from};
 pub(crate) use provenance::Provenance;
@@ -81,6 +83,13 @@ pub(crate) struct ReportRow {
     /// Reports exist but none finished the full pool (shown as ◐).
     #[serde(default)]
     incomplete: bool,
+    /// The serve that is up right now: its start (unix secs) from the
+    /// recipe's .lock ledger; None = not running.
+    #[serde(default)]
+    up: Option<i64>,
+    /// Lifetime launch count from the .lock ledger.
+    #[serde(default)]
+    runs: u64,
     /// 1 gold / 2 silver / 3 bronze — best lanes per (parent, nodes), by
     /// latest score. More nodes = more GPUs, so sizes don't compete.
     #[serde(default)]
@@ -313,7 +322,7 @@ fn scan_reports(recipe_dir: &Path, current_recipe_sha: &str) -> Vec<ReportEntry>
 /// Collect all discoverable recipes (cwd + recipe_sources) with their
 /// report history. First dir wins on name collisions (same precedence as
 /// `load_recipe`).
-pub(crate) fn collect(config: &Config) -> Vec<ReportRow> {
+pub(crate) fn collect(config: &Config, show_archived: bool) -> Vec<ReportRow> {
     let mut rows = Vec::new();
     let mut seen = HashSet::new();
     for dir in crate::config::recipe_dirs(config) {
@@ -337,9 +346,13 @@ pub(crate) fn collect(config: &Config) -> Vec<ReportRow> {
             let Ok(r) = toml::from_str::<Recipe>(&toml_str) else {
                 continue;
             };
+            if r.archived != show_archived {
+                continue; // archived recipes leave the default output
+            }
             let quant = detect_quant(&name, &r.model);
             let reports = scan_reports(&path, &sha256_hex(toml_str.as_bytes()));
             let incomplete = quality::incomplete(&reports);
+            let lock = crate::runlock::Lock::load(&path);
             rows.push(ReportRow {
                 name,
                 title: r.name,
@@ -353,6 +366,8 @@ pub(crate) fn collect(config: &Config) -> Vec<ReportRow> {
                 deploy: format!("{:?}", r.deploy).to_lowercase(),
                 port: r.port,
                 incomplete,
+                up: lock.running.as_ref().map(|r| r.start),
+                runs: lock.total,
                 medal: None,
                 reports,
             });
@@ -363,106 +378,27 @@ pub(crate) fn collect(config: &Config) -> Vec<ReportRow> {
     rows
 }
 
-fn latest_score(r: &ReportRow) -> Option<&ReportEntry> {
-    r.reports
-        .iter()
-        .rev()
-        .find(|e| e.pct.is_some() || e.points.is_some())
-}
-
-fn latest_perf(r: &ReportRow) -> Option<(&PerfCell, Option<&PerfCell>)> {
-    r.reports.iter().rev().find_map(|e| {
-        let c1 = e.perf.iter().find(|p| p.d == 0 && p.c == 1);
-        c1.map(|c1| (c1, e.perf.iter().find(|p| p.d == 0 && p.c == 2)))
-    })
-}
-
-/// Normalized score, whole numbers without decimals (81, not 81.0).
-fn fmt_pct(pct: f64) -> String {
-    if (pct - pct.trunc()).abs() < 1e-9 {
-        format!("{}", pct as i64)
-    } else {
-        format!("{pct:.1}")
+pub async fn run(
+    config: &Config,
+    serve: bool,
+    port: u16,
+    open: bool,
+    archive: Option<String>,
+    unarchive: Option<String>,
+    show_archived: bool,
+) -> Result<()> {
+    // archive bookkeeping first: flip the recipe's `archived` flag and stop
+    if let Some(name) = archive {
+        crate::runlock::set_archived(&name, &config.recipe_sources, true)?;
+        println!("archived: {name}");
+        return Ok(());
     }
-}
-
-fn fmt_tps(c: Option<&PerfCell>) -> String {
-    c.map(|c| format!("{:.1}", c.tg))
-        .unwrap_or_else(|| "—".into())
-}
-
-fn markdown(rows: &[ReportRow]) -> String {
-    let mut s = format!(
-        "# {}\n\n",
-        fluent_i18n::t!("report-title", { "count" => rows.len() })
-    );
-    s.push_str(&format!(
-        "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |\n",
-        t("report-col-model"),
-        t("report-col-quant"),
-        t("report-col-recipe"),
-        t("report-col-prov"),
-        t("report-col-engine"),
-        t("report-col-nodes"),
-        t("report-col-reports"),
-        t("report-col-last"),
-        t("report-col-points"),
-        t("report-col-score"),
-        t("report-col-tps1"),
-        t("report-col-tps2"),
-    ));
-    s.push_str("|---|---|---|---|---|---|---|---|---|---|---|---|\n");
-    for r in rows {
-        let last = r.reports.last();
-        let latest = latest_score(r);
-        let perf = latest_perf(r);
-        s.push_str(&format!(
-            "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |\n",
-            r.model,
-            r.quant,
-            r.name,
-            provenance::cell(r),
-            r.engine,
-            r.nodes
-                .map(|n| format!("{n}×{}", r.tp))
-                .unwrap_or_else(|| format!("1×{}", r.tp)),
-            r.reports.len(),
-            last.map(|e| match &e.time {
-                Some(t) => format!("{} {t}", e.date),
-                None => e.date.clone(),
-            })
-            .unwrap_or("—".into()),
-            latest
-                .and_then(|e| e.points.clone())
-                .unwrap_or_else(|| "—".into()),
-            latest
-                .and_then(|e| e.pct)
-                .map(fmt_pct)
-                .unwrap_or_else(|| "—".into()),
-            fmt_tps(perf.map(|(a, _)| a)),
-            fmt_tps(perf.and_then(|(_, b)| b)),
-        ));
+    if let Some(name) = unarchive {
+        crate::runlock::set_archived(&name, &config.recipe_sources, false)?;
+        println!("un-archived: {name}");
+        return Ok(());
     }
-    s
-}
-
-fn t(key: &str) -> String {
-    fluent_i18n::t!(key)
-}
-
-/// The interactive page: template with `{{DATA}}` replaced by the JSON
-/// rows (`<` escaped so `</script>` in data can't break out).
-fn html_page(rows: &[ReportRow]) -> String {
-    let json = serde_json::to_string(rows).unwrap_or_else(|_| "[]".into());
-    PAGE_TEMPLATE.replace("{{DATA}}", &json.replace('<', "\\u003c"))
-}
-
-/// Entry point: Markdown to stdout, or the interactive page on
-/// `http://127.0.0.1:<port>/` (`--open` also spawns the browser).
-/// The page is live: `/events` (SSE) pushes the rows JSON whenever the
-/// refresher detects a change in the `reports/` dirs.
-pub async fn run(config: &Config, serve: bool, port: u16, open: bool) -> Result<()> {
-    let rows = collect(config);
+    let rows = collect(config, show_archived);
     if !serve {
         print!("{}", markdown(&rows));
         return Ok(());
@@ -471,7 +407,12 @@ pub async fn run(config: &Config, serve: bool, port: u16, open: bool) -> Result<
     let initial = serde_json::to_string(&rows).unwrap_or_default();
     let listener = tokio::net::TcpListener::bind(("127.0.0.1", port)).await?;
     let addr = listener.local_addr()?;
-    let app = serve::router(std::sync::Arc::new(config.clone()), page, initial);
+    let app = serve::router(
+        std::sync::Arc::new(config.clone()),
+        page,
+        initial,
+        show_archived,
+    );
     if open {
         let url = format!("http://{addr}/");
         tokio::spawn(async move {

+ 33 - 3
src/report/page.html

@@ -77,6 +77,7 @@
       <option value="3">🥉 bronze</option>
     </select>
   </label>
+  <label><input type="checkbox" id="uponly"> ● running</label>
   <details id="cols"><summary>Columns ▾</summary><div id="colbox"></div></details>
   <label>Rows/page:
     <select id="size"><option>25</option><option selected>50</option><option>100</option><option value="0">All</option></select>
@@ -94,15 +95,26 @@
 "use strict";
 let DATA = {{DATA}};
 const COLS = [
-  ["name", "Recipe"], ["title", "Title"], ["model", "Model"], ["quant", "Quant"],
+  ["name", "Recipe"], ["state", "State"], ["uptime", "Uptime"], ["runs", "Launches"],
+  ["title", "Title"], ["model", "Model"], ["quant", "Quant"],
   ["engine", "Engine"], ["image", "Image"], ["nodes", "Nodes"], ["tp", "TP"],
   ["deploy", "Deploy"], ["port", "Port"], ["n", "Reports"], ["last", "Last bench"],
   ["points", "Points"], ["pct", "Score /100"], ["stars", "Bench"],
   ["tps1", "Tok/s c1"], ["tps2", "Tok/s c2"], ["parent", "Parent"],
 ];
-const NUM = new Set(["nodes", "tp", "port", "n", "pct", "tps1", "tps2", "stars"]);
+const NUM = new Set(["nodes", "tp", "port", "n", "pct", "tps1", "tps2", "stars", "state", "uptime", "runs"]);
 const MEDAL = { 1: "🥇 ", 2: "🥈 ", 3: "🥉 " };
 
+// run-ledger helpers: who is up right now, for how long (live — a 30 s tick
+// re-renders while anything is running)
+const nowSecs = () => Math.floor(Date.now() / 1000);
+const fmtUp = s => {
+  s = Math.max(0, s | 0);
+  const d = (s / 86400) | 0, h = ((s % 86400) / 3600) | 0, m = ((s % 3600) / 60) | 0;
+  return d ? d + "d " + h + "h" : h ? h + "h " + m + "m" : m ? m + "m" : s + "s";
+};
+const fmtSince = u => new Date(u * 1000).toISOString().slice(0, 16).replace("T", " ");
+
 // star grade of the latest completed bench. The 10-color palette covers
 // 0..100 a decade per color (color = which decade the score is in); the
 // star count shows the position WITHIN that decade (1 star = 2 points of
@@ -142,6 +154,8 @@ function cell(r, k) {
   if (k === "pct") { const e = latest(r, e => e.pct != null); return e ? e.pct : ""; }
   if (k === "tps1" || k === "tps2") { const e = latest(r, e => e.perf && e.perf.length); return tg(e, 0, k === "tps1" ? 1 : 2) ?? ""; }
   if (k === "stars") { const si = starInfo(r); return si ? si.pct : -1; } // sort by the absolute score
+  if (k === "state" || k === "uptime") return r.up != null ? nowSecs() - r.up : -1; // live uptime secs
+  if (k === "runs") return r.runs ?? 0;
   return r[k];
 }
 
@@ -286,6 +300,7 @@ const S = {
   closed: new Set(String(pick("closed", "")).split("|").map(decodeURIComponent).filter(Boolean)),
   openRow: pick("open", null),
   medal: pick("medal", ""),
+  uponly: pick("uponly", "") === "1",
   tp: TPVALS.includes(+(pick("tp", ""))) ? String(pick("tp", "")) : "",
 };
 document.getElementById("q").value = S.q;
@@ -293,6 +308,7 @@ document.getElementById("group").value = S.group;
 document.getElementById("size").value = String(S.size);
 document.getElementById("tp").value = S.tp;
 document.getElementById("medal").value = S.medal;
+document.getElementById("uponly").checked = S.uponly;
 function syncUrl() {
   const p = new URLSearchParams();
   if (S.q) p.set("q", S.q);
@@ -307,6 +323,7 @@ function syncUrl() {
   if (closed.length) p.set("closed", closed.join("|"));
   if (S.openRow) p.set("open", S.openRow);
   if (S.medal) p.set("medal", S.medal);
+  if (S.uponly) p.set("uponly", "1");
   if (S.tp) p.set("tp", S.tp);
   const qs = p.toString();
   history.replaceState(null, "", qs ? "?" + qs : location.pathname);
@@ -314,7 +331,7 @@ function syncUrl() {
     localStorage.setItem("report-view", JSON.stringify({
       q: S.q, group: S.group, sort: S.sort, dir: S.dir, size: S.size,
       hidden: [...S.hidden], closed: [...S.closed], openRow: S.openRow,
-      medal: S.medal, tp: S.tp,
+      medal: S.medal, tp: S.tp, uponly: S.uponly ? "1" : "",
     }));
   } catch { /* private mode etc. — URL still carries the state */ }
 }
@@ -324,6 +341,7 @@ const esc = s => String(s).replace(/[&<>"]/g, c => ({"&":"&amp;","<":"&lt;",">":
 function filtered() {
   const q = S.q.toLowerCase();
   return DATA.filter(r => (!S.tp || String(r.tp) === S.tp) &&
+    (!S.uponly || r.up != null) &&
     (!S.medal || (S.medal === "m" ? r.medal != null : String(r.medal) === S.medal)) &&
     (!q || (r.name + " " + r.title + " " + r.model + " " + r.parent + " " + r.quant + " " +
     r.engine + " " + r.image + " " + r.deploy + " " +
@@ -406,6 +424,14 @@ function render() {
             ? '<span class="dim" title="bench started, not finished">◐</span>'
             : '<span class="dim" title="no bench reports">—</span>'
           : `<span title="${si.pct}/100${pe && pe.points ? " (" + pe.points + ")" : ""}">${starGlyphs(si)}</span>`;
+      } else if (c[0] === "state") {
+        body = r.up != null
+          ? `<span title="up since ${fmtSince(r.up)} UTC"><span style="color:green">●</span><span class="dim"> up</span></span>`
+          : '<span class="dim" title="not running">○</span>';
+      } else if (c[0] === "uptime") {
+        body = r.up != null ? fmtUp(v) : '<span class="dim">—</span>';
+      } else if (c[0] === "runs") {
+        body = v > 0 ? esc(v) : '<span class="dim">0</span>';
       } else if (v === "" || v == null) {
         body = '<span class="dim">—</span>';
       } else if (c[0] === "name") {
@@ -475,6 +501,10 @@ document.getElementById("q").addEventListener("input", e => { S.q = e.target.val
 document.getElementById("group").addEventListener("change", e => { S.group = e.target.value; S.closed.clear(); S.page = 0; render(); });
 document.getElementById("tp").addEventListener("change", e => { S.tp = e.target.value; S.page = 0; render(); });
 document.getElementById("medal").addEventListener("change", e => { S.medal = e.target.value; S.page = 0; render(); });
+document.getElementById("uponly").addEventListener("change", e => { S.uponly = e.target.checked; S.page = 0; render(); });
+// live uptime: re-render while anything is running (the cell values are
+// computed at render time)
+setInterval(() => { if (DATA.some(r => r.up != null)) render(); }, 30000);
 document.getElementById("size").addEventListener("change", () => { S.page = 0; render(); });
 document.getElementById("prev").addEventListener("click", () => { if (S.page > 0) { S.page--; render(); } });
 document.getElementById("next").addEventListener("click", () => { S.page++; render(); });

+ 2 - 2
src/report/provenance.rs

@@ -132,7 +132,7 @@ mod tests {
         };
 
         // no drift: manifest sha == current recipe sha
-        let rows = collect(&mk_config());
+        let rows = collect(&mk_config(), false);
         let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap();
         let e = row.reports.iter().find(|e| e.label == "roce-run").unwrap();
         assert_eq!(e.provenance.image_sum, "8bb87348");
@@ -155,7 +155,7 @@ mod tests {
 
         // drift: the recipe changed since the bench
         fs::write(&recipe_path, recipe + "port = 30001\n").unwrap();
-        let rows = collect(&mk_config());
+        let rows = collect(&mk_config(), false);
         let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap();
         let e = row.reports.iter().find(|e| e.label == "roce-run").unwrap();
         assert!(e.provenance.recipe_drift, "recipe changed after the run");

+ 2 - 1
src/report/quality.rs

@@ -1,7 +1,8 @@
 //! Quality markers for collected recipes: bench-coverage stars and the
 //! per-category medal ranks.
 
-use super::{ReportEntry, ReportRow, latest_score};
+use super::render::latest_score;
+use super::{ReportEntry, ReportRow};
 
 /// Reports exist but none finished the full scenario pool.
 pub(crate) fn incomplete(reports: &[ReportEntry]) -> bool {

+ 104 - 0
src/report/render.rs

@@ -0,0 +1,104 @@
+//! Rendering for `sm121 report`: the Markdown table (stdout mode) and the
+//! embedded HTML page (serve mode), over the collected [`ReportRow`]s.
+
+use super::provenance;
+use super::{PAGE_TEMPLATE, PerfCell, ReportEntry, ReportRow};
+
+pub(crate) fn latest_score(r: &ReportRow) -> Option<&ReportEntry> {
+    r.reports
+        .iter()
+        .rev()
+        .find(|e| e.pct.is_some() || e.points.is_some())
+}
+
+pub(crate) fn latest_perf(r: &ReportRow) -> Option<(&PerfCell, Option<&PerfCell>)> {
+    r.reports.iter().rev().find_map(|e| {
+        let c1 = e.perf.iter().find(|p| p.d == 0 && p.c == 1);
+        c1.map(|c1| (c1, e.perf.iter().find(|p| p.d == 0 && p.c == 2)))
+    })
+}
+
+/// Normalized score, whole numbers without decimals (81, not 81.0).
+pub(crate) fn fmt_pct(pct: f64) -> String {
+    if (pct - pct.trunc()).abs() < 1e-9 {
+        format!("{}", pct as i64)
+    } else {
+        format!("{pct:.1}")
+    }
+}
+
+pub(crate) fn fmt_tps(c: Option<&PerfCell>) -> String {
+    c.map(|c| format!("{:.1}", c.tg))
+        .unwrap_or_else(|| "—".into())
+}
+
+pub(crate) fn markdown(rows: &[ReportRow]) -> String {
+    let mut s = format!(
+        "# {}\n\n",
+        fluent_i18n::t!("report-title", { "count" => rows.len() })
+    );
+    s.push_str(&format!(
+        "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |\n",
+        t("report-col-model"),
+        t("report-col-quant"),
+        t("report-col-recipe"),
+        t("report-col-prov"),
+        t("report-col-engine"),
+        t("report-col-nodes"),
+        t("report-col-reports"),
+        t("report-col-last"),
+        t("report-col-up"),
+        t("report-col-launches"),
+        t("report-col-points"),
+        t("report-col-score"),
+        t("report-col-tps1"),
+        t("report-col-tps2"),
+    ));
+    s.push_str("|---|---|---|---|---|---|---|---|---|---|---|---|\n");
+    for r in rows {
+        let last = r.reports.last();
+        let latest = latest_score(r);
+        let perf = latest_perf(r);
+        s.push_str(&format!(
+            "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |\n",
+            r.model,
+            r.quant,
+            r.name,
+            provenance::cell(r),
+            r.engine,
+            r.nodes
+                .map(|n| format!("{n}×{}", r.tp))
+                .unwrap_or_else(|| format!("1×{}", r.tp)),
+            r.reports.len(),
+            last.map(|e| match &e.time {
+                Some(t) => format!("{} {t}", e.date),
+                None => e.date.clone(),
+            })
+            .unwrap_or("—".into()),
+            r.up.map(|u| format!("● {}", crate::stamp::fmt_ts(u)))
+                .unwrap_or_else(|| "—".into()),
+            r.runs,
+            latest
+                .and_then(|e| e.points.clone())
+                .unwrap_or_else(|| "—".into()),
+            latest
+                .and_then(|e| e.pct)
+                .map(fmt_pct)
+                .unwrap_or_else(|| "—".into()),
+            fmt_tps(perf.map(|(a, _)| a)),
+            fmt_tps(perf.and_then(|(_, b)| b)),
+        ));
+    }
+    s
+}
+
+pub(crate) fn t(key: &str) -> String {
+    fluent_i18n::t!(key)
+}
+
+/// The interactive page: template with `{{DATA}}` replaced by the JSON
+/// rows (`<` escaped so `</script>` in data can't break out).
+pub(crate) fn html_page(rows: &[ReportRow]) -> String {
+    let json = serde_json::to_string(rows).unwrap_or_else(|_| "[]".into());
+    PAGE_TEMPLATE.replace("{{DATA}}", &json.replace('<', "\\u003c"))
+}

+ 4 - 3
src/report/serve.rs

@@ -70,6 +70,7 @@ pub(crate) async fn refresh_loop(
     cfg: Arc<Config>,
     tx: tokio::sync::watch::Sender<String>,
     keepalive: Duration,
+    archived: bool,
 ) {
     let (ev_tx, mut ev_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
     let watcher = notify::recommended_watcher(move |res: Result<notify::Event, notify::Error>| {
@@ -96,7 +97,7 @@ pub(crate) async fn refresh_loop(
         // debounce: let the burst settle, then drain the remaining events
         tokio::time::sleep(DEBOUNCE).await;
         while ev_rx.try_recv().is_ok() {}
-        let json = serde_json::to_string(&collect(&cfg)).unwrap_or_default();
+        let json = serde_json::to_string(&collect(&cfg, archived)).unwrap_or_default();
         if json != last {
             last = json;
             let _ = tx.send(last.clone());
@@ -291,9 +292,9 @@ async fn report_file(
 /// rows JSON snapshot used to seed the watch channel (from the same
 /// `collect` that rendered the page, so the first paint and the first SSE
 /// message agree).
-pub(crate) fn router(config: Arc<Config>, page: String, initial: String) -> Router {
+pub(crate) fn router(config: Arc<Config>, page: String, initial: String, archived: bool) -> Router {
     let (tx, rx) = tokio::sync::watch::channel(initial);
-    tokio::spawn(refresh_loop(config.clone(), tx, KEEPALIVE));
+    tokio::spawn(refresh_loop(config.clone(), tx, KEEPALIVE, archived));
     Router::new()
         .route(
             "/",

+ 6 - 7
src/report/tests.rs

@@ -1,9 +1,8 @@
 use super::parse::bench_complete;
 use super::quality::{award_medals, incomplete};
+use super::render::fmt_pct;
 use super::serve::load_report;
-use super::{
-    ReportRow, collect, detect_quant, extract_time, fmt_pct, html_page, parse_score, slash_pair,
-};
+use super::{ReportRow, collect, detect_quant, extract_time, html_page, parse_score, slash_pair};
 use crate::config::Config;
 use std::fs;
 use std::path::PathBuf;
@@ -132,7 +131,7 @@ fn collect_carries_file_and_time() {
     let dir = fixture();
     let config: Config =
         toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap();
-    let rows = collect(&config);
+    let rows = collect(&config, false);
     fs::remove_dir_all(&dir).ok();
     let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap();
     assert_eq!(row.reports[2].file, "2026-01-03_hardmode.md");
@@ -245,7 +244,7 @@ fn collect_reads_recipe_reports_and_dedups() {
     let dir = fixture();
     let config: Config =
         toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap();
-    let rows = collect(&config);
+    let rows = collect(&config, false);
     fs::remove_dir_all(&dir).ok();
     let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap();
     assert_eq!(row.title, "Test Recipe");
@@ -270,7 +269,7 @@ fn bench_triplet_groups_into_one_entry_with_json_authority() {
     let dir = fixture();
     let config: Config =
         toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap();
-    let rows = collect(&config);
+    let rows = collect(&config, false);
     fs::remove_dir_all(&dir).ok();
     let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap();
     let e = &row.reports[3];
@@ -432,7 +431,7 @@ async fn refresher_publishes_on_change_only() {
     let (tx, mut rx) = tokio::sync::watch::channel(String::new());
     // keepalive 100 ms: if the file watcher fires, the update lands via the
     // event path; if not, the keepalive picks it up — either way within 3 s
-    let h = tokio::spawn(refresh_loop(cfg, tx, Duration::from_millis(100)));
+    let h = tokio::spawn(refresh_loop(cfg, tx, Duration::from_millis(100), false));
     // the first collect runs immediately: the fixture's report is visible
     tokio::time::timeout(
         Duration::from_secs(3),

+ 3 - 3
src/report/ui_tests.rs

@@ -110,12 +110,12 @@ async fn report_playwright_checks() {
     let config: Config =
         toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap();
     let cfg = std::sync::Arc::new(config);
-    let rows = collect(&cfg);
+    let rows = collect(&cfg, false);
     let page = html_page(&rows);
     let initial = serde_json::to_string(&rows).unwrap();
     let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
     let addr = listener.local_addr().unwrap();
-    let app = serve::router(cfg, page, initial);
+    let app = serve::router(cfg, page, initial, false);
     let _h = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
     let reports = dir.join("test-recipe-nvfp4/reports");
     // tokio::process (not std::process::output — that would block this
@@ -170,7 +170,7 @@ fn markdown_lists_rows_and_scores() {
     )];
     let md = markdown(&rows);
     assert!(md.contains(
-        "| m/one | NVFP4 | r-a | — | vllm | 1×1 | 1 | 2026-01-02 | 159/176 | 90.4 | 27.4 | 33.3 |"
+        "| m/one | NVFP4 | r-a | — | vllm | 1×1 | 1 | 2026-01-02 | — | 0 | 159/176 | 90.4 | 27.4 | 33.3 |"
     ));
     let empty = markdown(&[]);
     assert!(empty.contains(fluent_i18n::t!("report-title", { "count" => 0 }).as_str()));

+ 228 - 0
src/runlock.rs

@@ -0,0 +1,228 @@
+//! Recipe state bookkeeping:
+//!
+//! **Run ledger** — `recipes/<name>/.lock` (gitignored, machine state):
+//! every `sm121 serve` opens a run (unix-epoch start, cluster), every
+//! verified `sm121 stop` closes it with its uptime; a serve over a stale
+//! open run (crash skipped `stop`) closes it first. `sm121 report` reads
+//! who is up right now, the current uptime and lifetime launch counts.
+//!
+//! **Archived flag** — `archived = true` in the recipe's own toml
+//! (round-trip-safe via `toml_edit`): archived recipes leave the report
+//! output until un-archived.
+
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+use std::path::{Path, PathBuf};
+
+/// One launch: unix-epoch seconds (UTC), open-ended while running.
+#[derive(Serialize, Deserialize, Clone)]
+pub(crate) struct RunEntry {
+    pub(crate) start: i64,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub(crate) end: Option<i64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub(crate) uptime_secs: Option<i64>,
+    /// cluster the run landed on (None for tmux deploys)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub(crate) cluster: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Default)]
+pub(crate) struct Lock {
+    /// every launch ever, newest last; capped to the newest [`HISTORY_CAP`]
+    #[serde(default)]
+    pub(crate) runs: Vec<RunEntry>,
+    /// the serve that is up right now (absent when stopped)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub(crate) running: Option<RunEntry>,
+    /// lifetime launch count — survives history capping
+    #[serde(default)]
+    pub(crate) total: u64,
+}
+
+const HISTORY_CAP: usize = 100;
+
+impl Lock {
+    pub(crate) fn load(dir: &Path) -> Self {
+        std::fs::read_to_string(dir.join(".lock"))
+            .ok()
+            .and_then(|s| serde_json::from_str(&s).ok())
+            .unwrap_or_default()
+    }
+
+    fn save(&self, dir: &Path) -> Result<()> {
+        std::fs::create_dir_all(dir)?;
+        std::fs::write(dir.join(".lock"), serde_json::to_string_pretty(self)?)?;
+        Ok(())
+    }
+}
+
+fn recipe_dir(recipe: &str, sources: &[String]) -> Result<PathBuf> {
+    crate::config::recipe_file(recipe, sources)
+        .and_then(|p| p.parent().map(Path::to_path_buf))
+        .ok_or_else(|| anyhow::anyhow!("recipe dir not found for '{recipe}'"))
+}
+
+/// `sm121 serve` went up: close a stale open run (a crash skipped `stop`),
+/// then open the new one. `cluster` is None for tmux deploys.
+pub(crate) fn mark_start(recipe: &str, sources: &[String], cluster: Option<&str>) -> Result<()> {
+    let dir = recipe_dir(recipe, sources)?;
+    let mut l = Lock::load(&dir);
+    let t = crate::stamp::now_secs();
+    if let Some(_stale) = l.running.take()
+        && let Some(last) = l.runs.last_mut()
+        && last.end.is_none()
+    {
+        last.end = Some(t);
+        last.uptime_secs = Some((t - last.start).max(0));
+    }
+    let run = RunEntry {
+        start: t,
+        end: None,
+        uptime_secs: None,
+        cluster: cluster.map(String::from),
+    };
+    l.runs.push(run.clone());
+    if l.runs.len() > HISTORY_CAP {
+        l.runs.drain(..l.runs.len() - HISTORY_CAP);
+    }
+    l.running = Some(run);
+    l.total += 1;
+    l.save(&dir)
+}
+
+/// `sm121 stop` verified the containers down: close the open run with its
+/// uptime. A stop without a tracked run (foreign serve, already-stopped)
+/// is a no-op.
+pub(crate) fn mark_stop(recipe: &str, sources: &[String]) -> Result<()> {
+    let dir = recipe_dir(recipe, sources)?;
+    let mut l = Lock::load(&dir);
+    if l.running.take().is_none() {
+        return Ok(());
+    }
+    let t = crate::stamp::now_secs();
+    if let Some(last) = l.runs.last_mut()
+        && last.end.is_none()
+    {
+        last.end = Some(t);
+        last.uptime_secs = Some((t - last.start).max(0));
+    }
+    l.save(&dir)
+}
+
+/// Flip the recipe's `archived` flag in its own toml (comment-preserving).
+pub(crate) fn set_archived(recipe: &str, sources: &[String], archived: bool) -> Result<()> {
+    let file = crate::config::recipe_file(recipe, sources)
+        .ok_or_else(|| anyhow::anyhow!("recipe file not found for '{recipe}'"))?;
+    let text = std::fs::read_to_string(&file)?;
+    let mut doc: toml_edit::DocumentMut = text.parse()?;
+    doc["archived"] = toml_edit::value(archived);
+    std::fs::write(&file, doc.to_string())?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn dir(tag: &str) -> PathBuf {
+        let d = std::env::temp_dir().join(format!("sm121-runlock-{}-{tag}", std::process::id()));
+        std::fs::create_dir_all(d.join("r-lock")).unwrap();
+        std::fs::write(
+            d.join("r-lock/recipe.toml"),
+            "engine = \"vllm\"\nname = \"T\"\n",
+        )
+        .unwrap();
+        d
+    }
+
+    fn sources(d: &Path) -> Vec<String> {
+        vec![d.to_string_lossy().to_string()]
+    }
+
+    #[test]
+    fn start_stop_roundtrip_and_counts() {
+        let d = dir("rt");
+        let src = sources(&d);
+        mark_start("r-lock", &src, Some("c1")).unwrap();
+        let l = Lock::load(&d.join("r-lock"));
+        assert_eq!(l.total, 1);
+        assert!(l.running.is_some());
+        assert_eq!(l.runs.len(), 1);
+        assert_eq!(l.runs[0].cluster.as_deref(), Some("c1"));
+        mark_stop("r-lock", &src).unwrap();
+        let l = Lock::load(&d.join("r-lock"));
+        assert!(l.running.is_none());
+        assert_eq!(l.runs[0].end, Some(l.runs[0].start), "instant stop");
+        assert_eq!(l.runs[0].uptime_secs, Some(0));
+        // stop without a tracked run: no-op
+        mark_stop("r-lock", &src).unwrap();
+        assert_eq!(Lock::load(&d.join("r-lock")).total, 1);
+        std::fs::remove_dir_all(&d).ok();
+    }
+
+    #[test]
+    fn start_over_stale_run_closes_it() {
+        let d = dir("stale");
+        let src = sources(&d);
+        mark_start("r-lock", &src, None).unwrap();
+        // rewind the open run's start so the stale-close uptime is positive
+        let p = d.join("r-lock/.lock");
+        let mut l = Lock::load(&d.join("r-lock"));
+        l.runs[0].start -= 100;
+        l.running.as_mut().unwrap().start -= 100;
+        std::fs::write(&p, serde_json::to_string(&l).unwrap()).unwrap();
+        // a new serve without a stop in between: the stale run closes
+        mark_start("r-lock", &src, Some("c2")).unwrap();
+        let l = Lock::load(&d.join("r-lock"));
+        assert_eq!(l.total, 2, "two launches");
+        assert_eq!(l.runs.len(), 2);
+        assert_eq!(l.runs[0].uptime_secs, Some(100), "stale closed");
+        assert!(l.runs[1].end.is_none(), "new run open");
+        assert_eq!(l.running.as_ref().unwrap().cluster.as_deref(), Some("c2"));
+        std::fs::remove_dir_all(&d).ok();
+    }
+
+    #[test]
+    fn history_is_capped_but_total_grows() {
+        let d = dir("cap");
+        let src = sources(&d);
+        for _ in 0..HISTORY_CAP + 5 {
+            mark_start("r-lock", &src, None).unwrap();
+            mark_stop("r-lock", &src).unwrap();
+        }
+        let l = Lock::load(&d.join("r-lock"));
+        assert_eq!(l.total, HISTORY_CAP as u64 + 5);
+        assert_eq!(l.runs.len(), HISTORY_CAP);
+        assert!(l.running.is_none());
+        std::fs::remove_dir_all(&d).ok();
+    }
+
+    #[test]
+    fn archived_flag_flips_round_trip() {
+        let d = dir("arch");
+        let src = sources(&d);
+        // recipe with comments — toml_edit must preserve them
+        let toml = d.join("r-lock/recipe.toml");
+        std::fs::write(
+            &toml,
+            "# keep me\nengine = \"vllm\" # inline\nname = \"T\"\n",
+        )
+        .unwrap();
+        set_archived("r-lock", &src, true).unwrap();
+        let after = std::fs::read_to_string(&toml).unwrap();
+        assert!(after.contains("archived = true"), "{after}");
+        assert!(after.contains("# keep me"), "comment lost: {after}");
+        assert!(
+            after.contains("engine = \"vllm\" # inline"),
+            "inline lost: {after}"
+        );
+        set_archived("r-lock", &src, false).unwrap();
+        assert!(
+            std::fs::read_to_string(&toml)
+                .unwrap()
+                .contains("archived = false")
+        );
+        std::fs::remove_dir_all(&d).ok();
+    }
+}

+ 75 - 0
src/stamp.rs

@@ -0,0 +1,75 @@
+//! UTC timestamps without a chrono dependency: Howard Hinnant's
+//! civil_from_days for the calendar math, unix-epoch seconds everywhere
+//! else (the run ledger stores epochs and renders through [`fmt_ts`]).
+
+use std::time::{SystemTime, UNIX_EPOCH};
+
+/// Unix-epoch seconds right now (UTC).
+pub(crate) fn now_secs() -> i64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_secs() as i64)
+        .unwrap_or(0)
+}
+
+/// `YYYY-MM-DD_HHMMSS` in UTC (bench report file names).
+pub(crate) fn utc_stamp() -> String {
+    let secs = now_secs();
+    let (h, m, s) = ((secs % 86400) / 3600, (secs % 3600) / 60, secs % 60);
+    let (y, mo, d) = civil_date(secs / 86400);
+    format!(
+        "{y:04}-{mo:02}-{d:02}_{h:02}{m:02}{s:02}",
+        y = y,
+        mo = mo,
+        d = d
+    )
+}
+
+/// `YYYY-MM-DD HH:MM` in UTC (run-ledger display).
+pub(crate) fn fmt_ts(secs: i64) -> String {
+    let (h, m, _) = ((secs % 86400) / 3600, (secs % 3600) / 60, secs % 60);
+    let (y, mo, d) = civil_date(secs / 86400);
+    format!("{y:04}-{mo:02}-{d:02} {h:02}:{m:02}")
+}
+
+/// Days since 1970-01-01 → (year, month, day); Howard Hinnant's civil_from_days.
+pub(crate) fn civil_date(z: i64) -> (i64, i64, i64) {
+    let z = z + 719_468;
+    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
+    let doe = z - era * 146_097;
+    let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
+    let y = yoe + era * 400;
+    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
+    let mp = (5 * doy + 2) / 153;
+    let d = doy - (153 * mp + 2) / 5 + 1;
+    let mo = if mp < 10 { mp + 3 } else { mp - 9 };
+    let y = if mo <= 2 { y + 1 } else { y };
+    (y, mo, d)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn stamp_format() {
+        let s = utc_stamp();
+        assert_eq!(s.len(), 17, "{s}");
+        assert_eq!(&s[4..5], "-");
+        assert_eq!(&s[7..8], "-");
+        assert_eq!(&s[10..11], "_");
+    }
+
+    #[test]
+    fn civil_date_known_values() {
+        assert_eq!(civil_date(0), (1970, 1, 1));
+        assert_eq!(civil_date(19_723), (2024, 1, 1));
+        assert_eq!(civil_date(20_818), (2026, 12, 31));
+    }
+
+    #[test]
+    fn fmt_ts_renders_date_and_minute() {
+        // 2026-09-14 10:03:00 UTC
+        assert_eq!(fmt_ts(1_789_380_180), "2026-09-14 10:03");
+    }
+}

+ 1 - 0
src/tmux/commands.rs

@@ -146,6 +146,7 @@ mod tests {
         Recipe {
             build_profile: None,
             parent_model: None,
+            archived: false,
             engine: "vllm".into(),
             name: "T".into(),
             docker_image: "img".into(),

+ 1 - 0
src/web/api.rs

@@ -63,6 +63,7 @@ pub(crate) async fn serve(
     let opts = crate::docker::ServeOpts {
         keep: req.keep,
         skip_before_serve: false,
+        no_wait_ready: false,
         dry_run: req.dry_run,
         auto_power_on: false,
         sync_image: true,