release: sm121 0.2.4 — cached liveness, TCP switch ping, bench log/job fixes

AGENTS.md

@@ -278,9 +278,11 @@ also **adopts** a manually staged tree without re-copying.
 **Cluster health is cluster-level** (`health`, web dashboard): a cluster is
 UP only when *all* its nodes are reachable AND its **blocking**
 `extra_entities` report `on`. HA entities are probed via HA REST; a
-`[mikrotik]` device referenced by name is probed through the **RouterOS API
-login** (logged in = on; unreachable or rejected login = off, like a dead
-switch — `health` warns with the reason). One offline node — or a blocking
+`[mikrotik]` device referenced by name is probed with a **plain TCP connect**
+(no RouterOS login) — reachable = on, unreachable/timeout = off, like a dead
+switch. Entity states are cached process-wide (HA 120 s, RouterOS 300 s) so the
+30 s dashboard poll does not authenticate the switch on every tick; a power
+action invalidates the cache so the next probe is fresh. One offline node — or a blocking
 switch off — takes the whole cluster down: containers on surviving nodes are
 not reported (serve/stop/jobs views all ignore them; `power on` stays
 available). `extra_entities` entries can opt out of gating with

CHANGELOG.md

@@ -3,6 +3,30 @@
 Semantic versioning for the launcher CLI (`sm121`). The add-on image has its
 own version and its own `sm121/CHANGELOG.md`; do not mix them.
 
+## 0.2.4 — 2026-10-29
+
+- **Liveness checks are cached and less aggressive.** Entity states are kept
+  process-wide (HA 120 s, RouterOS 300 s); the dashboard's 30 s poll no longer
+  re-authenticates every switch or re-fetches every HA entity on each tick.
+  A power action invalidates the cache so the next snapshot is fresh.
+- **Switch liveness is a plain TCP connect, not a RouterOS login.** `health`
+  and the dashboard only need to know whether the switch is reachable; the
+  RouterOS login was unnecessary API load. `power off` still logs in and sends
+  `/system/shutdown`.
+- **Bench log belongs to the current job.** `GET /api/bench/log` follows the
+  recipe's newest bench job and only accepts a log written at/after its start;
+  a previous campaign is labelled `previous`, and a fresh launch shows
+  `waiting` instead of a stale log.
+- **Job ids survive a web restart.** The id counter resumes past persisted
+  `j<hex>` ids, and `finish_job` updates the still-running job with that id,
+  not an older finished one.
+- **Bench append semantics.** `run_tool` always appends; `bench::run` truncates
+  the log once and writes launcher progress notes, so the dashboard window sees
+  readiness/notes and the depth suite shares one report log.
+- **Remote bench forces a pty.** `ssh -tt` avoids the "Pseudo-terminal will not
+  be allocated" path when the launcher has no terminal.
+- **Upstream watch** uses 100 commits and follows redirects.
+
 ## 0.2.3 — 2026-10-28
 
 - **FLARE topology/LED board: every rank node is busy.** `flare orbit status`

Cargo.lock

@@ -2941,7 +2941,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
 [[package]]
 name = "sm121"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "anyhow",
  "axum",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sm121"
-version = "0.2.3"
+version = "0.2.4"
 edition = "2024"
 license = "MIT"
 

scripts/flare-upstream-watch.sh

@@ -24,7 +24,7 @@ if [[ -f "$PIN_FILE" ]]; then
   read -r pin_sha pin_date <"$PIN_FILE" || true
 fi
 
-json="$(curl -fsS -m25 "$API/commits?per_page=40")"
+json="$(curl -fsSL -m25 "$API/commits?per_page=100")"
 head_sha="$(printf '%s' "$json" | python3 -c 'import json,sys; print(json.load(sys.stdin)[0]["sha"])')"
 head_date="$(printf '%s' "$json" | python3 -c 'import json,sys; d=json.load(sys.stdin)[0]["commit"]["committer"]; print(d["date"][:16])')"
 

src/bench.rs

@@ -184,23 +184,41 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
         return Ok(());
     }
 
+    fs::create_dir_all(&reports_dir)?;
+    // One truncate per run (run_tool appends, so its output and the notes
+    // below interleave in file order instead of clobbering each other).
+    fs::write(&log_path, "")?;
+    let n = |m: String| crate::bench_io::note(&log_path, &m);
+    n(format!(
+        "bench: {} · seed {} · concurrency {} · model {} · target {}",
+        stem, a.seed, a.concurrency, model, base_url
+    ));
     if let Some(host) = ssh_host {
+        n(format!(
+            "bench: harness host {host} ({})",
+            tool_scratch.display()
+        ));
         crate::bench_remote::prepare(host, &tool_scratch, &a.tokenizer)?;
     } else {
+        n(format!(
+            "bench: harness host (local) {bin} in {}",
+            tool_scratch.display()
+        ));
         fs::create_dir_all(&scratch)?;
     }
-    fs::create_dir_all(&reports_dir)?;
 
-    eprintln!("bench: waiting for {base_url} ...");
+    n(format!("bench: waiting for {base_url} ..."));
     crate::readiness::wait_ready(&base_url, crate::readiness::BENCH_WAIT_SECS).await?;
-    eprintln!("bench: API ready — running {} (seed {})", bin, a.seed);
+    n(format!(
+        "bench: API ready — running {bin} (seed {})",
+        a.seed
+    ));
 
     let code = run_tool(
         &bin,
         &args,
         &tool_scratch,
         &log_path,
-        false,
         ssh_host,
         a.cancel.as_ref(),
     )?;
@@ -222,7 +240,7 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
     // failure must not cost the finished quality run — warn, keep going.
     // Opt-in (`--depth`): a full sweep takes ~45 min on a 2x lane.
     if a.depth {
-        eprintln!(
+        n(format!(
             "bench: depth suite: {} @ c{}, {} repeats{}",
             crate::bench_depth::DEPTHS,
             crate::bench_depth::CONCURRENCY,
@@ -232,14 +250,13 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
             } else {
                 ""
             }
-        );
+        ));
         fs::create_dir_all(&depth_scratch)?;
         let dcode = run_tool(
             &bin,
             &depth_args,
             &depth_scratch,
             &log_path,
-            true,
             ssh_host,
             a.cancel.as_ref(),
         )?;
@@ -256,7 +273,7 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
             );
         }
     } else {
-        eprintln!("bench: depth suite skipped (opt-in: --depth)");
+        n(String::from("bench: depth suite skipped (opt-in: --depth)"));
     }
 
     // Remote host: download the json + the md from the remote scratch dir so
@@ -328,7 +345,10 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
         })
         .await?;
     }
-    eprintln!("bench: reports landed in {}", reports_dir.display());
+    n(format!(
+        "bench: reports landed in {}",
+        reports_dir.display()
+    ));
     Ok(())
 }
 
@@ -419,7 +439,6 @@ mod tests {
             ],
             &tmp,
             &log,
-            false,
             None,
             None,
         )
@@ -432,29 +451,21 @@ mod tests {
         assert!(!body.contains('\r'), "pty ONLCR leaked CRLF: {body:?}");
     }
 
-    /// The append mode must not truncate the main phase's log — the depth
-    /// suite shares it so one report carries both tables.
+    /// The depth suite shares the main phase's log (one report, both tables):
+    /// appending keeps the main rows instead of replacing them.
     #[test]
     fn run_tool_append_keeps_prior_output() {
         let tmp =
             std::env::temp_dir().join(format!("sm121-bench-append-test-{}", std::process::id()));
         fs::create_dir_all(&tmp).unwrap();
         let log = tmp.join("append.log");
-        let run = |a: &str, append| {
-            run_tool(
-                "sh",
-                &["-c".into(), a.into()],
-                &tmp,
-                &log,
-                append,
-                None,
-                None,
-            )
-            .unwrap()
-        };
-        assert_eq!(run("echo main", false), 0);
+        // run_tool always appends; the caller owns the one truncate.
+        fs::write(&log, "").unwrap();
+        let run =
+            |a: &str| run_tool("sh", &["-c".into(), a.into()], &tmp, &log, None, None).unwrap();
+        assert_eq!(run("echo main"), 0);
         assert!(fs::read_to_string(&log).unwrap().contains("main\n"));
-        assert_eq!(run("echo depth", true), 0);
+        assert_eq!(run("echo depth"), 0);
         let body = fs::read_to_string(&log).unwrap();
         let (i1, i2) = (body.find("main\n"), body.find("depth\n"));
         assert!(

src/bench_io.rs

@@ -70,3 +70,21 @@ pub fn latest_md(scratch: &Path) -> Option<PathBuf> {
     }
     best.map(|(_, p)| p)
 }
+
+/// Write a launcher-side progress note into the run log as well as stderr.
+///
+/// The dashboard's bench window tails that file: the harness suppresses its
+/// own progress output off a TTY (`TTY_COMPATIBLE=0`), and a remote run
+/// prints nothing locally until it ends — without these notes the window looks
+/// dead for the whole readiness wait and the run itself.
+pub fn note(log: &Path, msg: &str) {
+    eprintln!("{msg}");
+    use std::io::Write;
+    if let Ok(mut f) = std::fs::OpenOptions::new()
+        .create(true)
+        .append(true)
+        .open(log)
+    {
+        let _ = writeln!(f, "{msg}");
+    }
+}

src/bench_remote.rs

@@ -9,8 +9,9 @@ use std::process::Command;
 
 use anyhow::Result;
 
-/// Build the SSH command that runs the tool on the remote host. `ssh -t`
-/// allocates a pty (the rich Live display deadlocks on a non-tty pipe);
+/// Build the SSH command that runs the tool on the remote host. `ssh -tt`
+/// allocates a pty (the rich Live display deadlocks on a non-tty pipe, and a
+/// single `-t` is ignored when the launcher is not attached to a terminal);
 /// `stty -onlcr` keeps the log LF-only; TTY_COMPATIBLE/NO_COLOR match the
 /// local run. The `cwd` is a remote path embedded in the shell string.
 pub fn remote_command(host: &str, cwd: &Path, bin: &str, args: &[String]) -> Command {
@@ -22,7 +23,11 @@ pub fn remote_command(host: &str, cwd: &Path, bin: &str, args: &[String]) -> Com
         args.join(" ")
     );
     let mut c = Command::new("ssh");
-    c.arg("-t");
+    // `-tt` (not `-t`): force the pty even when the launcher has no terminal.
+    // With plain `-t` ssh prints "Pseudo-terminal will not be allocated …"
+    // and runs the harness without one — the bench window then shows that
+    // warning instead of the run.
+    c.arg("-tt");
     c.arg(host);
     c.arg(&remote);
     c

src/bench_target.rs

@@ -72,9 +72,11 @@ pub(crate) fn sanitize(label: &str) -> String {
 /// Run the bench tool in `cwd`, teeing stdout+stderr to the console and the
 /// log file; returns the exit code.
 ///
-/// `append_log = false` truncates the log (main campaign phase); `true`
-/// appends (the depth-suite phase shares one log so its rows land in the
-/// report's Throughput table).
+/// The log is always *appended*: `bench::run` truncates it once, before it
+/// writes its own progress notes — a truncating `run_tool` used to wipe those
+/// notes (the dashboard's bench window then stayed empty until the harness
+/// printed its final event). The depth-suite phase shares the same log so its
+/// rows land in the report's Throughput table.
 ///
 /// The child runs under a pty (`script -q /dev/null`) instead of raw pipes:
 /// tool-eval-bench's rich Live display deadlocks on a non-tty stdout pipe
@@ -93,18 +95,15 @@ pub(crate) fn run_tool(
     args: &[String],
     cwd: &Path,
     log_path: &Path,
-    append_log: bool,
     ssh_host: Option<&str>,
     cancel: Option<&Arc<AtomicBool>>,
 ) -> Result<i32> {
-    let file = Arc::new(Mutex::new(if append_log {
+    let file = Arc::new(Mutex::new(
         fs::OpenOptions::new()
             .create(true)
             .append(true)
-            .open(log_path)?
-    } else {
-        fs::File::create(log_path)?
-    }));
+            .open(log_path)?,
+    ));
     // The command: local (the TTY wrapper or direct exec) or remote (SSH to
     // the host — a pty via `ssh -t` keeps the rich Live display from
     // deadlocking on the non-tty pipe; `stty -onlcr` keeps the log LF-only;

src/entity_cache.rs

@@ -0,0 +1,94 @@
+//! Process-wide liveness cache for HA / RouterOS entity states.
+//!
+//! The dashboard's fleet probe runs every 30 s. Without a cache, every tick
+//! issues one HA `GET /api/states/...` per node/extra and — worse — a full
+//! RouterOS API login per switch. A dead switch must not be hammered, and a
+//! live one must not be asked to authenticate every 30 s. The cache keeps a
+//! state for a TTL (HA: 120 s, RouterOS: 300 s); a power action invalidates it
+//! so the next probe sees the real result immediately.
+
+use std::collections::HashMap;
+use std::sync::{LazyLock, Mutex};
+use std::time::{Duration, Instant};
+
+#[derive(Clone, Copy)]
+struct Entry {
+    value: Option<bool>,
+    checked: Instant,
+    ttl: Duration,
+}
+
+static CACHE: LazyLock<Mutex<HashMap<String, Entry>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+
+fn lock_cache() -> std::sync::MutexGuard<'static, HashMap<String, Entry>> {
+    CACHE
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner)
+}
+
+/// Fresh cached state (`true`/`false`) or `None` when absent/expired.
+pub fn fresh(key: &str) -> Option<Option<bool>> {
+    let mut cache = lock_cache();
+    match cache.get(key) {
+        Some(e) if e.checked.elapsed() < e.ttl => Some(e.value),
+        _ => {
+            cache.remove(key);
+            None
+        }
+    }
+}
+
+/// Store a state for `ttl`.
+pub fn store(key: &str, value: Option<bool>, ttl: Duration) {
+    let mut cache = lock_cache();
+    cache.insert(
+        key.to_string(),
+        Entry {
+            value,
+            checked: Instant::now(),
+            ttl,
+        },
+    );
+    // Keep the map bounded by dropping expired entries on write.
+    cache.retain(|_, e| e.checked.elapsed() < e.ttl);
+}
+
+/// Drop one cached entity (e.g. after a HA/ RouterOS toggle).
+pub fn invalidate(key: &str) {
+    lock_cache().remove(key);
+}
+
+/// Drop every entity (after a power flow; the dashboard's next tick re-probes).
+pub fn invalidate_all() {
+    lock_cache().clear();
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn fresh_store_and_expire() {
+        store("ha:switch.x", Some(true), Duration::from_secs(2));
+        assert_eq!(fresh("ha:switch.x"), Some(Some(true)));
+        // A zero TTL means "do not trust this entry".
+        store("ha:switch.y", Some(false), Duration::ZERO);
+        assert_eq!(fresh("ha:switch.y"), None);
+    }
+
+    #[test]
+    fn invalidation_forces_reprobe() {
+        store(
+            "mikrotik:sw:10.0.0.1",
+            Some(false),
+            Duration::from_secs(300),
+        );
+        assert_eq!(fresh("mikrotik:sw:10.0.0.1"), Some(Some(false)));
+        invalidate("mikrotik:sw:10.0.0.1");
+        assert_eq!(fresh("mikrotik:sw:10.0.0.1"), None);
+        store("ha:switch.z", Some(true), Duration::from_secs(120));
+        invalidate_all();
+        assert_eq!(fresh("ha:switch.z"), None);
+    }
+}

src/home_assistant.rs

@@ -12,6 +12,7 @@ use std::time::Duration;
 use tracing::{debug, info, warn};
 
 use crate::config::{Cluster, Config, HomeAssistantConfig};
+use crate::entity_cache as liveness;
 
 /// Process-wide shared HA client, built once. The fleet probe rebuilds HA
 /// state every 30 s across several clusters — a fresh `Client` per call
@@ -19,6 +20,12 @@ use crate::config::{Cluster, Config, HomeAssistantConfig};
 /// setup each time, ~2 GB/h of churn in the long-running dashboard.
 static SHARED_CLIENT: OnceLock<Option<Client>> = OnceLock::new();
 
+/// Entity states are cached because the dashboard probes every 30 s (and a
+/// switch must not be asked to authenticate on every tick). HA state changes
+/// within 2 min; RouterOS devices are checked at most every 5 min.
+const HA_TTL: Duration = Duration::from_secs(120);
+const MIKROTIK_TTL: Duration = Duration::from_secs(300);
+
 /// The shared client for long-lived callers (the web dashboard's probe
 /// loop); CLI paths (one-shot `power`) may keep using `Client::new`.
 pub fn shared_client(config: &Config) -> Option<Client> {
@@ -106,6 +113,7 @@ impl Client {
                 body.trim()
             );
         }
+        liveness::invalidate(&format!("ha:{}", entity_id));
         Ok(())
     }
 
@@ -231,19 +239,29 @@ pub async fn entity_state(
         .as_ref()
         .and_then(|m| m.devices.iter().find(|d| d.name == entity_id))
     {
-        return match crate::mikrotik::ping_device(dev).await {
-            Ok(()) => Some("on".to_string()),
+        let key = format!("mikrotik:{}:{}", dev.name, dev.address);
+        if let Some(Some(v)) = liveness::fresh(&key) {
+            return Some((if v { "on" } else { "off" }).to_string());
+        }
+        let v = match crate::mikrotik::ping_device(dev).await {
+            Ok(()) => true,
             Err(e) => {
                 warn!("RouterOS ping {entity_id} failed: {e}");
-                Some("off".to_string())
+                false
             }
         };
+        liveness::store(&key, Some(v), MIKROTIK_TTL);
+        return Some((if v { "on" } else { "off" }).to_string());
     }
     let client = client?;
-    client
-        .get_state(entity_id)
-        .await
-        .map(|s| if s == "on" { "on".into() } else { "off".into() })
+    let key = format!("ha:{}", entity_id);
+    if let Some(Some(v)) = liveness::fresh(&key) {
+        return Some((if v { "on" } else { "off" }).to_string());
+    }
+    let state = client.get_state(entity_id).await;
+    let v = state.as_deref() == Some("on");
+    liveness::store(&key, Some(v), HA_TTL);
+    Some((if v { "on" } else { "off" }).to_string())
 }
 
 /// Parse `switch.dgx1` into `("switch", "dgx1")` so callers can build

src/main.rs

@@ -11,7 +11,6 @@ static DHAT_ALLOC: dhat::Alloc = dhat::Alloc;
 #[cfg(feature = "mimalloc")]
 #[global_allocator]
 static MI_ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
-
 use crate::docker::sync;
 use anyhow::Result;
 use clap::Parser;
@@ -40,6 +39,7 @@ mod docker;
 mod doctor;
 mod ends;
 mod engine;
+mod entity_cache;
 mod exec;
 mod gconfig;
 mod handlers;

src/mikrotik.rs

@@ -17,20 +17,22 @@ use tracing::info;
 
 use crate::config::MikrotikDevice;
 
-/// Liveness probe for `health`: connect + login, nothing else.
-/// `Ok` = the router is up AND the configured credentials work;
-/// any failure (unreachable, timeout, rejected login) = the entity is off,
-/// exactly like an HA switch that does not answer.
+/// Liveness probe for `health`: a plain TCP connect, no login.
+/// RouterOS authentication is not needed to know the switch is alive; a full
+/// login per dashboard tick was hammering the API. `Ok` = the host/API port is
+/// reachable; any failure (unreachable, timeout) = the entity is off, exactly
+/// like an HA switch that does not answer.
 pub async fn ping_device(dev: &MikrotikDevice) -> Result<()> {
     let target = format!("{}:{}", dev.address, dev.api_port);
-    let mut s = tokio::time::timeout(Duration::from_secs(5), TcpStream::connect(&target))
-        .await
-        .with_context(|| format!("connecting to RouterOS at {}", target))?
-        .with_context(|| format!("connect to RouterOS at {}", target))?;
-    s.set_nodelay(true)?;
-    tokio::time::timeout(Duration::from_secs(10), login_stream(&mut s, dev))
-        .await
-        .map_err(|_| anyhow::anyhow!("RouterOS login timed out ({})", dev.name))?
+    let attempt = tokio::time::timeout(Duration::from_secs(3), TcpStream::connect(&target)).await;
+    match attempt {
+        Ok(Ok(stream)) => {
+            stream.set_nodelay(true)?;
+            Ok(())
+        }
+        Ok(Err(e)) => Err(anyhow::anyhow!("connect to RouterOS at {target}: {e}")),
+        Err(_) => Err(anyhow::anyhow!("RouterOS ping timed out ({target})")),
+    }
 }
 
 /// Connect + login, then shut the router down via `/system/shutdown`.
@@ -43,7 +45,9 @@ pub async fn shutdown_device(dev: &MikrotikDevice) -> Result<()> {
         .with_context(|| format!("connect to RouterOS at {}", target))?;
     s.set_nodelay(true)?;
     login_stream(&mut s, dev).await?;
-    shutdown_stream(&mut s, dev).await
+    shutdown_stream(&mut s, dev).await?;
+    crate::entity_cache::invalidate(&format!("mikrotik:{}:{}", dev.name, dev.address));
+    Ok(())
 }
 
 /// The login exchange on an authenticated-or-not stream: `/login` with
@@ -278,6 +282,27 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn ping_is_tcp_only() {
+        // A plain TCP connect is enough for liveness; no RouterOS login is
+        // sent. The mock listener accepts and closes.
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let port = listener.local_addr().unwrap().port();
+        let dev = MikrotikDevice {
+            name: "rt".into(),
+            address: "127.0.0.1".into(),
+            username: "x".into(),
+            password: "y".into(),
+            api_port: port,
+            on_entity: None,
+        };
+        let accept = tokio::spawn(async move {
+            let _ = listener.accept().await;
+        });
+        ping_device(&dev).await.unwrap();
+        accept.await.unwrap();
+    }
+
     #[tokio::test]
     async fn framing_roundtrip_all_bands() {
         for n in [0usize, 1, 127, 128, 16383, 16384, 2_097_151] {

src/power.rs

@@ -141,6 +141,7 @@ pub async fn power_off(
     if any_ssh_failed {
         warn!("{}", fluent_i18n::t!("power-off-ssh-failures"));
     }
+    crate::entity_cache::invalidate_all();
     Ok(())
 }
 
@@ -280,6 +281,7 @@ pub async fn power_on(
         "cluster",
     )
     .await;
+    crate::entity_cache::invalidate_all();
     Ok(report)
 }
 
@@ -401,6 +403,7 @@ pub async fn power_switch(
         )
         .await;
     }
+    crate::entity_cache::invalidate_all();
     Ok(())
 }
 

src/web/bench_api.rs

@@ -119,12 +119,49 @@ pub(crate) struct BenchLogReq {
     pub lines: Option<usize>,
 }
 
-/// `GET /api/bench/log?recipe=<name>` — the newest bench log for the recipe;
-/// the UI polls this in the bench window to show the harness early.
+/// The newest bench **job** for a recipe (running preferred, else the latest
+/// by start time). The bench window follows this job, not "whatever log the
+/// directory has": the previous campaign's log is older than the job and must
+/// never be shown as the current run.
+fn latest_bench_job(app: &AppState, recipe: &str) -> Option<super::jobs::Job> {
+    let jobs = app.jobs.lock().unwrap();
+    jobs.iter()
+        .filter(|j| j.kind == "bench" && j.recipe.as_deref() == Some(recipe))
+        .max_by_key(|j| j.started)
+        .cloned()
+}
+
+/// Pick the log file belonging to a job: only entries written at/after the
+/// job's start (a few seconds of slack for clock granularity) qualify.
+fn pick_log(entries: &[(u64, std::path::PathBuf)], cutoff: u64) -> Option<std::path::PathBuf> {
+    entries
+        .iter()
+        .filter(|(mtime, _)| *mtime + 5 >= cutoff)
+        .max_by_key(|(mtime, _)| *mtime)
+        .map(|(_, p)| p.clone())
+}
+
+/// `GET /api/bench/log?recipe=<name>` — the log of the recipe's **current**
+/// bench job, tailed for the bench window.
+///
+/// The window used to read the newest `.log` in `reports/`, so right after a
+/// launch it displayed the *previous* campaign (and stopped polling because
+/// that run had finished). The job's start time is the cutoff now; while the
+/// harness has not created its file yet the response is `waiting`, and the
+/// window keeps polling instead of declaring the run finished.
 pub(crate) async fn bench_log(
     app: State<AppState>,
     Query(req): Query<BenchLogReq>,
 ) -> Result<Json<serde_json::Value>, ApiError> {
+    let job = latest_bench_job(&app, &req.recipe);
+    let running = job
+        .as_ref()
+        .is_some_and(|j| j.status == super::jobs::JobStatus::Running);
+    let status = job
+        .as_ref()
+        .map(|j| format!("{:?}", j.status).to_lowercase());
+    let message = job.as_ref().map(|j| j.message.clone());
+    let cutoff = job.as_ref().map(|j| j.started).unwrap_or(0);
     let file =
         crate::config::recipe_file(&req.recipe, &app.config.recipe_sources).ok_or_else(|| {
             (
@@ -132,37 +169,46 @@ pub(crate) async fn bench_log(
                 format!("recipe '{}' not found", req.recipe),
             )
         })?;
-    let dir = file.parent().map(|p| p.join("reports"));
-    let Some(dir) = dir else {
-        return Err((
+    let dir = file.parent().map(|p| p.join("reports")).ok_or_else(|| {
+        (
             StatusCode::NOT_FOUND,
             "recipe has no reports dir".to_string(),
-        ));
-    };
-    let mut entries = tokio::fs::read_dir(&dir)
-        .await
-        .map_err(|e| (StatusCode::NOT_FOUND, format!("no reports dir: {e}")))?;
-    let mut latest: Option<(std::time::SystemTime, std::path::PathBuf)> = None;
-    while let Some(entry) = entries
-        .next_entry()
-        .await
-        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
-    {
-        let path = entry.path();
-        if path.extension() != Some(std::ffi::OsStr::new("log")) {
-            continue;
-        }
-        let md = entry
-            .metadata()
-            .await
-            .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
-        let m = md.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH);
-        if latest.as_ref().is_none_or(|(old, _)| m > *old) {
-            latest = Some((m, path));
+        )
+    })?;
+    let mut entries: Vec<(u64, std::path::PathBuf)> = Vec::new();
+    if let Ok(mut rd) = tokio::fs::read_dir(&dir).await {
+        while let Ok(Some(entry)) = rd.next_entry().await {
+            let path = entry.path();
+            if path.extension() != Some(std::ffi::OsStr::new("log")) {
+                continue;
+            }
+            let Ok(md) = entry.metadata().await else {
+                continue;
+            };
+            let mtime = md
+                .modified()
+                .ok()
+                .and_then(|m| m.duration_since(std::time::UNIX_EPOCH).ok())
+                .map(|d| d.as_secs())
+                .unwrap_or(0);
+            entries.push((mtime, path));
         }
     }
-    let Some((_, path)) = latest else {
-        return Err((StatusCode::NOT_FOUND, "no bench log yet".to_string()));
+    let picked = pick_log(&entries, cutoff);
+    let Some(path) = picked else {
+        // Job started but the harness has not created its log yet (the bench
+        // waits for API readiness first). Not an error — the window shows
+        // "waiting" and keeps polling.
+        return Ok(Json(serde_json::json!({
+            "waiting": true,
+            "previous": false,
+            "path": serde_json::Value::Null,
+            "lines": Vec::<String>::new(),
+            "job": job.as_ref().map(|j| j.id.clone()),
+            "running": running,
+            "status": status,
+            "message": message,
+        })));
     };
     let text = tokio::fs::read_to_string(&path)
         .await
@@ -170,20 +216,47 @@ pub(crate) async fn bench_log(
     let n = req.lines.unwrap_or(160).clamp(20, 1000);
     let all: Vec<&str> = text.lines().collect();
     let lines = all[all.len().saturating_sub(n)..].to_vec();
-    let (job, running) = {
-        let jobs = app.jobs.lock().unwrap();
-        let j = jobs
-            .iter()
-            .find(|j| j.kind == "bench" && j.recipe.as_deref() == Some(req.recipe.as_str()));
-        (
-            j.map(|j| j.id.clone()),
-            j.is_some_and(|j| j.status == super::jobs::JobStatus::Running),
-        )
-    };
     Ok(Json(serde_json::json!({
+        "waiting": false,
+        // No job at all: the pane shows the *previous* campaign's log, and the
+        // window says so instead of pretending it is the current run.
+        "previous": job.is_none(),
         "path": path.display().to_string(),
         "lines": lines,
-        "job": job,
+        "job": job.as_ref().map(|j| j.id.clone()),
         "running": running,
+        "status": status,
+        "message": message,
     })))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::pick_log;
+    use std::path::PathBuf;
+
+    fn entries() -> Vec<(u64, PathBuf)> {
+        vec![
+            (1_000, PathBuf::from("old.log")),
+            (2_000, PathBuf::from("older-but-newer.log")),
+        ]
+    }
+
+    #[test]
+    fn an_older_log_is_not_mistaken_for_the_current_run() {
+        // the job started at 3000 — both files predate it
+        assert_eq!(pick_log(&entries(), 3_000), None);
+    }
+
+    #[test]
+    fn the_log_written_after_the_job_start_wins() {
+        let mut e = entries();
+        e.push((3_100, PathBuf::from("current.log")));
+        assert_eq!(pick_log(&e, 3_000), Some(PathBuf::from("current.log")));
+        // a file created in the same second as the job counts (slack)
+        assert_eq!(
+            pick_log(&[(2_998, PathBuf::from("same-second.log"))], 3_000),
+            Some(PathBuf::from("same-second.log"))
+        );
+    }
+}

src/web/jobs.rs

@@ -48,6 +48,24 @@ fn next_id() -> String {
     format!("j{:x}", COUNTER.fetch_add(1, Ordering::Relaxed) + 1)
 }
 
+/// Resume the id counter past the persisted jobs. Without this a restarted
+/// web process re-issues `j1`, `j2`, … and `finish_job` updates the OLD job
+/// with the same id — the new one then stays `running` forever (the dashboard
+/// showed a bench "running" that had already finished, and the bench window
+/// displayed the previous campaign's log).
+fn resume_counter(jobs: &[Job]) {
+    COUNTER.fetch_max(max_suffix(jobs), Ordering::Relaxed);
+}
+
+/// Highest numeric suffix among `j<hex>` ids (0 when there are none).
+fn max_suffix(jobs: &[Job]) -> u64 {
+    jobs.iter()
+        .filter_map(|j| j.id.strip_prefix('j'))
+        .filter_map(|hex| u64::from_str_radix(hex, 16).ok())
+        .max()
+        .unwrap_or(0)
+}
+
 fn emit(
     app: &AppState,
     id: &str,
@@ -113,14 +131,20 @@ pub fn new_job(
 pub fn finish_job(app: &AppState, id: &str, ok: bool, message: String) {
     let target = {
         let mut jobs = app.jobs.lock().unwrap();
-        if let Some(job) = jobs.iter_mut().find(|j| j.id == id) {
-            job.status = if ok { JobStatus::Ok } else { JobStatus::Failed };
-            job.message = message.clone();
+        // Prefer the running job: ids can repeat in a registry written by an
+        // older build, and marking a finished job again left the real one
+        // stuck at `running`.
+        let idx = jobs
+            .iter()
+            .rposition(|j| j.id == id && j.status == JobStatus::Running)
+            .or_else(|| jobs.iter().rposition(|j| j.id == id));
+        if let Some(i) = idx {
+            jobs[i].status = if ok { JobStatus::Ok } else { JobStatus::Failed };
+            jobs[i].message = message.clone();
         }
         trim_finished(&mut jobs);
         save_locked(&jobs);
-        jobs.iter()
-            .find(|j| j.id == id)
+        idx.and_then(|i| jobs.get(i))
             .map(|j| (j.kind.clone(), j.cluster.clone(), j.recipe.clone()))
     };
     if let Some((kind, cluster, recipe)) = target {
@@ -264,6 +288,7 @@ pub(crate) fn load_from(path: Option<&std::path::Path>, app: &AppState) {
             return;
         }
     };
+    resume_counter(&loaded);
     let mut jobs = app.jobs.lock().unwrap();
     *jobs = loaded
         .into_iter()
@@ -296,3 +321,37 @@ fn save_locked(jobs: &[Job]) {
         let _ = std::fs::rename(&tmp, &path);
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn job(id: &str, kind: &str, status: JobStatus) -> Job {
+        Job {
+            id: id.into(),
+            kind: kind.into(),
+            cluster: "c".into(),
+            recipe: Some("r".into()),
+            status,
+            message: String::new(),
+            started: 1,
+            cancel: None,
+        }
+    }
+
+    #[test]
+    fn counter_resumes_past_persisted_ids() {
+        // A restarted web process must not re-issue `j1` while a persisted
+        // `j1` is still in the registry: `finish_job` would hit the old job
+        // and leave the new one running forever. `je` (14) must win over `j9`,
+        // so the next id is `jf`.
+        let jobs = [
+            job("j1", "power", JobStatus::Ok),
+            job("j9", "serve", JobStatus::Ok),
+            job("je", "bench", JobStatus::Ok),
+        ];
+        assert_eq!(max_suffix(&jobs), 14);
+        assert_eq!(max_suffix(&[]), 0);
+        assert_eq!(max_suffix(&[job("jx", "bench", JobStatus::Ok)]), 0);
+    }
+}

src/web/ui.html

@@ -2352,6 +2352,9 @@ async function openBench(recipeArg) {
     persist();
     startBtn.disabled = true;
     status.textContent = "starting…";
+    // Drop the previous campaign's text: the new job's log only exists once
+    // the harness starts, and a stale pane looks like this run already ran.
+    pre.textContent = "";
     try {
       const j = await api("/api/bench", payload);
       saveBenchJob(j.job, recipe);
@@ -2376,24 +2379,48 @@ async function openBench(recipeArg) {
       if (!res.ok) status.textContent = "cancel error: " + (j.error || res.status);
     } catch (e) { status.textContent = "cancel error: " + e.message; }
   };
+  // The poll follows the recipe's current bench JOB: the server only serves a
+  // log written after that job started, so a fresh launch never displays the
+  // previous campaign. While the harness has not created its file yet the
+  // response is `waiting` — keep polling, never declare the run finished.
   async function pollBenchLog() {
     if (!recipeSel.value) return;
     try {
       const res = await fetch("/api/bench/log?recipe=" + encodeURIComponent(recipeSel.value) + "&lines=220", { headers: AUTH() });
-      if (res.status === 404) { status.textContent = "waiting for the first log line…"; return; }
-      const j = await res.json();
-      if (!res.ok) { status.textContent = "log error: " + (j.error || res.status); return; }
+      const j = await res.json().catch(() => ({}));
+      if (!res.ok) { status.textContent = "log: " + (j.error || j.message || res.status); return; }
       if (j.job && !benchJobId) { benchJobId = j.job; saveBenchJob(j.job, recipeSel.value); }
-      pre.textContent = (j.lines || []).join("\n");
-      const atBottom = pre.scrollTop + pre.clientHeight >= pre.scrollHeight - 24;
-      if (atBottom) pre.scrollTop = pre.scrollHeight;
-      status.textContent = j.running ? "running — " + j.path : "finished — " + j.path;
+      if (!j.waiting) {
+        pre.textContent = (j.lines || []).join("\n");
+        const atBottom = pre.scrollTop + pre.clientHeight >= pre.scrollHeight - 24;
+        if (atBottom) pre.scrollTop = pre.scrollHeight;
+      }
       if (j.running) {
         cancelBtn.disabled = false;
-      } else {
+        startBtn.disabled = true;
+        status.textContent = j.waiting
+          ? "job " + (j.job || "?") + " running — waiting for the harness log…"
+          : "running — " + j.path;
+        return; // keep polling
+      }
+      if (j.previous && j.path) {
+        // no job for this recipe: the pane holds the LAST campaign's log
         cancelBtn.disabled = true;
         startBtn.disabled = false;
+        status.textContent = "last run (no active job) — " + j.path;
         stopBenchPoll();
+        return;
+      }
+      // No live job: either it finished (status/message set) or none exists.
+      cancelBtn.disabled = true;
+      startBtn.disabled = false;
+      stopBenchPoll();
+      if (j.job) {
+        status.textContent = "finished (" + (j.status || "?") + ")" +
+          (j.message && j.message !== "ok" ? ": " + j.message : "") +
+          (j.path ? " — " + j.path : "");
+      } else {
+        status.textContent = "no bench job for this recipe yet";
       }
     } catch (e) { status.textContent = "log error: " + e.message; }
   }
@@ -2404,8 +2431,15 @@ async function openBench(recipeArg) {
   }
   startBtn.onclick = startBench;
   // Reopening a closed window with a running job restores the live view.
+  // A stored job id from a previous page load: resume the live view only if
+  // the server still has that job (otherwise the window would poll a finished
+  // run forever).
   const savedJob = jobSaved.job;
-  if (savedJob) { benchJobId = savedJob; cancelBtn.disabled = true; startBenchPoll(); }
+  if (savedJob && recipeSel.value) {
+    benchJobId = savedJob;
+    cancelBtn.disabled = true;
+    startBenchPoll();
+  }
   saveWinState();
   return handle;
 }