memory: mimalloc allocator feature (add-on builds), lazy report refresh (file-event-driven, fleet cross-check rides the dashboard bus), SSH key cache, graceful web shutdown (SIGTERM) + dhat profiler feature — the Pi OOM-kills (2.7 GiB RSS) traced to musl's never-trimming allocator ratcheting on per-tick transient spikes

Cargo.lock

@@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "addr2line"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
+dependencies = [
+ "gimli",
+]
+
 [[package]]
 name = "adler2"
 version = "2.0.1"
@@ -218,6 +227,21 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "backtrace"
+version = "0.3.76"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
+dependencies = [
+ "addr2line",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+ "windows-link",
+]
+
 [[package]]
 name = "base16ct"
 version = "1.0.0"
@@ -653,6 +677,22 @@ dependencies = [
  "cipher",
 ]
 
+[[package]]
+name = "dhat"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "98cd11d84628e233de0ce467de10b8633f4ddaecafadefc86e13b84b8739b827"
+dependencies = [
+ "backtrace",
+ "lazy_static",
+ "mintex",
+ "parking_lot",
+ "rustc-hash 1.1.0",
+ "serde",
+ "serde_json",
+ "thousands",
+]
+
 [[package]]
 name = "digest"
 version = "0.11.3"
@@ -847,7 +887,7 @@ dependencies = [
  "fluent-syntax",
  "intl-memoizer",
  "intl_pluralrules",
- "rustc-hash",
+ "rustc-hash 2.1.2",
  "self_cell",
  "smallvec",
  "unic-langid",
@@ -1108,6 +1148,12 @@ dependencies = [
  "polyval",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.32.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
+
 [[package]]
 name = "globset"
 version = "0.4.18"
@@ -1644,6 +1690,15 @@ version = "0.2.186"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.49"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "libredox"
 version = "0.1.17"
@@ -1719,6 +1774,15 @@ version = "2.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
+[[package]]
+name = "mimalloc"
+version = "0.1.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
+dependencies = [
+ "libmimalloc-sys",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -1735,6 +1799,12 @@ dependencies = [
  "simd-adler32",
 ]
 
+[[package]]
+name = "mintex"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c505b3e17ed6b70a7ed2e67fbb2c560ee327353556120d6e72f5232b6880d536"
+
 [[package]]
 name = "mio"
 version = "1.2.1"
@@ -1848,6 +1918,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "object"
+version = "0.37.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.21.4"
@@ -2133,7 +2212,7 @@ dependencies = [
  "pin-project-lite",
  "quinn-proto",
  "quinn-udp",
- "rustc-hash",
+ "rustc-hash 2.1.2",
  "rustls",
  "socket2",
  "thiserror",
@@ -2154,7 +2233,7 @@ dependencies = [
  "lru-slab",
  "rand 0.9.4",
  "ring",
- "rustc-hash",
+ "rustc-hash 2.1.2",
  "rustls",
  "rustls-pki-types",
  "slab",
@@ -2458,6 +2537,18 @@ dependencies = [
  "wasm-bindgen-futures",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb"
+
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.2"
@@ -2856,12 +2947,14 @@ dependencies = [
  "axum",
  "base64",
  "clap",
+ "dhat",
  "dirs",
  "flate2",
  "fluent-i18n",
  "futures",
  "getrandom 0.3.4",
  "md5 0.7.0",
+ "mimalloc",
  "notify",
  "reqwest",
  "russh",
@@ -3062,6 +3155,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "thousands"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820"
+
 [[package]]
 name = "thread_local"
 version = "1.1.9"
@@ -3318,7 +3417,7 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90"
 dependencies = [
- "rustc-hash",
+ "rustc-hash 2.1.2",
 ]
 
 [[package]]

Cargo.toml

@@ -4,6 +4,18 @@ version = "0.1.0"
 edition = "2024"
 license = "MIT"
 
+[features]
+# Opt-in heap profiler (dhat): builds with a counting global allocator and
+# dumps per-callsite retained-heap stats on graceful exit (dhat.json in cwd).
+# Use for leak hunting only — never ship it (measured allocation overhead).
+# (Mutually exclusive with `mimalloc` — both install a global allocator.)
+dhat = ["dep:dhat"]
+# mimalloc as the global allocator (the add-on image build uses it): unlike
+# musl's default malloc it aggressively returns memory to the OS, which the
+# long-running `sm121 web` on the 4 GB Pi needs — musl's never-trimming
+# allocator ratchets RSS up with every transient spike (the 2.7 GiB OOMs).
+mimalloc = ["dep:mimalloc"]
+
 [dependencies]
 clap = { version = "4.6", features = ["derive"] }
 tokio = { version = "1", features = ["full"] }
@@ -28,3 +40,6 @@ flate2 = "1"
 tar = "0.4"
 fluent-i18n = "0.1.0"
 axum = "0.8"
+
+dhat = { version = "0.3", optional = true }
+mimalloc = { version = "0.1", optional = true }

src/main.rs

@@ -1,4 +1,17 @@
 #![deny(clippy::wildcard_imports)]
+// Opt-in heap profiler (the `dhat` feature): a counting global allocator that
+// dumps per-callsite retained-heap stats on graceful exit. Debug builds only
+// in practice — the normal release build never pays for it.
+#[cfg(feature = "dhat")]
+#[global_allocator]
+static DHAT_ALLOC: dhat::Alloc = dhat::Alloc;
+
+// mimalloc (add-on image builds): trims back to the OS — see the feature
+// doc in Cargo.toml for why the Pi needs it.
+#[cfg(feature = "mimalloc")]
+#[global_allocator]
+static MI_ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
 use crate::docker::sync;
 use anyhow::Result;
 use clap::Parser;

src/report/mod.rs

@@ -430,6 +430,7 @@ pub async fn run(
         initial,
         show_archived,
         "",
+        None, // CLI `report --serve`: no dashboard bus — the cross-check probes directly
     );
     if open {
         let url = format!("http://{addr}/");

src/report/serve.rs

@@ -65,18 +65,22 @@ fn is_relevant(ev: &notify::Event) -> bool {
     })
 }
 
-/// Background refresher: watch the recipe dirs, re-collect after a debounced
-/// filesystem event (or the keepalive safety net), publish the rows JSON on
-/// the watch channel only when it changed. The first collect runs
-/// immediately so clients see current data without waiting. If the watcher
-/// cannot be created (unsupported FS), the keepalive alone keeps the data
-/// fresh.
+/// Background refresher — lazy by design (the web process must stay small;
+/// a full collect scans every recipe + report file): a full re-collect runs
+/// ONLY on a debounced filesystem event (or once at startup); the keepalive
+/// tick just re-runs the live stale cross-check (the only row field that
+/// changes without a file change) and only when a run is open — otherwise it
+/// does zero work. The cross-check rides on the main dashboard's fleet
+/// probe (`fleet_rx` bus State events — no extra SSH) and falls back to a
+/// direct probe when no bus is wired (CLI `report --serve`).
 pub(crate) async fn refresh_loop(
     cfg: Arc<Config>,
     tx: tokio::sync::watch::Sender<String>,
     keepalive: Duration,
     archived: bool,
+    fleet_rx: Option<tokio::sync::broadcast::Receiver<crate::web::events::Event>>,
 ) {
+    let mut fleet_rx = fleet_rx;
     let (ev_tx, mut ev_rx) = tokio::sync::mpsc::unbounded_channel::<()>();
     let watcher = notify::recommended_watcher(move |res: Result<notify::Event, notify::Error>| {
         if let Ok(ev) = res
@@ -94,47 +98,111 @@ pub(crate) async fn refresh_loop(
         }
     }
     let mut last = String::new();
+    let mut rows: Vec<crate::report::ReportRow> = Vec::new();
+    let mut fleet: Option<crate::web::events::Snapshot> = None;
+    let cluster_nodes: HashMap<String, Vec<String>> = cfg
+        .clusters
+        .iter()
+        .map(|c| {
+            (
+                c.name.clone(),
+                c.nodes.iter().map(|n| n.name.clone()).collect(),
+            )
+        })
+        .collect();
+    let has_open_run = |rows: &[crate::report::ReportRow]| {
+        rows.iter()
+            .any(|r| r.up.is_some() && r.up_cluster.is_some())
+    };
     loop {
         tokio::select! {
-            _ = ev_rx.recv() => {}
-            _ = tokio::time::sleep(keepalive) => {}
+            // Filesystem event (debounced) → the only full-rescan trigger.
+            _ = ev_rx.recv() => {
+                // debounce: let the burst settle, then drain the remaining events
+                tokio::time::sleep(DEBOUNCE).await;
+                while ev_rx.try_recv().is_ok() {}
+                rows = collect(&cfg, archived);
+                cross_check(&cfg, &mut rows, &cluster_nodes, &mut fleet).await;
+                let json = serde_json::to_string(&rows).unwrap_or_default();
+                if json != last {
+                    last = json;
+                    let _ = tx.send(last.clone());
+                }
+            }
+            // Bus State event (the main dashboard's 30 s fleet probe) →
+            // just track the fleet; the keepalive tick consumes it.
+            ev = async {
+                match fleet_rx.as_mut() {
+                    Some(rx) => rx.recv().await,
+                    None => std::future::pending::<Result<
+                        crate::web::events::Event,
+                        tokio::sync::broadcast::error::RecvError,
+                    > >().await,
+                }
+            } => {
+                if let Ok(crate::web::events::Event::State { snapshot, .. }) = ev {
+                    fleet = Some(snapshot);
+                }
+            }
+            // Keepalive safety net (missed watcher events): full collect only
+            // while uncollected (startup); afterwards only the cheap
+            // cross-check — and only when a run is open.
+            _ = tokio::time::sleep(keepalive) => {
+                if rows.is_empty() {
+                    rows = collect(&cfg, archived);
+                    cross_check(&cfg, &mut rows, &cluster_nodes, &mut fleet).await;
+                } else if has_open_run(&rows) {
+                    cross_check(&cfg, &mut rows, &cluster_nodes, &mut fleet).await;
+                } else {
+                    continue; // nothing to do — zero work, zero allocation
+                }
+                let json = serde_json::to_string(&rows).unwrap_or_default();
+                if json != last {
+                    last = json;
+                    let _ = tx.send(last.clone());
+                }
+            }
         }
-        // debounce: let the burst settle, then drain the remaining events
-        tokio::time::sleep(DEBOUNCE).await;
-        while ev_rx.try_recv().is_ok() {}
-        let mut rows = collect(&cfg, archived);
-        // Live cross-check: an open ledger run with no running container
-        // anywhere (all its nodes reachable) is marked stale — otherwise a
-        // killed serve or a node that died before `stop` could verify would
-        // show as "up" forever. No open run with a cluster → no probe.
-        if rows
-            .iter()
-            .any(|r| r.up.is_some() && r.up_cluster.is_some())
-        {
-            let fleet = crate::web::probe::probe(&cfg).await;
-            let mut cluster_nodes: HashMap<String, Vec<String>> = HashMap::new();
-            for c in &cfg.clusters {
-                cluster_nodes.insert(
-                    c.name.clone(),
-                    c.nodes.iter().map(|n| n.name.clone()).collect(),
-                );
+    }
+}
+
+/// Reconcile the open-ledger rows against the live fleet: prefer the main
+/// dashboard's last fleet snapshot (no extra SSH); probe directly only when
+/// no snapshot has arrived yet (CLI `report --serve`, first 30 s of web).
+async fn cross_check(
+    cfg: &Config,
+    rows: &mut [crate::report::ReportRow],
+    cluster_nodes: &HashMap<String, Vec<String>>,
+    fleet: &mut Option<crate::web::events::Snapshot>,
+) {
+    let (reachable, live) = match fleet {
+        Some(snap) => {
+            let mut reachable: HashMap<String, bool> = HashMap::new();
+            let mut live: std::collections::HashSet<String> = std::collections::HashSet::new();
+            for c in &snap.clusters {
+                for n in &c.nodes {
+                    reachable.insert(n.name.clone(), n.reachable);
+                    for ct in &n.containers {
+                        live.insert(ct.recipe.clone());
+                    }
+                }
             }
+            (reachable, live)
+        }
+        None => {
+            let f = crate::web::probe::probe(cfg).await;
             let mut reachable: HashMap<String, bool> = HashMap::new();
             let mut live: std::collections::HashSet<String> = std::collections::HashSet::new();
-            for (node, st) in &fleet.nodes {
+            for (node, st) in &f.nodes {
                 reachable.insert(node.clone(), st.reachable);
                 for c in &st.containers {
                     live.insert(c.recipe.clone());
                 }
             }
-            crate::report::stale::mark_stale(&mut rows, &cluster_nodes, &reachable, &live);
+            (reachable, live)
         }
-        let json = serde_json::to_string(&rows).unwrap_or_default();
-        if json != last {
-            last = json;
-            let _ = tx.send(last.clone());
-        }
-    }
+    };
+    crate::report::stale::mark_stale(rows, cluster_nodes, &reachable, &live);
 }
 
 /// `GET /events` — the latest rows JSON on connect, then one message per
@@ -389,9 +457,16 @@ pub(crate) fn router(
     initial: String,
     archived: bool,
     base: &str,
+    fleet_rx: Option<tokio::sync::broadcast::Receiver<crate::web::events::Event>>,
 ) -> Router {
     let (tx, rx) = tokio::sync::watch::channel(initial);
-    tokio::spawn(refresh_loop(config.clone(), tx, KEEPALIVE, archived));
+    tokio::spawn(refresh_loop(
+        config.clone(),
+        tx,
+        KEEPALIVE,
+        archived,
+        fleet_rx,
+    ));
     Router::new()
         .route(
             "/",

src/report/tests.rs

@@ -413,7 +413,13 @@ async fn refresher_publishes_on_change_only() {
     let (tx, mut rx) = tokio::sync::watch::channel(String::new());
     // keepalive 100 ms: if the file watcher fires, the update lands via the
     // event path; if not, the keepalive picks it up — either way within 3 s
-    let h = tokio::spawn(refresh_loop(cfg, tx, Duration::from_millis(100), false));
+    let h = tokio::spawn(refresh_loop(
+        cfg,
+        tx,
+        Duration::from_millis(100),
+        false,
+        None,
+    ));
     // the first collect runs immediately: the fixture's report is visible
     tokio::time::timeout(
         Duration::from_secs(3),

src/report/ui_tests.rs

@@ -125,7 +125,7 @@ async fn report_playwright_checks() {
     let initial = serde_json::to_string(&rows).unwrap();
     let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
     let addr = listener.local_addr().unwrap();
-    let app = serve::router(cfg, page, initial, false, "");
+    let app = serve::router(cfg, page, initial, false, "", None);
     let _h = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
     let reports = dir.join("test-recipe-nvfp4/reports");
     // Serialize with the web dashboard's Playwright suite (shared browser).

src/ssh.rs

@@ -40,7 +40,7 @@ async fn ssh_session_with_timeout(
              config nor a cluster source provides them (see `sm121 config path`)"
         );
     }
-    let key = load_secret_key(&config.ssh_key_path)?;
+    let key = cached_secret_key(&config.ssh_key_path)?;
     let ssh_config = russh::client::Config {
         inactivity_timeout,
         ..Default::default()
@@ -76,10 +76,7 @@ async fn ssh_session_with_timeout(
     ))??;
 
     session
-        .authenticate_publickey(
-            &config.ssh_user,
-            PrivateKeyWithHashAlg::new(Arc::new(key), None),
-        )
+        .authenticate_publickey(&config.ssh_user, PrivateKeyWithHashAlg::new(key, None))
         .await?;
 
     Ok(session)
@@ -270,7 +267,7 @@ async fn fetch_host_key_data(config: &Config, node: &Node) -> Result<Vec<u8>> {
         }
     }
 
-    let key = load_secret_key(&config.ssh_key_path)?;
+    let key = cached_secret_key(&config.ssh_key_path)?;
     let ssh_config = russh::client::Config::default();
     let handler = CaptureHandler {
         captured: captured_clone,
@@ -290,10 +287,7 @@ async fn fetch_host_key_data(config: &Config, node: &Node) -> Result<Vec<u8>> {
     ))??;
 
     session
-        .authenticate_publickey(
-            &config.ssh_user,
-            PrivateKeyWithHashAlg::new(Arc::new(key), None),
-        )
+        .authenticate_publickey(&config.ssh_user, PrivateKeyWithHashAlg::new(key, None))
         .await?;
 
     captured
@@ -445,6 +439,25 @@ pub fn load_secret_key(path: &Path) -> Result<russh::keys::PrivateKey> {
     russh_load_secret_key(path, None).context("Failed to load SSH key")
 }
 
+/// Parsed private keys, cached by canonical path. The web dashboard opens a
+/// fresh SSH session per node every 30 s (the fleet probe) — re-reading and
+/// re-parsing the key file on every session was pure waste; the key is
+/// static per deployment, so parse it once and share the `Arc`.
+static KEY_CACHE: std::sync::LazyLock<
+    std::sync::Mutex<std::collections::HashMap<std::path::PathBuf, Arc<russh::keys::PrivateKey>>>,
+> = std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashMap::new()));
+
+fn cached_secret_key(path: &Path) -> Result<Arc<russh::keys::PrivateKey>> {
+    let mut cache = KEY_CACHE.lock().unwrap();
+    let key_path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
+    if let Some(k) = cache.get(&key_path) {
+        return Ok(Arc::clone(k));
+    }
+    let k = Arc::new(load_secret_key(path)?);
+    cache.insert(key_path, Arc::clone(&k));
+    Ok(k)
+}
+
 /// Handler that verifies the server's host key against an expected fingerprint.
 pub struct VerifyingHandler {
     pub expected_key: Arc<std::sync::Mutex<Option<String>>>,

src/web/mod.rs

@@ -284,6 +284,7 @@ pub fn router(app: AppState) -> Router {
         serde_json::to_string(&rows).unwrap_or_default(),
         false,
         "/recipes",
+        Some(app.bus.subscribe()),
     )
     .layer(middleware::from_fn_with_state(app.clone(), token_guard));
     Router::new()
@@ -362,7 +363,20 @@ pub async fn run(
         });
     }
     tracing::info!("sm121 web: http://{addr}/ (poll every {POLL_SECS}s)");
-    axum::serve(listener, router(app)).await?;
+    // Graceful shutdown: SIGTERM/SIGINT finish in-flight requests, then the
+    // process exits cleanly (container stops, dhat heap dump on exit).
+    let shutdown = async {
+        let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
+            .expect("SIGTERM handler");
+        tokio::select! {
+            _ = tokio::signal::ctrl_c() => {},
+            _ = term.recv() => {},
+        }
+        tracing::info!("sm121 web: shutdown signal — finishing in-flight requests");
+    };
+    axum::serve(listener, router(app))
+        .with_graceful_shutdown(shutdown)
+        .await?;
     Ok(())
 }