cache: 'sm121 cache status|doctor' — the shared HF cache, read-only first

The cache lives on the NAS and an add-on on a Pi has no mount, so the command
reaches it over SSH through the cluster: [cache].node pins the node (default:
rank 0 of --cluster), [cache].hf_home the root. 'status' lists cached repos with
their sizes and the room left (--json for the dashboard); 'doctor' reports
whether hfrs/uvx/hf/flare exist on that node, that the root is writable and how
much room is left — the checks the update job silently needs today.

'update'/'prune' are deliberately not in this commit: read-only first.

src/after_stop.rs

@@ -166,6 +166,7 @@ mod tests {
 
     fn test_config() -> Config {
         Config {
+            cache: None,
             flare: None,
             ssh_user: "u".to_string(),
             ssh_key_path: "/k".into(),

src/before_serve_tests.rs

@@ -33,6 +33,7 @@ fn test_cluster() -> Cluster {
 
 fn test_config(before_serve: HashMap<String, crate::config::BeforeServeTable>) -> Config {
     Config {
+        cache: None,
         flare: None,
         ssh_user: "u".to_string(),
         ssh_key_path: "/k".into(),

src/cache.rs

@@ -0,0 +1,175 @@
+//! Shared Hugging Face cache management: `sm121 cache status|doctor`.
+//!
+//! The cache lives on the NAS and is reached over SSH through the cluster (an
+//! add-on on a Pi has no NAS mount, so this is the only path that works
+//! everywhere). A node that holds the mount runs the actual inspection —
+//! `[cache].node` pins it, otherwise the rank-0 node of `--cluster` is used.
+//!
+//! Read-only here: `status` reports what is cached and how much room is left,
+//! `doctor` reports whether the tooling (`hfrs`, `uvx`, `flare`) that the
+//! update job needs is actually present. Destructive steps (`update`, `prune`)
+//! are separate subcommands so they never run by accident.
+
+use anyhow::Result;
+use serde_json::json;
+
+use crate::config::Config;
+
+/// Env/CLI knobs for the cache commands.
+pub struct CacheOpts {
+    pub cluster: Option<String>,
+    pub json: bool,
+    pub dry_run: bool,
+}
+
+/// The node that can see the cache: `[cache].node`, else rank 0 of the chosen
+/// cluster, else rank 0 of the first cluster.
+fn cache_node(config: &Config, cluster: Option<&str>) -> Result<crate::config::Node> {
+    if let Some(name) = config.cache.as_ref().and_then(|c| c.node.clone()) {
+        return config
+            .clusters
+            .iter()
+            .flat_map(|c| c.nodes.iter())
+            .find(|n| n.name == name)
+            .cloned()
+            .ok_or_else(|| anyhow::anyhow!("[cache].node '{name}' is not a configured node"));
+    }
+    let cl = match cluster {
+        Some(name) => crate::config::select_cluster_by_name(config, name)?,
+        None => config
+            .clusters
+            .first()
+            .cloned()
+            .ok_or_else(|| anyhow::anyhow!("no clusters configured"))?,
+    };
+    cl.nodes
+        .iter()
+        .find(|n| n.rank == 0)
+        .or_else(|| cl.nodes.first())
+        .cloned()
+        .ok_or_else(|| anyhow::anyhow!("cluster '{}' has no nodes", cl.name))
+}
+
+/// HF cache root (config `[cache].hf_home`, else the ssh user's default).
+pub fn hf_home(config: &Config) -> String {
+    config
+        .cache
+        .as_ref()
+        .and_then(|c| c.hf_home.clone())
+        .unwrap_or_else(|| config.resolved_hf_cache_dir().display().to_string())
+}
+
+/// CLI dispatch (`sm121 cache <subcommand>`).
+pub async fn dispatch(
+    config: &Config,
+    cmd: &crate::cli::CacheCommands,
+    dry_run: bool,
+) -> Result<()> {
+    use crate::cli::CacheCommands as C;
+    let (cluster, json) = match cmd {
+        C::Status { cluster, json } | C::Doctor { cluster, json } => (cluster.clone(), *json),
+    };
+    let opts = CacheOpts {
+        cluster,
+        json,
+        dry_run,
+    };
+    match cmd {
+        C::Status { .. } => status(config, &opts).await,
+        C::Doctor { .. } => doctor(config, &opts).await,
+    }
+}
+
+/// `sm121 cache status` — repos in the cache, their sizes, and the room left.
+pub async fn status(config: &Config, opts: &CacheOpts) -> Result<()> {
+    let node = cache_node(config, opts.cluster.as_deref())?;
+    let home = hf_home(config);
+    let root = crate::docker::shq(&home);
+    let cmd = format!(
+        "echo \"__HOME {home}\"; \
+         df -BG --output=size,used,avail {root} 2>/dev/null | tail -1; \
+         du -s --block-size=1 {root}/hub/*--* 2>/dev/null | awk '{{print $1, $2}}'"
+    );
+    let out = crate::ssh::run_ssh(config, &node, cmd, opts.dry_run).await?;
+    let mut free = String::new();
+    let mut repos: Vec<(u64, String)> = Vec::new();
+    for line in out.lines() {
+        if let Some(rest) = line.strip_prefix("__HOME ") {
+            if opts.json {
+                free = rest.to_string();
+            } else {
+                println!("cache: {rest} (on {})", node.name);
+            }
+            continue;
+        }
+        let mut it = line.split_whitespace();
+        if let (Some(bytes), Some(path)) = (it.next(), it.next())
+            && let Ok(b) = bytes.parse::<u64>()
+        {
+            let name = path.rsplit('/').next().unwrap_or(path).to_string();
+            repos.push((b, name));
+        }
+    }
+    repos.sort_by_key(|(b, _)| std::cmp::Reverse(*b));
+    if opts.json {
+        let items: Vec<_> = repos
+            .iter()
+            .map(|(b, n)| json!({"repo": n, "bytes": b, "gib": (*b as f64) / (1u64 << 30) as f64}))
+            .collect();
+        println!(
+            "{}",
+            serde_json::to_string_pretty(&json!({
+                "node": node.name, "hf_home": home, "df": free, "repos": items,
+            }))?
+        );
+        return Ok(());
+    }
+    let total: u64 = repos.iter().map(|(b, _)| b).sum();
+    println!("{:<58} {:>9}", "repo", "GiB");
+    for (b, n) in &repos {
+        println!("{:<58} {:>9.1}", n, *b as f64 / (1u64 << 30) as f64);
+    }
+    println!(
+        "{:<58} {:>9.1}",
+        format!("— {} repos", repos.len()),
+        total as f64 / (1u64 << 30) as f64
+    );
+    Ok(())
+}
+
+/// `sm121 cache doctor` — is the tooling the maintenance job needs present?
+pub async fn doctor(config: &Config, opts: &CacheOpts) -> Result<()> {
+    let node = cache_node(config, opts.cluster.as_deref())?;
+    let home = hf_home(config);
+    let cmd = format!(
+        "for t in hfrs uvx flare hf; do printf '%s=%s\\n' \"$t\" \"$(command -v $t || echo MISSING)\"; done; \
+         [ -d {root} ] && echo \"hf_home=ok\" || echo \"hf_home=MISSING\"; \
+         touch {root}/.sm121-write-test 2>/dev/null && echo write=ok && rm -f {root}/.sm121-write-test || echo write=FAILED; \
+         df -BG --output=avail {root} 2>/dev/null | tail -1 | sed 's/^/free=/'",
+        root = crate::docker::shq(&home)
+    );
+    let out = crate::ssh::run_ssh(config, &node, cmd, opts.dry_run).await?;
+    if opts.json {
+        let map: serde_json::Map<String, serde_json::Value> = out
+            .lines()
+            .filter_map(|l| l.split_once('='))
+            .map(|(k, v)| (k.trim().to_string(), json!(v.trim())))
+            .collect();
+        println!(
+            "{}",
+            serde_json::to_string_pretty(&json!({"node": node.name, "checks": map}))?
+        );
+        return Ok(());
+    }
+    println!(
+        "cache doctor on {} ({}), keep_revs={:?}:",
+        node.name,
+        home,
+        config.cache.as_ref().and_then(|c| c.keep_revs)
+    );
+    for line in out.lines() {
+        let ok = !line.contains("MISSING") && !line.contains("FAILED");
+        println!("  {} {}", if ok { "✓" } else { "✗" }, line);
+    }
+    Ok(())
+}

src/cli.rs

@@ -309,6 +309,13 @@ pub enum Commands {
     #[command(subcommand)]
     Runlock(RunlockCommands),
 
+    /// Shared Hugging Face cache on the NAS: `status` / `doctor` (read-only
+    /// here; update/prune are separate subcommands).
+    Cache {
+        #[command(subcommand)]
+        cmd: CacheCommands,
+    },
+
     /// Manage the configured git recipe repos (clone/pull cache, archive
     /// registry). `sm121 repos` lists state; `sm121 repos sync [--force]`
     /// re-pulls.
@@ -439,3 +446,24 @@ pub enum Commands {
 #[cfg(test)]
 #[path = "cli_tests.rs"]
 mod tests;
+
+/// `sm121 cache …`
+#[derive(Subcommand)]
+pub enum CacheCommands {
+    /// What is cached and how much room is left (read-only)
+    Status {
+        /// Cluster whose rank-0 node inspects the cache
+        #[arg(long)]
+        cluster: Option<String>,
+        /// Machine-readable output (for the dashboard)
+        #[arg(long)]
+        json: bool,
+    },
+    /// Is the tooling the maintenance job needs present on that node?
+    Doctor {
+        #[arg(long)]
+        cluster: Option<String>,
+        #[arg(long)]
+        json: bool,
+    },
+}

src/cluster_state/tests.rs

@@ -190,6 +190,7 @@ fn select_smallest_with_enough_free() {
     );
 
     let cfg = Config {
+        cache: None,
         flare: None,
         ssh_user: "u".to_string(),
         ssh_key_path: "/k".into(),
@@ -232,6 +233,7 @@ fn select_smallest_with_enough_free() {
 fn select_cluster_not_found_errors() {
     let state = ClusterState::default();
     let cfg = Config {
+        cache: None,
         flare: None,
         ssh_user: "u".to_string(),
         ssh_key_path: "/k".into(),
@@ -266,6 +268,7 @@ fn no_cluster_has_enough_free_errors() {
         },
     );
     let cfg = Config {
+        cache: None,
         flare: None,
         ssh_user: "u".to_string(),
         ssh_key_path: "/k".into(),
@@ -304,6 +307,7 @@ fn single_node_allows_cluster_without_nccl() {
         },
     );
     let cfg = Config {
+        cache: None,
         flare: None,
         ssh_user: "u".to_string(),
         ssh_key_path: "/k".into(),
@@ -348,6 +352,7 @@ fn multi_node_requires_nccl() {
         },
     );
     let cfg = Config {
+        cache: None,
         flare: None,
         ssh_user: "u".to_string(),
         ssh_key_path: "/k".into(),

src/config.rs

@@ -56,6 +56,7 @@ mod staging;
 pub use staging::{Staging, StagingFormat, StagingKind, StagingVerify};
 mod types;
 pub use types::*;
+pub(crate) mod cache_settings;
 pub(crate) mod flare_cluster;
 mod hf_cache;
 

src/config/cache_settings.rs

@@ -0,0 +1,16 @@
+//! `[cache]` — how `sm121 cache` reaches the shared Hugging Face cache.
+
+/// `[cache]` — how `sm121 cache` reaches the shared Hugging Face cache.
+#[derive(Clone, Debug, Default, serde::Deserialize)]
+pub struct CacheSettings {
+    /// Node that holds the mount and runs the maintenance (default: rank 0 of
+    /// `--cluster`, else of the first cluster).
+    #[serde(default)]
+    pub node: Option<String>,
+    /// Cache root on that node (default: the ssh user's `~/.cache/huggingface`).
+    #[serde(default)]
+    pub hf_home: Option<String>,
+    /// Revisions kept per repo by `cache prune`.
+    #[serde(default)]
+    pub keep_revs: Option<u32>,
+}

src/config/tests.rs

@@ -30,6 +30,7 @@ fn tcls() -> Cluster {
 
 fn tcfg() -> Config {
     Config {
+        cache: None,
         flare: None,
         ssh_user: "u".to_string(),
         ssh_key_path: "/k".into(),
@@ -51,7 +52,6 @@ fn tcfg() -> Config {
         registry: None,
     }
 }
-
 #[test]
 fn registry_section_parses_and_defaults_to_none() {
     let with_reg: Config =

src/config/types.rs

@@ -70,9 +70,13 @@ pub struct Config {
     pub exec_aliases: HashMap<String, String>,
     #[serde(default)]
     pub home_assistant: Option<HomeAssistantConfig>,
-    /// RouterOS devices for graceful `power off` (see [`MikrotikConfig`]).
+    /// RouterOS devices for graceful `power off` ([`MikrotikConfig`]).
     #[serde(default)]
     pub mikrotik: Option<MikrotikConfig>,
+    /// Shared HF cache management (`sm121 cache`): where it lives, which node
+    /// may touch it, how many revisions to keep.
+    #[serde(default)]
+    pub cache: Option<super::cache_settings::CacheSettings>,
     /// FLARE deploy settings (private: the publisher acceptance key must never
     /// reach the shared recipe repo).
     #[serde(default)]
@@ -483,8 +487,7 @@ pub struct FlareSettings {
     /// the machine it was written for.
     #[serde(default)]
     pub clusters: std::collections::HashMap<String, super::flare_cluster::FlareCluster>,
-    /// Local package mirror dir on the controller (see `native::flare`):
-    /// `serve` copies the package there and launches from the local copy.
+    /// Local package mirror dir on the controller (see `native::flare`).
     #[serde(default)]
     pub local_dir: Option<String>,
     /// `[cluster.<name>]` in `~/.config/flare/orbit.toml` (default `sparks`).

src/detect.rs

@@ -143,6 +143,7 @@ mod tests {
             entity: None,
         };
         let config = crate::config::Config {
+            cache: None,
             flare: None,
             ssh_user: "u".into(),
             ssh_key_path: std::path::PathBuf::from("/p"),

src/docker/command_tests.rs

@@ -37,6 +37,7 @@ fn tnccl() -> NcclConfig {
 
 fn minimal_config() -> Config {
     Config {
+        cache: None,
         flare: None,
         ssh_user: "u".into(),
         ssh_key_path: "/tmp/key".into(),

src/docker/mirror.rs

@@ -188,6 +188,7 @@ mod tests {
 
     fn cfg() -> Config {
         Config {
+            cache: None,
             flare: None,
             ssh_user: "u".into(),
             ssh_key_path: "/tmp/k".into(),

src/exec.rs

@@ -159,6 +159,7 @@ mod tests {
             .map(|(k, v)| (k.to_string(), v.to_string()))
             .collect();
         Config {
+            cache: None,
             flare: None,
             ssh_user: "u".to_string(),
             ssh_key_path: "/k".into(),

src/main.rs

@@ -29,6 +29,7 @@ mod bench_report;
 mod bench_target;
 mod build_info;
 mod builtin_scripts;
+mod cache;
 mod cli;
 mod cli_sub;
 mod cluster_state;
@@ -379,6 +380,7 @@ async fn main() -> Result<()> {
             .await?;
         }
         Commands::Runlock(cmd) => crate::runlock::run(&config, &cmd)?,
+        Commands::Cache { cmd } => crate::cache::dispatch(&config, &cmd, dry_run).await?,
         Commands::Repos(a) => {
             let cmd = a.command.clone().unwrap_or(crate::cli::ReposCommands::List);
             crate::recipe_repos::run(&config, &cmd, dry_run).await?

src/orchestrate/tests.rs

@@ -80,6 +80,7 @@ fn test_config(clusters: Vec<Cluster>) -> Config {
         nccl.insert(c.name.clone(), test_nccl());
     }
     Config {
+        cache: None,
         flare: None,
         ssh_user: "u".to_string(),
         ssh_key_path: "/k".into(),

src/promote_tests.rs

@@ -84,6 +84,7 @@ fn config_with_repos(urls: &[String]) -> Config {
         })
         .collect();
     Config {
+        cache: None,
         flare: None,
         ssh_user: "u".to_string(),
         ssh_key_path: "/k".into(),

src/staging.rs

@@ -270,6 +270,7 @@ mod tests {
 
     fn cfg() -> Config {
         Config {
+            cache: None,
             flare: None,
             ssh_user: "u".into(),
             ssh_key_path: "/k".into(),

src/web/events.rs

@@ -189,6 +189,7 @@ mod tests {
 
     fn config() -> Config {
         Config {
+            cache: None,
             flare: None,
             ssh_user: "u".into(),
             ssh_key_path: "/k".into(),

src/web/tests.rs

@@ -19,6 +19,7 @@ pub(crate) async fn call(addr: SocketAddr, path: &str) -> reqwest::Response {
 /// machine-independent (no real SSH, no hardware).
 pub(crate) fn fake_config() -> Config {
     Config {
+        cache: None,
         flare: None,
         ssh_user: "u".into(),
         ssh_key_path: "/nonexistent/key".into(),

src/web/topo.rs

@@ -168,6 +168,7 @@ mod tests {
     #[test]
     fn derive_topo_fabric_dac_lan() {
         let config = Config {
+            cache: None,
             flare: None,
             ssh_user: "u".into(),
             ssh_key_path: "/k".into(),