|
|
@@ -5,6 +5,11 @@ use crate::config::Config;
|
|
|
use crate::ssh::run_ssh;
|
|
|
use std::collections::HashMap;
|
|
|
|
|
|
+/// Nodes already warned as unreachable (the probe runs every 30 s —
|
|
|
+/// log only the reachable → unreachable transition, clear on recovery).
|
|
|
+static LAST_WARNED: std::sync::LazyLock<tokio::sync::Mutex<std::collections::HashSet<String>>> =
|
|
|
+ std::sync::LazyLock::new(|| tokio::sync::Mutex::new(std::collections::HashSet::new()));
|
|
|
+
|
|
|
/// A running sm121-labeled container on a node.
|
|
|
#[derive(Clone, Debug, serde::Serialize)]
|
|
|
pub struct ContainerSnap {
|
|
|
@@ -132,6 +137,7 @@ pub async fn probe(config: &Config) -> Fleet {
|
|
|
});
|
|
|
}
|
|
|
}
|
|
|
+ LAST_WARNED.lock().await.remove(&name);
|
|
|
out.insert(
|
|
|
name,
|
|
|
WebNodeState {
|
|
|
@@ -141,8 +147,13 @@ pub async fn probe(config: &Config) -> Fleet {
|
|
|
},
|
|
|
);
|
|
|
}
|
|
|
- Ok((name, Err(_))) => {
|
|
|
- out.insert(name, WebNodeState::default());
|
|
|
+ Ok((name, Err(e))) => {
|
|
|
+ out.insert(name.clone(), WebNodeState::default());
|
|
|
+ // Warn only on the reachable → unreachable transition (the
|
|
|
+ // web polls every 30 s — a WARN per poll would be spam).
|
|
|
+ if LAST_WARNED.lock().await.insert(name.clone()) {
|
|
|
+ tracing::warn!("node probe failed: {name}: {e:#}");
|
|
|
+ }
|
|
|
}
|
|
|
Err(e) => {
|
|
|
tracing::debug!("web probe task panicked: {}", e);
|