Browse Source

Log probe SSH failures (previously swallowed as Err(_))

A node silently going unreachable made the dashboard show False with no
reason anywhere. cluster_state::probe warns per call (one-shot CLI
paths); the web probe warns only on the reachable → unreachable
transition (a static set cleared on recovery) so the 30 s poll does not
spam.
Alexander Korolev 1 week ago
parent
commit
e00a59ec49
2 changed files with 16 additions and 4 deletions
  1. 3 2
      src/cluster_state.rs
  2. 13 2
      src/web/probe.rs

+ 3 - 2
src/cluster_state.rs

@@ -97,15 +97,16 @@ impl ClusterState {
                         },
                     );
                 }
-                Ok((name, Err(_))) => {
+                Ok((name, Err(e))) => {
                     // SSH failure → unreachable, cannot be deployed on
                     nodes.insert(
-                        name,
+                        name.clone(),
                         NodeState {
                             reachable: false,
                             occupied: false,
                         },
                     );
+                    warn!("node probe failed: {name}: {e:#}");
                 }
                 Err(e) => {
                     // A probe task panicked (JoinError) — log it so a node

+ 13 - 2
src/web/probe.rs

@@ -5,6 +5,11 @@ use crate::config::Config;
 use crate::ssh::run_ssh;
 use std::collections::HashMap;
 
+/// Nodes already warned as unreachable (the probe runs every 30 s —
+/// log only the reachable → unreachable transition, clear on recovery).
+static LAST_WARNED: std::sync::LazyLock<tokio::sync::Mutex<std::collections::HashSet<String>>> =
+    std::sync::LazyLock::new(|| tokio::sync::Mutex::new(std::collections::HashSet::new()));
+
 /// A running sm121-labeled container on a node.
 #[derive(Clone, Debug, serde::Serialize)]
 pub struct ContainerSnap {
@@ -132,6 +137,7 @@ pub async fn probe(config: &Config) -> Fleet {
                         });
                     }
                 }
+                LAST_WARNED.lock().await.remove(&name);
                 out.insert(
                     name,
                     WebNodeState {
@@ -141,8 +147,13 @@ pub async fn probe(config: &Config) -> Fleet {
                     },
                 );
             }
-            Ok((name, Err(_))) => {
-                out.insert(name, WebNodeState::default());
+            Ok((name, Err(e))) => {
+                out.insert(name.clone(), WebNodeState::default());
+                // Warn only on the reachable → unreachable transition (the
+                // web polls every 30 s — a WARN per poll would be spam).
+                if LAST_WARNED.lock().await.insert(name.clone()) {
+                    tracing::warn!("node probe failed: {name}: {e:#}");
+                }
             }
             Err(e) => {
                 tracing::debug!("web probe task panicked: {}", e);