feat: health checks NCCL socket interface status per node

- Verifies socket_ifname (from NCCL config) is UP on each online node
- Shows [net: socket:DOWN] per node when interface is down
- Cluster header shows 🌐 interface down when any node has net issues
- 4nodes-switch: socket:DOWN on reachable nodes (switch is off)
- 2nodes-dac: socket:UP, no warnings

src/docker/health.rs

@@ -13,12 +13,15 @@ struct NodeHealth {
     ib_ip: String,
     online: bool,
     containers: Vec<String>,
+    net_up: bool,
+    net_details: Vec<String>,
 }
 
 /// Checks Docker health — shows node status and containers belonging to this cluster.
 pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) -> Result<()> {
     let nodes = &cluster.nodes;
     let cluster_label = &cluster.name;
+    let nccl = config.nccl.get(cluster_label);
 
     // Check HA cluster-level entities (switch, etc.) in parallel.
     let ha_handle = if let Some(ref ha_cfg) = config.home_assistant {
@@ -38,18 +41,32 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
         None
     };
 
+    // Build the network check command snippet for this cluster's NCCL interfaces.
+    let net_script = nccl.map(|n| {
+        let socket = &n.socket_ifname;
+        format!(
+            "echo '---NET---'; echo -n 'socket:' && (ip -o link show dev {socket} 2>/dev/null | grep -qi 'state UP' && echo UP || echo DOWN); echo '---END---'"
+        )
+    });
+
     // SSH into each node in parallel.
     let mut handles = Vec::new();
-    for node in &cluster.nodes {
+    for node in nodes {
         let config = config.clone();
         let node = node.clone();
         let name = node.name.clone();
         let ib_ip = node.ib_ip.clone();
         let cl = cluster_label.clone();
-        let cmd = format!(
+        let ns = net_script.clone();
+        let docker_cmd = format!(
             "docker ps --filter label=sm121.cluster={cl} \
              --format '{{{{.Names}}}}\t{{{{.Status}}}}\t{{{{.Image}}}}' 2>&1"
         );
+        let cmd = if let Some(ref script) = ns {
+            format!("{docker_cmd}; {script}")
+        } else {
+            docker_cmd
+        };
         handles.push(tokio::spawn(async move {
             let result = run_ssh(&config, &node, cmd, dry_run).await;
             (name, ib_ip, result)
@@ -61,20 +78,45 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
     for h in futures::future::join_all(handles).await {
         match h {
             Ok((name, ib_ip, Ok(output))) => {
-                let lines: Vec<String> = output
-                    .lines()
-                    .filter(|l| {
-                        !l.is_empty()
-                            && !l.contains("permission denied")
-                            && !l.starts_with("docker:")
-                    })
-                    .map(String::from)
-                    .collect();
+                let mut net_up = true;
+                let mut net_details = Vec::new();
+                let mut lines: Vec<String> = Vec::new();
+                let mut in_net = false;
+                for line in output.lines() {
+                    let line = line.trim();
+                    if line.is_empty()
+                        || line.contains("permission denied")
+                        || line.starts_with("docker:")
+                    {
+                        continue;
+                    }
+                    if line == "---NET---" {
+                        in_net = true;
+                        continue;
+                    }
+                    if line == "---END---" {
+                        in_net = false;
+                        continue;
+                    }
+                    if in_net {
+                        net_details.push(line.to_string());
+                        if line.ends_with("DOWN")
+                            || line.ends_with("missing")
+                            || line.ends_with("error")
+                        {
+                            net_up = false;
+                        }
+                        continue;
+                    }
+                    lines.push(line.to_string());
+                }
                 node_health.push(NodeHealth {
                     name,
                     ib_ip,
                     online: true,
                     containers: lines,
+                    net_up,
+                    net_details,
                 });
             }
             Ok((name, ib_ip, Err(_))) => {
@@ -83,6 +125,8 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
                     ib_ip,
                     online: false,
                     containers: vec![],
+                    net_up: false,
+                    net_details: vec![],
                 });
             }
             Err(join_err) => eprintln!("task join: {}", join_err),
@@ -99,10 +143,13 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
     let online = node_health.iter().filter(|n| n.online).count();
     let offline = total - online;
     let container_count: usize = node_health.iter().map(|n| n.containers.len()).sum();
+    let net_ok = node_health.iter().all(|n| !n.online || n.net_up);
 
     print!("=== {} === {} nodes", cluster_label, total);
     if !ha_ok && !cluster.extra_entities.is_empty() {
         print!(" 🚫 switch off");
+    } else if !net_ok {
+        print!(" 🌐 interface down");
     } else if online == total {
         print!(" ✅ all online");
     } else {
@@ -117,10 +164,17 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
     for nh in &node_health {
         if !nh.online {
             eprintln!("  ✗ {} ({}) — offline", nh.name, nh.ib_ip);
-        } else if nh.containers.is_empty() {
-            println!("  ○ {} ({})", nh.name, nh.ib_ip);
+            continue;
+        }
+        let net_status = if !nh.net_up {
+            format!(" [net: {}]", nh.net_details.join(", "))
+        } else {
+            String::new()
+        };
+        if nh.containers.is_empty() {
+            println!("  ○ {} ({}){}", nh.name, nh.ib_ip, net_status);
         } else {
-            println!("  ● {} ({})", nh.name, nh.ib_ip);
+            println!("  ● {} ({}){}", nh.name, nh.ib_ip, net_status);
             for line in &nh.containers {
                 println!("    {line}");
             }