|
@@ -1,90 +1,165 @@
|
|
|
|
|
+//! Cluster health — shows node status, running containers, and recipe
|
|
|
|
|
+//! deployments across all clusters.
|
|
|
|
|
+
|
|
|
use crate::config::{Cluster, Config};
|
|
use crate::config::{Cluster, Config};
|
|
|
-use crate::ssh::run_ssh_with_retry;
|
|
|
|
|
|
|
+use crate::ssh::run_ssh;
|
|
|
use anyhow::Result;
|
|
use anyhow::Result;
|
|
|
|
|
+use std::collections::HashMap;
|
|
|
|
|
+
|
|
|
|
|
+/// Per-node result from the parallel SSH sweep.
|
|
|
|
|
+struct NodeHealth {
|
|
|
|
|
+ name: String,
|
|
|
|
|
+ ib_ip: String,
|
|
|
|
|
+ online: bool,
|
|
|
|
|
+ containers: Vec<String>,
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
-/// Checks Docker health on a cluster — shows all containers and marks ours.
|
|
|
|
|
|
|
+/// Checks Docker health — shows node status and containers belonging to this cluster.
|
|
|
pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) -> Result<()> {
|
|
pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) -> Result<()> {
|
|
|
- println!(
|
|
|
|
|
- "{}",
|
|
|
|
|
- fluent_i18n::t!("cluster-section", { "name" => &cluster.name })
|
|
|
|
|
- );
|
|
|
|
|
|
|
+ let nodes = &cluster.nodes;
|
|
|
|
|
+ let cluster_label = &cluster.name;
|
|
|
let mut handles = Vec::new();
|
|
let mut handles = Vec::new();
|
|
|
- for node in &cluster.nodes {
|
|
|
|
|
|
|
+ for node in nodes {
|
|
|
let config = config.clone();
|
|
let config = config.clone();
|
|
|
let node = node.clone();
|
|
let node = node.clone();
|
|
|
- let dry = dry_run;
|
|
|
|
|
|
|
+ let name = node.name.clone();
|
|
|
|
|
+ let ib_ip = node.ib_ip.clone();
|
|
|
|
|
+ let cl = cluster_label.clone();
|
|
|
|
|
+ let cmd = format!(
|
|
|
|
|
+ "docker ps --filter label=sm121.cluster={cl} \
|
|
|
|
|
+ --format '{{{{.Names}}}}\t{{{{.Status}}}}\t{{{{.Image}}}}' 2>&1"
|
|
|
|
|
+ );
|
|
|
handles.push(tokio::spawn(async move {
|
|
handles.push(tokio::spawn(async move {
|
|
|
- let result = run_ssh_with_retry(
|
|
|
|
|
- &config,
|
|
|
|
|
- &node,
|
|
|
|
|
- "docker ps --format '{{.Names}}\t{{.Status}}\t{{.Image}}' 2>&1; echo '---SM121---'; docker ps --filter label=sm121.recipe --format '{{.Names}}' 2>&1"
|
|
|
|
|
- .to_string(),
|
|
|
|
|
- dry,
|
|
|
|
|
- 1,
|
|
|
|
|
- )
|
|
|
|
|
- .await;
|
|
|
|
|
- (node.name, node.ib_ip, result)
|
|
|
|
|
|
|
+ let result = run_ssh(&config, &node, cmd, dry_run).await;
|
|
|
|
|
+ (name, ib_ip, result)
|
|
|
}));
|
|
}));
|
|
|
}
|
|
}
|
|
|
- for result in futures::future::join_all(handles).await {
|
|
|
|
|
- match result {
|
|
|
|
|
|
|
+
|
|
|
|
|
+ let mut node_health: Vec<NodeHealth> = Vec::new();
|
|
|
|
|
+ for h in futures::future::join_all(handles).await {
|
|
|
|
|
+ match h {
|
|
|
Ok((name, ib_ip, Ok(output))) => {
|
|
Ok((name, ib_ip, Ok(output))) => {
|
|
|
- println!("{} ({})", name, ib_ip);
|
|
|
|
|
- let parts: Vec<&str> = output.splitn(2, "---SM121---").collect();
|
|
|
|
|
- let all_lines: Vec<&str> = parts
|
|
|
|
|
- .first()
|
|
|
|
|
- .map(|s| s.lines().filter(|l| !l.is_empty()).collect())
|
|
|
|
|
- .unwrap_or_default();
|
|
|
|
|
- let sm121_names: Vec<&str> = parts
|
|
|
|
|
- .get(1)
|
|
|
|
|
- .map(|s| s.lines().filter(|l| !l.is_empty()).collect())
|
|
|
|
|
- .unwrap_or_default();
|
|
|
|
|
- if dry_run && all_lines.is_empty() {
|
|
|
|
|
- continue;
|
|
|
|
|
- }
|
|
|
|
|
- if output.starts_with("docker:") || output.contains("permission denied") {
|
|
|
|
|
- println!(" {}", fluent_i18n::t!("docker-unavailable"));
|
|
|
|
|
- if let Some(line) = output.lines().next() {
|
|
|
|
|
- let err = line.to_string();
|
|
|
|
|
- println!(
|
|
|
|
|
- " └─ {}",
|
|
|
|
|
- fluent_i18n::t!("docker-error-line", { "err" => &err })
|
|
|
|
|
- );
|
|
|
|
|
- }
|
|
|
|
|
- } else if all_lines.is_empty() {
|
|
|
|
|
- println!(" {}", fluent_i18n::t!("health-no-containers"));
|
|
|
|
|
- } else {
|
|
|
|
|
- println!(
|
|
|
|
|
- " {}",
|
|
|
|
|
- fluent_i18n::t!("container-count", { "count" => all_lines.len() })
|
|
|
|
|
- );
|
|
|
|
|
- for line in &all_lines {
|
|
|
|
|
- let name_col = line.split('\t').next().unwrap_or("");
|
|
|
|
|
- let tag = if sm121_names.contains(&name_col) {
|
|
|
|
|
- "sm121"
|
|
|
|
|
- } else {
|
|
|
|
|
- "other"
|
|
|
|
|
- };
|
|
|
|
|
- println!(" {tag:5} {line}");
|
|
|
|
|
- }
|
|
|
|
|
- if sm121_names.is_empty() {
|
|
|
|
|
- println!(" {}", fluent_i18n::t!("no-sm121-containers"));
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ let lines: Vec<String> = output
|
|
|
|
|
+ .lines()
|
|
|
|
|
+ .filter(|l| {
|
|
|
|
|
+ !l.is_empty()
|
|
|
|
|
+ && !l.contains("permission denied")
|
|
|
|
|
+ && !l.starts_with("docker:")
|
|
|
|
|
+ })
|
|
|
|
|
+ .map(String::from)
|
|
|
|
|
+ .collect();
|
|
|
|
|
+ node_health.push(NodeHealth {
|
|
|
|
|
+ name,
|
|
|
|
|
+ ib_ip,
|
|
|
|
|
+ online: true,
|
|
|
|
|
+ containers: lines,
|
|
|
|
|
+ });
|
|
|
|
|
+ }
|
|
|
|
|
+ Ok((name, ib_ip, Err(_))) => {
|
|
|
|
|
+ node_health.push(NodeHealth {
|
|
|
|
|
+ name,
|
|
|
|
|
+ ib_ip,
|
|
|
|
|
+ online: false,
|
|
|
|
|
+ containers: vec![],
|
|
|
|
|
+ });
|
|
|
|
|
+ }
|
|
|
|
|
+ Err(join_err) => eprintln!("task join: {}", join_err),
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ let total = nodes.len();
|
|
|
|
|
+ let online = node_health.iter().filter(|n| n.online).count();
|
|
|
|
|
+ let offline = total - online;
|
|
|
|
|
+ let container_count: usize = node_health.iter().map(|n| n.containers.len()).sum();
|
|
|
|
|
+
|
|
|
|
|
+ print!("=== {} === {} nodes", cluster_label, total);
|
|
|
|
|
+ if online == total {
|
|
|
|
|
+ print!(" ✅ all online");
|
|
|
|
|
+ } else {
|
|
|
|
|
+ print!(" ({} online, {} offline)", online, offline);
|
|
|
|
|
+ }
|
|
|
|
|
+ if container_count > 0 {
|
|
|
|
|
+ println!(", {} container(s)", container_count);
|
|
|
|
|
+ } else {
|
|
|
|
|
+ println!(", no containers");
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ for nh in &node_health {
|
|
|
|
|
+ if !nh.online {
|
|
|
|
|
+ eprintln!(" ✗ {} ({}) — offline", nh.name, nh.ib_ip);
|
|
|
|
|
+ } else if nh.containers.is_empty() {
|
|
|
|
|
+ println!(" ○ {} ({})", nh.name, nh.ib_ip);
|
|
|
|
|
+ } else {
|
|
|
|
|
+ println!(" ● {} ({})", nh.name, nh.ib_ip);
|
|
|
|
|
+ for line in &nh.containers {
|
|
|
|
|
+ println!(" {line}");
|
|
|
}
|
|
}
|
|
|
- Ok((name, ib_ip, Err(e))) => {
|
|
|
|
|
- println!(
|
|
|
|
|
- "{}",
|
|
|
|
|
- fluent_i18n::t!("ssh-error", { "node" => &name, "ip" => &ib_ip, "error" => &e.to_string() })
|
|
|
|
|
- );
|
|
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ Ok(())
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/// Quick overview — recipes running on each cluster.
|
|
|
|
|
+pub async fn health_summary(config: &Config) -> Result<()> {
|
|
|
|
|
+ let mut seen = std::collections::HashSet::new();
|
|
|
|
|
+ let mut handles = Vec::new();
|
|
|
|
|
+ for cluster in &config.clusters {
|
|
|
|
|
+ for node in &cluster.nodes {
|
|
|
|
|
+ if seen.contains(&node.ssh_ip) {
|
|
|
|
|
+ continue;
|
|
|
}
|
|
}
|
|
|
- Err(join_err) => {
|
|
|
|
|
- println!(
|
|
|
|
|
- "{}",
|
|
|
|
|
- fluent_i18n::t!("task-join-error", { "error" => &join_err.to_string() })
|
|
|
|
|
- );
|
|
|
|
|
|
|
+ seen.insert(node.ssh_ip.clone());
|
|
|
|
|
+ let config = config.clone();
|
|
|
|
|
+ let node = node.clone();
|
|
|
|
|
+ let name = node.name.clone();
|
|
|
|
|
+ let cmd = "docker ps --filter label=sm121.recipe \
|
|
|
|
|
+ --format '{{.Label \"sm121.recipe\"}}|{{.Label \"sm121.cluster\"}}'"
|
|
|
|
|
+ .to_string();
|
|
|
|
|
+ handles.push(tokio::spawn(async move {
|
|
|
|
|
+ let out = run_ssh(&config, &node, cmd, false)
|
|
|
|
|
+ .await
|
|
|
|
|
+ .unwrap_or_default();
|
|
|
|
|
+ (name, out)
|
|
|
|
|
+ }));
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ let mut recipe_nodes: HashMap<String, std::collections::HashSet<String>> = HashMap::new();
|
|
|
|
|
+ for h in futures::future::join_all(handles).await {
|
|
|
|
|
+ let (node_name, output) = h.unwrap_or_default();
|
|
|
|
|
+ for line in output.lines() {
|
|
|
|
|
+ let line = line.trim();
|
|
|
|
|
+ if line.is_empty() {
|
|
|
|
|
+ continue;
|
|
|
}
|
|
}
|
|
|
|
|
+ let mut parts = line.splitn(2, '|');
|
|
|
|
|
+ let recipe = parts.next().unwrap_or("").to_string();
|
|
|
|
|
+ if recipe.is_empty() {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+ recipe_nodes
|
|
|
|
|
+ .entry(recipe)
|
|
|
|
|
+ .or_default()
|
|
|
|
|
+ .insert(node_name.clone());
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
+
|
|
|
|
|
+ if recipe_nodes.is_empty() {
|
|
|
|
|
+ println!("\nNo sm121 containers running.");
|
|
|
|
|
+ return Ok(());
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ println!("\nRunning recipes:");
|
|
|
|
|
+ for (recipe, nodes) in &recipe_nodes {
|
|
|
|
|
+ let mut sorted: Vec<_> = nodes.iter().collect();
|
|
|
|
|
+ sorted.sort();
|
|
|
|
|
+ let node_list: Vec<&str> = sorted.iter().map(|s| s.as_str()).collect();
|
|
|
|
|
+ println!(
|
|
|
|
|
+ " {} — {} node(s): {}",
|
|
|
|
|
+ recipe,
|
|
|
|
|
+ node_list.len(),
|
|
|
|
|
+ node_list.join(", ")
|
|
|
|
|
+ );
|
|
|
|
|
+ }
|
|
|
Ok(())
|
|
Ok(())
|
|
|
}
|
|
}
|