소스 검색

feat: health command shows all containers, marks ours (sm121)

Previously
=== Кластер: 4nodes-switch ===
dgx1.lan (192.168.200.15)
  Всего контейнеров: 1
  other vllm_node	Up 5 hours	vllm-node-dsv4-b12x-fix
  (нет контейнеров sm121)
dgx2.lan (192.168.200.13)
  Всего контейнеров: 1
  other vllm_node	Up 5 hours	vllm-node-dsv4-b12x-fix
  (нет контейнеров sm121)
dgx3.lan (192.168.200.17)
  Всего контейнеров: 1
  other vllm_node	Up 5 hours	vllm-node-dsv4-b12x-fix
  (нет контейнеров sm121)
dgx4.lan (192.168.200.19)
  Всего контейнеров: 1
  other vllm_node	Up 5 hours	vllm-node-dsv4-b12x-fix
  (нет контейнеров sm121)

=== Кластер: 2nodes-dac ===
dgx1.lan (192.168.100.14)
  Всего контейнеров: 1
  other vllm_node	Up 5 hours	vllm-node-dsv4-b12x-fix
  (нет контейнеров sm121)
dgx2.lan (192.168.100.10)
  Всего контейнеров: 1
  other vllm_node	Up 5 hours	vllm-node-dsv4-b12x-fix
  (нет контейнеров sm121) only showed containers with the sm121
label. Now it runs  for all containers, plus a second query
for sm121-labeled ones, and marks each line with  or .

Example output:
  dgx1 (192.168.200.15)
    Всего контейнеров: 3
    other  my-other-container  Up 2 hours  nginx:latest
    sm121  sm121_glm51_0       Up 2 hours  vllm/vllm-openai:glm51-cu130
    sm121  sm121_glm51_1       Up 2 hours  vllm/vllm-openai:glm51-cu130
Alexander Korolev 4 달 전
부모
커밋
91e273a649
1개의 변경된 파일과 35개의 추가작업 그리고 27개의 파일을 삭제
  1. 35 27
      src/docker/mod.rs

+ 35 - 27
src/docker/mod.rs

@@ -145,7 +145,7 @@ pub async fn stop_cluster(
     Ok(())
     Ok(())
 }
 }
 
 
-/// Checks Docker container health on a cluster.
+/// Checks Docker health on a cluster — shows all containers and marks ours.
 pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) -> Result<()> {
 pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) -> Result<()> {
     println!("\n=== Кластер: {} ===", cluster.name);
     println!("\n=== Кластер: {} ===", cluster.name);
     let mut handles = Vec::new();
     let mut handles = Vec::new();
@@ -154,10 +154,11 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
         let node = node.clone();
         let node = node.clone();
         let dry = dry_run;
         let dry = dry_run;
         handles.push(tokio::spawn(async move {
         handles.push(tokio::spawn(async move {
+            // Query all containers + mark sm121-managed ones
             let result = run_ssh_with_retry(
             let result = run_ssh_with_retry(
                 &config,
                 &config,
                 &node,
                 &node,
-                "docker ps --filter \"label=sm121\" --format '{{.Names}} {{.Status}}' 2>&1"
+                "docker ps --format '{{.Names}}\t{{.Status}}\t{{.Image}}' 2>&1; echo '---SM121---'; docker ps --filter label=sm121 --format '{{.Names}}' 2>&1"
                     .to_string(),
                     .to_string(),
                 dry,
                 dry,
                 1,
                 1,
@@ -166,48 +167,55 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
             (node.name, node.ib_ip, result)
             (node.name, node.ib_ip, result)
         }));
         }));
     }
     }
-    let mut all_healthy = true;
     for result in futures::future::join_all(handles).await {
     for result in futures::future::join_all(handles).await {
         match result {
         match result {
-            Ok((name, ib_ip, Ok(check))) => {
-                print!("{} ({}): ", name, ib_ip);
-                if check.is_empty() && dry_run {
+            Ok((name, ib_ip, Ok(output))) => {
+                println!("{} ({})", name, ib_ip);
+                let parts: Vec<&str> = output.splitn(2, "---SM121---").collect();
+                let all_lines: Vec<&str> = parts
+                    .first()
+                    .map(|s| s.lines().filter(|l| !l.is_empty()).collect())
+                    .unwrap_or_default();
+                let sm121_names: Vec<&str> = parts
+                    .get(1)
+                    .map(|s| s.lines().filter(|l| !l.is_empty()).collect())
+                    .unwrap_or_default();
+
+                if dry_run && all_lines.is_empty() {
                     continue;
                     continue;
                 }
                 }
-                if check.starts_with("docker:") || check.contains("permission denied") {
-                    println!("✗ Docker недоступен");
-                    if let Some(line) = check.lines().next() {
-                        println!("  └─ {}", line);
+
+                if output.starts_with("docker:") || output.contains("permission denied") {
+                    println!("  ✗ Docker недоступен");
+                    if let Some(line) = output.lines().next() {
+                        println!("    └─ {}", line);
                     }
                     }
-                    all_healthy = false;
+                } else if all_lines.is_empty() {
+                    println!("  (no containers)");
                 } else {
                 } else {
-                    let containers: Vec<&str> = check.lines().filter(|l| !l.is_empty()).collect();
-                    if containers.is_empty() {
-                        println!("✗ Нет контейнеров");
-                        all_healthy = false;
-                    } else {
-                        println!("✓ {} контейнер(ов)", containers.len());
-                        for c in containers {
-                            println!("  └─ {}", c);
-                        }
+                    println!("  Всего контейнеров: {}", all_lines.len());
+                    for line in &all_lines {
+                        let name_col = line.split('\t').next().unwrap_or("");
+                        let tag = if sm121_names.contains(&name_col) {
+                            "sm121"
+                        } else {
+                            "other"
+                        };
+                        println!("  {tag:5} {line}");
+                    }
+                    if sm121_names.is_empty() {
+                        println!("  (нет контейнеров sm121)");
                     }
                     }
                 }
                 }
             }
             }
             Ok((name, ib_ip, Err(e))) => {
             Ok((name, ib_ip, Err(e))) => {
                 println!("{} ({}): ✗ SSH ошибка: {}", name, ib_ip, e);
                 println!("{} ({}): ✗ SSH ошибка: {}", name, ib_ip, e);
-                all_healthy = false;
             }
             }
             Err(join_err) => {
             Err(join_err) => {
                 println!("✗ Task join error: {}", join_err);
                 println!("✗ Task join error: {}", join_err);
-                all_healthy = false;
             }
             }
         }
         }
     }
     }
-    if !dry_run && all_healthy {
-        info!("✅ Кластер здоров!");
-    } else if !dry_run {
-        info!("⚠️ Кластер имеет проблемы");
-    }
     Ok(())
     Ok(())
 }
 }