|
|
@@ -0,0 +1,176 @@
|
|
|
+//! Tmux deployment discovery for the `health` command.
|
|
|
+//!
|
|
|
+//! The `health` command is Docker-only by default. This module adds
|
|
|
+//! tmux awareness: it scans `recipes/` for tmux-deployed recipes,
|
|
|
+//! checks each node for active sessions, and reports status alongside
|
|
|
+//! the Docker container listing.
|
|
|
+
|
|
|
+use tracing::warn;
|
|
|
+
|
|
|
+use crate::config::{Cluster, Config, DeployTarget, Recipe};
|
|
|
+use crate::health;
|
|
|
+use crate::tmux::TmuxManager;
|
|
|
+
|
|
|
+/// Lists all recipe directory names in `recipes/`.
|
|
|
+pub fn list_recipe_dirs() -> Vec<String> {
|
|
|
+ let mut recipes = Vec::new();
|
|
|
+ if let Ok(entries) = std::fs::read_dir("recipes") {
|
|
|
+ for entry in entries.flatten() {
|
|
|
+ if entry.path().is_dir() {
|
|
|
+ let name = entry.file_name();
|
|
|
+ let Some(name) = name.to_str() else {
|
|
|
+ continue;
|
|
|
+ };
|
|
|
+ recipes.push(name.to_string());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ recipes.sort();
|
|
|
+ recipes
|
|
|
+}
|
|
|
+
|
|
|
+/// Finds all recipes with `deploy = "tmux"`.
|
|
|
+pub async fn find_tmux_recipes() -> Vec<(String, Recipe)> {
|
|
|
+ let dirs = list_recipe_dirs();
|
|
|
+ let mut recipes = Vec::new();
|
|
|
+ for dir in dirs {
|
|
|
+ let Ok(recipe) = crate::config::load_recipe(&dir).await else {
|
|
|
+ continue;
|
|
|
+ };
|
|
|
+ if recipe.deploy == DeployTarget::Tmux {
|
|
|
+ recipes.push((dir, recipe));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ recipes
|
|
|
+}
|
|
|
+
|
|
|
+/// Health entry for a single tmux deployment on a node.
|
|
|
+pub struct TmuxHealthEntry {
|
|
|
+ pub recipe_display: String,
|
|
|
+ pub node_name: String,
|
|
|
+ pub session_exists: bool,
|
|
|
+ pub process_running: bool,
|
|
|
+ pub api_ok: bool,
|
|
|
+}
|
|
|
+
|
|
|
+/// Checks tmux health for all tmux-deployed recipes on a cluster.
|
|
|
+///
|
|
|
+/// For each recipe with `deploy = "tmux"`, runs a deep health check
|
|
|
+/// (session existence, process liveness, GPU, API) on every node.
|
|
|
+/// Returns one entry per node per recipe.
|
|
|
+pub async fn tmux_health_for_cluster(config: &Config, cluster: &Cluster) -> Vec<TmuxHealthEntry> {
|
|
|
+ let tmux_recipes = find_tmux_recipes().await;
|
|
|
+ let manager = TmuxManager;
|
|
|
+ let mut entries = Vec::new();
|
|
|
+
|
|
|
+ for (_dir, recipe) in tmux_recipes {
|
|
|
+ let report = health::check_cluster_health(&manager, config, cluster, &recipe, false).await;
|
|
|
+ match report {
|
|
|
+ Ok(report) => {
|
|
|
+ for node_health in &report.nodes {
|
|
|
+ entries.push(TmuxHealthEntry {
|
|
|
+ recipe_display: recipe.name.clone(),
|
|
|
+ node_name: node_health.name.clone(),
|
|
|
+ session_exists: node_health.session_exists,
|
|
|
+ process_running: node_health.process_running,
|
|
|
+ api_ok: node_health.port_accessible,
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Err(e) => warn!("Tmux health check failed for '{}': {}", recipe.name, e),
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ entries
|
|
|
+}
|
|
|
+
|
|
|
+/// Prints tmux health entries to stdout.
|
|
|
+///
|
|
|
+/// Shows nothing if there are no tmux-deployed recipes.
|
|
|
+pub fn print_tmux_health(entries: &[TmuxHealthEntry]) {
|
|
|
+ if entries.is_empty() {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ println!();
|
|
|
+ println!("{}", fluent_i18n::t!("tmux-health-section"));
|
|
|
+ for entry in entries {
|
|
|
+ let status = if entry.session_exists && entry.process_running {
|
|
|
+ let api_str = if entry.api_ok {
|
|
|
+ fluent_i18n::t!("tmux-api-ok")
|
|
|
+ } else {
|
|
|
+ fluent_i18n::t!("tmux-api-down")
|
|
|
+ };
|
|
|
+ fluent_i18n::t!("tmux-health-entry-running", {
|
|
|
+ "recipe" => &entry.recipe_display,
|
|
|
+ "node" => &entry.node_name,
|
|
|
+ "api" => &api_str,
|
|
|
+ })
|
|
|
+ } else if entry.session_exists {
|
|
|
+ fluent_i18n::t!("tmux-health-entry-degraded", {
|
|
|
+ "recipe" => &entry.recipe_display,
|
|
|
+ "node" => &entry.node_name,
|
|
|
+ })
|
|
|
+ } else {
|
|
|
+ fluent_i18n::t!("tmux-health-entry-stopped", {
|
|
|
+ "recipe" => &entry.recipe_display,
|
|
|
+ "node" => &entry.node_name,
|
|
|
+ })
|
|
|
+ };
|
|
|
+ println!(" {}", status);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+#[cfg(test)]
|
|
|
+mod tests {
|
|
|
+ use super::*;
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn list_recipe_dirs_finds_directories() {
|
|
|
+ let dirs = list_recipe_dirs();
|
|
|
+ // Should find at least the laguna recipe
|
|
|
+ assert!(dirs.iter().any(|d| d == "laguna-s2.1-nvfp4"));
|
|
|
+ // Should be sorted
|
|
|
+ let sorted = dirs.windows(2).all(|w| w[0] <= w[1]);
|
|
|
+ assert!(sorted);
|
|
|
+ }
|
|
|
+
|
|
|
+ #[tokio::test]
|
|
|
+ async fn find_tmux_recipes_finds_tmux_deployed() {
|
|
|
+ // This test runs in the project directory, so it should find
|
|
|
+ // the laguna recipe which has deploy = "tmux"
|
|
|
+ let recipes = find_tmux_recipes().await;
|
|
|
+ assert!(recipes.iter().any(|(dir, _)| dir == "laguna-s2.1-nvfp4"));
|
|
|
+ for (_dir, recipe) in &recipes {
|
|
|
+ assert_eq!(recipe.deploy, DeployTarget::Tmux);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn print_tmux_health_empty_is_noop() {
|
|
|
+ // Should not panic or print anything
|
|
|
+ print_tmux_health(&[]);
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn print_tmux_health_with_entries() {
|
|
|
+ let entries = vec![
|
|
|
+ TmuxHealthEntry {
|
|
|
+ recipe_display: "Test Recipe".to_string(),
|
|
|
+ node_name: "dgx1".to_string(),
|
|
|
+ session_exists: true,
|
|
|
+ process_running: true,
|
|
|
+ api_ok: true,
|
|
|
+ },
|
|
|
+ TmuxHealthEntry {
|
|
|
+ recipe_display: "Test Recipe".to_string(),
|
|
|
+ node_name: "dgx2".to_string(),
|
|
|
+ session_exists: false,
|
|
|
+ process_running: false,
|
|
|
+ api_ok: false,
|
|
|
+ },
|
|
|
+ ];
|
|
|
+ // Should not panic
|
|
|
+ print_tmux_health(&entries);
|
|
|
+ }
|
|
|
+}
|
