| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216 |
- //! Tmux deployment discovery for the `health` command.
- //!
- //! The `health` command is Docker-only by default. This module adds
- //! tmux awareness: it scans `recipes/` for tmux-deployed recipes,
- //! checks each node for active sessions, and reports status alongside
- //! the Docker container listing.
- use std::collections::HashSet;
- use tracing::warn;
- use crate::config::{Cluster, Config, DeployTarget, Recipe};
- use crate::health;
- use crate::tmux::TmuxManager;
- /// Lists all recipe directory names in `recipes/`.
- pub fn list_recipe_dirs() -> Vec<String> {
- let mut recipes = Vec::new();
- if let Ok(entries) = std::fs::read_dir("recipes") {
- for entry in entries.flatten() {
- if entry.path().is_dir() {
- let name = entry.file_name();
- let Some(name) = name.to_str() else {
- continue;
- };
- recipes.push(name.to_string());
- }
- }
- }
- recipes.sort();
- recipes
- }
- /// Finds all recipes with `deploy = "tmux"` (in `./recipes`, resolved via
- /// `load_recipe` with the given extra sources).
- pub async fn find_tmux_recipes(sources: &[String]) -> Vec<(String, Recipe)> {
- let dirs = list_recipe_dirs();
- let mut recipes = Vec::new();
- for dir in dirs {
- let Ok(recipe) = crate::config::load_recipe(&dir, sources).await else {
- continue;
- };
- if recipe.deploy == DeployTarget::Tmux {
- recipes.push((dir, recipe));
- }
- }
- recipes
- }
- /// Health entry for a single tmux deployment on a node.
- pub struct TmuxHealthEntry {
- pub recipe_display: String,
- pub node_name: String,
- pub session_exists: bool,
- pub process_running: bool,
- pub api_ok: bool,
- }
- /// Checks tmux health for all tmux-deployed recipes on a cluster.
- ///
- /// For each recipe with `deploy = "tmux"`, runs a deep health check
- /// (session existence, process liveness, GPU, API) on every node.
- /// Returns one entry per node per recipe.
- pub async fn tmux_health_for_cluster(config: &Config, cluster: &Cluster) -> Vec<TmuxHealthEntry> {
- let tmux_recipes = find_tmux_recipes(&config.recipe_sources).await;
- let manager = TmuxManager;
- let mut entries = Vec::new();
- for (_dir, recipe) in tmux_recipes {
- let report = health::check_cluster_health(&manager, config, cluster, &recipe, false).await;
- match report {
- Ok(report) => {
- for node_health in &report.nodes {
- entries.push(TmuxHealthEntry {
- recipe_display: recipe.name.clone(),
- node_name: node_health.name.clone(),
- session_exists: node_health.session_exists,
- process_running: node_health.process_running,
- api_ok: node_health.port_accessible,
- });
- }
- }
- Err(e) => warn!("Tmux health check failed for '{}': {}", recipe.name, e),
- }
- }
- entries
- }
- /// Prints tmux health entries to stdout.
- ///
- /// Only shows sessions that are running or degraded (session exists).
- /// If all sessions are stopped, prints a single summary line.
- /// Shows nothing if there are no tmux-deployed recipes at all.
- pub fn print_tmux_health(entries: &[TmuxHealthEntry]) {
- if entries.is_empty() {
- return;
- }
- // Only show running/degraded sessions — skip stopped ones to keep output readable.
- let active: Vec<&TmuxHealthEntry> = entries.iter().filter(|e| e.session_exists).collect();
- if active.is_empty() {
- // All sessions stopped — show a one-line summary.
- let recipe_count = entries
- .iter()
- .map(|e| e.recipe_display.as_str())
- .collect::<HashSet<_>>()
- .len();
- println!();
- println!(
- " {}",
- fluent_i18n::t!("tmux-health-all-stopped", { "count" => recipe_count })
- );
- return;
- }
- println!();
- println!("{}", fluent_i18n::t!("tmux-health-section"));
- for entry in active {
- let status = if entry.process_running {
- let api_str = if entry.api_ok {
- fluent_i18n::t!("tmux-api-ok")
- } else {
- fluent_i18n::t!("tmux-api-down")
- };
- fluent_i18n::t!("tmux-health-entry-running", {
- "recipe" => &entry.recipe_display,
- "node" => &entry.node_name,
- "api" => &api_str,
- })
- } else {
- fluent_i18n::t!("tmux-health-entry-degraded", {
- "recipe" => &entry.recipe_display,
- "node" => &entry.node_name,
- })
- };
- println!(" {}", status);
- }
- }
- #[cfg(test)]
- mod tests {
- use super::*;
- #[test]
- fn list_recipe_dirs_finds_directories() {
- // Recipe data lives in git.kilork.org/sm121/recipes now — tests
- // seed their own fixture instead of relying on repo data.
- let fixture = "recipes/ztest-list-fixture";
- std::fs::create_dir_all(fixture).unwrap();
- let dirs = list_recipe_dirs();
- let sorted = dirs.windows(2).all(|w| w[0] <= w[1]);
- let _ = std::fs::remove_dir(fixture);
- assert!(dirs.iter().any(|d| d == "ztest-list-fixture"));
- assert!(sorted);
- }
- #[tokio::test]
- async fn find_tmux_recipes_finds_tmux_deployed() {
- let fixture = "recipes/ztest-tmux-fixture";
- std::fs::create_dir_all(fixture).unwrap();
- std::fs::write(
- format!("{fixture}/recipe.toml"),
- "name = \"ztest-tmux-fixture\"\ndeploy = \"tmux\"\n",
- )
- .unwrap();
- let recipes = find_tmux_recipes(&[]).await;
- let _ = std::fs::remove_dir_all(fixture);
- assert!(recipes.iter().any(|(dir, _)| dir == "ztest-tmux-fixture"));
- for (_dir, recipe) in &recipes {
- assert_eq!(recipe.deploy, DeployTarget::Tmux);
- }
- }
- #[test]
- fn print_tmux_health_empty_is_noop() {
- // Should not panic or print anything
- print_tmux_health(&[]);
- }
- #[test]
- fn print_tmux_health_all_stopped_shows_summary() {
- let entries = vec![TmuxHealthEntry {
- recipe_display: "Test Recipe".to_string(),
- node_name: "dgx1".to_string(),
- session_exists: false,
- process_running: false,
- api_ok: false,
- }];
- // Should not panic — prints summary line
- print_tmux_health(&entries);
- }
- #[test]
- fn print_tmux_health_running_shows_entry() {
- let entries = vec![
- TmuxHealthEntry {
- recipe_display: "Test Recipe".to_string(),
- node_name: "dgx1".to_string(),
- session_exists: true,
- process_running: true,
- api_ok: true,
- },
- TmuxHealthEntry {
- recipe_display: "Test Recipe".to_string(),
- node_name: "dgx2".to_string(),
- session_exists: false,
- process_running: false,
- api_ok: false,
- },
- ];
- // Should not panic — prints running entry, skips stopped
- print_tmux_health(&entries);
- }
- }
|