discover.rs 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216
  1. //! Tmux deployment discovery for the `health` command.
  2. //!
  3. //! The `health` command is Docker-only by default. This module adds
  4. //! tmux awareness: it scans `recipes/` for tmux-deployed recipes,
  5. //! checks each node for active sessions, and reports status alongside
  6. //! the Docker container listing.
  7. use std::collections::HashSet;
  8. use tracing::warn;
  9. use crate::config::{Cluster, Config, DeployTarget, Recipe};
  10. use crate::health;
  11. use crate::tmux::TmuxManager;
  12. /// Lists all recipe directory names in `recipes/`.
  13. pub fn list_recipe_dirs() -> Vec<String> {
  14. let mut recipes = Vec::new();
  15. if let Ok(entries) = std::fs::read_dir("recipes") {
  16. for entry in entries.flatten() {
  17. if entry.path().is_dir() {
  18. let name = entry.file_name();
  19. let Some(name) = name.to_str() else {
  20. continue;
  21. };
  22. recipes.push(name.to_string());
  23. }
  24. }
  25. }
  26. recipes.sort();
  27. recipes
  28. }
  29. /// Finds all recipes with `deploy = "tmux"` (in `./recipes`, resolved via
  30. /// `load_recipe` with the given extra sources).
  31. pub async fn find_tmux_recipes(sources: &[String]) -> Vec<(String, Recipe)> {
  32. let dirs = list_recipe_dirs();
  33. let mut recipes = Vec::new();
  34. for dir in dirs {
  35. let Ok(recipe) = crate::config::load_recipe(&dir, sources).await else {
  36. continue;
  37. };
  38. if recipe.deploy == DeployTarget::Tmux {
  39. recipes.push((dir, recipe));
  40. }
  41. }
  42. recipes
  43. }
  44. /// Health entry for a single tmux deployment on a node.
  45. pub struct TmuxHealthEntry {
  46. pub recipe_display: String,
  47. pub node_name: String,
  48. pub session_exists: bool,
  49. pub process_running: bool,
  50. pub api_ok: bool,
  51. }
  52. /// Checks tmux health for all tmux-deployed recipes on a cluster.
  53. ///
  54. /// For each recipe with `deploy = "tmux"`, runs a deep health check
  55. /// (session existence, process liveness, GPU, API) on every node.
  56. /// Returns one entry per node per recipe.
  57. pub async fn tmux_health_for_cluster(config: &Config, cluster: &Cluster) -> Vec<TmuxHealthEntry> {
  58. let tmux_recipes = find_tmux_recipes(&config.recipe_sources).await;
  59. let manager = TmuxManager;
  60. let mut entries = Vec::new();
  61. for (_dir, recipe) in tmux_recipes {
  62. let report = health::check_cluster_health(&manager, config, cluster, &recipe, false).await;
  63. match report {
  64. Ok(report) => {
  65. for node_health in &report.nodes {
  66. entries.push(TmuxHealthEntry {
  67. recipe_display: recipe.name.clone(),
  68. node_name: node_health.name.clone(),
  69. session_exists: node_health.session_exists,
  70. process_running: node_health.process_running,
  71. api_ok: node_health.port_accessible,
  72. });
  73. }
  74. }
  75. Err(e) => warn!("Tmux health check failed for '{}': {}", recipe.name, e),
  76. }
  77. }
  78. entries
  79. }
  80. /// Prints tmux health entries to stdout.
  81. ///
  82. /// Only shows sessions that are running or degraded (session exists).
  83. /// If all sessions are stopped, prints a single summary line.
  84. /// Shows nothing if there are no tmux-deployed recipes at all.
  85. pub fn print_tmux_health(entries: &[TmuxHealthEntry]) {
  86. if entries.is_empty() {
  87. return;
  88. }
  89. // Only show running/degraded sessions — skip stopped ones to keep output readable.
  90. let active: Vec<&TmuxHealthEntry> = entries.iter().filter(|e| e.session_exists).collect();
  91. if active.is_empty() {
  92. // All sessions stopped — show a one-line summary.
  93. let recipe_count = entries
  94. .iter()
  95. .map(|e| e.recipe_display.as_str())
  96. .collect::<HashSet<_>>()
  97. .len();
  98. println!();
  99. println!(
  100. " {}",
  101. fluent_i18n::t!("tmux-health-all-stopped", { "count" => recipe_count })
  102. );
  103. return;
  104. }
  105. println!();
  106. println!("{}", fluent_i18n::t!("tmux-health-section"));
  107. for entry in active {
  108. let status = if entry.process_running {
  109. let api_str = if entry.api_ok {
  110. fluent_i18n::t!("tmux-api-ok")
  111. } else {
  112. fluent_i18n::t!("tmux-api-down")
  113. };
  114. fluent_i18n::t!("tmux-health-entry-running", {
  115. "recipe" => &entry.recipe_display,
  116. "node" => &entry.node_name,
  117. "api" => &api_str,
  118. })
  119. } else {
  120. fluent_i18n::t!("tmux-health-entry-degraded", {
  121. "recipe" => &entry.recipe_display,
  122. "node" => &entry.node_name,
  123. })
  124. };
  125. println!(" {}", status);
  126. }
  127. }
  128. #[cfg(test)]
  129. mod tests {
  130. use super::*;
  131. #[test]
  132. fn list_recipe_dirs_finds_directories() {
  133. // Recipe data lives in git.kilork.org/sm121/recipes now — tests
  134. // seed their own fixture instead of relying on repo data.
  135. let fixture = "recipes/ztest-list-fixture";
  136. std::fs::create_dir_all(fixture).unwrap();
  137. let dirs = list_recipe_dirs();
  138. let sorted = dirs.windows(2).all(|w| w[0] <= w[1]);
  139. let _ = std::fs::remove_dir(fixture);
  140. assert!(dirs.iter().any(|d| d == "ztest-list-fixture"));
  141. assert!(sorted);
  142. }
  143. #[tokio::test]
  144. async fn find_tmux_recipes_finds_tmux_deployed() {
  145. let fixture = "recipes/ztest-tmux-fixture";
  146. std::fs::create_dir_all(fixture).unwrap();
  147. std::fs::write(
  148. format!("{fixture}/recipe.toml"),
  149. "name = \"ztest-tmux-fixture\"\ndeploy = \"tmux\"\n",
  150. )
  151. .unwrap();
  152. let recipes = find_tmux_recipes(&[]).await;
  153. let _ = std::fs::remove_dir_all(fixture);
  154. assert!(recipes.iter().any(|(dir, _)| dir == "ztest-tmux-fixture"));
  155. for (_dir, recipe) in &recipes {
  156. assert_eq!(recipe.deploy, DeployTarget::Tmux);
  157. }
  158. }
  159. #[test]
  160. fn print_tmux_health_empty_is_noop() {
  161. // Should not panic or print anything
  162. print_tmux_health(&[]);
  163. }
  164. #[test]
  165. fn print_tmux_health_all_stopped_shows_summary() {
  166. let entries = vec![TmuxHealthEntry {
  167. recipe_display: "Test Recipe".to_string(),
  168. node_name: "dgx1".to_string(),
  169. session_exists: false,
  170. process_running: false,
  171. api_ok: false,
  172. }];
  173. // Should not panic — prints summary line
  174. print_tmux_health(&entries);
  175. }
  176. #[test]
  177. fn print_tmux_health_running_shows_entry() {
  178. let entries = vec![
  179. TmuxHealthEntry {
  180. recipe_display: "Test Recipe".to_string(),
  181. node_name: "dgx1".to_string(),
  182. session_exists: true,
  183. process_running: true,
  184. api_ok: true,
  185. },
  186. TmuxHealthEntry {
  187. recipe_display: "Test Recipe".to_string(),
  188. node_name: "dgx2".to_string(),
  189. session_exists: false,
  190. process_running: false,
  191. api_ok: false,
  192. },
  193. ];
  194. // Should not panic — prints running entry, skips stopped
  195. print_tmux_health(&entries);
  196. }
  197. }