exec.rs 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229
  1. use anyhow::{Context, Result, anyhow};
  2. use tracing::{info, warn};
  3. use crate::config::{Cluster, Config, Recipe, select_cluster};
  4. use std::collections::HashSet;
  5. /// Resolve the target cluster for `cargo run exec` from `--cluster` or
  6. /// `--recipe` (exactly one of which must be given).
  7. pub async fn resolve_target(
  8. config: &Config,
  9. cluster_name: Option<&str>,
  10. recipe_name: Option<&str>,
  11. ) -> Result<(Cluster, Option<Recipe>)> {
  12. match (cluster_name, recipe_name) {
  13. (Some(_), Some(_)) => {
  14. anyhow::bail!(fluent_i18n::t!("exec-both-specified"))
  15. }
  16. (None, None) => {
  17. anyhow::bail!(fluent_i18n::t!("exec-missing"))
  18. }
  19. (Some(name), None) => {
  20. let cluster = crate::config::select_cluster_by_name(config, name)?;
  21. Ok((cluster, None))
  22. }
  23. (None, Some(rname)) => {
  24. let recipe = crate::config::load_recipe(rname, &config.recipe_sources).await?;
  25. let cname = recipe.cluster.as_deref().unwrap_or("auto");
  26. let (cluster, _) =
  27. select_cluster(config, cname, recipe.required_nodes, &HashSet::new())
  28. .with_context(|| format!("resolving cluster for recipe '{}'", rname))?;
  29. Ok((cluster, Some(recipe)))
  30. }
  31. }
  32. }
  33. /// Run `command` on every node of `cluster` (or a single node if `node`
  34. /// is given) and stream the output. Used by the `cargo run exec`
  35. /// subcommand to support mass-execution flows like `sudo shutdown now`.
  36. pub async fn run(
  37. config: &Config,
  38. cluster: &Cluster,
  39. node_name: Option<&str>,
  40. command: &str,
  41. dry_run: bool,
  42. ) -> Result<()> {
  43. let targets: Vec<_> = match node_name {
  44. Some(name) => cluster
  45. .nodes
  46. .iter()
  47. .filter(|n| n.name == name)
  48. .cloned()
  49. .collect(),
  50. None => cluster.nodes.clone(),
  51. };
  52. if targets.is_empty() {
  53. if let Some(n) = node_name {
  54. let name = n.to_string();
  55. anyhow::bail!(
  56. fluent_i18n::t!("exec-node-not-found", { "name" => &name, "cluster" => &cluster.name })
  57. );
  58. }
  59. anyhow::bail!(fluent_i18n::t!("exec-cluster-empty", { "name" => &cluster.name }));
  60. }
  61. let cmd = command.to_string();
  62. info!(
  63. "{}",
  64. fluent_i18n::t!("exec-header", { "cmd" => &cmd, "nodes" => targets.len() })
  65. );
  66. let mut handles = Vec::with_capacity(targets.len());
  67. for node in targets {
  68. let config = config.clone();
  69. let command = command.to_string();
  70. let dry = dry_run;
  71. handles.push(crate::locale::spawn_with_locale(async move {
  72. let name = node.name.clone();
  73. let result = crate::ssh::run_ssh(&config, &node, command.clone(), dry).await;
  74. (name, command, result)
  75. }));
  76. }
  77. let mut any_failed = false;
  78. for h in handles {
  79. let (name, command, result) = h.await?;
  80. match result {
  81. Ok(output) => {
  82. if !output.trim().is_empty() {
  83. println!("{}", fluent_i18n::t!("exec-section", { "name" => &name }));
  84. print!("{}", output);
  85. if !output.ends_with('\n') {
  86. println!();
  87. }
  88. }
  89. }
  90. Err(e) => {
  91. any_failed = true;
  92. warn!(
  93. "{}",
  94. fluent_i18n::t!("exec-failed", { "name" => &name, "cmd" => &command, "error" => &e.to_string() })
  95. );
  96. }
  97. }
  98. }
  99. if any_failed {
  100. warn!("{}", fluent_i18n::t!("exec-completed-errors"));
  101. }
  102. Ok(())
  103. }
  104. /// Resolve the command to execute for `cargo run exec`.
  105. ///
  106. /// - `--alias <name>` looks up `name` in `config.exec_aliases` and returns
  107. /// the configured shell command. Errors if not found.
  108. /// - Otherwise joins the trailing positional args into a single command
  109. /// line (must be non-empty).
  110. ///
  111. /// Exactly one of `alias_name` or `command_args` must be provided.
  112. pub fn resolve_command(
  113. config: &Config,
  114. alias_name: Option<&str>,
  115. command_args: &[String],
  116. ) -> Result<String> {
  117. let has_alias = alias_name.is_some();
  118. let has_args = !command_args.is_empty();
  119. match (has_alias, has_args) {
  120. (true, false) => {
  121. let name = alias_name.expect("checked by has_alias").to_string();
  122. config.exec_aliases.get(&name).cloned().ok_or_else(|| {
  123. let mut known: Vec<String> = config.exec_aliases.keys().cloned().collect();
  124. known.sort();
  125. if known.is_empty() {
  126. anyhow!(fluent_i18n::t!("exec-unknown-alias-empty", { "name" => &name }))
  127. } else {
  128. anyhow!(fluent_i18n::t!("exec-unknown-alias-list", { "name" => &name, "aliases" => &known.join(", ") }))
  129. }
  130. })
  131. }
  132. (true, true) => {
  133. anyhow::bail!(fluent_i18n::t!("exec-cannot-use-alias"))
  134. }
  135. (false, false) => {
  136. anyhow::bail!(fluent_i18n::t!("exec-must-specify"))
  137. }
  138. (false, true) => Ok(command_args.join(" ")),
  139. }
  140. }
  141. #[cfg(test)]
  142. mod tests {
  143. use super::*;
  144. use std::collections::HashMap;
  145. fn test_config(aliases: &[(&str, &str)]) -> Config {
  146. let exec_aliases: HashMap<String, String> = aliases
  147. .iter()
  148. .map(|(k, v)| (k.to_string(), v.to_string()))
  149. .collect();
  150. Config {
  151. ssh_user: "u".to_string(),
  152. ssh_key_path: "/k".into(),
  153. clusters: vec![],
  154. nccl: HashMap::new(),
  155. hf_cache_dir: None,
  156. init_port: 25000,
  157. before_serve: HashMap::new(),
  158. exec_aliases,
  159. home_assistant: None,
  160. mikrotik: None,
  161. drop_caches: true,
  162. mount_aliases: HashMap::new(),
  163. recipe_sources: Vec::new(),
  164. docker_image_sources: Vec::new(),
  165. cluster_sources: Vec::new(),
  166. global_mods_dir: None,
  167. }
  168. }
  169. #[test]
  170. fn resolve_command_alias() {
  171. let cfg = test_config(&[("gpu-lock", "sudo nvidia-smi -lgc 300,2300")]);
  172. let cmd = resolve_command(&cfg, Some("gpu-lock"), &[]).unwrap();
  173. assert_eq!(cmd, "sudo nvidia-smi -lgc 300,2300");
  174. }
  175. #[test]
  176. fn resolve_command_unknown_alias_lists_known() {
  177. let cfg = test_config(&[("gpu-lock", "a"), ("shutdown", "b")]);
  178. let err = resolve_command(&cfg, Some("nope"), &[]).unwrap_err();
  179. let msg = err.to_string();
  180. assert!(msg.contains("Unknown alias @nope"), "got: {msg}");
  181. assert!(msg.contains("gpu-lock"), "got: {msg}");
  182. assert!(msg.contains("shutdown"), "got: {msg}");
  183. }
  184. #[test]
  185. fn resolve_command_unknown_alias_no_aliases() {
  186. let cfg = test_config(&[]);
  187. let err = resolve_command(&cfg, Some("anything"), &[]).unwrap_err();
  188. let msg = err.to_string();
  189. assert!(msg.contains("no aliases configured"), "got: {msg}");
  190. }
  191. #[test]
  192. fn resolve_command_no_alias_no_args_errors() {
  193. let cfg = test_config(&[]);
  194. let err = resolve_command(&cfg, None, &[]).unwrap_err();
  195. let msg = err.to_string();
  196. assert!(msg.contains("Must specify"), "got: {msg}");
  197. }
  198. #[test]
  199. fn resolve_command_alias_and_args_errors() {
  200. let cfg = test_config(&[("gpu", "a")]);
  201. let err = resolve_command(&cfg, Some("gpu"), &["ls".into()]).unwrap_err();
  202. let msg = err.to_string();
  203. assert!(msg.contains("Cannot use --alias together"), "got: {msg}");
  204. }
  205. #[test]
  206. fn resolve_command_literal_join() {
  207. let cfg = test_config(&[]);
  208. let cmd =
  209. resolve_command(&cfg, None, &["echo".into(), "hello".into(), "world".into()]).unwrap();
  210. assert_eq!(cmd, "echo hello world");
  211. }
  212. }