main.rs 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500
  1. #![deny(clippy::wildcard_imports)]
  2. use anyhow::Result;
  3. use clap::Parser;
  4. fluent_i18n::i18n!("locales", fallback = "en");
  5. mod after_stop;
  6. mod before_serve;
  7. mod builtin_scripts;
  8. mod cli;
  9. mod cluster_state;
  10. mod config;
  11. mod detect;
  12. mod docker;
  13. mod doctor;
  14. mod ends;
  15. mod engine_flags;
  16. mod exec;
  17. mod health;
  18. mod home_assistant;
  19. mod host_keys;
  20. mod locale;
  21. mod orchestrate;
  22. mod power;
  23. mod pre_serve;
  24. mod ssh;
  25. mod tmux;
  26. use crate::cli::{Cli, Commands, PowerAction};
  27. use crate::config::{
  28. Cluster, Config, DeployTarget, default_config_dir, load_recipe, resolve_cluster,
  29. select_cluster_by_name,
  30. };
  31. use crate::ssh::fetch_host_key;
  32. use std::collections::HashSet;
  33. /// Resolve the path to the config file.
  34. ///
  35. /// Search order:
  36. /// 1. `--config <path>` from CLI (always wins; error if missing)
  37. /// 2. `./config.toml` in cwd
  38. /// 3. `<default_config_dir>/config.toml`
  39. ///
  40. /// Returns an error if nothing is found, with a hint pointing the user
  41. /// to copy the example config to one of the two paths.
  42. fn resolve_config_path(cli_config: &str) -> std::path::PathBuf {
  43. use std::path::PathBuf;
  44. let explicit = PathBuf::from(cli_config);
  45. // User passed a non-default path: use it as-is.
  46. if cli_config != "config.toml" {
  47. return explicit;
  48. }
  49. // Default value (or no flag): try cwd first, then default folder.
  50. if explicit.exists() {
  51. return explicit;
  52. }
  53. if let Some(dir) = default_config_dir() {
  54. let candidate = dir.join("config.toml");
  55. if candidate.exists() {
  56. return candidate;
  57. }
  58. }
  59. // Fall through; the loader will report a clean error.
  60. explicit
  61. }
  62. #[tokio::main]
  63. async fn main() -> Result<()> {
  64. // Locale: check LANG env var first (macOS sys-locale ignores it).
  65. // Normalize "ru_RU.UTF-8" → "ru-RU" for unic-langid.
  66. let locale = std::env::var("LANG")
  67. .ok()
  68. .map(|l| l.split('.').next().unwrap_or(&l).replace('_', "-"));
  69. let _ = fluent_i18n::set_locale(locale.as_deref());
  70. // Capture for spawned tasks — set_locale is thread-local, so tokio
  71. // workers miss it. spawn_with_locale re-applies from this global.
  72. locale::set_global_locale(locale.clone());
  73. tracing_subscriber::fmt()
  74. .with_env_filter(
  75. tracing_subscriber::EnvFilter::try_from_default_env()
  76. .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
  77. )
  78. .init();
  79. let cli = Cli::parse();
  80. let config_path = resolve_config_path(&cli.config);
  81. let raw_config = tokio::fs::read_to_string(&config_path).await?;
  82. let config: Config = toml::from_str(&raw_config)?;
  83. let dry_run = cli.dry_run;
  84. match cli.command {
  85. Commands::Serve {
  86. recipe,
  87. cluster,
  88. node,
  89. keep,
  90. skip_before_serve,
  91. follow,
  92. auto_power_on,
  93. sync_image,
  94. sync_image_source,
  95. build_node,
  96. force_build,
  97. force,
  98. solo,
  99. mirror,
  100. fit,
  101. clear,
  102. } => {
  103. handle_serve(
  104. &config,
  105. &recipe,
  106. &cluster,
  107. node,
  108. docker::ServeOpts {
  109. keep,
  110. skip_before_serve,
  111. dry_run,
  112. auto_power_on,
  113. sync_image,
  114. sync_image_source,
  115. build_node,
  116. force_build,
  117. force,
  118. solo,
  119. mirror,
  120. fit,
  121. clear,
  122. },
  123. follow,
  124. dry_run,
  125. )
  126. .await?;
  127. }
  128. Commands::Build {
  129. recipe,
  130. build_node,
  131. cluster,
  132. force_build,
  133. } => {
  134. handle_build_cmd(
  135. &config,
  136. &recipe,
  137. &build_node,
  138. &cluster,
  139. force_build,
  140. dry_run,
  141. )
  142. .await?;
  143. }
  144. Commands::Logs {
  145. recipe: recipe_name,
  146. cluster,
  147. follow,
  148. node,
  149. } => {
  150. handle_logs(&config, &recipe_name, &cluster, follow, node, dry_run).await?;
  151. }
  152. Commands::Stop {
  153. recipe: recipe_name,
  154. cluster,
  155. node,
  156. power_off,
  157. } => {
  158. handle_stop(&config, &recipe_name, &cluster, &node, power_off, dry_run).await?;
  159. }
  160. Commands::Health {
  161. cluster: cluster_name,
  162. fingerprint,
  163. } => {
  164. handle_health(&config, cluster_name, fingerprint).await?;
  165. }
  166. Commands::Doctor {
  167. recipe,
  168. cluster,
  169. repair,
  170. } => {
  171. doctor::doctor(&config, &recipe, &cluster, repair, dry_run).await?;
  172. }
  173. Commands::Tmux(tmux_cmd) => {
  174. crate::tmux::handle_tmux(&config, tmux_cmd, dry_run).await?;
  175. }
  176. Commands::InitHostKeys { cluster } => {
  177. host_keys::init_host_keys(
  178. &config,
  179. cluster.as_deref(),
  180. config_path.to_str().unwrap_or("config.toml"),
  181. )
  182. .await?;
  183. }
  184. Commands::Exec {
  185. cluster,
  186. recipe,
  187. node,
  188. alias,
  189. command,
  190. } => {
  191. handle_exec(&config, cluster, recipe, node, alias, command, dry_run).await?;
  192. }
  193. Commands::Power { action } => {
  194. handle_power(&config, action, dry_run).await?;
  195. }
  196. }
  197. Ok(())
  198. }
  199. /// Handle `serve` command: launch inference containers on a cluster.
  200. ///
  201. /// Dispatches to `docker::serve` or `tmux::serve` based on the recipe's
  202. /// `deploy` field. Docker-only CLI opts (sync_image, build_node,
  203. /// force_build, keep, auto_power_on) are ignored with a warning when
  204. /// `deploy = "tmux"`.
  205. async fn handle_serve(
  206. config: &Config,
  207. recipe: &str,
  208. cluster: &str,
  209. node: Option<String>,
  210. opts: docker::ServeOpts,
  211. follow: bool,
  212. dry_run: bool,
  213. ) -> Result<()> {
  214. let recipe_obj = load_recipe(recipe).await?;
  215. match recipe_obj.deploy {
  216. DeployTarget::Docker => {
  217. let cluster = docker::serve(config, recipe, cluster, node.as_deref(), opts).await?;
  218. if follow && !dry_run {
  219. tracing::info!("{}", fluent_i18n::t!("container-log-header"));
  220. docker::show_logs(config, &cluster, recipe, true, node.as_deref(), dry_run).await?;
  221. }
  222. }
  223. DeployTarget::Tmux => {
  224. // Warn about docker-only CLI opts that are silently ignored.
  225. let ignored: Vec<&str> = [
  226. (opts.sync_image, "--sync-image"),
  227. (opts.sync_image_source.is_some(), "--sync-image-source"),
  228. (opts.build_node.is_some(), "--build-node"),
  229. (opts.force_build, "--force-build"),
  230. (opts.keep, "--keep"),
  231. (opts.solo.is_some(), "--solo/--no-solo"),
  232. (opts.mirror.is_some(), "--mirror"),
  233. (opts.fit.is_some(), "--fit"),
  234. ]
  235. .iter()
  236. .filter_map(|(set, flag)| if *set { Some(*flag) } else { None })
  237. .collect();
  238. if !ignored.is_empty() {
  239. tracing::warn!(
  240. "{}",
  241. fluent_i18n::t!("tmux-deploy-ignored-opts", { "opts" => &ignored.join(", ") })
  242. );
  243. }
  244. crate::tmux::serve::tmux_serve(
  245. config,
  246. recipe,
  247. cluster,
  248. opts.skip_before_serve,
  249. dry_run,
  250. opts.auto_power_on,
  251. opts.force,
  252. )
  253. .await?;
  254. if follow && !dry_run {
  255. tracing::info!(
  256. "{}",
  257. fluent_i18n::t!("tmux-following-logs", { "name" => recipe.to_string() })
  258. );
  259. crate::tmux::ops::tmux_logs(config, recipe, cluster, true, dry_run).await?;
  260. }
  261. }
  262. }
  263. Ok(())
  264. }
  265. /// Handle `build` command: build a Docker image on a remote node.
  266. async fn handle_build_cmd(
  267. config: &Config,
  268. recipe: &str,
  269. build_node: &str,
  270. cluster: &str,
  271. force_build: bool,
  272. dry_run: bool,
  273. ) -> Result<()> {
  274. let recipe_obj = crate::config::load_recipe(recipe).await?;
  275. let cluster = crate::config::resolve_build_cluster(config, cluster, build_node)?;
  276. let opts = docker::ServeOpts {
  277. keep: false,
  278. skip_before_serve: true,
  279. dry_run,
  280. auto_power_on: false,
  281. sync_image: false,
  282. sync_image_source: None,
  283. build_node: Some(build_node.to_string()),
  284. force_build,
  285. force: false,
  286. solo: None,
  287. mirror: None,
  288. fit: None,
  289. clear: false,
  290. };
  291. docker::build::handle_build(config, &cluster, recipe, &recipe_obj, &opts, build_node).await?;
  292. Ok(())
  293. }
  294. /// Handle `logs` command: show container logs.
  295. async fn handle_logs(
  296. config: &Config,
  297. recipe_name: &str,
  298. cluster: &str,
  299. follow: bool,
  300. node: Option<String>,
  301. dry_run: bool,
  302. ) -> Result<()> {
  303. let cluster = resolve_cluster_for_op(config, recipe_name, cluster).await?;
  304. docker::show_logs(
  305. config,
  306. &cluster,
  307. recipe_name,
  308. follow,
  309. node.as_deref(),
  310. dry_run,
  311. )
  312. .await?;
  313. Ok(())
  314. }
  315. /// Handle `stop` command: stop containers and optionally power off.
  316. async fn handle_stop(
  317. config: &Config,
  318. recipe_name: &str,
  319. cluster: &str,
  320. node: &[String],
  321. power_off: bool,
  322. dry_run: bool,
  323. ) -> Result<()> {
  324. let (stopped_cluster, stopped_nodes): (Cluster, Vec<String>) = if !node.is_empty() {
  325. let cluster = resolve_cluster_for_op(config, recipe_name, cluster).await?;
  326. let nodes = node.to_vec();
  327. docker::stop_cluster(config, &cluster, recipe_name, &nodes, dry_run).await?;
  328. (cluster, nodes)
  329. } else {
  330. let variants = docker::deploy::discover_recipe_deployments(config, recipe_name).await?;
  331. if variants.is_empty() {
  332. anyhow::bail!("No running containers found for recipe '{}'", recipe_name);
  333. }
  334. if variants.len() == 1 {
  335. let v = &variants[0];
  336. let cluster = select_cluster_by_name(config, &v.cluster)?;
  337. let nodes = v.nodes.clone();
  338. docker::stop_cluster(config, &cluster, recipe_name, &nodes, dry_run).await?;
  339. (cluster, nodes)
  340. } else {
  341. let mut msg = format!(
  342. "Multiple deployments of '{}' found — use --node to select:\n",
  343. recipe_name
  344. );
  345. for (i, v) in variants.iter().enumerate() {
  346. let nodes = v
  347. .nodes
  348. .iter()
  349. .map(|n| format!("--node {}", n))
  350. .collect::<Vec<_>>()
  351. .join(" ");
  352. msg.push_str(&format!(
  353. " {}: cluster '{}' with {}\n",
  354. i + 1,
  355. v.cluster,
  356. nodes
  357. ));
  358. }
  359. anyhow::bail!(msg);
  360. }
  361. };
  362. // Run recipe `after_stop` hooks (e.g. restore_dgx_services) on the
  363. // nodes that were actually stopped. Best-effort against the loaded
  364. // recipe; if the recipe file is unavailable, skip silently.
  365. if let Ok(recipe) = crate::config::load_recipe(recipe_name).await {
  366. crate::after_stop::run(config, &stopped_cluster, &recipe, dry_run).await?;
  367. }
  368. if power_off {
  369. crate::power::power_off(config, &stopped_cluster, &stopped_nodes, dry_run).await?;
  370. }
  371. Ok(())
  372. }
  373. /// Handle `health` command: check cluster health or show host key fingerprints.
  374. async fn handle_health(
  375. config: &Config,
  376. cluster_name: Option<String>,
  377. fingerprint: bool,
  378. ) -> Result<()> {
  379. if fingerprint {
  380. let clusters_to_check: Vec<Cluster> = match &cluster_name {
  381. Some(name) => vec![select_cluster_by_name(config, name)?],
  382. None => config.clusters.clone(),
  383. };
  384. for cluster in &clusters_to_check {
  385. println!(
  386. "\n{}",
  387. fluent_i18n::t!("cluster-section", { "name" => &cluster.name })
  388. );
  389. for node in &cluster.nodes {
  390. match fetch_host_key(config, node).await {
  391. Ok(fp) => println!(
  392. "{}",
  393. fluent_i18n::t!("host-key-line", { "name" => &node.name, "ip" => &node.ssh_ip, "key" => &fp })
  394. ),
  395. Err(e) => println!(
  396. "{}",
  397. fluent_i18n::t!("host-key-error", { "name" => &node.name, "ip" => &node.ssh_ip, "error" => &e.to_string() })
  398. ),
  399. }
  400. }
  401. }
  402. } else {
  403. match cluster_name {
  404. Some(ref name) => {
  405. let cluster = select_cluster_by_name(config, name)?;
  406. docker::health::health_check(config, &cluster, false).await?;
  407. let tmux_entries = tmux::discover::tmux_health_for_cluster(config, &cluster).await;
  408. tmux::discover::print_tmux_health(&tmux_entries);
  409. }
  410. None => {
  411. for cluster in &config.clusters {
  412. docker::health::health_check(config, cluster, false).await?;
  413. let tmux_entries =
  414. tmux::discover::tmux_health_for_cluster(config, cluster).await;
  415. tmux::discover::print_tmux_health(&tmux_entries);
  416. }
  417. docker::health::health_summary(config).await?;
  418. }
  419. }
  420. }
  421. Ok(())
  422. }
  423. /// Handle `exec` command: run a command on cluster nodes via SSH.
  424. async fn handle_exec(
  425. config: &Config,
  426. cluster: Option<String>,
  427. recipe: Option<String>,
  428. node: Option<String>,
  429. alias: Option<String>,
  430. command: Vec<String>,
  431. dry_run: bool,
  432. ) -> Result<()> {
  433. let (cluster, _recipe) =
  434. exec::resolve_target(config, cluster.as_deref(), recipe.as_deref()).await?;
  435. let cmd_str = exec::resolve_command(config, alias.as_deref(), &command)?;
  436. exec::run(config, &cluster, node.as_deref(), &cmd_str, dry_run).await?;
  437. Ok(())
  438. }
  439. /// Handle `power` command: power on/off/switch nodes.
  440. async fn handle_power(config: &Config, action: PowerAction, dry_run: bool) -> Result<()> {
  441. match action {
  442. PowerAction::Off { cluster, node } => {
  443. let cluster = select_cluster_by_name(config, &cluster)?;
  444. let nodes = node.into_iter().collect::<Vec<_>>();
  445. power::power_off(config, &cluster, &nodes, dry_run).await?;
  446. }
  447. PowerAction::On { cluster, node } => {
  448. let cluster = select_cluster_by_name(config, &cluster)?;
  449. let nodes = node.into_iter().collect::<Vec<_>>();
  450. power::power_on(config, &cluster, &nodes, dry_run).await?;
  451. }
  452. PowerAction::Switch { from, to } => {
  453. let from = select_cluster_by_name(config, &from)?;
  454. let to = select_cluster_by_name(config, &to)?;
  455. power::power_switch(config, &from, &to, dry_run).await?;
  456. }
  457. }
  458. Ok(())
  459. }
  460. /// Resolve a cluster for container operations (stop/logs).
  461. /// When `auto`, tries live container label lookup first, then falls back
  462. /// to config-based auto-selection (for dry-run / offline scenarios).
  463. async fn resolve_cluster_for_op(
  464. config: &Config,
  465. recipe_name: &str,
  466. cluster_name: &str,
  467. ) -> Result<Cluster> {
  468. if cluster_name != "auto" {
  469. let recipe = load_recipe(recipe_name).await?;
  470. let (cluster, _) = resolve_cluster(config, &recipe, cluster_name, &HashSet::new())?;
  471. return Ok(cluster);
  472. }
  473. // Try live container label lookup first
  474. if let Ok(c) = docker::resolve_cluster_by_container_label(config, recipe_name).await {
  475. return Ok(c.clone());
  476. }
  477. // Fall back to config-based auto (dry-run / containers not running)
  478. let recipe = load_recipe(recipe_name).await?;
  479. let (cluster, _) = resolve_cluster(config, &recipe, "auto", &HashSet::new())?;
  480. Ok(cluster)
  481. }