main.rs 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464
  1. #![deny(clippy::wildcard_imports)]
  2. use anyhow::Result;
  3. use clap::Parser;
  4. fluent_i18n::i18n!("locales", fallback = "en");
  5. mod after_stop;
  6. mod before_serve;
  7. mod bench;
  8. mod builtin_scripts;
  9. mod cli;
  10. mod cluster_state;
  11. mod config;
  12. mod detect;
  13. mod docker;
  14. mod doctor;
  15. mod ends;
  16. mod engine_flags;
  17. mod exec;
  18. mod gconfig;
  19. mod health;
  20. mod home_assistant;
  21. mod host_keys;
  22. mod locale;
  23. mod orchestrate;
  24. mod power;
  25. mod pre_serve;
  26. mod publish;
  27. mod registry;
  28. mod report;
  29. mod ssh;
  30. mod tmux;
  31. mod web;
  32. use crate::cli::{Cli, Commands, OutputFormat, PowerAction};
  33. use crate::config::{
  34. Cluster, Config, DeployTarget, load_recipe, resolve_cluster, select_cluster_by_name,
  35. };
  36. use crate::ssh::fetch_host_key;
  37. use std::collections::HashSet;
  38. #[tokio::main]
  39. async fn main() -> Result<()> {
  40. // Locale: check LANG env var first (macOS sys-locale ignores it).
  41. // Normalize "ru_RU.UTF-8" → "ru-RU" for unic-langid.
  42. let locale = std::env::var("LANG")
  43. .ok()
  44. .map(|l| l.split('.').next().unwrap_or(&l).replace('_', "-"));
  45. let _ = fluent_i18n::set_locale(locale.as_deref());
  46. // Capture for spawned tasks — set_locale is thread-local, so tokio
  47. // workers miss it. spawn_with_locale re-applies from this global.
  48. locale::set_global_locale(locale.clone());
  49. tracing_subscriber::fmt()
  50. .with_env_filter(
  51. tracing_subscriber::EnvFilter::try_from_default_env()
  52. .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
  53. )
  54. .init();
  55. let cli = Cli::parse();
  56. let (config_path, raw_config) = crate::gconfig::read_config(&cli.config).await?;
  57. let mut config: Config = toml::from_str(&raw_config)?;
  58. crate::config::merge_cluster_sources(&mut config).await?;
  59. let dry_run = cli.dry_run;
  60. // `config` subcommands are pure path/file utilities: dispatch them
  61. // before cluster_sources merging so they work even with broken sources.
  62. if let Commands::Config(cmd) = &cli.command {
  63. return crate::gconfig::run(cmd, &cli.config, dry_run);
  64. }
  65. match cli.command {
  66. Commands::Serve {
  67. recipe,
  68. cluster,
  69. node,
  70. keep,
  71. skip_before_serve,
  72. no_follow,
  73. auto_power_on,
  74. sync_image,
  75. sync_image_source,
  76. build_node,
  77. force_build,
  78. force,
  79. solo,
  80. mirror,
  81. fit,
  82. clear,
  83. } => {
  84. handle_serve(
  85. &config,
  86. &recipe,
  87. &cluster,
  88. node,
  89. docker::ServeOpts {
  90. keep,
  91. skip_before_serve,
  92. dry_run,
  93. auto_power_on,
  94. sync_image,
  95. sync_image_source,
  96. build_node,
  97. force_build,
  98. force,
  99. solo,
  100. mirror,
  101. fit,
  102. clear,
  103. },
  104. !no_follow,
  105. dry_run,
  106. )
  107. .await?;
  108. }
  109. Commands::Build {
  110. recipe,
  111. build_node,
  112. cluster,
  113. force_build,
  114. } => {
  115. handle_build_cmd(
  116. &config,
  117. &recipe,
  118. &build_node,
  119. &cluster,
  120. force_build,
  121. dry_run,
  122. )
  123. .await?;
  124. }
  125. Commands::Logs {
  126. recipe: recipe_name,
  127. cluster,
  128. follow,
  129. node,
  130. } => {
  131. handle_logs(&config, &recipe_name, &cluster, follow, node, dry_run).await?;
  132. }
  133. Commands::Stop {
  134. recipe: recipe_name,
  135. cluster,
  136. node,
  137. power_off,
  138. no_wait,
  139. } => {
  140. crate::docker::stop::handle(
  141. &config,
  142. &recipe_name,
  143. &cluster,
  144. &node,
  145. !no_wait,
  146. power_off,
  147. dry_run,
  148. )
  149. .await?;
  150. }
  151. Commands::Bench {
  152. recipe,
  153. cluster,
  154. label,
  155. concurrency,
  156. seed,
  157. } => {
  158. let args = crate::bench::Args {
  159. recipe,
  160. cluster,
  161. label,
  162. concurrency,
  163. seed,
  164. };
  165. crate::bench::run(&config, &args, dry_run).await?;
  166. }
  167. Commands::Health {
  168. cluster: cluster_name,
  169. fingerprint,
  170. format,
  171. } => {
  172. handle_health(&config, cluster_name, fingerprint, format).await?;
  173. }
  174. Commands::Doctor {
  175. recipe,
  176. cluster,
  177. repair,
  178. } => {
  179. doctor::doctor(&config, &recipe, &cluster, repair, dry_run).await?;
  180. }
  181. Commands::Publish(a) => {
  182. publish::publish(
  183. &config,
  184. &a.recipe,
  185. a.cluster.as_deref(),
  186. a.out,
  187. a.gist,
  188. a.registry.as_deref(),
  189. a.token.as_deref(),
  190. )
  191. .await?;
  192. }
  193. Commands::Registry(cmd) => registry::handle(cmd).await?,
  194. Commands::Tmux(tmux_cmd) => {
  195. crate::tmux::handle_tmux(&config, tmux_cmd, dry_run).await?;
  196. }
  197. Commands::InitHostKeys { cluster } => {
  198. host_keys::init_host_keys(
  199. &config,
  200. cluster.as_deref(),
  201. config_path.to_str().unwrap_or("config.toml"),
  202. )
  203. .await?;
  204. }
  205. Commands::Config(_) => unreachable!("dispatched above, before merging"),
  206. Commands::Web {
  207. port,
  208. bind,
  209. token,
  210. open,
  211. } => {
  212. crate::web::run(config, port, bind, token, open).await?;
  213. }
  214. Commands::Exec {
  215. cluster,
  216. recipe,
  217. node,
  218. alias,
  219. command,
  220. } => {
  221. handle_exec(&config, cluster, recipe, node, alias, command, dry_run).await?;
  222. }
  223. Commands::Power { action } => {
  224. handle_power(&config, action, dry_run).await?;
  225. }
  226. Commands::Report { serve, port, open } => {
  227. crate::report::run(&config, serve || open, port, open).await?;
  228. }
  229. }
  230. Ok(())
  231. }
  232. /// Handle `serve` command: launch inference containers on a cluster.
  233. ///
  234. /// Dispatches to `docker::serve` or `tmux::serve` based on the recipe's
  235. /// `deploy` field. Docker-only CLI opts (sync_image, build_node,
  236. /// force_build, keep, auto_power_on) are ignored with a warning when
  237. /// `deploy = "tmux"`.
  238. pub(crate) async fn handle_serve(
  239. config: &Config,
  240. recipe: &str,
  241. cluster: &str,
  242. node: Option<String>,
  243. opts: docker::ServeOpts,
  244. follow: bool,
  245. dry_run: bool,
  246. ) -> Result<()> {
  247. let recipe_obj = load_recipe(recipe, &config.recipe_sources).await?;
  248. match recipe_obj.deploy {
  249. DeployTarget::Docker => {
  250. let cluster = docker::serve(config, recipe, cluster, node.as_deref(), opts).await?;
  251. if follow && !dry_run {
  252. tracing::info!("{}", fluent_i18n::t!("container-log-header"));
  253. docker::show_logs(config, &cluster, recipe, true, node.as_deref(), dry_run).await?;
  254. }
  255. }
  256. DeployTarget::Tmux => {
  257. // Warn about docker-only CLI opts that are silently ignored.
  258. let ignored: Vec<&str> = [
  259. (opts.sync_image, "--sync-image"),
  260. (opts.sync_image_source.is_some(), "--sync-image-source"),
  261. (opts.build_node.is_some(), "--build-node"),
  262. (opts.force_build, "--force-build"),
  263. (opts.keep, "--keep"),
  264. (opts.solo.is_some(), "--solo/--no-solo"),
  265. (opts.mirror.is_some(), "--mirror"),
  266. (opts.fit.is_some(), "--fit"),
  267. ]
  268. .iter()
  269. .filter_map(|(set, flag)| if *set { Some(*flag) } else { None })
  270. .collect();
  271. if !ignored.is_empty() {
  272. tracing::warn!(
  273. "{}",
  274. fluent_i18n::t!("tmux-deploy-ignored-opts", { "opts" => &ignored.join(", ") })
  275. );
  276. }
  277. crate::tmux::serve::tmux_serve(
  278. config,
  279. recipe,
  280. cluster,
  281. opts.skip_before_serve,
  282. dry_run,
  283. opts.auto_power_on,
  284. opts.force,
  285. )
  286. .await?;
  287. if follow && !dry_run {
  288. tracing::info!(
  289. "{}",
  290. fluent_i18n::t!("tmux-following-logs", { "name" => recipe.to_string() })
  291. );
  292. crate::tmux::ops::tmux_logs(config, recipe, cluster, true, dry_run).await?;
  293. }
  294. }
  295. }
  296. Ok(())
  297. }
  298. /// Handle `build` command: build a Docker image on a remote node.
  299. async fn handle_build_cmd(
  300. config: &Config,
  301. recipe: &str,
  302. build_node: &str,
  303. cluster: &str,
  304. force_build: bool,
  305. dry_run: bool,
  306. ) -> Result<()> {
  307. let recipe_obj = crate::config::load_recipe(recipe, &config.recipe_sources).await?;
  308. let cluster = crate::config::resolve_build_cluster(config, cluster, build_node)?;
  309. let opts = docker::ServeOpts {
  310. keep: false,
  311. skip_before_serve: true,
  312. dry_run,
  313. auto_power_on: false,
  314. sync_image: false,
  315. sync_image_source: None,
  316. build_node: Some(build_node.to_string()),
  317. force_build,
  318. force: false,
  319. solo: None,
  320. mirror: None,
  321. fit: None,
  322. clear: false,
  323. };
  324. docker::build::handle_build(config, &cluster, recipe, &recipe_obj, &opts, build_node).await?;
  325. Ok(())
  326. }
  327. /// Handle `logs` command: show container logs.
  328. async fn handle_logs(
  329. config: &Config,
  330. recipe_name: &str,
  331. cluster: &str,
  332. follow: bool,
  333. node: Option<String>,
  334. dry_run: bool,
  335. ) -> Result<()> {
  336. let cluster = resolve_cluster_for_op(config, recipe_name, cluster).await?;
  337. docker::show_logs(
  338. config,
  339. &cluster,
  340. recipe_name,
  341. follow,
  342. node.as_deref(),
  343. dry_run,
  344. )
  345. .await?;
  346. Ok(())
  347. }
  348. /// Handle `health` command: check cluster health or show host key fingerprints.
  349. async fn handle_health(
  350. config: &Config,
  351. cluster_name: Option<String>,
  352. fingerprint: bool,
  353. format: OutputFormat,
  354. ) -> Result<()> {
  355. if fingerprint {
  356. let clusters_to_check: Vec<Cluster> = match &cluster_name {
  357. Some(name) => vec![select_cluster_by_name(config, name)?],
  358. None => config.clusters.clone(),
  359. };
  360. for cluster in &clusters_to_check {
  361. println!(
  362. "\n{}",
  363. fluent_i18n::t!("cluster-section", { "name" => &cluster.name })
  364. );
  365. for node in &cluster.nodes {
  366. match fetch_host_key(config, node).await {
  367. Ok(fp) => println!(
  368. "{}",
  369. fluent_i18n::t!("host-key-line", { "name" => &node.name, "ip" => &node.ssh_ip, "key" => &fp })
  370. ),
  371. Err(e) => println!(
  372. "{}",
  373. fluent_i18n::t!("host-key-error", { "name" => &node.name, "ip" => &node.ssh_ip, "error" => &e.to_string() })
  374. ),
  375. }
  376. }
  377. }
  378. } else {
  379. match cluster_name {
  380. Some(ref name) => {
  381. let cluster = select_cluster_by_name(config, name)?;
  382. docker::health::health_check(config, &cluster, false, format).await?;
  383. }
  384. None => {
  385. let fleet = docker::health::collect(config, None, false).await?;
  386. docker::health::print_fleet(&fleet, format);
  387. }
  388. }
  389. }
  390. Ok(())
  391. }
  392. /// Handle `exec` command: run a command on cluster nodes via SSH.
  393. async fn handle_exec(
  394. config: &Config,
  395. cluster: Option<String>,
  396. recipe: Option<String>,
  397. node: Option<String>,
  398. alias: Option<String>,
  399. command: Vec<String>,
  400. dry_run: bool,
  401. ) -> Result<()> {
  402. let (cluster, _recipe) =
  403. exec::resolve_target(config, cluster.as_deref(), recipe.as_deref()).await?;
  404. let cmd_str = exec::resolve_command(config, alias.as_deref(), &command)?;
  405. exec::run(config, &cluster, node.as_deref(), &cmd_str, dry_run).await?;
  406. Ok(())
  407. }
  408. /// Handle `power` command: power on/off/switch nodes.
  409. async fn handle_power(config: &Config, action: PowerAction, dry_run: bool) -> Result<()> {
  410. match action {
  411. PowerAction::Off { cluster, node } => {
  412. let cluster = select_cluster_by_name(config, &cluster)?;
  413. let nodes = node.into_iter().collect::<Vec<_>>();
  414. power::power_off(config, &cluster, &nodes, dry_run).await?;
  415. }
  416. PowerAction::On { cluster, node } => {
  417. let cluster = select_cluster_by_name(config, &cluster)?;
  418. let nodes = node.into_iter().collect::<Vec<_>>();
  419. power::power_on(config, &cluster, &nodes, dry_run).await?;
  420. }
  421. PowerAction::Switch { from, to } => {
  422. let from = select_cluster_by_name(config, &from)?;
  423. let to = select_cluster_by_name(config, &to)?;
  424. power::power_switch(config, &from, &to, dry_run).await?;
  425. }
  426. }
  427. Ok(())
  428. }
  429. /// Resolve a cluster for container operations (stop/logs).
  430. /// When `auto`, tries live container label lookup first, then falls back
  431. /// to config-based auto-selection (for dry-run / offline scenarios).
  432. pub(crate) async fn resolve_cluster_for_op(
  433. config: &Config,
  434. recipe_name: &str,
  435. cluster_name: &str,
  436. ) -> Result<Cluster> {
  437. if cluster_name != "auto" {
  438. let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
  439. let (cluster, _) = resolve_cluster(config, &recipe, cluster_name, &HashSet::new())?;
  440. return Ok(cluster);
  441. }
  442. // Try live container label lookup first
  443. if let Ok(c) = docker::resolve_cluster_by_container_label(config, recipe_name).await {
  444. return Ok(c.clone());
  445. }
  446. // Fall back to config-based auto (dry-run / containers not running)
  447. let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
  448. let (cluster, _) = resolve_cluster(config, &recipe, "auto", &HashSet::new())?;
  449. Ok(cluster)
  450. }