main.rs 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498
  1. #![deny(clippy::wildcard_imports)]
  2. use crate::docker::sync;
  3. use anyhow::Result;
  4. use clap::Parser;
  5. fluent_i18n::i18n!("locales", fallback = "en");
  6. mod after_stop;
  7. mod ansi;
  8. mod before_serve;
  9. mod bench;
  10. mod bench_artifacts;
  11. mod bench_depth;
  12. mod bench_report;
  13. mod builtin_scripts;
  14. mod cli;
  15. mod cluster_state;
  16. mod config;
  17. mod detect;
  18. mod docker;
  19. mod doctor;
  20. mod ends;
  21. mod engine_flags;
  22. mod exec;
  23. mod gconfig;
  24. mod health;
  25. mod home_assistant;
  26. mod host_keys;
  27. mod locale;
  28. mod mikrotik;
  29. mod orchestrate;
  30. mod power;
  31. mod pre_serve;
  32. mod publish;
  33. mod publish_render;
  34. mod readiness;
  35. mod registry;
  36. mod report;
  37. mod runlock;
  38. mod ssh;
  39. mod staging;
  40. mod stamp;
  41. mod tmux;
  42. mod web;
  43. use crate::cli::{Cli, Commands, OutputFormat, PowerAction};
  44. use crate::config::{
  45. Cluster, Config, DeployTarget, load_recipe, resolve_cluster, select_cluster_by_name,
  46. };
  47. use crate::ssh::fetch_host_key;
  48. use std::collections::HashSet;
  49. #[tokio::main]
  50. async fn main() -> Result<()> {
  51. // Locale: check LANG env var first (macOS sys-locale ignores it).
  52. // Normalize "ru_RU.UTF-8" → "ru-RU" for unic-langid.
  53. let locale = std::env::var("LANG")
  54. .ok()
  55. .map(|l| l.split('.').next().unwrap_or(&l).replace('_', "-"));
  56. let _ = fluent_i18n::set_locale(locale.as_deref());
  57. locale::set_global_locale(locale.clone()); // tokio workers miss the thread-local; spawn_with_locale re-applies
  58. tracing_subscriber::fmt()
  59. .with_env_filter(
  60. tracing_subscriber::EnvFilter::try_from_default_env()
  61. .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
  62. )
  63. .init();
  64. let cli = Cli::parse();
  65. let (config_path, raw_config) = crate::gconfig::read_config(&cli.config).await?;
  66. let mut config: Config = toml::from_str(&raw_config)?;
  67. crate::config::merge_cluster_sources(&mut config).await?;
  68. let dry_run = cli.dry_run;
  69. // `config` subcommands are pure path/file utilities: dispatch them
  70. // before cluster_sources merging so they work even with broken sources.
  71. if let Commands::Config(cmd) = &cli.command {
  72. return crate::gconfig::run(cmd, &cli.config, dry_run);
  73. }
  74. match cli.command {
  75. Commands::Serve {
  76. recipe,
  77. cluster,
  78. node,
  79. keep,
  80. skip_before_serve,
  81. no_follow,
  82. no_wait,
  83. auto_power_on,
  84. sync_image,
  85. sync_image_source,
  86. build_node,
  87. force_build,
  88. force,
  89. solo,
  90. mirror,
  91. hf_cache,
  92. stage,
  93. fit,
  94. clear,
  95. no_push,
  96. } => {
  97. handle_serve(
  98. &config,
  99. &recipe,
  100. &cluster,
  101. node,
  102. docker::ServeOpts {
  103. keep,
  104. skip_before_serve,
  105. no_wait_ready: no_wait,
  106. dry_run,
  107. auto_power_on,
  108. sync_image,
  109. sync_image_source,
  110. no_push,
  111. build_node,
  112. force_build,
  113. force,
  114. solo,
  115. mirror,
  116. hf_cache,
  117. stage,
  118. fit,
  119. clear,
  120. },
  121. !no_follow,
  122. dry_run,
  123. )
  124. .await?;
  125. }
  126. Commands::SyncImage(a) => {
  127. sync::handle_sync_image(&config, &a.recipe, &a.cluster, a.source.as_deref(), dry_run)
  128. .await?;
  129. }
  130. Commands::Build {
  131. recipe,
  132. build_node,
  133. cluster,
  134. force_build,
  135. no_push,
  136. } => {
  137. docker::build::handle_build_cmd(
  138. &config,
  139. &recipe,
  140. &build_node,
  141. &cluster,
  142. force_build,
  143. no_push,
  144. dry_run,
  145. )
  146. .await?;
  147. }
  148. Commands::Logs {
  149. recipe: recipe_name,
  150. cluster,
  151. follow,
  152. node,
  153. } => {
  154. handle_logs(&config, &recipe_name, &cluster, follow, node, dry_run).await?;
  155. }
  156. Commands::Stop {
  157. recipe: recipe_name,
  158. cluster,
  159. node,
  160. power_off,
  161. no_wait,
  162. } => {
  163. crate::docker::stop::handle(
  164. &config,
  165. &recipe_name,
  166. &cluster,
  167. &node,
  168. !no_wait,
  169. power_off,
  170. dry_run,
  171. )
  172. .await?;
  173. }
  174. Commands::Bench {
  175. recipe,
  176. cluster,
  177. label,
  178. concurrency,
  179. seed,
  180. parallel,
  181. perf_only,
  182. tokenizer,
  183. } => {
  184. let args = crate::bench::Args {
  185. recipe,
  186. cluster,
  187. label,
  188. concurrency,
  189. seed,
  190. parallel,
  191. perf_only,
  192. tokenizer,
  193. };
  194. crate::bench::run(&config, &args, dry_run).await?;
  195. }
  196. Commands::Health {
  197. cluster: cluster_name,
  198. fingerprint,
  199. format,
  200. } => {
  201. handle_health(&config, cluster_name, fingerprint, format).await?;
  202. }
  203. Commands::Doctor {
  204. recipe,
  205. cluster,
  206. repair,
  207. } => {
  208. doctor::doctor(&config, &recipe, &cluster, repair, dry_run).await?;
  209. }
  210. Commands::Publish(a) => {
  211. publish::publish(
  212. &config,
  213. &a.recipe,
  214. a.cluster.as_deref(),
  215. a.out,
  216. a.gist,
  217. &publish::RecipeRegistryOpts {
  218. url: a.registry.clone(),
  219. token: a.token.clone(),
  220. },
  221. publish::PushOpts {
  222. push: a.push,
  223. node: a.node.clone(),
  224. force: a.force,
  225. },
  226. )
  227. .await?;
  228. }
  229. Commands::Registry(cmd) => registry::handle(cmd).await?,
  230. Commands::Tmux(tmux_cmd) => {
  231. crate::tmux::handle_tmux(&config, tmux_cmd, dry_run).await?;
  232. }
  233. Commands::InitHostKeys { cluster } => {
  234. host_keys::init_host_keys(
  235. &config,
  236. cluster.as_deref(),
  237. config_path.to_str().unwrap_or("config.toml"),
  238. )
  239. .await?;
  240. }
  241. Commands::Config(_) => unreachable!("dispatched above, before merging"),
  242. Commands::Web {
  243. port,
  244. bind,
  245. token,
  246. open,
  247. } => {
  248. crate::web::run(config, port, bind, token, open).await?;
  249. }
  250. Commands::Exec {
  251. cluster,
  252. recipe,
  253. node,
  254. alias,
  255. command,
  256. } => {
  257. handle_exec(&config, cluster, recipe, node, alias, command, dry_run).await?;
  258. }
  259. Commands::Power { action } => {
  260. handle_power(&config, action, dry_run).await?;
  261. }
  262. Commands::Report {
  263. serve,
  264. port,
  265. open,
  266. archive,
  267. unarchive,
  268. archived,
  269. } => {
  270. crate::report::run(
  271. &config,
  272. serve || open,
  273. port,
  274. open,
  275. archive,
  276. unarchive,
  277. archived,
  278. )
  279. .await?;
  280. }
  281. Commands::Runlock(cmd) => crate::runlock::run(&config, &cmd)?,
  282. }
  283. Ok(())
  284. }
  285. /// Handle `serve` command: launch inference containers on a cluster.
  286. ///
  287. /// Dispatches to `docker::serve` or `tmux::serve` based on the recipe's
  288. /// `deploy` field. Docker-only CLI opts (sync_image, build_node,
  289. /// force_build, keep, auto_power_on) are ignored with a warning when
  290. /// `deploy = "tmux"`.
  291. pub(crate) async fn handle_serve(
  292. config: &Config,
  293. recipe: &str,
  294. cluster: &str,
  295. node: Option<String>,
  296. opts: docker::ServeOpts,
  297. follow: bool,
  298. dry_run: bool,
  299. ) -> Result<()> {
  300. let recipe_obj = load_recipe(recipe, &config.recipe_sources).await?;
  301. match recipe_obj.deploy {
  302. DeployTarget::Docker => {
  303. let cluster = docker::serve(config, recipe, cluster, node.as_deref(), opts).await?;
  304. if follow && !dry_run {
  305. tracing::info!("{}", fluent_i18n::t!("container-log-header"));
  306. docker::logs::follow_with_liveness(
  307. config,
  308. &cluster,
  309. recipe,
  310. recipe_obj.port,
  311. node.as_deref(),
  312. dry_run,
  313. )
  314. .await?;
  315. }
  316. }
  317. DeployTarget::Tmux => {
  318. // Warn about docker-only CLI opts that are silently ignored.
  319. let ignored: Vec<&str> = [
  320. (opts.sync_image, "--sync-image"),
  321. (opts.sync_image_source.is_some(), "--sync-image-source"),
  322. (opts.build_node.is_some(), "--build-node"),
  323. (opts.force_build, "--force-build"),
  324. (opts.keep, "--keep"),
  325. (opts.solo.is_some(), "--solo/--no-solo"),
  326. (opts.mirror.is_some(), "--mirror"),
  327. (opts.fit.is_some(), "--fit"),
  328. ]
  329. .iter()
  330. .filter_map(|(set, flag)| if *set { Some(*flag) } else { None })
  331. .collect();
  332. if !ignored.is_empty() {
  333. tracing::warn!(
  334. "{}",
  335. fluent_i18n::t!("tmux-deploy-ignored-opts", { "opts" => &ignored.join(", ") })
  336. );
  337. }
  338. crate::tmux::serve::tmux_serve(
  339. config,
  340. recipe,
  341. cluster,
  342. opts.skip_before_serve,
  343. dry_run,
  344. opts.auto_power_on,
  345. opts.force,
  346. )
  347. .await?;
  348. if !dry_run {
  349. // no readiness probe for tmux; cluster stays None (unresolved here)
  350. crate::runlock::mark_start(recipe, &config.recipe_sources, None)?;
  351. }
  352. if follow && !dry_run {
  353. tracing::info!(
  354. "{}",
  355. fluent_i18n::t!("tmux-following-logs", { "name" => recipe.to_string() })
  356. );
  357. crate::tmux::ops::tmux_logs(config, recipe, cluster, true, dry_run).await?;
  358. }
  359. }
  360. }
  361. Ok(())
  362. }
  363. /// Handle `logs` command: show container logs.
  364. async fn handle_logs(
  365. config: &Config,
  366. recipe_name: &str,
  367. cluster: &str,
  368. follow: bool,
  369. node: Option<String>,
  370. dry_run: bool,
  371. ) -> Result<()> {
  372. let cluster = resolve_cluster_for_op(config, recipe_name, cluster).await?;
  373. docker::show_logs(
  374. config,
  375. &cluster,
  376. recipe_name,
  377. follow,
  378. node.as_deref(),
  379. dry_run,
  380. )
  381. .await?;
  382. Ok(())
  383. }
  384. /// Handle `health` command: check cluster health or show host key fingerprints.
  385. async fn handle_health(
  386. config: &Config,
  387. cluster_name: Option<String>,
  388. fingerprint: bool,
  389. format: OutputFormat,
  390. ) -> Result<()> {
  391. if fingerprint {
  392. let clusters_to_check: Vec<Cluster> = match &cluster_name {
  393. Some(name) => vec![select_cluster_by_name(config, name)?],
  394. None => config.clusters.clone(),
  395. };
  396. for cluster in &clusters_to_check {
  397. println!(
  398. "\n{}",
  399. fluent_i18n::t!("cluster-section", { "name" => &cluster.name })
  400. );
  401. for node in &cluster.nodes {
  402. match fetch_host_key(config, node).await {
  403. Ok(fp) => println!(
  404. "{}",
  405. fluent_i18n::t!("host-key-line", { "name" => &node.name, "ip" => &node.ssh_ip, "key" => &fp })
  406. ),
  407. Err(e) => println!(
  408. "{}",
  409. fluent_i18n::t!("host-key-error", { "name" => &node.name, "ip" => &node.ssh_ip, "error" => &e.to_string() })
  410. ),
  411. }
  412. }
  413. }
  414. } else {
  415. match cluster_name {
  416. Some(ref name) => {
  417. let cluster = select_cluster_by_name(config, name)?;
  418. docker::health::health_check(config, &cluster, false, format).await?;
  419. }
  420. None => {
  421. let fleet = docker::health::collect(config, None, false).await?;
  422. docker::health::print_fleet(&fleet, format);
  423. }
  424. }
  425. }
  426. Ok(())
  427. }
  428. /// Handle `exec` command: run a command on cluster nodes via SSH.
  429. async fn handle_exec(
  430. config: &Config,
  431. cluster: Option<String>,
  432. recipe: Option<String>,
  433. node: Option<String>,
  434. alias: Option<String>,
  435. command: Vec<String>,
  436. dry_run: bool,
  437. ) -> Result<()> {
  438. let (cluster, _recipe) =
  439. exec::resolve_target(config, cluster.as_deref(), recipe.as_deref()).await?;
  440. let cmd_str = exec::resolve_command(config, alias.as_deref(), &command)?;
  441. exec::run(config, &cluster, node.as_deref(), &cmd_str, dry_run).await?;
  442. Ok(())
  443. }
  444. /// Handle `power` command: power on/off/switch nodes.
  445. async fn handle_power(config: &Config, action: PowerAction, dry_run: bool) -> Result<()> {
  446. match action {
  447. PowerAction::Off { cluster, node } => {
  448. let cluster = select_cluster_by_name(config, &cluster)?;
  449. let nodes = node.into_iter().collect::<Vec<_>>();
  450. power::power_off(config, &cluster, &nodes, dry_run).await?;
  451. }
  452. PowerAction::On { cluster, node } => {
  453. let cluster = select_cluster_by_name(config, &cluster)?;
  454. let nodes = node.into_iter().collect::<Vec<_>>();
  455. power::power_on(config, &cluster, &nodes, dry_run).await?;
  456. }
  457. PowerAction::Switch { from, to } => {
  458. let from = select_cluster_by_name(config, &from)?;
  459. let to = select_cluster_by_name(config, &to)?;
  460. power::power_switch(config, &from, &to, dry_run).await?;
  461. }
  462. }
  463. Ok(())
  464. }
  465. /// Resolve a cluster for container operations (stop/logs).
  466. /// When `auto`, tries live container label lookup first, then falls back
  467. /// to config-based auto-selection (for dry-run / offline scenarios).
  468. pub(crate) async fn resolve_cluster_for_op(
  469. config: &Config,
  470. recipe_name: &str,
  471. cluster_name: &str,
  472. ) -> Result<Cluster> {
  473. if cluster_name != "auto" {
  474. let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
  475. let (cluster, _) = resolve_cluster(config, &recipe, cluster_name, &HashSet::new())?;
  476. return Ok(cluster);
  477. }
  478. // Try live container label lookup first
  479. if let Ok(c) = docker::resolve_cluster_by_container_label(config, recipe_name).await {
  480. return Ok(c.clone());
  481. }
  482. // Fall back to config-based auto (dry-run / containers not running)
  483. let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
  484. let (cluster, _) = resolve_cluster(config, &recipe, "auto", &HashSet::new())?;
  485. Ok(cluster)
  486. }