recipe_repos.rs 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408
  1. //! Recipe repos: git repos holding `recipes/` (+ `docker-images/` build
  2. //! contexts), cloned/pulled by the launcher into a local cache and merged
  3. //! into the recipe search path (first repo wins on a name collision — see
  4. //! `Config.recipe_repos`).
  5. //!
  6. //! Archive state lives in a separate JSON registry (`repos.json` next to
  7. //! the global config): a `[[recipe_repos]]` entry can be archived without
  8. //! editing the config (`sm121 archive <url>` / `unarchive`), and the next
  9. //! sync skips archived repos. A repo that is unreachable at sync time
  10. //! degrades to a warning — a dead playground repo must not take down
  11. //! production serving.
  12. use crate::cli::ReposCommands;
  13. use crate::config::{RecipeRepo, default_config_dir};
  14. use anyhow::{Context, Result};
  15. use serde::{Deserialize, Serialize};
  16. use sha2::{Digest, Sha256};
  17. use std::path::{Path, PathBuf};
  18. use tracing::{info, warn};
  19. /// Stamp file written into a synced checkout after each successful
  20. /// clone/pull (unix-epoch seconds; shown by `sm121 repos`).
  21. pub const LAST_SYNC_FILE: &str = ".sm121-last-sync";
  22. /// Per-repo cache dir root: `~/.cache/sm121/recipe-repos`.
  23. pub fn cache_base() -> PathBuf {
  24. dirs::home_dir()
  25. .unwrap_or_else(|| PathBuf::from("."))
  26. .join(".cache/sm121/recipe-repos")
  27. }
  28. /// `sha256(url)` hex (full; callers truncate for dir names). Stable per
  29. /// URL — the cache dir is keyed on the repo URL, not the hostname.
  30. pub(crate) fn url_hash(url: &str) -> String {
  31. let digest = Sha256::digest(url.as_bytes());
  32. digest.iter().map(|b| format!("{:02x}", b)).collect()
  33. }
  34. /// Cache checkout dir for one repo: `<base>/<sha256(url)16>`.
  35. pub fn repo_cache_dir(base: &Path, url: &str) -> PathBuf {
  36. let h = url_hash(url);
  37. base.join(&h[..16])
  38. }
  39. /// Run `git <args>` locally, capturing output. Stdout on success; an
  40. /// anyhow error carrying the stderr tail on failure.
  41. pub(crate) fn git(args: &[&str]) -> Result<String> {
  42. let out = std::process::Command::new("git")
  43. .args(args)
  44. .output()
  45. .with_context(|| {
  46. format!(
  47. "failed to run git (first arg: '{}')",
  48. args.first().unwrap_or(&"")
  49. )
  50. })?;
  51. if !out.status.success() {
  52. let err = String::from_utf8_lossy(&out.stderr).trim().to_string();
  53. anyhow::bail!(fluent_i18n::t!(
  54. "git-failed",
  55. { "cmd" => args.first().copied().unwrap_or("").to_string(), "err" => err }
  56. ));
  57. }
  58. Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
  59. }
  60. /// Clone URL with the optional token embedded (HTTPS only — for ssh URLs
  61. /// the token is a no-op and the URL is used as-is). The token lives only
  62. /// in the cached checkout's `.git/config`, never in a recipe repo.
  63. fn auth_url(repo: &RecipeRepo) -> String {
  64. if let Some(t) = &repo.token
  65. && (repo.url.starts_with("https://") || repo.url.starts_with("http://"))
  66. {
  67. return repo.url.replace("//", &format!("//{t}@"));
  68. }
  69. repo.url.clone()
  70. }
  71. /// The push URL for a repo (the token embedded for HTTPS pushes) — used
  72. /// by `promote` so the push authenticates without a credential store.
  73. pub fn push_url(repo: &RecipeRepo) -> String {
  74. auth_url(repo)
  75. }
  76. /// Clone (first time) or `pull --ff-only` (subsequent) one repo into its
  77. /// cache dir; stamp `LAST_SYNC_FILE` on success. `force` re-clones from
  78. /// scratch (drops the cached checkout first).
  79. pub fn sync_one(url: &str, dir: &Path, force: bool) -> Result<()> {
  80. if force && dir.exists() {
  81. std::fs::remove_dir_all(dir)
  82. .with_context(|| format!("removing cached checkout {}", dir.display()))?;
  83. }
  84. if dir.join(".git").is_dir() {
  85. git(&["-C", &dir.display().to_string(), "pull", "--ff-only"])?;
  86. } else {
  87. if dir.exists() {
  88. // partial clone from an interrupted run — start clean
  89. std::fs::remove_dir_all(dir)?;
  90. }
  91. if let Some(parent) = dir.parent() {
  92. std::fs::create_dir_all(parent)?;
  93. }
  94. git(&["clone", "--", url, &dir.display().to_string()])?;
  95. }
  96. let ts = crate::stamp::now_secs().to_string();
  97. let stamp_path = dir.join(LAST_SYNC_FILE);
  98. std::fs::write(&stamp_path, ts)
  99. .with_context(|| format!("stamping {}", stamp_path.display()))?;
  100. Ok(())
  101. }
  102. /// The archive registry: which `[[recipe_repos]]` URLs are currently
  103. /// archived (skipped by sync). Missing file → a fresh empty registry; a
  104. /// corrupt file is moved to `repos.json.bak` and replaced (the launcher
  105. /// never crashes on the registry).
  106. #[derive(Clone, Debug, Default, Deserialize, Serialize)]
  107. pub struct ReposRegistry {
  108. #[serde(default)]
  109. pub archived: Vec<ArchiveEntry>,
  110. }
  111. #[derive(Clone, Debug, Deserialize, Serialize)]
  112. pub struct ArchiveEntry {
  113. pub url: String,
  114. /// Unix-epoch seconds the entry was archived.
  115. #[serde(default)]
  116. pub at: i64,
  117. }
  118. impl ReposRegistry {
  119. pub fn is_archived(&self, url: &str) -> bool {
  120. self.archived.iter().any(|e| e.url == url)
  121. }
  122. }
  123. /// `repos.json` next to the global config (`~/.config/sm121/` on Linux,
  124. /// `~/Library/Application Support/sm121/` on macOS).
  125. pub fn registry_path() -> Option<PathBuf> {
  126. default_config_dir().map(|d| d.join("repos.json"))
  127. }
  128. /// Load the registry at `path` (missing → fresh; corrupt → `.bak` + fresh
  129. /// + a warning).
  130. pub fn load_registry(path: &Path) -> ReposRegistry {
  131. match std::fs::read_to_string(path) {
  132. Ok(s) => match serde_json::from_str(&s) {
  133. Ok(r) => r,
  134. Err(e) => {
  135. warn!(
  136. "{}",
  137. fluent_i18n::t!(
  138. "repos-registry-corrupt",
  139. { "path" => path.display().to_string(), "err" => e.to_string(), "bak" => path.with_extension("json.bak").display().to_string() }
  140. )
  141. );
  142. let bak = path.with_extension("json.bak");
  143. if std::fs::copy(path, &bak).is_err() {
  144. warn!(
  145. "{}",
  146. fluent_i18n::t!("repos-registry-backup-failed", { "path" => path.display().to_string() })
  147. );
  148. }
  149. ReposRegistry::default()
  150. }
  151. },
  152. Err(_) => ReposRegistry::default(),
  153. }
  154. }
  155. pub fn save_registry(path: &Path, reg: &ReposRegistry) -> Result<()> {
  156. if let Some(parent) = path.parent() {
  157. std::fs::create_dir_all(parent)?;
  158. }
  159. std::fs::write(path, serde_json::to_string_pretty(reg)?)?;
  160. Ok(())
  161. }
  162. /// `sm121 archive <url>`: flip the entry in `repos.json` (idempotent).
  163. pub fn archive_at(path: &Path, url: &str) -> Result<()> {
  164. let mut reg = load_registry(path);
  165. if !reg.is_archived(url) {
  166. reg.archived.push(ArchiveEntry {
  167. url: url.to_string(),
  168. at: crate::stamp::now_secs(),
  169. });
  170. save_registry(path, &reg)?;
  171. }
  172. Ok(())
  173. }
  174. /// `sm121 unarchive <url>`: drop the entry (a no-op when absent).
  175. pub fn unarchive_at(path: &Path, url: &str) -> Result<()> {
  176. let mut reg = load_registry(path);
  177. let before = reg.archived.len();
  178. reg.archived.retain(|e| e.url != url);
  179. if reg.archived.len() != before {
  180. save_registry(path, &reg)?;
  181. }
  182. Ok(())
  183. }
  184. /// Clone/pull every non-archived `repos` entry (list order) into `base`,
  185. /// best-effort: an unreachable repo warns and is skipped, the reachable
  186. /// ones still land. Returns the checkout roots in list order.
  187. pub fn sync_repos(
  188. repos: &[RecipeRepo],
  189. base: &Path,
  190. registry: &ReposRegistry,
  191. force: bool,
  192. ) -> Vec<PathBuf> {
  193. let mut roots = Vec::new();
  194. for repo in repos {
  195. if registry.is_archived(&repo.url) {
  196. info!(
  197. "{}",
  198. fluent_i18n::t!("recipe-repo-archived-skip", { "url" => &repo.url })
  199. );
  200. continue;
  201. }
  202. let dir = repo_cache_dir(base, &repo.url);
  203. match sync_one(&auth_url(repo), &dir, force) {
  204. Ok(()) => {
  205. info!(
  206. "{}",
  207. fluent_i18n::t!("recipe-repo-synced", { "url" => &repo.url, "dir" => dir.display().to_string() })
  208. );
  209. roots.push(dir);
  210. }
  211. Err(e) => {
  212. warn!(
  213. "{}",
  214. fluent_i18n::t!("recipe-repo-unreachable", { "url" => &repo.url, "err" => e.to_string() })
  215. );
  216. }
  217. }
  218. }
  219. roots
  220. }
  221. /// Clone/pull the `Config.recipe_repos` entries into the default cache
  222. /// base (blocking git work off the async runtime). Best-effort: per-repo
  223. /// failures are warnings, never errors — returns the reachable roots.
  224. pub async fn sync_recipe_repos(config: &crate::config::Config) -> Vec<PathBuf> {
  225. let repos = config.recipe_repos.clone();
  226. let reg_path = registry_path();
  227. let base = cache_base();
  228. let registry = reg_path.as_deref().map(load_registry).unwrap_or_default();
  229. tokio::task::spawn_blocking(move || {
  230. crate::locale::apply_global_locale();
  231. let _ = std::fs::create_dir_all(&base);
  232. sync_repos(&repos, &base, &registry, false)
  233. })
  234. .await
  235. .unwrap_or_default()
  236. }
  237. /// Clone/pull ONE configured repo into its cache dir (hard failure —
  238. /// unlike the best-effort batch sync). Used by `promote` to make sure
  239. /// both checkouts exist before planning the move.
  240. pub fn sync_repo(repo: &RecipeRepo) -> Result<()> {
  241. let base = cache_base();
  242. let _ = std::fs::create_dir_all(&base);
  243. let dir = repo_cache_dir(&base, &repo.url);
  244. sync_one(&auth_url(repo), &dir, false)
  245. }
  246. /// Count the recipe dirs (`<root>/recipes/<name>/recipe.toml`) in a synced
  247. /// checkout — shown by `sm121 repos`.
  248. fn recipe_count(root: &Path) -> Option<usize> {
  249. let dir = root.join("recipes");
  250. let Ok(entries) = std::fs::read_dir(&dir) else {
  251. return None;
  252. };
  253. Some(
  254. entries
  255. .flatten()
  256. .filter(|e| e.path().join("recipe.toml").is_file())
  257. .count(),
  258. )
  259. }
  260. /// `sm121 repos` / `sm121 repos sync` / `sm121 archive` / `sm121 unarchive`
  261. /// dispatch (offline except `sync`, which shells out to git locally).
  262. pub async fn run(config: &crate::config::Config, cmd: &ReposCommands, dry_run: bool) -> Result<()> {
  263. match cmd {
  264. ReposCommands::List => {
  265. if config.recipe_repos.is_empty() {
  266. println!("{}", fluent_i18n::t!("repos-none-configured"));
  267. return Ok(());
  268. }
  269. let reg_path = registry_path().unwrap_or_else(|| PathBuf::from("repos.json"));
  270. let reg = load_registry(&reg_path);
  271. let base = cache_base();
  272. println!(
  273. "{:<52} {:<22} {:<18} {n}",
  274. fluent_i18n::t!("repos-col-repo"),
  275. fluent_i18n::t!("repos-col-state"),
  276. fluent_i18n::t!("repos-col-last"),
  277. n = fluent_i18n::t!("repos-col-count")
  278. );
  279. for repo in &config.recipe_repos {
  280. let dir = repo_cache_dir(&base, &repo.url);
  281. let state = if reg.is_archived(&repo.url) {
  282. fluent_i18n::t!("repos-state-archived")
  283. } else if dir.join(".git").is_dir() {
  284. fluent_i18n::t!("repos-state-synced")
  285. } else {
  286. fluent_i18n::t!("repos-state-not-cloned")
  287. };
  288. let last = std::fs::read_to_string(dir.join(LAST_SYNC_FILE))
  289. .ok()
  290. .map(|s| crate::stamp::fmt_ts(s.trim().parse().unwrap_or(0)))
  291. .unwrap_or_else(|| "—".to_string());
  292. let n = recipe_count(&dir)
  293. .map(|n| n.to_string())
  294. .unwrap_or_else(|| "—".into());
  295. println!("{:<52} {:<22} {:<18} {n}", repo.url, state, last);
  296. }
  297. }
  298. ReposCommands::Sync {
  299. force,
  300. watch,
  301. interval,
  302. } => {
  303. let force = *force;
  304. if dry_run {
  305. for repo in &config.recipe_repos {
  306. let dir = repo_cache_dir(&cache_base(), &repo.url);
  307. if force {
  308. println!(
  309. "{}",
  310. fluent_i18n::t!("repos-sync-dry-reclone", { "url" => &repo.url, "dir" => dir.display().to_string() })
  311. );
  312. } else {
  313. println!(
  314. "{}",
  315. fluent_i18n::t!("repos-sync-dry-sync", { "url" => &repo.url, "dir" => dir.display().to_string() })
  316. );
  317. }
  318. }
  319. return Ok(());
  320. }
  321. let interval_min = *interval;
  322. loop {
  323. let reg_path = registry_path().unwrap_or_else(|| PathBuf::from("repos.json"));
  324. let reg = load_registry(&reg_path);
  325. let base = cache_base();
  326. let repos = config.recipe_repos.clone();
  327. let n = repos.len();
  328. let roots =
  329. tokio::task::spawn_blocking(move || sync_repos(&repos, &base, &reg, force))
  330. .await
  331. .unwrap_or_default();
  332. if roots.is_empty() {
  333. warn!("{}", fluent_i18n::t!("repos-none-synced"));
  334. }
  335. if !watch {
  336. break;
  337. }
  338. info!(
  339. "{}",
  340. fluent_i18n::t!("recipe-repo-watch", { "n" => n, "interval" => interval_min })
  341. );
  342. tokio::time::sleep(std::time::Duration::from_secs(interval_min as u64 * 60)).await;
  343. }
  344. }
  345. }
  346. Ok(())
  347. }
  348. /// `sm121 archive <url>` / `sm121 unarchive <url>` — flip the entry in the
  349. /// `repos.json` registry (creates the file if missing).
  350. pub fn archive(url: &str, unarchive: bool, dry_run: bool) -> Result<()> {
  351. let path = registry_path()
  352. .ok_or_else(|| anyhow::anyhow!(fluent_i18n::t!("archive-config-dir-fail")))?;
  353. if dry_run {
  354. if unarchive {
  355. println!(
  356. "{}",
  357. fluent_i18n::t!("archive-dry-unarchive", { "url" => url.to_string(), "path" => path.display().to_string() })
  358. );
  359. } else {
  360. println!(
  361. "{}",
  362. fluent_i18n::t!("archive-dry-archive", { "url" => url.to_string(), "path" => path.display().to_string() })
  363. );
  364. }
  365. return Ok(());
  366. }
  367. if unarchive {
  368. unarchive_at(&path, url)?;
  369. println!(
  370. "{}",
  371. fluent_i18n::t!("unarchive-done", { "url" => url.to_string(), "path" => path.display().to_string() })
  372. );
  373. } else {
  374. archive_at(&path, url)?;
  375. println!(
  376. "{}",
  377. fluent_i18n::t!("archive-done", { "url" => url.to_string(), "path" => path.display().to_string() })
  378. );
  379. }
  380. Ok(())
  381. }
  382. #[cfg(test)]
  383. #[path = "recipe_repos_tests.rs"]
  384. mod tests;