|
|
@@ -0,0 +1,236 @@
|
|
|
+//! Image build-dir resolution and reproducible checksums.
|
|
|
+//!
|
|
|
+//! `sm121 build` has to work from anywhere: a launcher workspace
|
|
|
+//! (`docker-images/<dir>` + `docker_image_sources`), or a published
|
|
|
+//! bundle clone (git.kilork.org/sm121/recipes, flat `<name>/` layout).
|
|
|
+//! The checksum over the build dir is what gets stamped as the
|
|
|
+//! `sm121.checksum` label and compared by `docker/stale.rs`.
|
|
|
+
|
|
|
+use anyhow::{Context, Result};
|
|
|
+use sha2::{Digest, Sha256};
|
|
|
+use std::collections::{BTreeMap, HashMap};
|
|
|
+use std::io::Read;
|
|
|
+use std::path::Path;
|
|
|
+
|
|
|
+/// Compute a SHA-256 checksum over all files in `dir` (recursive).
|
|
|
+/// Sorted by relative path to ensure reproducibility.
|
|
|
+pub fn compute_dir_checksum(dir: &Path) -> Result<String> {
|
|
|
+ let mut hasher = Sha256::new();
|
|
|
+ let mut entries: Vec<_> = Vec::new();
|
|
|
+
|
|
|
+ for entry in walkdir::WalkDir::new(dir).sort_by_file_name() {
|
|
|
+ let entry = entry?;
|
|
|
+ if entry.file_type().is_dir() {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ let rel = entry
|
|
|
+ .path()
|
|
|
+ .strip_prefix(dir)
|
|
|
+ .unwrap_or(entry.path())
|
|
|
+ .to_string_lossy()
|
|
|
+ .to_string();
|
|
|
+ entries.push((rel, entry.path().to_path_buf()));
|
|
|
+ }
|
|
|
+ entries.sort_by(|a, b| a.0.cmp(&b.0));
|
|
|
+
|
|
|
+ for (rel, path) in &entries {
|
|
|
+ let mut file = std::fs::File::open(path)?;
|
|
|
+ let mut buf = Vec::new();
|
|
|
+ file.read_to_end(&mut buf)?;
|
|
|
+ hasher.update(rel.as_bytes());
|
|
|
+ hasher.update(b"\0");
|
|
|
+ hasher.update(&buf);
|
|
|
+ }
|
|
|
+
|
|
|
+ let hash = hasher.finalize();
|
|
|
+ Ok(hash.iter().map(|b| format!("{:02x}", b)).collect())
|
|
|
+}
|
|
|
+
|
|
|
+/// Parse a `versions.env` file into a sorted map. Only `KEY=VAL` lines are
|
|
|
+/// captured; comments (`#`) and blank lines are dropped. The value is
|
|
|
+/// everything after the first `=` (so values may contain `=`), with one
|
|
|
+/// level of matching `"` or `'` quotes stripped — quoted and unquoted forms
|
|
|
+/// of the same value must resolve (and checksum) identically.
|
|
|
+pub fn parse_versions_env(raw: &str) -> BTreeMap<String, String> {
|
|
|
+ let mut pins = BTreeMap::new();
|
|
|
+ for line in raw.lines() {
|
|
|
+ let line = line.trim();
|
|
|
+ if line.is_empty() || line.starts_with('#') {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if let Some((k, v)) = line.split_once('=') {
|
|
|
+ let k = k.trim();
|
|
|
+ if !k.is_empty() {
|
|
|
+ pins.insert(k.to_string(), unquote(v.trim()));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ pins
|
|
|
+}
|
|
|
+
|
|
|
+/// Strip one level of matching single/double quotes (build.sh does the
|
|
|
+/// same; keeping the two parsers in agreement keeps checksums stable).
|
|
|
+fn unquote(v: &str) -> String {
|
|
|
+ let b = v.as_bytes();
|
|
|
+ let quoted = b.len() >= 2
|
|
|
+ && (b[0] == b'"' && *b.last().unwrap() == b'"'
|
|
|
+ || b[0] == b'\'' && *b.last().unwrap() == b'\'');
|
|
|
+ if quoted {
|
|
|
+ v[1..v.len() - 1].to_string()
|
|
|
+ } else {
|
|
|
+ v.to_string()
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/// Serialize a pin map back to `versions.env` text (sorted by key; one
|
|
|
+/// `KEY=VAL` per line). Comments are not preserved — this is a generated
|
|
|
+/// artifact written to the remote build dir, not the source of truth.
|
|
|
+pub fn serialize_versions_env(pins: &BTreeMap<String, String>) -> String {
|
|
|
+ let mut out = String::from(
|
|
|
+ "# Merged versions.env — generated by sm121 (recipe [build.versions] overrides applied)\n",
|
|
|
+ );
|
|
|
+ for (k, v) in pins {
|
|
|
+ out.push_str(k);
|
|
|
+ out.push('=');
|
|
|
+ out.push_str(v);
|
|
|
+ out.push('\n');
|
|
|
+ }
|
|
|
+ out
|
|
|
+}
|
|
|
+
|
|
|
+/// Compute the build checksum, folding in recipe `[build.versions]` overrides.
|
|
|
+///
|
|
|
+/// Returns `(merged_versions_env, checksum)`:
|
|
|
+/// - No overrides -> `(None, dir_checksum)`: the image's `versions.env` is
|
|
|
+/// uploaded as-is and the checksum is unchanged (existing images skip
|
|
|
+/// rebuild, backward-compatible).
|
|
|
+/// - Overrides present -> `(Some(merged), combined)`: `merged` is the image's
|
|
|
+/// `versions.env` with recipe pins applied (recipe wins) for the launcher
|
|
|
+/// to write to the remote build dir; `combined` hashes the dir checksum
|
|
|
+/// together with the merged env so any override change forces a rebuild.
|
|
|
+pub(crate) fn build_checksum(
|
|
|
+ local_dir: &Path,
|
|
|
+ overrides: &HashMap<String, String>,
|
|
|
+) -> Result<(Option<String>, String)> {
|
|
|
+ let dir_checksum = compute_dir_checksum(local_dir)?;
|
|
|
+ if overrides.is_empty() {
|
|
|
+ return Ok((None, dir_checksum));
|
|
|
+ }
|
|
|
+ let env_path = local_dir.join("versions.env");
|
|
|
+ let raw = std::fs::read_to_string(&env_path).context(format!(
|
|
|
+ "recipe carries [build.versions] overrides but '{}' has no versions.env",
|
|
|
+ env_path.display()
|
|
|
+ ))?;
|
|
|
+ let mut pins = parse_versions_env(&raw);
|
|
|
+ for (k, v) in overrides {
|
|
|
+ pins.insert(k.clone(), v.clone());
|
|
|
+ }
|
|
|
+ let merged = serialize_versions_env(&pins);
|
|
|
+ let mut h = Sha256::new();
|
|
|
+ h.update(dir_checksum.as_bytes());
|
|
|
+ h.update(b"\0");
|
|
|
+ h.update(merged.as_bytes());
|
|
|
+ let combined: String = h.finalize().iter().map(|b| format!("{:02x}", b)).collect();
|
|
|
+ Ok((Some(merged), combined))
|
|
|
+}
|
|
|
+
|
|
|
+/// Where a recipe's image is built from.
|
|
|
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
|
+pub enum BuildSource {
|
|
|
+ /// Full build context: `build.sh` + `Dockerfile` + versions.env.
|
|
|
+ Workspace,
|
|
|
+ /// Published bundle (flattened; `build.sh`/`Dockerfile` at bundle root).
|
|
|
+ Bundle,
|
|
|
+}
|
|
|
+
|
|
|
+/// Resolve a recipe's image build dir. Search order:
|
|
|
+/// 1. `./docker-images/<dir>` (launcher workspace) — needs `build.sh`
|
|
|
+/// 2. each `docker_image_sources` entry `S`: `S/docker-images/<dir>`, then
|
|
|
+/// `S/<dir>` (a repo root containing `docker-images/`, or a dir of
|
|
|
+/// image dirs)
|
|
|
+/// 3. `./recipes/<name>/` then `./<name>/` carrying a bundled `build.sh`
|
|
|
+/// (published bundles from the recipe registry are self-contained)
|
|
|
+///
|
|
|
+/// Errors name every path tried.
|
|
|
+pub fn resolve_build_dir(
|
|
|
+ base: &Path,
|
|
|
+ recipe_name: &str,
|
|
|
+ image_dir: &str,
|
|
|
+ sources: &[String],
|
|
|
+) -> Result<(std::path::PathBuf, BuildSource)> {
|
|
|
+ let mut tried: Vec<String> = Vec::new();
|
|
|
+ let workspace = base.join("docker-images").join(image_dir);
|
|
|
+ if workspace.join("build.sh").is_file() {
|
|
|
+ return Ok((workspace, BuildSource::Workspace));
|
|
|
+ }
|
|
|
+ tried.push(workspace.to_string_lossy().into());
|
|
|
+ for src in sources {
|
|
|
+ let root = crate::config::local_path(src);
|
|
|
+ for cand in [
|
|
|
+ root.join("docker-images").join(image_dir),
|
|
|
+ root.join(image_dir),
|
|
|
+ ] {
|
|
|
+ if cand.join("build.sh").is_file() {
|
|
|
+ return Ok((cand, BuildSource::Workspace));
|
|
|
+ }
|
|
|
+ tried.push(cand.to_string_lossy().into());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for bun in [
|
|
|
+ base.join("recipes").join(recipe_name),
|
|
|
+ base.join(recipe_name),
|
|
|
+ ] {
|
|
|
+ if bun.join("build.sh").is_file() && bun.join("Dockerfile").is_file() {
|
|
|
+ return Ok((bun, BuildSource::Bundle));
|
|
|
+ }
|
|
|
+ tried.push(bun.to_string_lossy().into());
|
|
|
+ }
|
|
|
+ anyhow::bail!(
|
|
|
+ "no build context for image dir '{}' of recipe '{}'; searched (each needs build.sh; bundles also need a Dockerfile): {}",
|
|
|
+ image_dir,
|
|
|
+ recipe_name,
|
|
|
+ tried.join(", ")
|
|
|
+ )
|
|
|
+}
|
|
|
+
|
|
|
+#[cfg(test)]
|
|
|
+mod tests {
|
|
|
+ use super::*;
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn parse_versions_env_quotes_are_normalized() {
|
|
|
+ let raw = "A=\"x:tag@sha256:abc\"\nB='y'\nC=z\nE=\"half\nF='weird\"\n";
|
|
|
+ let pins = parse_versions_env(raw);
|
|
|
+ assert_eq!(pins.get("A"), Some(&"x:tag@sha256:abc".to_string()));
|
|
|
+ assert_eq!(pins.get("B"), Some(&"y".to_string()));
|
|
|
+ assert_eq!(pins.get("C"), Some(&"z".to_string()));
|
|
|
+ assert_eq!(pins.get("E"), Some(&"\"half".to_string()));
|
|
|
+ assert_eq!(pins.get("F"), Some(&"'weird\"".to_string()));
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn parse_versions_env_skips_comments_and_blanks() {
|
|
|
+ let raw = "# comment\n\nKEY=val\nOTHER=a=b\n SPACED = x \nnoeq\n";
|
|
|
+ let pins = parse_versions_env(raw);
|
|
|
+ assert_eq!(pins.get("KEY"), Some(&"val".to_string()));
|
|
|
+ assert_eq!(pins.get("OTHER"), Some(&"a=b".to_string()));
|
|
|
+ assert_eq!(pins.get("SPACED"), Some(&"x".to_string()));
|
|
|
+ assert!(!pins.contains_key("noeq"));
|
|
|
+ assert_eq!(pins.len(), 3);
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn resolve_falls_back_to_bundle_layout() {
|
|
|
+ let base = std::env::temp_dir().join(format!("sm121-resolve-test-{}", std::process::id()));
|
|
|
+ let bundle = base.join("recipes").join("test-resolve-recipe-x");
|
|
|
+ std::fs::create_dir_all(&bundle).unwrap();
|
|
|
+ std::fs::write(bundle.join("build.sh"), "").unwrap();
|
|
|
+ std::fs::write(bundle.join("Dockerfile"), "").unwrap();
|
|
|
+ let (dir, src) =
|
|
|
+ resolve_build_dir(&base, "test-resolve-recipe-x", "test-resolve-recipe-x", &[])
|
|
|
+ .unwrap();
|
|
|
+ assert_eq!(src, BuildSource::Bundle);
|
|
|
+ assert_eq!(dir, bundle);
|
|
|
+ let _ = std::fs::remove_dir_all(&base);
|
|
|
+ }
|
|
|
+}
|
kilork