//! Image build-dir resolution and reproducible checksums.
//!
//! `sm121 build` has to work from anywhere: a launcher workspace
//! (`docker-images/
` + `docker_image_sources`), or a published
//! bundle clone (git.kilork.org/sm121/recipes, flat `/` layout).
//! The checksum over the build dir is what gets stamped as the
//! `sm121.checksum` label and compared by `docker/stale.rs`.
use anyhow::{Context, Result};
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, HashMap};
use std::io::Read;
use std::path::{Path, PathBuf};
/// Compute a SHA-256 checksum over all files in `dir` (recursive).
/// Sorted by relative path to ensure reproducibility.
/// Flatten an image overlay into `dst` and rewrite its Dockerfile:
/// the `patches/` subtree plus every sub-directory the Dockerfile
/// COPY/ADD-references is flattened to the bundle root (gists cannot
/// carry directories). Name collisions rename the flattened file to
/// `{dir}-{name}` and the COPY line is rewritten to the new name; a
/// collision under a glob source aborts the flatten for that directory
/// (it stays nested, `nested = true` — the gist flow refuses it).
/// Unreferenced sub-directories are always preserved nested.
pub fn flatten_overlay(src_dir: &Path, dst: &Path, dockerfile: &str) -> Result<(String, bool)> {
use std::collections::BTreeSet;
// 1) Sub-directories referenced by COPY/ADD sources (incl. continuations).
let mut referenced: BTreeSet = BTreeSet::new();
let mut in_copy = false;
for line in dockerfile.lines() {
let t = line.trim_start();
let is_copy = in_copy || t.starts_with("COPY ") || t.starts_with("ADD ");
in_copy = is_copy && t.ends_with('\\');
if !is_copy {
continue;
}
let body = t.trim_end_matches('\\');
let toks: Vec<&str> = body.split_whitespace().skip(1).collect();
for tok in &toks[..toks.len().saturating_sub(1)] {
if tok.starts_with('-') {
continue; // --from=… etc.
}
let parts: Vec<&str> = tok.split('/').collect();
// parts[0].is_empty() => absolute path (/tmp/…) — not a build-dir ref
if parts.len() >= 2 && !parts[0].is_empty() && src_dir.join(parts[0]).is_dir() {
referenced.insert(parts[0].to_string());
}
}
}
let flatten_dirs: BTreeSet = referenced
.iter()
.cloned()
.chain(std::iter::once("patches".to_string()))
.collect();
// 2) Root files first (they define the collision set), except the driver.
for entry in std::fs::read_dir(src_dir)? {
let entry = entry?;
if entry.file_type()?.is_file() {
let name = entry.file_name().to_string_lossy().to_string();
if name != "build.sh" && name != "Dockerfile" {
std::fs::copy(entry.path(), dst.join(&name))?;
}
}
}
// 3) Flatten the referenced dirs (+ patches/); everything else nests.
let mut nested = false;
let mut flattened: BTreeSet = BTreeSet::new(); // dirs actually flattened
let mut renames: BTreeMap = BTreeMap::new(); // "dir/file" -> new name
let mut globs: BTreeSet = BTreeSet::new(); // dirs whose COPY source is a glob
for line in dockerfile.lines() {
let t = line.trim_start();
if t.starts_with("COPY ") || t.starts_with("ADD ") {
for tok in t.split_whitespace().skip(1) {
if tok.starts_with('-') {
continue;
}
let parts: Vec<&str> = tok.split('/').collect();
if parts.len() >= 2 && flatten_dirs.contains(parts[0]) && tok.contains('*') {
globs.insert(parts[0].to_string());
}
}
}
}
for dir in flatten_dirs.iter() {
let dir_path = src_dir.join(dir);
if !dir_path.is_dir() {
continue; // referenced but absent — COPY will fail in docker, not here
}
let files: Vec = std::fs::read_dir(&dir_path)?
.filter_map(|e| e.ok())
.filter(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false))
.map(|e| e.path())
.collect();
let collides = files
.iter()
.any(|f| dst.join(f.file_name().unwrap()).exists());
if collides && globs.contains(dir) {
nested = true;
std::fs::create_dir_all(dst.join(dir))?;
copy_tree(&dir_path, &dst.join(dir))?; // rename under a glob is impossible
continue;
}
for f in files {
let name = f.file_name().unwrap().to_string_lossy().to_string();
let key = format!("{dir}/{name}");
let new = if dst.join(&name).exists() {
let renamed = format!("{dir}-{name}");
renames.insert(key, renamed.clone());
renamed
} else {
name
};
std::fs::copy(f, dst.join(&new))?;
}
flattened.insert(dir.clone());
}
for entry in std::fs::read_dir(src_dir)? {
let entry = entry?;
let name = entry.file_name().to_string_lossy().to_string();
if !entry.file_type()?.is_dir() || flatten_dirs.contains(&name) {
continue;
}
// profiles/ is launcher metadata — its pins are already merged into
// the bundle's versions.env, so it is not re-shipped.
if name == "profiles" {
continue;
}
nested = true;
std::fs::create_dir_all(dst.join(&name))?;
copy_tree(&entry.path(), &dst.join(&name))?;
}
// 4) Rewrite COPY/ADD sources for the flattened dirs (incl. continuations).
let mut in_copy = false;
let mut dst_done = false; // destination token already seen on an earlier line
let rewritten = dockerfile
.lines()
.map(|line| {
let t = line.trim_start();
let is_copy = in_copy || t.starts_with("COPY ") || t.starts_with("ADD ");
in_copy = is_copy && t.ends_with('\\');
if !is_copy {
return line.to_string();
}
let head = &line[..line.len() - t.len()]; // preserve indentation
let is_first = t.starts_with("COPY ") || t.starts_with("ADD ");
let toks: Vec<&str> = t.split_whitespace().collect();
let n = toks.len();
if is_first {
// command + at least one src + a dest on this line -> all
// continuation tokens are sources
dst_done = toks[1..]
.iter()
.filter(|t| **t != "\\" && !t.starts_with('-'))
.count()
>= 2;
}
let rewritten_toks: Vec = toks
.iter()
.enumerate()
.map(|(i, tok)| {
let is_dest = !is_first && !dst_done && i == n.saturating_sub(1);
if (is_first && i == 0) || is_dest || tok.starts_with('-') {
return tok.to_string();
}
let parts: Vec<&str> = tok.split('/').collect();
if parts.len() < 2 || !flattened.contains(parts[0]) {
return tok.to_string();
}
if tok.contains('*') {
return parts[1..].join("/"); // glob: drop the dir prefix
}
let key = (*tok).to_string();
renames
.get(&key)
.cloned()
.unwrap_or_else(|| parts[1].to_string())
})
.collect();
format!("{}{}", head, rewritten_toks.join(" "))
})
.collect::>()
.join("\n")
+ if dockerfile.ends_with('\n') { "\n" } else { "" };
Ok((rewritten, nested))
}
fn copy_tree(src_dir: &Path, dst_dir: &Path) -> Result<()> {
for entry in std::fs::read_dir(src_dir)? {
let entry = entry?;
let ft = entry.file_type()?;
let target = dst_dir.join(entry.file_name());
if ft.is_dir() {
std::fs::create_dir_all(&target)?;
copy_tree(&entry.path(), &target)?;
} else if ft.is_file() {
std::fs::copy(entry.path(), &target)?;
}
}
Ok(())
}
pub fn compute_dir_checksum(dir: &Path) -> Result {
let mut hasher = Sha256::new();
let mut entries: Vec<_> = Vec::new();
for entry in walkdir::WalkDir::new(dir).sort_by_file_name() {
let entry = entry?;
if entry.file_type().is_dir() {
continue;
}
let rel = entry
.path()
.strip_prefix(dir)
.unwrap_or(entry.path())
.to_string_lossy()
.to_string();
entries.push((rel, entry.path().to_path_buf()));
}
entries.sort_by(|a, b| a.0.cmp(&b.0));
for (rel, path) in &entries {
let mut file = std::fs::File::open(path)?;
let mut buf = Vec::new();
file.read_to_end(&mut buf)?;
hasher.update(rel.as_bytes());
hasher.update(b"\0");
hasher.update(&buf);
}
let hash = hasher.finalize();
Ok(hash.iter().map(|b| format!("{:02x}", b)).collect())
}
/// Parse a `versions.env` file into a sorted map. Only `KEY=VAL` lines are
/// captured; comments (`#`) and blank lines are dropped. The value is
/// everything after the first `=` (so values may contain `=`), with one
/// level of matching `"` or `'` quotes stripped — quoted and unquoted forms
/// of the same value must resolve (and checksum) identically.
pub fn parse_versions_env(raw: &str) -> BTreeMap {
let mut pins = BTreeMap::new();
for line in raw.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
if let Some((k, v)) = line.split_once('=') {
let k = k.trim();
if !k.is_empty() {
pins.insert(k.to_string(), unquote(v.trim()));
}
}
}
pins
}
/// Strip one level of matching single/double quotes (build.sh does the
/// same; keeping the two parsers in agreement keeps checksums stable).
fn unquote(v: &str) -> String {
let b = v.as_bytes();
let quoted = b.len() >= 2
&& (b[0] == b'"' && *b.last().unwrap() == b'"'
|| b[0] == b'\'' && *b.last().unwrap() == b'\'');
if quoted {
v[1..v.len() - 1].to_string()
} else {
v.to_string()
}
}
/// Serialize a pin map back to `versions.env` text (sorted by key; one
/// `KEY=VAL` per line). Comments are not preserved — this is a generated
/// artifact written to the remote build dir, not the source of truth.
pub fn serialize_versions_env(pins: &BTreeMap) -> String {
let mut out = String::from(
"# Merged versions.env — generated by sm121 (profile + recipe [build.versions] applied)\n",
);
for (k, v) in pins {
out.push_str(k);
out.push('=');
out.push_str(v);
out.push('\n');
}
out
}
/// Compute the build checksum, folding in the version profile and recipe
/// `[build.versions]` overrides.
///
/// Precedence: image `versions.env` < profile file < recipe overrides.
/// Returns `(merged_versions_env, checksum)`:
/// - No profile and no overrides -> `(None, dir_checksum)`: the image's
/// `versions.env` is uploaded as-is and the checksum is unchanged
/// (existing images skip rebuild, backward-compatible).
/// - Profile and/or overrides -> `(Some(merged), combined)`: `merged` is the
/// image's `versions.env` with the profile pins and recipe pins applied
/// (recipe wins) for the launcher to write to the remote build dir;
/// `combined` hashes the dir checksum together with the merged env so any
/// pin change forces a rebuild. Profile files live inside the build dir, so
/// editing one changes the dir checksum too (double-protected).
pub(crate) fn build_checksum(
local_dir: &Path,
profile: Option<&str>,
overrides: &HashMap,
) -> Result<(Option, String)> {
let dir_checksum = compute_dir_checksum(local_dir)?;
if profile.is_none() && overrides.is_empty() {
return Ok((None, dir_checksum));
}
let env_path = local_dir.join("versions.env");
let raw = std::fs::read_to_string(&env_path).context(format!(
"a version profile / [build.versions] override requires '{}' to exist",
env_path.display()
))?;
let mut pins = parse_versions_env(&raw);
if let Some(name) = profile {
let profile_path = local_dir.join("profiles").join(format!("{name}.env"));
let raw = std::fs::read_to_string(&profile_path).with_context(|| {
let available = std::fs::read_dir(local_dir.join("profiles"))
.map(|d| {
d.filter_map(|e| {
e.ok()
.filter(|e| e.path().extension().is_some_and(|x| x == "env"))
.map(|e| e.file_name().to_string_lossy().into_owned())
})
.collect::>()
.join(", ")
})
.unwrap_or_default();
format!(
"build profile '{}' not found at {} (available: [{}])",
name,
profile_path.display(),
available
)
})?;
for (k, v) in parse_versions_env(&raw) {
pins.insert(k, v);
}
}
for (k, v) in overrides {
pins.insert(k.clone(), v.clone());
}
let merged = serialize_versions_env(&pins);
let mut h = Sha256::new();
h.update(dir_checksum.as_bytes());
h.update(b"\0");
h.update(merged.as_bytes());
let combined: String = h.finalize().iter().map(|b| format!("{:02x}", b)).collect();
Ok((Some(merged), combined))
}
/// Where a recipe's image is built from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BuildSource {
/// Full build context: `build.sh` + `Dockerfile` + versions.env.
Workspace,
/// Published bundle (flattened; `build.sh`/`Dockerfile` at bundle root).
Bundle,
}
/// Resolve a recipe's image build dir. Search order:
/// 1. `./docker-images/` (launcher workspace) — needs `build.sh`
/// 2. each `docker_image_sources` entry `S`: `S/docker-images/`, then
/// `S/` (a repo root containing `docker-images/`, or a dir of
/// image dirs)
/// 3. `./recipes//` then `.//` carrying a bundled `build.sh`
/// (published bundles from the recipe registry are self-contained)
///
/// Errors name every path tried.
pub fn resolve_build_dir(
base: &Path,
recipe_name: &str,
image_dir: &str,
sources: &[String],
) -> Result<(std::path::PathBuf, BuildSource)> {
let mut tried: Vec = Vec::new();
let workspace = base.join("docker-images").join(image_dir);
if workspace.join("build.sh").is_file() {
return Ok((workspace, BuildSource::Workspace));
}
tried.push(workspace.to_string_lossy().into());
for src in sources {
let root = crate::config::local_path(src);
for cand in [
root.join("docker-images").join(image_dir),
root.join(image_dir),
] {
if cand.join("build.sh").is_file() {
return Ok((cand, BuildSource::Workspace));
}
tried.push(cand.to_string_lossy().into());
}
}
for bun in [
base.join("recipes").join(recipe_name),
base.join(recipe_name),
] {
if bun.join("build.sh").is_file() && bun.join("Dockerfile").is_file() {
return Ok((bun, BuildSource::Bundle));
}
tried.push(bun.to_string_lossy().into());
}
anyhow::bail!(
"no build context for image dir '{}' of recipe '{}'; searched (each needs build.sh; bundles also need a Dockerfile): {}",
image_dir,
recipe_name,
tried.join(", ")
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_versions_env_quotes_are_normalized() {
let raw = "A=\"x:tag@sha256:abc\"\nB='y'\nC=z\nE=\"half\nF='weird\"\n";
let pins = parse_versions_env(raw);
assert_eq!(pins.get("A"), Some(&"x:tag@sha256:abc".to_string()));
assert_eq!(pins.get("B"), Some(&"y".to_string()));
assert_eq!(pins.get("C"), Some(&"z".to_string()));
assert_eq!(pins.get("E"), Some(&"\"half".to_string()));
assert_eq!(pins.get("F"), Some(&"'weird\"".to_string()));
}
#[test]
fn parse_versions_env_skips_comments_and_blanks() {
let raw = "# comment\n\nKEY=val\nOTHER=a=b\n SPACED = x \nnoeq\n";
let pins = parse_versions_env(raw);
assert_eq!(pins.get("KEY"), Some(&"val".to_string()));
assert_eq!(pins.get("OTHER"), Some(&"a=b".to_string()));
assert_eq!(pins.get("SPACED"), Some(&"x".to_string()));
assert!(!pins.contains_key("noeq"));
assert_eq!(pins.len(), 3);
}
#[test]
fn resolve_falls_back_to_bundle_layout() {
let base = std::env::temp_dir().join(format!("sm121-resolve-test-{}", std::process::id()));
let bundle = base.join("recipes").join("test-resolve-recipe-x");
std::fs::create_dir_all(&bundle).unwrap();
std::fs::write(bundle.join("build.sh"), "").unwrap();
std::fs::write(bundle.join("Dockerfile"), "").unwrap();
let (dir, src) =
resolve_build_dir(&base, "test-resolve-recipe-x", "test-resolve-recipe-x", &[])
.unwrap();
assert_eq!(src, BuildSource::Bundle);
assert_eq!(dir, bundle);
let _ = std::fs::remove_dir_all(&base);
}
}