|
@@ -1,13 +1,11 @@
|
|
|
use super::ServeOpts;
|
|
use super::ServeOpts;
|
|
|
|
|
+use super::build_dir::{build_checksum, resolve_build_dir};
|
|
|
use super::shq;
|
|
use super::shq;
|
|
|
use crate::config::{Cluster, Config, Node, Recipe};
|
|
use crate::config::{Cluster, Config, Node, Recipe};
|
|
|
use crate::ssh::run_ssh;
|
|
use crate::ssh::run_ssh;
|
|
|
-use anyhow::{Context, Result};
|
|
|
|
|
|
|
+use anyhow::Result;
|
|
|
use base64::Engine;
|
|
use base64::Engine;
|
|
|
use base64::engine::general_purpose::STANDARD;
|
|
use base64::engine::general_purpose::STANDARD;
|
|
|
-use sha2::{Digest, Sha256};
|
|
|
|
|
-use std::collections::{BTreeMap, HashMap};
|
|
|
|
|
-use std::io::Read;
|
|
|
|
|
use std::path::Path;
|
|
use std::path::Path;
|
|
|
use tracing::info;
|
|
use tracing::info;
|
|
|
|
|
|
|
@@ -16,112 +14,6 @@ use tracing::info;
|
|
|
/// otherwise loop the 30s poll forever, hanging the launcher.
|
|
/// otherwise loop the 30s poll forever, hanging the launcher.
|
|
|
pub const DEFAULT_MAX_BUILD_SECS: u64 = 90 * 60;
|
|
pub const DEFAULT_MAX_BUILD_SECS: u64 = 90 * 60;
|
|
|
|
|
|
|
|
-/// Compute a SHA-256 checksum over all files in `dir` (recursive).
|
|
|
|
|
-/// Sorted by relative path to ensure reproducibility.
|
|
|
|
|
-pub fn compute_dir_checksum(dir: &Path) -> Result<String> {
|
|
|
|
|
- let mut hasher = Sha256::new();
|
|
|
|
|
- let mut entries: Vec<_> = Vec::new();
|
|
|
|
|
-
|
|
|
|
|
- for entry in walkdir::WalkDir::new(dir).sort_by_file_name() {
|
|
|
|
|
- let entry = entry?;
|
|
|
|
|
- if entry.file_type().is_dir() {
|
|
|
|
|
- continue;
|
|
|
|
|
- }
|
|
|
|
|
- let rel = entry
|
|
|
|
|
- .path()
|
|
|
|
|
- .strip_prefix(dir)
|
|
|
|
|
- .unwrap_or(entry.path())
|
|
|
|
|
- .to_string_lossy()
|
|
|
|
|
- .to_string();
|
|
|
|
|
- entries.push((rel, entry.path().to_path_buf()));
|
|
|
|
|
- }
|
|
|
|
|
- entries.sort_by(|a, b| a.0.cmp(&b.0));
|
|
|
|
|
-
|
|
|
|
|
- for (rel, path) in &entries {
|
|
|
|
|
- let mut file = std::fs::File::open(path)?;
|
|
|
|
|
- let mut buf = Vec::new();
|
|
|
|
|
- file.read_to_end(&mut buf)?;
|
|
|
|
|
- hasher.update(rel.as_bytes());
|
|
|
|
|
- hasher.update(b"\0");
|
|
|
|
|
- hasher.update(&buf);
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- let hash = hasher.finalize();
|
|
|
|
|
- Ok(hash.iter().map(|b| format!("{:02x}", b)).collect())
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
-/// Parse a `versions.env` file into a sorted map. Only `KEY=VAL` lines are
|
|
|
|
|
-/// captured; comments (`#`) and blank lines are dropped. The value is
|
|
|
|
|
-/// everything after the first `=` (so values may contain `=`).
|
|
|
|
|
-fn parse_versions_env(raw: &str) -> BTreeMap<String, String> {
|
|
|
|
|
- let mut pins = BTreeMap::new();
|
|
|
|
|
- for line in raw.lines() {
|
|
|
|
|
- let line = line.trim();
|
|
|
|
|
- if line.is_empty() || line.starts_with('#') {
|
|
|
|
|
- continue;
|
|
|
|
|
- }
|
|
|
|
|
- if let Some((k, v)) = line.split_once('=') {
|
|
|
|
|
- let k = k.trim();
|
|
|
|
|
- if !k.is_empty() {
|
|
|
|
|
- pins.insert(k.to_string(), v.trim().to_string());
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
- pins
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
-/// Serialize a pin map back to `versions.env` text (sorted by key; one
|
|
|
|
|
-/// `KEY=VAL` per line). Comments are not preserved — this is a generated
|
|
|
|
|
-/// artifact written to the remote build dir, not the source of truth.
|
|
|
|
|
-fn serialize_versions_env(pins: &BTreeMap<String, String>) -> String {
|
|
|
|
|
- let mut out = String::from(
|
|
|
|
|
- "# Merged versions.env \u{2014} generated by sm121 (recipe [build.versions] overrides applied)\n",
|
|
|
|
|
- );
|
|
|
|
|
- for (k, v) in pins {
|
|
|
|
|
- out.push_str(k);
|
|
|
|
|
- out.push('=');
|
|
|
|
|
- out.push_str(v);
|
|
|
|
|
- out.push('\n');
|
|
|
|
|
- }
|
|
|
|
|
- out
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
-/// Compute the build checksum, folding in recipe `[build.versions]` overrides.
|
|
|
|
|
-///
|
|
|
|
|
-/// Returns `(merged_versions_env, checksum)`:
|
|
|
|
|
-/// - No overrides -> `(None, dir_checksum)`: the image's `versions.env` is
|
|
|
|
|
-/// uploaded as-is and the checksum is unchanged (existing images skip
|
|
|
|
|
-/// rebuild, backward-compatible).
|
|
|
|
|
-/// - Overrides present -> `(Some(merged), combined)`: `merged` is the image's
|
|
|
|
|
-/// `versions.env` with recipe pins applied (recipe wins) for the launcher
|
|
|
|
|
-/// to write to the remote build dir; `combined` hashes the dir checksum
|
|
|
|
|
-/// together with the merged env so any override change forces a rebuild.
|
|
|
|
|
-pub(crate) fn build_checksum(
|
|
|
|
|
- local_dir: &Path,
|
|
|
|
|
- overrides: &HashMap<String, String>,
|
|
|
|
|
-) -> Result<(Option<String>, String)> {
|
|
|
|
|
- let dir_checksum = compute_dir_checksum(local_dir)?;
|
|
|
|
|
- if overrides.is_empty() {
|
|
|
|
|
- return Ok((None, dir_checksum));
|
|
|
|
|
- }
|
|
|
|
|
- let env_path = local_dir.join("versions.env");
|
|
|
|
|
- let raw = std::fs::read_to_string(&env_path).context(format!(
|
|
|
|
|
- "recipe carries [build.versions] overrides but '{}' has no versions.env",
|
|
|
|
|
- env_path.display()
|
|
|
|
|
- ))?;
|
|
|
|
|
- let mut pins = parse_versions_env(&raw);
|
|
|
|
|
- for (k, v) in overrides {
|
|
|
|
|
- pins.insert(k.clone(), v.clone());
|
|
|
|
|
- }
|
|
|
|
|
- let merged = serialize_versions_env(&pins);
|
|
|
|
|
- let mut h = Sha256::new();
|
|
|
|
|
- h.update(dir_checksum.as_bytes());
|
|
|
|
|
- h.update(b"\0");
|
|
|
|
|
- h.update(merged.as_bytes());
|
|
|
|
|
- let combined: String = h.finalize().iter().map(|b| format!("{:02x}", b)).collect();
|
|
|
|
|
- Ok((Some(merged), combined))
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
/// Start a Docker image build on a remote node in a tmux session.
|
|
/// Start a Docker image build on a remote node in a tmux session.
|
|
|
/// After the build completes, stamps the checksum label on the image.
|
|
/// After the build completes, stamps the checksum label on the image.
|
|
|
pub async fn build_image_on_node(
|
|
pub async fn build_image_on_node(
|
|
@@ -297,6 +189,7 @@ pub async fn build_image_on_node(
|
|
|
/// the build was skipped (checksum match) or it was a dry-run. Callers use the
|
|
/// the build was skipped (checksum match) or it was a dry-run. Callers use the
|
|
|
/// return value to treat the freshly built image as the authoritative sync
|
|
/// return value to treat the freshly built image as the authoritative sync
|
|
|
/// source instead of re-running the "most recently created" heuristic.
|
|
/// source instead of re-running the "most recently created" heuristic.
|
|
|
|
|
+/// Where a recipe's image is built from.
|
|
|
pub async fn handle_build(
|
|
pub async fn handle_build(
|
|
|
config: &Config,
|
|
config: &Config,
|
|
|
cluster: &Cluster,
|
|
cluster: &Cluster,
|
|
@@ -307,6 +200,14 @@ pub async fn handle_build(
|
|
|
) -> Result<bool> {
|
|
) -> Result<bool> {
|
|
|
if opts.dry_run {
|
|
if opts.dry_run {
|
|
|
let dir = recipe.docker_image_dir.as_deref().unwrap_or(recipe_name);
|
|
let dir = recipe.docker_image_dir.as_deref().unwrap_or(recipe_name);
|
|
|
|
|
+ let review = resolve_build_dir(
|
|
|
|
|
+ Path::new("."),
|
|
|
|
|
+ recipe_name,
|
|
|
|
|
+ dir,
|
|
|
|
|
+ &config.docker_image_sources,
|
|
|
|
|
+ )
|
|
|
|
|
+ .map(|(p, s)| format!("{} [{:?}]", p.display(), s))
|
|
|
|
|
+ .unwrap_or_else(|e| e.to_string());
|
|
|
let overrides = if recipe.build.versions.is_empty() {
|
|
let overrides = if recipe.build.versions.is_empty() {
|
|
|
String::new()
|
|
String::new()
|
|
|
} else {
|
|
} else {
|
|
@@ -316,8 +217,8 @@ pub async fn handle_build(
|
|
|
)
|
|
)
|
|
|
};
|
|
};
|
|
|
println!(
|
|
println!(
|
|
|
- "[dry-run] would build '{}' on '{}' from docker-images/{}/build.sh{}",
|
|
|
|
|
- recipe.docker_image, build_node_name, dir, overrides
|
|
|
|
|
|
|
+ "[dry-run] would build '{}' on '{}' from {}{}",
|
|
|
|
|
+ recipe.docker_image, build_node_name, review, overrides
|
|
|
);
|
|
);
|
|
|
return Ok(false);
|
|
return Ok(false);
|
|
|
}
|
|
}
|
|
@@ -325,13 +226,12 @@ pub async fn handle_build(
|
|
|
.docker_image_dir
|
|
.docker_image_dir
|
|
|
.clone()
|
|
.clone()
|
|
|
.unwrap_or_else(|| recipe_name.to_string());
|
|
.unwrap_or_else(|| recipe_name.to_string());
|
|
|
- let local_dir = std::path::Path::new("docker-images").join(&image_dir);
|
|
|
|
|
- if !local_dir.join("build.sh").exists() {
|
|
|
|
|
- anyhow::bail!(
|
|
|
|
|
- "No build system at '{}' (missing build.sh)",
|
|
|
|
|
- local_dir.display()
|
|
|
|
|
- );
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ let (local_dir, _source) = resolve_build_dir(
|
|
|
|
|
+ Path::new("."),
|
|
|
|
|
+ recipe_name,
|
|
|
|
|
+ &image_dir,
|
|
|
|
|
+ &config.docker_image_sources,
|
|
|
|
|
+ )?;
|
|
|
let build_node = cluster
|
|
let build_node = cluster
|
|
|
.nodes
|
|
.nodes
|
|
|
.iter()
|
|
.iter()
|
|
@@ -412,7 +312,8 @@ fn format_elapsed(d: std::time::Duration) -> String {
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
#[cfg(test)]
|
|
|
mod tests {
|
|
mod tests {
|
|
|
- use super::{build_checksum, format_elapsed, parse_versions_env, serialize_versions_env};
|
|
|
|
|
|
|
+ use super::format_elapsed;
|
|
|
|
|
+ use crate::docker::build_dir::{build_checksum, parse_versions_env, serialize_versions_env};
|
|
|
use std::collections::HashMap;
|
|
use std::collections::HashMap;
|
|
|
use std::fs;
|
|
use std::fs;
|
|
|
use std::path::PathBuf;
|
|
use std::path::PathBuf;
|
|
@@ -447,6 +348,21 @@ mod tests {
|
|
|
assert_eq!(pins.len(), 3);
|
|
assert_eq!(pins.len(), 3);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ #[test]
|
|
|
|
|
+ fn parse_versions_env_quotes_are_normalized() {
|
|
|
|
|
+ // quoted / unquoted / single-quoted forms of the same value must
|
|
|
|
|
+ // resolve identically (stable checksums across edits); inner quotes
|
|
|
|
|
+ // and non-matching outer quotes stay verbatim.
|
|
|
|
|
+ let raw = "A=\"x:tag@sha256:abc\"\nB='y'\nC=z\nD=\"m@n\"'\"op\"'\nE=\"half\nF='weird\"\n";
|
|
|
|
|
+ let pins = parse_versions_env(raw);
|
|
|
|
|
+ assert_eq!(pins.get("A"), Some(&"x:tag@sha256:abc".to_string()));
|
|
|
|
|
+ assert_eq!(pins.get("B"), Some(&"y".to_string()));
|
|
|
|
|
+ assert_eq!(pins.get("C"), Some(&"z".to_string()));
|
|
|
|
|
+ assert_eq!(pins.get("D"), Some(&"\"m@n\"'\"op\"'".to_string())); // mismatched outer quotes stay
|
|
|
|
|
+ assert_eq!(pins.get("E"), Some(&"\"half".to_string()));
|
|
|
|
|
+ assert_eq!(pins.get("F"), Some(&"'weird\"".to_string()));
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
#[test]
|
|
#[test]
|
|
|
fn serialize_versions_env_sorted_one_per_line() {
|
|
fn serialize_versions_env_sorted_one_per_line() {
|
|
|
let mut pins = std::collections::BTreeMap::new();
|
|
let mut pins = std::collections::BTreeMap::new();
|