|
|
@@ -14,19 +14,25 @@ use std::path::{Path, PathBuf};
|
|
|
|
|
|
/// Compute a SHA-256 checksum over all files in `dir` (recursive).
|
|
|
/// Sorted by relative path to ensure reproducibility.
|
|
|
-/// Flatten an image overlay into `dst` and rewrite its Dockerfile:
|
|
|
-/// the `patches/` subtree plus every sub-directory the Dockerfile
|
|
|
-/// COPY/ADD-references is flattened to the bundle root (gists cannot
|
|
|
-/// carry directories). Name collisions rename the flattened file to
|
|
|
-/// `{dir}-{name}` and the COPY line is rewritten to the new name; a
|
|
|
-/// collision under a glob source aborts the flatten for that directory
|
|
|
-/// (it stays nested, `nested = true` — the gist flow refuses it).
|
|
|
-/// Unreferenced sub-directories are always preserved nested.
|
|
|
+/// Flatten an image overlay into `dst` and rewrite its Dockerfile.
|
|
|
+///
|
|
|
+/// Only files the build actually references are shipped: every COPY/ADD
|
|
|
+/// source of the Dockerfile (incl. continuations), plus `versions.env`
|
|
|
+/// (read by the build driver from the context). Referenced files from a
|
|
|
+/// sub-directory are flattened to the bundle root (gists cannot carry
|
|
|
+/// directories); name collisions rename the flattened file to `{dir}-{name}`
|
|
|
+/// and the COPY line is rewritten to the new name. A collision under a
|
|
|
+/// glob source cannot be renamed — that directory stays nested
|
|
|
+/// (`nested = true`, the gist flow refuses it). Unreferenced files and
|
|
|
+/// directories (e.g. `profiles/` launcher metadata, leftover sources) are
|
|
|
+/// not shipped at all.
|
|
|
pub fn flatten_overlay(src_dir: &Path, dst: &Path, dockerfile: &str) -> Result<(String, bool)> {
|
|
|
use std::collections::BTreeSet;
|
|
|
|
|
|
- // 1) Sub-directories referenced by COPY/ADD sources (incl. continuations).
|
|
|
- let mut referenced: BTreeSet<String> = BTreeSet::new();
|
|
|
+ // 1) Collect COPY/ADD sources: root names, and per-dir exact names + glob patterns.
|
|
|
+ let mut referenced_root: BTreeSet<String> = BTreeSet::new();
|
|
|
+ let mut exact: BTreeMap<String, BTreeSet<String>> = BTreeMap::new(); // dir -> rel paths
|
|
|
+ let mut globs: BTreeMap<String, Vec<String>> = BTreeMap::new(); // dir -> patterns
|
|
|
let mut in_copy = false;
|
|
|
for line in dockerfile.lines() {
|
|
|
let t = line.trim_start();
|
|
|
@@ -35,75 +41,100 @@ pub fn flatten_overlay(src_dir: &Path, dst: &Path, dockerfile: &str) -> Result<(
|
|
|
if !is_copy {
|
|
|
continue;
|
|
|
}
|
|
|
+ let is_first = t.starts_with("COPY ") || t.starts_with("ADD ");
|
|
|
let body = t.trim_end_matches('\\');
|
|
|
- let toks: Vec<&str> = body.split_whitespace().skip(1).collect();
|
|
|
- for tok in &toks[..toks.len().saturating_sub(1)] {
|
|
|
+ let toks: Vec<&str> = body.split_whitespace().collect();
|
|
|
+ let n = toks.len();
|
|
|
+ // first line: [cmd, sources…, dest]; continuation: sources only
|
|
|
+ let (lo, hi) = if is_first {
|
|
|
+ (1, n.saturating_sub(1))
|
|
|
+ } else {
|
|
|
+ (0, n)
|
|
|
+ };
|
|
|
+ for tok in &toks[lo..hi] {
|
|
|
if tok.starts_with('-') {
|
|
|
continue; // --from=… etc.
|
|
|
}
|
|
|
let parts: Vec<&str> = tok.split('/').collect();
|
|
|
// parts[0].is_empty() => absolute path (/tmp/…) — not a build-dir ref
|
|
|
- if parts.len() >= 2 && !parts[0].is_empty() && src_dir.join(parts[0]).is_dir() {
|
|
|
- referenced.insert(parts[0].to_string());
|
|
|
+ if parts.len() == 1 {
|
|
|
+ referenced_root.insert(tok.to_string());
|
|
|
+ } else if !parts[0].is_empty() && src_dir.join(parts[0]).is_dir() {
|
|
|
+ let rel = parts[1..].join("/");
|
|
|
+ if tok.contains('*') || tok.contains('?') {
|
|
|
+ globs.entry(parts[0].to_string()).or_default().push(rel);
|
|
|
+ } else {
|
|
|
+ exact.entry(parts[0].to_string()).or_default().insert(rel);
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- let flatten_dirs: BTreeSet<String> = referenced
|
|
|
- .iter()
|
|
|
- .cloned()
|
|
|
- .chain(std::iter::once("patches".to_string()))
|
|
|
- .collect();
|
|
|
+ // patches/ is always flattenable even without an explicit reference.
|
|
|
+ let dirs: Vec<String> = {
|
|
|
+ let mut set: BTreeSet<String> = exact.keys().cloned().collect();
|
|
|
+ for k in globs.keys() {
|
|
|
+ set.insert(k.clone());
|
|
|
+ }
|
|
|
+ set.insert("patches".to_string());
|
|
|
+ set.into_iter().collect()
|
|
|
+ };
|
|
|
|
|
|
- // 2) Root files first (they define the collision set), except the driver.
|
|
|
+ // 2) Root files: only referenced ones (+ versions.env, the build driver's
|
|
|
+ // build-arg source) — leftover top-level files are not shipped.
|
|
|
for entry in std::fs::read_dir(src_dir)? {
|
|
|
let entry = entry?;
|
|
|
if entry.file_type()?.is_file() {
|
|
|
let name = entry.file_name().to_string_lossy().to_string();
|
|
|
- if name != "build.sh" && name != "Dockerfile" {
|
|
|
+ if name == "build.sh" || name == "Dockerfile" {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if name == "versions.env" || referenced_root.contains(&name) {
|
|
|
std::fs::copy(entry.path(), dst.join(&name))?;
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- // 3) Flatten the referenced dirs (+ patches/); everything else nests.
|
|
|
+ // 3) Flatten the referenced dirs: only the referenced files.
|
|
|
let mut nested = false;
|
|
|
let mut flattened: BTreeSet<String> = BTreeSet::new(); // dirs actually flattened
|
|
|
let mut renames: BTreeMap<String, String> = BTreeMap::new(); // "dir/file" -> new name
|
|
|
- let mut globs: BTreeSet<String> = BTreeSet::new(); // dirs whose COPY source is a glob
|
|
|
- for line in dockerfile.lines() {
|
|
|
- let t = line.trim_start();
|
|
|
- if t.starts_with("COPY ") || t.starts_with("ADD ") {
|
|
|
- for tok in t.split_whitespace().skip(1) {
|
|
|
- if tok.starts_with('-') {
|
|
|
- continue;
|
|
|
- }
|
|
|
- let parts: Vec<&str> = tok.split('/').collect();
|
|
|
- if parts.len() >= 2 && flatten_dirs.contains(parts[0]) && tok.contains('*') {
|
|
|
- globs.insert(parts[0].to_string());
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- for dir in flatten_dirs.iter() {
|
|
|
+ for dir in dirs.iter() {
|
|
|
let dir_path = src_dir.join(dir);
|
|
|
if !dir_path.is_dir() {
|
|
|
- continue; // referenced but absent — COPY will fail in docker, not here
|
|
|
+ continue; // referenced but absent — the COPY will fail in docker, not here
|
|
|
}
|
|
|
+ let is_glob_dir = globs.contains_key(dir);
|
|
|
let files: Vec<PathBuf> = std::fs::read_dir(&dir_path)?
|
|
|
.filter_map(|e| e.ok())
|
|
|
.filter(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false))
|
|
|
.map(|e| e.path())
|
|
|
.collect();
|
|
|
- let collides = files
|
|
|
+ let needed = |f: &Path| {
|
|
|
+ let rel = f
|
|
|
+ .strip_prefix(&dir_path)
|
|
|
+ .unwrap_or(f)
|
|
|
+ .to_string_lossy()
|
|
|
+ .to_string();
|
|
|
+ exact
|
|
|
+ .get(dir)
|
|
|
+ .map(|s| s.contains(&rel))
|
|
|
+ .unwrap_or(false)
|
|
|
+ || globs
|
|
|
+ .get(dir)
|
|
|
+ .map(|pats| pats.iter().any(|p| glob_match(p, &rel)))
|
|
|
+ .unwrap_or(false)
|
|
|
+ };
|
|
|
+ let needed_files: Vec<PathBuf> = files.iter().filter(|f| needed(f)).cloned().collect();
|
|
|
+ let collides = needed_files
|
|
|
.iter()
|
|
|
.any(|f| dst.join(f.file_name().unwrap()).exists());
|
|
|
- if collides && globs.contains(dir) {
|
|
|
+ if collides && is_glob_dir {
|
|
|
nested = true;
|
|
|
std::fs::create_dir_all(dst.join(dir))?;
|
|
|
copy_tree(&dir_path, &dst.join(dir))?; // rename under a glob is impossible
|
|
|
continue;
|
|
|
}
|
|
|
- for f in files {
|
|
|
+ for f in needed_files {
|
|
|
let name = f.file_name().unwrap().to_string_lossy().to_string();
|
|
|
let key = format!("{dir}/{name}");
|
|
|
let new = if dst.join(&name).exists() {
|
|
|
@@ -117,21 +148,6 @@ pub fn flatten_overlay(src_dir: &Path, dst: &Path, dockerfile: &str) -> Result<(
|
|
|
}
|
|
|
flattened.insert(dir.clone());
|
|
|
}
|
|
|
- for entry in std::fs::read_dir(src_dir)? {
|
|
|
- let entry = entry?;
|
|
|
- let name = entry.file_name().to_string_lossy().to_string();
|
|
|
- if !entry.file_type()?.is_dir() || flatten_dirs.contains(&name) {
|
|
|
- continue;
|
|
|
- }
|
|
|
- // profiles/ is launcher metadata — its pins are already merged into
|
|
|
- // the bundle's versions.env, so it is not re-shipped.
|
|
|
- if name == "profiles" {
|
|
|
- continue;
|
|
|
- }
|
|
|
- nested = true;
|
|
|
- std::fs::create_dir_all(dst.join(&name))?;
|
|
|
- copy_tree(&entry.path(), &dst.join(&name))?;
|
|
|
- }
|
|
|
|
|
|
// 4) Rewrite COPY/ADD sources for the flattened dirs (incl. continuations).
|
|
|
let mut in_copy = false;
|
|
|
@@ -203,6 +219,36 @@ fn copy_tree(src_dir: &Path, dst_dir: &Path) -> Result<()> {
|
|
|
Ok(())
|
|
|
}
|
|
|
|
|
|
+/// Minimal glob match for COPY sources: `*` = any run of chars, `?` = one char.
|
|
|
+fn glob_match(pat: &str, name: &str) -> bool {
|
|
|
+ let p: Vec<char> = pat.chars().collect();
|
|
|
+ let n: Vec<char> = name.chars().collect();
|
|
|
+ let (mut i, mut j) = (0usize, 0usize);
|
|
|
+ let (mut star_i, mut star_j) = (0usize, 0usize);
|
|
|
+ let mut has_star = false;
|
|
|
+ while j < n.len() {
|
|
|
+ if i < p.len() && (p[i] == '?' || p[i] == n[j]) {
|
|
|
+ i += 1;
|
|
|
+ j += 1;
|
|
|
+ } else if i < p.len() && p[i] == '*' {
|
|
|
+ has_star = true;
|
|
|
+ star_i = i;
|
|
|
+ star_j = j;
|
|
|
+ i += 1;
|
|
|
+ } else if has_star {
|
|
|
+ star_j += 1;
|
|
|
+ i = star_i + 1;
|
|
|
+ j = star_j;
|
|
|
+ } else {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ while i < p.len() && p[i] == '*' {
|
|
|
+ i += 1;
|
|
|
+ }
|
|
|
+ i == p.len()
|
|
|
+}
|
|
|
+
|
|
|
pub fn compute_dir_checksum(dir: &Path) -> Result<String> {
|
|
|
let mut hasher = Sha256::new();
|
|
|
let mut entries: Vec<_> = Vec::new();
|
|
|
@@ -416,6 +462,17 @@ pub fn resolve_build_dir(
|
|
|
mod tests {
|
|
|
use super::*;
|
|
|
|
|
|
+ #[test]
|
|
|
+ fn glob_match_basics() {
|
|
|
+ assert!(glob_match("patch_vllm_*.py", "patch_vllm_topk.py"));
|
|
|
+ assert!(glob_match("patch_vllm_*.py", "patch_vllm_a.py"));
|
|
|
+ assert!(!glob_match("patch_vllm_*.py", "other.py"));
|
|
|
+ assert!(glob_match("a?c", "abc"));
|
|
|
+ assert!(!glob_match("a?c", "ac"));
|
|
|
+ assert!(glob_match("x", "x"));
|
|
|
+ assert!(glob_match("*", "anything/at-all"));
|
|
|
+ }
|
|
|
+
|
|
|
#[test]
|
|
|
fn parse_versions_env_quotes_are_normalized() {
|
|
|
let raw = "A=\"x:tag@sha256:abc\"\nB='y'\nC=z\nE=\"half\nF='weird\"\n";
|
