|
|
@@ -183,6 +183,16 @@ class Stager:
|
|
|
"outside" the tree and turn them into duplicated real files."""
|
|
|
return os.path.normpath(os.path.join(os.path.dirname(p), os.readlink(p)))
|
|
|
|
|
|
+ def resolve_local(self, path):
|
|
|
+ """Normalize a candidate mirror path: follow one symlink hop (old
|
|
|
+ repo-blob layout) as long as the resolved path stays inside the
|
|
|
+ mirror; None if it escapes or the link is dangling."""
|
|
|
+ if os.path.islink(path):
|
|
|
+ path = os.path.realpath(path)
|
|
|
+ if not path.startswith(self.dst.rstrip(os.sep) + os.sep):
|
|
|
+ return None
|
|
|
+ return os.path.normpath(path)
|
|
|
+
|
|
|
def stage_links(self, links):
|
|
|
for p, rel in links:
|
|
|
target = self.link_target(p)
|
|
|
@@ -294,32 +304,62 @@ class Stager:
|
|
|
target = self.link_target(p)
|
|
|
if not os.path.exists(target):
|
|
|
fail(f"dangling link in source snapshot: {p}")
|
|
|
- mirror = os.path.normpath(
|
|
|
- os.path.join(self.dst, os.path.relpath(target, self.src)))
|
|
|
+ mirror = os.path.join(self.dst, os.path.relpath(target, self.src))
|
|
|
else:
|
|
|
target = p
|
|
|
- mirror = os.path.normpath(
|
|
|
- os.path.join(self.dst, os.path.relpath(p, self.src)))
|
|
|
+ mirror = os.path.join(self.dst, os.path.relpath(p, self.src))
|
|
|
dst_entry = os.path.join(export_root, rel)
|
|
|
entries += 1
|
|
|
- if os.path.isdir(mirror):
|
|
|
+ # the mirror tree may use either hub layout (hub/blobs/xx/sha)
|
|
|
+ # or the older repo-blob layout (models--X/blobs/sha) behind a
|
|
|
+ # symlink — normalize to a real local file if it resolves
|
|
|
+ # inside the mirror.
|
|
|
+ local = self.resolve_local(mirror)
|
|
|
+ # skip-listed subtrees (page15) must never be materialized as
|
|
|
+ # hardlinks: they were not staged, so which nodes happen to
|
|
|
+ # hold them is history. Emit them uniformly as per-file
|
|
|
+ # binds (local copy when verified, else source) — the export
|
|
|
+ # tree is then identical across nodes.
|
|
|
+ if any(part in self.skip for part in rel.split(os.sep)):
|
|
|
+ if os.path.exists(dst_entry) or os.path.islink(dst_entry):
|
|
|
+ os.remove(dst_entry) # stale hardlink from an older pass
|
|
|
+ if local is not None and os.path.isfile(local) \
|
|
|
+ and (self.verify != "sample-hash"
|
|
|
+ or sample_hash(local) == sample_hash(target)):
|
|
|
+ bind_src = local
|
|
|
+ else:
|
|
|
+ bind_src = target
|
|
|
+ os.makedirs(os.path.dirname(dst_entry), exist_ok=True)
|
|
|
+ if not os.path.exists(dst_entry):
|
|
|
+ # mountpoint placeholder: the per-file bind lands on
|
|
|
+ # it (runc cannot create it when the export dir is
|
|
|
+ # mounted read-only, and the runtime guards its
|
|
|
+ # weight shards behind a read-only filesystem).
|
|
|
+ open(dst_entry, "wb").close()
|
|
|
+ manifest_lines.append(f"{bind_src}\t{os.path.join(container, rel)}")
|
|
|
+ manifest += 1
|
|
|
+ continue
|
|
|
+ if local is not None and os.path.isdir(local):
|
|
|
os.makedirs(dst_entry, exist_ok=True)
|
|
|
continue
|
|
|
- if os.path.exists(mirror) and not os.path.islink(mirror):
|
|
|
+ if local is not None and os.path.isfile(local):
|
|
|
+ if (self.verify == "sample-hash"
|
|
|
+ and sample_hash(local) != sample_hash(target)):
|
|
|
+ fail(f"mirror blob {local} no longer matches source for {p}")
|
|
|
if (os.path.exists(dst_entry) and not os.path.islink(dst_entry)
|
|
|
- and os.path.samefile(mirror, dst_entry)):
|
|
|
+ and os.path.samefile(local, dst_entry)):
|
|
|
links += 1
|
|
|
continue
|
|
|
os.makedirs(os.path.dirname(dst_entry), exist_ok=True)
|
|
|
if os.path.exists(dst_entry) or os.path.islink(dst_entry):
|
|
|
os.remove(dst_entry)
|
|
|
try:
|
|
|
- os.link(mirror, dst_entry)
|
|
|
+ os.link(local, dst_entry)
|
|
|
except OSError as e:
|
|
|
- fail(f"hardlink {dst_entry} -> {mirror}: {e}")
|
|
|
+ fail(f"hardlink {dst_entry} -> {local}: {e}")
|
|
|
links += 1
|
|
|
else:
|
|
|
- # source-only entry (skip-listed or never staged): bind it
|
|
|
+ # source-only entry (never staged outside skip — bind source)
|
|
|
manifest_lines.append(f"{target}\t{os.path.join(container, rel)}")
|
|
|
manifest += 1
|
|
|
print(f"STAGED export {name} entries={entries} links={links} "
|
