Переглянути джерело

staging(hub-mirror): resolve source links syntactically, not realpath

realpath() resolved symlinked src roots (~/.cache/huggingface -> NAS) and
flung blob targets outside src, hitting the materialize-as-file branch:
weight files were duplicated into the snapshot tree (147-175 GiB/node of
dead weight, per-serve copy loops). Composed-model technique: keep a common
root, join normpath(dirname, readlink). Fix verified E2E (symlinked-src
fixture) + live on dgx1/dgx2 (verify-only, 0 copies).
Alexander Korolev 3 тижнів тому
батько
коміт
f9582dd215
1 змінених файлів з 12 додано та 4 видалено
  1. 12 4
      src/staging/scripts/hub_mirror.py

+ 12 - 4
src/staging/scripts/hub_mirror.py

@@ -175,9 +175,17 @@ class Stager:
                     self.ensure_file(p, os.path.join(self.dst, rel))
         return links
 
+    def link_target(self, p):
+        """Resolve one link entry SYNTACTICALLY within the shared root: the
+        composed-model technique — paths keep a common root so relative
+        refs survive a symlinked src (e.g. ~/.cache/huggingface -> NAS).
+        realpath() would resolve the src root itself, fling blob targets
+        "outside" the tree and turn them into duplicated real files."""
+        return os.path.normpath(os.path.join(os.path.dirname(p), os.readlink(p)))
+
     def stage_links(self, links):
         for p, rel in links:
-            target = os.path.realpath(p)
+            target = self.link_target(p)
             if not os.path.exists(target):
                 fail(f"dangling link in source: {p}")
             mapped_rel = os.path.relpath(target, self.src)
@@ -237,14 +245,14 @@ class Stager:
         for p, rel in links:
             parts = rel.split(os.sep)
             if len(parts) > 3 and parts[2] == "snapshots" and os.path.islink(p):
-                t = os.path.realpath(p)
+                t = self.link_target(p)
                 if os.path.isfile(t):
                     refset.add(os.path.basename(t))
         kept = []
         for p, rel in links:
             parts = rel.split(os.sep)
             if (len(parts) >= 4 and parts[2] == "blobs" and os.path.islink(p)
-                    and os.path.basename(os.path.realpath(p)) not in refset):
+                    and os.path.basename(self.link_target(p)) not in refset):
                 self.skipped_refs += 1
                 continue
             kept.append((p, rel))
@@ -283,7 +291,7 @@ class Stager:
                     stack.append(p)
                     continue
                 if stat.S_ISLNK(st.st_mode):
-                    target = os.path.realpath(p)
+                    target = self.link_target(p)
                     if not os.path.exists(target):
                         fail(f"dangling link in source snapshot: {p}")
                     mirror = os.path.normpath(