Explorar el Código

r27 fetch-deps: recursive gitlink population (FlashMLA/qutlass/FlashKDA cutlass vendors; -d stub trap fixed)

Alexander Korolev hace 1 mes
padre
commit
35d8f090b2
Se han modificado 1 ficheros con 31 adiciones y 10 borrados
  1. 31 10
      docker-images/glm53-jovian-r27/fetch-deps.sh

+ 31 - 10
docker-images/glm53-jovian-r27/fetch-deps.sh

@@ -72,16 +72,37 @@ fetch https://github.com/vllm-project/tml-fa4.git \
 fetch https://github.com/triton-lang/triton.git v3.5.1 "$DEPS/triton"
 
 # ---- 3. Sub-module populations the tree's install steps require ----
-# FA vendors its own cutlass via a gitlink at csrc/cutlass; MSA vendors one
-# at python/fmha_sm100/cutlass. Empty dirs abort the cmake install steps
-# (r18 lesson: the staging once shipped stubs — builds died at INSTALL).
-fa_cutlass="$(git -C "$DEPS/flash-attention" ls-tree HEAD csrc/cutlass | awk '{print $3}')"
-[ -n "$fa_cutlass" ] || { echo "GATE FAILED: FA cutlass gitlink missing" >&2; exit 1; }
-fetch https://github.com/nvidia/cutlass.git "$fa_cutlass" "$DEPS/flash-attention/csrc/cutlass"
-
-msa_cutlass="$(git -C "$DEPS/MSA" ls-tree HEAD python/fmha_sm100/cutlass | awk '{print $3}')"
-[ -n "$msa_cutlass" ] || { echo "GATE FAILED: MSA cutlass gitlink missing" >&2; exit 1; }
-fetch https://github.com/nvidia/cutlass.git "$msa_cutlass" "$DEPS/MSA/python/fmha_sm100/cutlass"
+# Several deps vendor cutlass via a 160000 gitlink and their cmake install
+# steps require the tree to EXIST (r18 lesson: an empty stub dies at cmake
+# INSTALL — and r27's FlashMLA addendum died at compile). Generic rule: walk
+# every dep's tree recursively and populate each gitlink at its pinned SHA.
+# AMD-only trees (composable_kernel, aiter) are skipped explicitly — the
+# CUDA cmake path never references them. All gitlinks so far vendor
+# nvidia/cutlass; anything else aborts loudly, keeping the contract explicit.
+populate_gitlinks() {
+    local d path sha
+    for d in flash-attention MSA FlashMLA FlashKDA qutlass tml-fa4; do
+        while IFS=$'\t' read -r path sha; do
+            [ -n "${path:-}" ] || continue
+            case "$path" in
+                */composable_kernel|composable_kernel|*/aiter|aiter)
+                    echo "[deps] skip AMD-only gitlink $path (CUDA path unused)"; continue ;;
+            esac
+            case "$(basename "$path")" in
+                cutlass) ;;    # expected vendor
+                *) echo "ABORT: unhandled gitlink '$path' in $d (populate explicitly)" >&2; exit 1 ;;
+            esac
+            # a gitlink path EXISTS in the worktree as an EMPTY dir (git's
+            # clone behavior) — the gate is "did WE populate it", .git marker.
+            # Skipping on plain -d reproduces the r18 empty-stub failure.
+            if [ ! -e "$DEPS/$d/$path/.git" ]; then
+                fetch https://github.com/nvidia/cutlass.git "$sha" "$DEPS/$d/$path"
+            fi
+        done < <(git -C "$DEPS/$d" ls-tree -r -t HEAD | awk '$1=="160000" {print $4"\t"$3}')
+    done
+    echo "[deps] gitlink populations verified"
+}
+populate_gitlinks
 
 # ---- 4. Triton into the layout the Dockerfile/build expects ----
 mv "$DEPS/triton/python/triton_kernels/triton_kernels" "$DEPS/triton_kernels"