r12 clean build: SHA-pinned PR heads (550 closed), deepgemm pre-clone, lil launch-policy diff recorded

docker-images/glm53-jovian-r12/Dockerfile

@@ -62,28 +62,42 @@ RUN --mount=type=cache,id=uv-cache,target=/root/.cache/uv \
 
 # The fork's r12 composition: base branch + sequential merge of the open
 # PR heads (ascending). The PR head pseudo-refs are GitHub's stable refs.
-FROM torch-base AS vllm-builder
+FROM torch-base AS deepgemm-builder
+ARG DEEPGEMM_REPO DEEPGEMM_REF DEEPGEMM_BRANCH
+RUN set -eux; \
+    git config --global protocol.version 0; \
+    git clone --branch ${DEEPGEMM_BRANCH} ${DEEPGEMM_REPO} /deps/deepgemm; \
+    cd /deps/deepgemm; \
+    git checkout ${DEEPGEMM_REF}; \
+    git submodule update --init --depth 1 -- third-party/cutlass third-party/fmt
+
+FROM deepgemm-builder AS vllm-builder
 ARG VLLM_REPO
 ARG VLLM_BASE_REF
-ARG VLLM_PRS
+ARG VLLM_PR_HEADS
 ARG SOURCE_DATE_EPOCH
 ARG BUILD_JOBS
-ENV TORCH_CUDA_ARCH_LIST=12.1a
+ENV TORCH_CUDA_ARCH_LIST=12.1a GIT_PROTOCOL=0 DEEPGEMM_SRC_DIR=/deps/deepgemm
 RUN set -eux; \
     git config --global user.email build@sm121.lan; \
     git config --global user.name sm121-builder; \
+    git config --global protocol.version 0; \
     git clone --recursive ${VLLM_REPO} /workspace/vllm; \
     cd /workspace/vllm; \
     git checkout ${VLLM_BASE_REF}; \
-    for pr in ${VLLM_PRS}; do \
-        git fetch origin "refs/pull/$pr/head:prs/$pr"; \
-        git merge --no-ff -m "PR $pr" "prs/$pr"; \
+    # PR head commits fetched by SHA (refs/pull/N/head vanishes when a PR
+    # closes — 550 closed unmerged; raw-SHA fetches stay available).
+    for pair in ${VLLM_PR_HEADS}; do \
+        pr="${pair%%:*}"; sha="${pair#*:}"; \
+        git fetch origin "${sha}:refs/prs/${pr}"; \
+        git merge --no-ff -m "PR ${pr}" "refs/prs/${pr}"; \
     done; \
     git rev-parse HEAD
 WORKDIR /workspace/vllm
 RUN --mount=type=cache,id=uv-cache,target=/root/.cache/uv \
     --mount=type=cache,id=ccache,target=/root/.ccache \
-    _j="${BUILD_JOBS:-$(nproc)}" \
+    uv pip install setuptools-rust setuptools-scm \
+    && _j="${BUILD_JOBS:-$(nproc)}" \
     && export MAX_JOBS="${_j}" CMAKE_BUILD_PARALLEL_LEVEL="${_j}" \
     && python3 use_existing_torch.py \
     && uv build --no-build-isolation --wheel . --out-dir=/wheels -v
@@ -96,6 +110,7 @@ ARG SOURCE_DATE_EPOCH
 RUN set -eux; \
     git config --global user.email build@sm121.lan; \
     git config --global user.name sm121-builder; \
+    git config --global protocol.version 0; \
     git clone ${B12X_REPO} /workspace/b12x; \
     cd /workspace/b12x; \
     git checkout ${B12X_BASE_REF}; \

docker-images/glm53-jovian-r12/versions.env

@@ -22,15 +22,23 @@ TORCHAUDIO_VERSION=2.11.0
 # fetches cubins when the package is absent).
 FLASHINFER_VERSION=0.6.17
 
-# vLLM — local-inference-lab fork, dev/jovian-judgement base + PR heads
+# vLLM — local-inference-lab fork, dev/jovian-judgement base + PR heads.
+# PR 550 closed (unmerged), which deleted its refs/pull/550/head pseudo-ref,
+# so pin the PR head commits by SHA (GitHub keeps the objects).
 VLLM_REPO=https://github.com/local-inference-lab/vllm.git
 VLLM_BASE_REF=54f6e9826c20ef06ed65d838c0ad497ad0abdecf
-VLLM_PRS="550 552 553 554 555 557"
+VLLM_PR_HEADS="550:52006e858766911cebccc8a7b2b21a260f5c97f2 552:65ef1cd4d1d595a6b54698e3c2983ed4af421445 553:c6b1530e895e5188b6f57109db1f609f5615ce19 554:88cff3d5a98b95df9ceeff77aa676ef2f18a03b6 555:78bfa29002d4d7a2d6c16877074af449ba1b0ce7 557:3ef77b3125253d6f797f1524f9a52ae478dad4f2"
 
 # B12X — master + PR heads (pure-python wheel; CuTe-DSL JIT kernels)
 B12X_REPO=https://github.com/local-inference-lab/b12x.git
 B12X_BASE_REF=139e04048bc3bb4f7210c99e7184d8d2f0e345e7
 B12X_PRS="260 267 268 269 270"
 
+# DeepGEMM — fetched by vllm's cmake; pre-cloned (raw-SHA fetches of a
+# non-default-branch tip 404 over GitHub's git protocol)
+DEEPGEMM_REPO=https://github.com/deepseek-ai/DeepGEMM.git
+DEEPGEMM_BRANCH=nv_dev
+DEEPGEMM_REF=8b1392b978f5a03c828dd1711090d7fb50958b8a
+
 # Target GPU architecture (DGX Spark = sm_121a)
 TORCH_CUDA_ARCH_LIST=12.1a

specs/tech-architecture/glm53-recipe-state.md

@@ -384,3 +384,42 @@ NVFP4-Spark boot on it (recipe glm53-flash-nvfp4-spark, retargeted):
   Build running on dgx3 (tmux r12build → /tmp/r12build2.log), same tag
   glm53-jovian-r12:local (running serve keeps the old image; re-verify lane
   after build).
+
+## 2026-09-02 (cont 2): lil — the lab's own launcher, read
+
+local-inference-lab/lil = standalone Go launcher (their sm121 equivalent):
+reads each HF model repo's lil.yaml at runtime + embedded family policy +
+discovered topology (local or spark-rdma via SSH), renders/validates/runs a
+typed vLLM invocation. `lil render <model>` prints the exact env + command.
+
+GLM family policy (configs/models/_bases.yaml) — the lab's canonical
+launch for these checkpoints, vs our spark recipe today:
+- --attention-backend B12X, --linear-backend b12x, --moe-backend b12x
+  (we have only the global attention flag; linear/moe backends MISSING)
+- spec config: {"method":"mtp","num_speculative_tokens":5 (lil.yaml
+  mtp_tokens), "moe_backend":<quant-mapped: nvfp4→b12x, mxfp8/bf16→triton>,
+  "attention_backend":"B12X"} — we use k=3 and no moe_backend. MTP:3 is the
+  r12 x86-qualified mode; the checkpoint's manifest default is 5.
+- --load-format instanttensor (+INSTANTTENSOR_BUFFER_SIZE/CHUNK_SIZE env)
+- --mamba-cache-mode align, --max-num-seqs 8, --max-num-batched-tokens 4096,
+  --max-model-len auto
+- --reasoning-parser glm45, --tool-call-parser glm47
+- DFlash2 (the GLM-side second speculator, method "dflash") needs the
+  separate draft checkpoint local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8.
+
+TODO: after the clean image lands, align the spark recipe to the lil policy
+(moe/linear backend b12x, spec moe_backend, instanttensor, mamba-cache-mode,
+parsers) and A/B k=3 vs k=5.
+
+## Clean r12 build: fix log (builds 2-9)
+1. flashinfer: whl-index path dead → flashinfer-python 0.6.17 from PyPI
+   (fork's setup.py skips flashinfer-cubin at build; runtime fetches cubins).
+2. git protocol v2 flake in-container → GIT_PROTOCOL=0 + protocol.version 0.
+3. setuptools: fork build needs setuptools-rust + setuptools-scm (uv hint).
+4. DeepGEMM FetchContent: pinned SHA on non-default nv_dev branch → GitHub
+   404s raw-SHA fetch → DEEPGEMM_SRC_DIR pre-clone stage (cmake escape hatch).
+5. PR 550 closed (unmerged) → refs/pull/550/head deleted; all PR heads now
+   pinned by commit SHA (vllm 550-557, b12x 260-270). Raw-SHA fetches
+   verified working.
+6. Operator rule: no build while a serve container runs (OOM risk on 128 GB
+   unified) — spark lane stopped for the build window.