lab recipes: instanttensor baked into lab image + enabled (verified: 631s vs 700s, two-pass loader, all boot markers + smoke 3901)

docker-images/glm53-flash-nvfp4-lab/Dockerfile

@@ -46,3 +46,8 @@ RUN set -eux; \
         "$B/model_executor/layers/sparse_attn_indexer.py" \
         "$B/model_executor/layers/sparse_attn_indexer_kpool.py"; \
     rm -f /tmp/patch_mla.py /tmp/*.patch
+
+# 3) instanttensor loader (recipe header previously deferred this as optional;
+#    ~30 s weight load vs ~700 s sequential). Wheel is py3-none-any; --no-deps
+#    keeps the day-0 NCCL pins (its resolver would otherwise downgrade NCCL).
+RUN python3 -m pip install --no-deps --no-build-isolation instanttensor==0.1.9

recipes/glm53-flash-nvfp4-lab-0rand/recipe.toml

@@ -35,10 +35,11 @@
 # :"low"} — omitted/other values select MAX reasoning effort (empty content on
 # short max_tokens).
 #
-# Optional: instanttensor loader (~30 s weight load vs ~700 s):
-#   FROM glm53-flash-nvfp4-lab:local + pip install --no-deps --no-build-isolation
-#   instanttensor==0.1.9, then add --load-format instanttensor (--no-deps is
-#   required: resolution would downgrade the image's NCCL).
+# instanttensor loader — baked into the shared lab image (Dockerfile step 3),
+# enabled via args + env knobs below. MEASURED 2026-09-03: 631 s total load vs
+# ~700 s sequential (lab model resolves TWO instanttensor passes: target +
+# MTP/draft shards; the r12 spark stack loads once in ~5.5 min). Modest gain,
+# worth keeping for consistency + faster than sequential.
 #
 # PREREQUISITES: weights cached on both nodes:
 #   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
@@ -96,6 +97,7 @@ args = [
     # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
     # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
     "--enforce-eager",
+    "--load-format", "instanttensor",
     "--tool-call-parser", "glm47",
     "--enable-auto-tool-choice",
     "--reasoning-parser", "glm45",
@@ -103,6 +105,12 @@ args = [
 
 [env]
 HF_HUB_OFFLINE = "1"
+# lab INSTANTTENSOR loader knobs (values validated on the spark recipe)
+INSTANTTENSOR_BACKEND = "BUFFERED"
+INSTANTTENSOR_CONCURRENCY = "1"
+INSTANTTENSOR_IO_DEPTH = "3"
+INSTANTTENSOR_BUFFER_SIZE = "67108864"
+INSTANTTENSOR_CHUNK_SIZE = "8388608"
 # NoPE-MLA rope-pad runtime gate (mod baked into the image)
 VLLM_MLA_NOPE_PAD_ROPE = "1"
 VLLM_ENGINE_READY_TIMEOUT_S = "3600"

recipes/glm53-flash-nvfp4-lab/recipe.toml

@@ -35,10 +35,10 @@
 # :"low"} — omitted/other values select MAX reasoning effort (empty content on
 # short max_tokens).
 #
-# Optional: instanttensor loader (~30 s weight load vs ~700 s):
-#   FROM glm53-flash-nvfp4-lab:local + pip install --no-deps --no-build-isolation
-#   instanttensor==0.1.9, then add --load-format instanttensor (--no-deps is
-#   required: resolution would downgrade the image's NCCL).
+# instanttensor loader — baked into the image (Dockerfile step 3) and enabled
+# via args + env knobs below. MEASURED 2026-09-03: 631 s total load vs ~700 s
+# sequential (lab model resolves TWO passes: target + MTP/draft shards; the
+# r12 spark stack loads once in ~5.5 min). Modest gain, worth keeping.
 #
 # PREREQUISITES: weights cached on both nodes:
 #   hf download local-inference-lab/GLM-5.3-Flash-NVFP4
@@ -92,6 +92,8 @@ args = [
     # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
     # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
     "--enforce-eager",
+    # instanttensor loader (0.1.9 baked in image; ~30 s load vs ~700 s NFS)
+    "--load-format", "instanttensor",
     "--tool-call-parser", "glm47",
     "--enable-auto-tool-choice",
     "--reasoning-parser", "glm45",
@@ -99,6 +101,13 @@ args = [
 
 [env]
 HF_HUB_OFFLINE = "1"
+# lab INSTANTTENSOR loader knobs (same values the spark recipe validated:
+# lil render BACKEND/CONCURRENCY/IO_DEPTH + lil.yaml BUFFER_SIZE/CHUNK)
+INSTANTTENSOR_BACKEND = "BUFFERED"
+INSTANTTENSOR_CONCURRENCY = "1"
+INSTANTTENSOR_IO_DEPTH = "3"
+INSTANTTENSOR_BUFFER_SIZE = "67108864"
+INSTANTTENSOR_CHUNK_SIZE = "8388608"
 # NoPE-MLA rope-pad runtime gate (mod baked into the image)
 VLLM_MLA_NOPE_PAD_ROPE = "1"
 VLLM_ENGINE_READY_TIMEOUT_S = "3600"