persist JIT caches per node (flashinfer/torchinductor/vllm mounts in all glm53 lanes)

recipes/glm53-flash-nvfp4-eugr/recipe.toml

@@ -123,3 +123,19 @@ INSTANTTENSOR_BUFFER_SIZE = "67108864"
 INSTANTTENSOR_CHUNK_SIZE = "8388608"
 INSTANTTENSOR_CONCURRENCY = "1"
 INSTANTTENSOR_IO_DEPTH = "3"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"

recipes/glm53-flash-nvfp4-r17abi/recipe.toml

@@ -186,3 +186,19 @@ OMP_NUM_THREADS = "2"
 [[extra_mounts]]
 host = "~/sm121-cache/glm53-jit-121f/b12x"
 container = "/root/.cache/b12x"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"

recipes/glm53-flash-nvfp4-r18-4x-dcp4/recipe.toml

@@ -188,3 +188,19 @@ OMP_NUM_THREADS = "2"
 [[extra_mounts]]
 host = "~/sm121-cache/glm53-jit-121f/b12x"
 container = "/root/.cache/b12x"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"

recipes/glm53-flash-nvfp4-r18-4x-nvfp4kv/recipe.toml

@@ -187,3 +187,19 @@ OMP_NUM_THREADS = "2"
 [[extra_mounts]]
 host = "~/sm121-cache/glm53-jit-121f/b12x"
 container = "/root/.cache/b12x"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"

recipes/glm53-flash-nvfp4-r18-4x/recipe.toml

@@ -186,3 +186,19 @@ OMP_NUM_THREADS = "2"
 [[extra_mounts]]
 host = "~/sm121-cache/glm53-jit-121f/b12x"
 container = "/root/.cache/b12x"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"

recipes/glm53-flash-nvfp4-r18-dflash2/recipe.toml

@@ -193,3 +193,19 @@ OMP_NUM_THREADS = "2"
 [[extra_mounts]]
 host = "~/sm121-cache/glm53-jit-121f/b12x"
 container = "/root/.cache/b12x"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"

recipes/glm53-flash-nvfp4-r18/recipe.toml

@@ -186,3 +186,19 @@ OMP_NUM_THREADS = "2"
 [[extra_mounts]]
 host = "~/sm121-cache/glm53-jit-121f/b12x"
 container = "/root/.cache/b12x"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"

recipes/glm53-flash-nvfp4-redhat/recipe.toml

@@ -103,3 +103,19 @@ TORCH_CUDA_ARCH_LIST = "12.1a"
 CUTE_DSL_ARCH = "sm_121a"
 OMP_NUM_THREADS = "4"
 PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"

specs/tech-architecture/glm53-recipe-state.md

@@ -795,3 +795,20 @@ now, no mixing:
   (new) → r18 tag/dir; `glm53-flash-nvfp4-r17abi` unchanged → r12 tag/dir.
 - Source tarballs (vllm-src/b12x-src) are local build artifacts — now
   gitignored; deps-src already was.
+
+## JIT cache persistence (2026-09-06, operator direction)
+
+Every fresh container re-pays ~20 min of flashinfer/torch JIT compilation
+(cold `/root/.cache` per container — seen on every lane all night). All glm53
+lanes now persist the JIT caches per node (local FS, not NAS) via plain
+`[[extra_mounts]]`:
+- `~/sm121-cache/jit-121f/flashinfer` → `/root/.cache/flashinfer`
+- `~/sm121-cache/jit-121f/torchinductor` → `/root/.cache/torchinductor`
+- `~/sm121-cache/jit-121f/vllm` → `/root/.cache/vllm`
+Host dirs are SHARED between lanes of the same arch — each cache
+self-namespaces by its engine-version key internally (flashinfer keys by
+pkg version + arch; torchinductor by torch version; vllm by compile input
+hash), so mixing r17/r18/lab/eugr in one dir is safe. The pre-existing
+`glm53-jit-121f/b12x` mount is untouched (keeps its accumulated cache).
+First serve of each image pays the compile once per node; later serves
+should start ~20 min faster.