dsv41-flash build fixes: cutlass extraction, build tools, NVRTC, no-GPU gate

- cutlass tarball extracts into an existing dir (strip-components scatters
  otherwise); cmake/ninja/g++ + NVRTC resolution (their build_stable_ext.sh)
- _C_stable_libtorch gate = file existence, not import (dlopen needs the
  GPU driver's libcuda.so.1, absent in build containers)

docker-images/dsv41-flash/Dockerfile

@@ -48,19 +48,21 @@ RUN set -eux; \
 # stable-only build: external_projects includes commented out, cutlass fetched
 # at the pinned tag, Release build of the single C extension target.
 RUN set -eux; \
+    apt-get update -qq && apt-get install -y --no-install-recommends cmake ninja-build g++ >/dev/null && rm -rf /var/lib/apt/lists/*; \
     cd /src/vllm; \
-    mkdir -p build/_deps; \
-    curl -sL -m 600 ${CUTLASS_URL} | tar xz -C build/_deps --strip-components=1 && \
-        mv build/_deps/cutlass build/_deps/cutlass-src 2>/dev/null || true; \
+    mkdir -p build/_deps/cutlass-src; \
+    curl -sL -m 600 ${CUTLASS_URL} | tar xz -C build/_deps/cutlass-src --strip-components=1; \
     ls build/_deps/cutlass-src/include/cutlass/cutlass.h >/dev/null; \
     cp CMakeLists.txt CMakeLists.txt.orig; \
     sed -i -E "s|^(\s*)include\(cmake/external_projects/|\1# STABLE-ONLY: include(cmake/external_projects/|" CMakeLists.txt; \
     PYPATH=$(python3 -c "import sys;print(':'.join(p for p in sys.path if p))"); \
     TORCH_PREFIX=$(python3 -c "import torch;print(torch.utils.cmake_prefix_path)"); \
+    NVRTC=$(ls /usr/local/cuda/lib64/libnvrtc.so /usr/local/cuda/lib64/libnvrtc.so.* /usr/local/lib/python3.12/dist-packages/nvidia/*/lib/libnvrtc.so* /usr/lib/aarch64-linux-gnu/libnvrtc.so* 2>/dev/null | head -1); \
+    test -n "${NVRTC}"; \
     cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DVLLM_TARGET_DEVICE=cuda \
         -DVLLM_PYTHON_EXECUTABLE=$(which python3) -DVLLM_PYTHON_PATH="$PYPATH" \
         -DFETCHCONTENT_BASE_DIR=build/_deps -DFETCHCONTENT_SOURCE_DIR_CUTLASS=build/_deps/cutlass-src \
-        -DCMAKE_PREFIX_PATH="$TORCH_PREFIX" -DNVCC_THREADS=2 -DTORCH_CUDA_ARCH_LIST=12.1a; \
+        -DCMAKE_PREFIX_PATH="$TORCH_PREFIX" -DNVCC_THREADS=2 -DCUDA_nvrtc_LIBRARY="$NVRTC" -DTORCH_CUDA_ARCH_LIST=12.1a; \
     cmake --build build --target _C_stable_libtorch -j 20; \
     find build vllm -maxdepth 2 -name "_C_stable_libtorch*.so" | tee /tmp/stable_ext.so; \
     # place the built extension in the package dir (vLLM imports vllm._C_stable_libtorch)
@@ -72,7 +74,10 @@ RUN set -eux; \
     cp -a /src/vllm/vllm/. ${SITE}/; \
     find ${SITE} -name "__pycache__" -type d -prune -exec rm -rf {} +; \
     python3 -c "import vllm; print('vllm', vllm.__version__)"; \
-    python3 -c "import vllm._C_stable_libtorch as m; print('stable ext ok:', m.__file__)"
+    # existence check only: dlopen of the CUDA extension needs libcuda.so.1
+    # (GPU driver), which the build container does not have — the real import
+    # happens at serve time on the GPU node
+    ls ${SITE}/_C_stable_libtorch*.so
 
 # ---- FlashInfer 0.7.0rc1 from pinned source (0.6.18 has no SM120 sparse-MLA) ----
 RUN set -eux; \
@@ -136,7 +141,7 @@ PY
 # ---- final gates ----
 RUN set -eux; \
     python3 -c "import vllm; print('vllm', vllm.__version__)"; \
-    python3 -c "import vllm._C_stable_libtorch; print('stable ext import ok')"; \
+    ls /usr/local/lib/python3.12/dist-packages/vllm/_C_stable_libtorch*.so; \
     python3 -c "import flashinfer; print('flashinfer', flashinfer.__version__)"; \
     grep -q "DSV41_ENGRAM_DISK" /usr/local/lib/python3.12/dist-packages/vllm/models/deepseek_v4_1/common/engram.py; \
     echo "=== image gates passed ==="

recipes/dsv41-flash-4x/recipe.toml

@@ -0,0 +1,91 @@
+# DeepSeek-V4.1-Flash (552B MoE, MXFP4 experts, Engram-on-disk) — 4x DGX Spark.
+#
+# Image: dsv41-flash:local (docker-images/dsv41-flash/ — tonyd2wild overlay
+# chain as clone-at-pin build: vllm dsv41-feat@e47aa780 + sm121
+# _C_stable_libtorch + FlashInfer 0.7.0rc1 + 7 boot-10 patches baked in,
+# MIT provenance).
+#
+# Model: deepseek-ai/DeepSeek-V4.1-Flash (476 GB on the NFS HF cache,
+# refs/main). The two 203 GB FP8 Engram n-gram tables are NEVER loaded into
+# HBM — the baked engram.py patch keeps them on disk (DSV41_ENGRAM_DISK=1)
+# and reads rows on demand. Node-local sparse copies of each rank's rows
+# (48 GB, engram-local.json) accelerate decode once created (DSV41_ENGRAM_DIR
+# below, added after the copies exist — first boot reads straight from NFS).
+#
+# Config mirrors their boot-10 serving setup (launch/dsv41-tp4.sh):
+#   CUDA graphs FULL_AND_PIECEWISE + exact DSpark capture sizes (no padded
+#   rows — padded speculative batches hang SM120 sparse MLA, FlashInfer
+#   #5015), DSpark k=5 with the checkpoint's own draft layers, GMU 0.80,
+#   300K ctx (KV pool ~1.07M tokens = 3.57x headroom), tools + vision on,
+#   thinking off by default.
+
+engine = "vllm"
+name = "DeepSeek-V4.1-Flash MXFP4 (dsv41 img, 4x, DSpark k=5, Engram-on-disk)"
+docker_image = "dsv41-flash:local"
+docker_image_dir = "dsv41-flash"
+model = "deepseek-ai/DeepSeek-V4.1-Flash"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+entrypoint = ["vllm", "serve"]
+
+args = [
+    "--served-model-name", "dsv41",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    "--gpu-memory-utilization", "0.80",
+    "--max-model-len", "300000",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "8192",
+    # boot-4 fix: the V4 indexer backend refuses the smallest-listed block
+    "--block-size", "128",
+    # boot-10: exact capture sizes for DSpark k=5, SEQS=8 (multiples of 5
+    # and 6 up to 48 — every decode batch gets an exact FULL graph, no padding)
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[5,6,10,12,15,18,20,24,25,30,35,36,40,42,48]}'""",
+    "--speculative-config", """
+'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}'""",
+    "--tool-call-parser", "deepseek_v41",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "deepseek_v41",
+    "--default-chat-template-kwargs", """
+'{"thinking": false}'""",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+# Engram tables stay on disk (the baked engram.py patch); 32 parallel row
+# reads, 16 MB chunks (their launch defaults)
+DSV41_ENGRAM_DISK = "1"
+DSV41_ENGRAM_DISK_THREADS = "32"
+DSV41_ENGRAM_DISK_CHUNK = "16"
+# CUDA graphs: the Engram rows are staged in prepare_inputs (baked
+# model_state.py prestage patch) so the whole decode step is capturable
+VLLM_USE_BREAKABLE_CUDAGRAPH = "1"
+VLLM_HAS_FLASHINFER_CUBIN = "1"
+VLLM_USE_RUST_FRONTEND = "0"
+# runtime JIT wedge countermeasure (their boot-3 lesson): kernels are
+# prebuilt in the image, but any residual JIT must not OOM the box
+MAX_JOBS = "2"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+OMP_NUM_THREADS = "2"
+
+# JIT caches persist per-node (self-namespace by engine version key; the
+# image prebakes the flashinfer 0.7.0rc1 SM120 kernels, first boots re-JIT
+# the rest)
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"