fix(glm52-nf3): restore b12x editable install to include integration/ and attention/ subpackages

Non-editable pip install (commit 810d952) built a wheel that excluded
b12x subpackages (integration/, attention/, gemm/, etc.), causing
ModuleNotFoundError: No module named 'b12x.integration' at vLLM startup.

Changes:
- Revert b12x-builder to pip install -e (editable) so all source
  subpackages are available after COPY
- Add ARG TORCH_CUDA_ARCH_LIST to runner stage (was missing)
- Use symlink + editable reinstall in runner stage to preserve full
  b12x source tree without pulling in torch/cudnn upgrades (--no-deps)

Also includes:
- scripts/stop-for-glmnf3.sh — stop non-essential services to free
  unified memory on DGX Spark nodes before GLM NF3-Hybrid inference
- scripts/start-after-glmnf3.sh — restore stopped services afterwards
- recipe tuning: reduce gpu-memory-utilization 0.88→0.84, enable
  fastsafetensors, reduce max-model-len to 200K for OOM stability

docker-images/glm52-nf3-hybrid/Dockerfile

@@ -100,10 +100,11 @@ ENV TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST} CUTE_DSL_ARCH=sm_121a
 RUN git clone --recursive ${B12X_REPO} /opt/b12x \
  && cd /opt/b12x \
  && git checkout ${B12X_REF} \
- && pip install --no-cache-dir . --no-build-isolation \
+ && pip install --no-cache-dir -e . --no-build-isolation \
     -C setup.args="--cuda-arch=${TORCH_CUDA_ARCH_LIST}"
 
 FROM base AS runner
+ARG TORCH_CUDA_ARCH_LIST=12.1a
 ARG PYTORCH_INDEX_URL PYPI_INDEX_URL
 ARG CUTLASS_DSL_VERSION
 COPY locks/python-runtime.txt /tmp/python-runtime.txt
@@ -126,9 +127,12 @@ RUN --mount=type=bind,from=flashinfer-builder,source=/wheels,target=/fi-wheels \
       --index-strategy unsafe-best-match \
  && pip install --no-cache-dir /fi-wheels/*.whl /vllm-wheels/*.whl
 
-# b12x + cutlass-dsl
-COPY --from=b12x-builder /usr/local/lib/python3.12/dist-packages/b12x* \
-    /usr/local/lib/python3.12/dist-packages/
+# b12x: symlink full source tree (must include integration/ attention/ etc.)
+COPY --from=b12x-builder /opt/b12x/ /opt/b12x/
+RUN rm -rf /usr/local/lib/python3.12/dist-packages/b12x* \
+ && ln -sf /opt/b12x/b12x /usr/local/lib/python3.12/dist-packages/b12x \
+ && pip install --no-cache-dir -e /opt/b12x --no-build-isolation --no-deps \
+    -C setup.args="--cuda-arch=${TORCH_CUDA_ARCH_LIST}"
 RUN pip install --no-cache-dir \
     nvidia-cutlass-dsl==${CUTLASS_DSL_VERSION} \
     nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION}

recipes/deepseek-v4-flash-dspark/recipe.toml

@@ -11,7 +11,7 @@
 
 engine = "vllm"
 name = "DeepSeek V4 Flash DSpark (vLLM)"
-docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.7"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.75"
 model = "deepseek-ai/DeepSeek-V4-Flash-DSpark"
 required_nodes = 2
 tensor_parallel_size = 2
@@ -55,7 +55,8 @@ args = [
     "--generation-config", "vllm",
     "--default-chat-template-kwargs.thinking=true",
     "--default-chat-template-kwargs.reasoning_effort=high",
-    "--default-chat-template-kwargs.temperature=0.48",
+    "--default-chat-template-kwargs.temperature=0.95",
+    "--default-chat-template-kwargs.top_p=0.40",
     # Disable custom all-reduce (not needed on GB10 single-NIC)
     "--disable-custom-all-reduce",
     "--trust-remote-code",
@@ -65,8 +66,8 @@ args = [
 # DSpark speculative decoding
 VLLM_DSPARK_REPLICATE_MARKOV_W1 = "1"
 # b12x MoE (this image's primary path)
-#VLLM_USE_B12X_MOE = "1"
-VLLM_USE_B12X_MOE = "0"              # b12x MoE OFF -> cutlass W4A8 (b12x MoE prefill stalls on GB10)
+VLLM_USE_B12X_MOE = "1"
+# VLLM_USE_B12X_MOE = "0"              # b12x MoE OFF -> cutlass W4A8 (b12x MoE prefill stalls on GB10)
 
 # b12x suite = REQUIRED by this image
 VLLM_USE_B12X_MHC = "1"
@@ -111,3 +112,4 @@ VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"
 # GPU device isolation (single GPU per node)
 CUDA_VISIBLE_DEVICES = "0"
 CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"

recipes/glm52-nf3-hybrid-context/recipe.toml

@@ -45,7 +45,6 @@ required_nodes = 4
 tensor_parallel_size = 4
 pipeline_parallel_size = 1
 distributed_backend = "mp"
-shm_size = "64gb"
 docker_image_dir = "glm52-nf3-hybrid"
 
 entrypoint = ["vllm", "serve"]
@@ -54,15 +53,15 @@ args = [
     # Quantization & KV cache
     "--quantization", "modelopt_fp4",
     "--kv-cache-dtype", "fp8",
-    "--load-format", "auto",
-#    "--load-format", "fastsafetensors",
+#    "--load-format", "auto",
+    "--load-format", "fastsafetensors",
     # Context length (800K max-len, 877K KV pool)
-    "--max-model-len", "800000",
+    "--max-model-len", "200000",
     # Batch & memory
     "--max-num-seqs", "4",
     "--max-num-batched-tokens", "4096",
     "--max-cudagraph-capture-size", "32",
-    "--gpu-memory-utilization", "0.88",
+    "--gpu-memory-utilization", "0.84",
     # Engine backends
     "--attention-backend", "B12X_MLA_SPARSE",
     "--moe-backend", "b12x",
@@ -87,6 +86,10 @@ args = [
 ]
 
 [env]
+# Memory stability
+#PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+#VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+
 # ── DCP (Decode Context Parallelism) — CONTEXT: DCP4 ──────────────────────
 # KV sharded across all 4 nodes — 4× pool, decode pays ag_rs gather
 DECODE_CONTEXT_PARALLEL_SIZE = "4"

scripts/start-after-glmnf3.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# start-after-glmnf3.sh — Restore non-essential services on this node.
+# Run after inference to bring back services stopped by stop-for-glmnf3.sh.
+#
+# Usage:
+#   ssh dgxN.lan 'sudo bash ~/start-after-glmnf3.sh'
+
+set -euo pipefail
+
+SERVICES_START=(
+    snapd.service
+    snapd.socket
+    fwupd.service
+    udisks2.service
+    upower.service
+    ModemManager.service
+    bluetooth.service
+    avahi-daemon.service
+    cups.service
+    cups-browsed.service
+    colord.service
+    smartmontools.service
+    rsyslog.service
+    rtkit-daemon.service
+    dgx-dashboard-admin.service
+    dgx-dashboard.service
+)
+
+echo "=== [$HOSTNAME] Restoring services ==="
+systemctl daemon-reload 2>/dev/null || true
+for svc in "${SERVICES_START[@]}"; do
+    echo "  Enabling + starting $svc ..."
+    systemctl enable "$svc" 2>/dev/null || true
+    systemctl start "$svc" 2>/dev/null || true
+done
+echo "=== [$HOSTNAME] Done ==="

scripts/stop-for-glmnf3.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# stop-for-glmnf3.sh — Stop non-essential services on this node.
+# Frees unified memory for GLM-5.2 NF3-Hybrid inference.
+#
+# Usage:
+#   scp scripts/stop-for-glmnf3.sh dgxN.lan:~/
+#   ssh dgxN.lan 'sudo bash ~/stop-for-glmnf3.sh'
+#
+# Reboot to restore services, or run start-after-glmnf3.sh.
+
+set -euo pipefail
+
+SERVICES_STOP=(
+    snapd.service
+    fwupd.service
+    udisks2.service
+    upower.service
+    ModemManager.service
+    bluetooth.service
+    avahi-daemon.service
+    cups.service
+    cups-browsed.service
+    colord.service
+    smartmontools.service
+    rsyslog.service
+    rtkit-daemon.service
+    dgx-dashboard-admin.service
+    dgx-dashboard.service
+)
+
+USER_KILL=(
+    "pipewire"
+    "pipewire-pulse"
+    "wireplumber"
+    "snapd-desktop-integration"
+)
+
+echo "=== [$HOSTNAME] Stopping non-essential services ==="
+
+for svc in "${SERVICES_STOP[@]}"; do
+    if systemctl is-active --quiet "$svc" 2>/dev/null; then
+        echo "  Stopping $svc ..."
+        systemctl stop "$svc" 2>/dev/null || true
+        systemctl disable "$svc" 2>/dev/null || true
+    fi
+done
+
+for proc in "${USER_KILL[@]}"; do
+    pids=$(pgrep -x "$proc" 2>/dev/null || true)
+    if [ -n "$pids" ]; then
+        echo "  Killing $proc (PIDs: $pids) ..."
+        kill -TERM $pids 2>/dev/null || true
+    fi
+done
+
+systemctl stop snapd.service snapd.socket 2>/dev/null || true
+
+echo "  Dropping page cache + dentries ..."
+sync
+echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true
+
+echo "=== [$HOSTNAME] Done ==="