r17 composition ship anchor: backend-flags are the quality toxin (bisect 46/47/74 vs 87); knobs env clean; graphs+MTP fixed on public r17 refs; ledger

docker-images/glm53-jovian-r12/Dockerfile

@@ -80,7 +80,7 @@ COPY deps-src.tar.gz /tmp/deps-src.tar.gz
 RUN set -eux; \
     mkdir -p /deps2; \
     tar xzf /tmp/deps-src.tar.gz -C /deps2; \
-    mv /deps2/triton/python/triton_kernels /deps2/triton_kernels; \
+    mv /deps2/triton/python/triton_kernels/triton_kernels /deps2/triton_kernels; \
     rm -rf /deps2/triton /tmp/deps-src.tar.gz
 
 FROM deepgemm-builder AS vllm-builder
@@ -90,20 +90,17 @@ ARG VLLM_PR_HEADS
 ARG SOURCE_DATE_EPOCH
 ARG BUILD_JOBS
 ENV TORCH_CUDA_ARCH_LIST=12.1a GIT_PROTOCOL=0 DEEPGEMM_SRC_DIR=/deps/deepgemm
+# Pre-staged r12 source tree (vllm-src.tar.gz in the build context, assembled
+# on the build host: base VLLM_BASE_REF + VLLM_PR_HEADS merged in order, fresh
+# lightweight .git). The node's github.com git access is WAF-blocked
+# (401 on git-upload-pack; curl GET works) so no in-build git is possible.
+COPY vllm-src.tar.gz /tmp/vllm-src.tar.gz
 RUN set -eux; \
-    git config --global user.email build@sm121.lan; \
-    git config --global user.name sm121-builder; \
-    git config --global protocol.version 0; \
-    git clone --recursive ${VLLM_REPO} /workspace/vllm; \
+    mkdir -p /workspace/vllm; \
+    tar xzf /tmp/vllm-src.tar.gz -C /workspace/vllm; \
+    rm -f /tmp/vllm-src.tar.gz; \
+    git config --global --add safe.directory /workspace/vllm; \
     cd /workspace/vllm; \
-    git checkout ${VLLM_BASE_REF}; \
-    # PR head commits fetched by SHA (refs/pull/N/head vanishes when a PR
-    # closes — 550 closed unmerged; raw-SHA fetches stay available).
-    for pair in ${VLLM_PR_HEADS}; do \
-        pr="${pair%%:*}"; sha="${pair#*:}"; \
-        git fetch origin "${sha}:refs/prs/${pr}"; \
-        git merge --no-ff -m "PR ${pr}" "refs/prs/${pr}"; \
-    done; \
     git rev-parse HEAD
 WORKDIR /workspace/vllm
 # eugr/spark-vllm-docker build-time patches (main, 2026-09-02): the lab's
@@ -138,10 +135,16 @@ RUN VLLM_PRESERVE_SM12X_TARGET=1 \
 COPY --from=deps-src /deps2 /deps2
 RUN --mount=type=cache,id=uv-cache,target=/root/.cache/uv \
     --mount=type=cache,id=ccache,target=/root/.ccache \
+    --mount=type=cache,id=cargo-registry,target=/root/.cargo/registry \
+    --mount=type=cache,id=rust-target,target=/workspace/vllm/vllm/target \
     git config --global protocol.version 0 \
     && uv pip install setuptools-rust setuptools-scm \
+    && (command -v cargo >/dev/null 2>&1 || curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable --no-modify-path) \
+    && export PATH=/root/.cargo/bin:$PATH \
+    && (command -v protoc >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y --no-install-recommends protobuf-compiler libprotobuf-dev >/dev/null && rm -rf /var/lib/apt/lists/*)) \
     && _j="${BUILD_JOBS:-$(nproc)}" \
     && export GIT_PROTOCOL=0 MAX_JOBS="${_j}" CMAKE_BUILD_PARALLEL_LEVEL="${_j}" \
+        VLLM_REQUIRE_RUST_FRONTEND=1 CARGO_BUILD_JOBS="${_j}" \
         TRITON_KERNELS_SRC_DIR=/deps2/triton_kernels \
         FMHA_SM100_SRC_DIR=/deps2/MSA \
         FLASH_MLA_SRC_DIR=/deps2/FlashMLA \
@@ -150,6 +153,7 @@ RUN --mount=type=cache,id=uv-cache,target=/root/.cache/uv \
         QUTLASS_SRC_DIR=/deps2/qutlass \
         TML_FA4_SRC_DIR=/deps2/tml-fa4 \
         VLLM_CUTLASS_SRC_DIR=/deps2/cutlass \
+        CUTLASS_INCLUDE_DIR_PRE=/deps2/cutlass/include \
     && python3 use_existing_torch.py \
     && uv build --no-build-isolation --wheel . --out-dir=/wheels -v
 
@@ -158,17 +162,15 @@ ARG B12X_REPO
 ARG B12X_BASE_REF
 ARG B12X_PRS
 ARG SOURCE_DATE_EPOCH
+# Pre-staged r12 b12x tree (same reason as vllm: node github WAF block);
+# base B12X_BASE_REF + B12X_PRS merged on the build host.
+COPY b12x-src.tar.gz /tmp/b12x-src.tar.gz
 RUN set -eux; \
-    git config --global user.email build@sm121.lan; \
-    git config --global user.name sm121-builder; \
-    git config --global protocol.version 0; \
-    git clone ${B12X_REPO} /workspace/b12x; \
+    mkdir -p /workspace/b12x; \
+    tar xzf /tmp/b12x-src.tar.gz -C /workspace/b12x; \
+    rm -f /tmp/b12x-src.tar.gz; \
+    git config --global --add safe.directory /workspace/b12x; \
     cd /workspace/b12x; \
-    git checkout ${B12X_BASE_REF}; \
-    for pr in ${B12X_PRS}; do \
-        git fetch origin "refs/pull/$pr/head:prs/$pr"; \
-        git merge --no-ff -m "PR $pr" "prs/$pr"; \
-    done; \
     git rev-parse HEAD
 WORKDIR /workspace/b12x
 RUN --mount=type=cache,id=uv-cache,target=/root/.cache/uv \

recipes/glm53-flash-nvfp4-spark/recipe.toml

@@ -1,39 +1,73 @@
-# GLM-5.3-Flash NVFP4-Spark — local-inference-lab quant, r12 fork — 2× DGX
-# Spark, TP=2. The lab's own "DSpark" checkpoint (speculative MTP layers
-# embedded, no separate draft download).
+# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
 #
-# local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (rev 06972cdee7280cec849aedf05a02062b3e870246):
-# NVFP4 routed experts (layers 3+, group-16) + NVFP4 MTP expert layer in the
-# separate model-mtp-0000x.safetensors shards (mtp_tokens: 5 per their
-# lil.yaml), amax/nonexpert/inputscales side shards, Glm5NextForConditionalGeneration.
-#
-# Image: glm53-jovian-r12:local (docker-images/glm53-jovian-r12/) — clean
-# solar-pattern build of the r12 stack: local-inference-lab/vllm
-# dev/jovian-judgement (base 54f6e98 + PR 550/552/553/554/555/557 by SHA) +
-# b12x 139e040 + PR 260/267/268/269/270, torch 2.13 cu130, flashinfer 0.6.17.
+# local-inference-lab mixed quant: NVFP4 routed experts (layers 3–44,
+# group-16) + MXFP8 MTP experts (layer 45) + W4A4 input scales. Best agent
+# quant so far: tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8).
+# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
+#        (~186 GB → 92.72 GiB weights/node; KV pinned at 10 GiB, see below)
 #
-# Launch flags mirror the lab's lil launcher policy for this family
-# (local-inference-lab/lil, configs/models/_bases.yaml glm base + lil.yaml):
-# attention B12X, linear/moe backend b12x, MTP spec with moe_backend b12x,
-# instanttensor load, block 256, max-num-seqs 8, batched 4096, glm45/glm47
-# parsers, mamba-cache-mode align.
+# Image: glm53-flash-nvfp4-lab:local (docker-images/glm53-flash-nvfp4-lab/ —
+# day-0 arm64 image + NoPE-MLA mod + the four FujitsuPolycom lab-checkpoint
+# patches: model naming shim, modelopt MTP namespace fix, persistent_topk
+# CC-12.x guards). Their launch config is mirrored below.
+# PROVENANCE: https://github.com/FujitsuPolycom/glm53-flash-tp2-spark
+#             (Apache-2.0; see patches/UPSTREAM.md) + forum 381350 #41.
 #
-# Boot success markers: "Model loading took ~93 GiB" (instanttensor: ~1 min)
-# → "GPU KV cache size" → "Application startup complete".
+# Deviations from their launch script (intentional):
+#   * no --kda-prefill-backend flashkda (needs their base-image patch — skipped)
+#   * --kv-cache-dtype fp8_ds_mla instead of `fp8` (same packed layout; this
+#     spelling is what the NoPE-MLA mod booted with on the LibertAIDAI stack)
+#   * --language-model-only + no mm-renderer patch/env (text-only)
+#   * no bundled NCCL/LD_PRELOAD, no NCCL_* fabric block (our switched fabric
+#     is configured via [nccl.ds4] + launcher GID auto-detect)
+#   * no --moe-backend marlin: their proven stack uses the default backend for
+#     THIS quant and reports clean evals. Sanity-check output for token loops
+#     anyway (the silent FP4 MoE corruption gotcha) before trusting a new node.
+#   * flashinfer autotune disabled via --kernel-config (their proven boot);
+#     VLLM_ENGINE_READY_TIMEOUT_S stays high regardless.
 #
-# JIT cache: the flashinfer/b12x JIT caches are MOUNTED from
-# ~/sm121-cache/glm53-jit-121f (see [[extra_mounts]] at the bottom, same
-# pattern as the dsv4 recipe) — pre-baked on both nodes so boot skips the
-# ~15 min first-boot JIT (and the nvcc OOM risk; MAX_JOBS=2 stays as guard).
+# Boot success markers (engine log, in order):
+#   "Model loading took 92.72 GiB" → "[quantprobe] ... algo=MXFP8" (MTP fix
+#   active; algo=None means broken) → "GPU KV cache size: 1,164,369 tokens" →
+#   "Application startup complete". ~700 s sequential weight load.
 #
 # Test request: chat completions need "chat_template_kwargs":{"reasoning_effort"
 # :"low"} — omitted/other values select MAX reasoning effort (empty content on
 # short max_tokens).
 #
+# Optional: instanttensor loader (~30 s weight load vs ~700 s):
+#   FROM glm53-flash-nvfp4-lab:local + pip install --no-deps --no-build-isolation
+#   instanttensor==0.1.9, then add --load-format instanttensor (--no-deps is
+#   required: resolution would downgrade the image's NCCL).
+#
 # PREREQUISITES: weights cached on both nodes:
-#   ~/dsv4vision-hf is for DeepSeek — for GLM the launcher resolves the repo
-#   from the node's HF cache (NAS hub: 175 GiB) or a local NVMe copy.
-#   hf download local-inference-lab/GLM-5.3-Flash-NVFP4-Spark
+# [BISECT A2 2026-09-03] EXACT 90/100 recipe (236ce5a) + instanttensor + JIT
+# mounts, on the CURRENT eugr-patched image. Isolates image lineage from the
+# execution-env/flag cluster (AOT/V2/b12x/card-env). Comparator: 124/138.
+# GLM-5.3-Flash NVFP4-Spark — local-inference-lab quant with the speculative
+# layers embedded — 2× DGX Spark, TP=2.
+#
+# local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (rev 06972cdee7280cec849aedf05a02062b3e870246):
+# NVFP4 routed experts (layers 3+, group-16) + NVFP4 MTP expert layer in the
+# separate model-mtp-0000x.safetensors shards (mtp_tokens: 5 per their
+# lil.yaml), amax/nonexpert/inputscales side shards, Glm5NextForConditionalGeneration.
+# THE "DSpark speculative embedded" checkpoint — no separate DFlash2 download
+# needed; lil.yaml: default_speculator mtp, dflash2_tokens 7 (r12 doc fr: their
+# standalone-Q: MTP:3 254 tok/s CC1, 333 Sieve, 94 steps/s).
+#
+# Image: glm53-flash-nvfp4-lab:local (same proven day-0 arm64 stack + NoPE-MLA
+# mod + modelopt MTP namespace patch used by the 92/100 lab lane). The MTP
+# layer here is NVFP4 (not MXFP8) — the quantprobe marker may print
+# algo=NVFP4/- different; algo=None = patch missed (fall back to 4p67 image).
+#
+# Deviations from lab (unchanged mechanics): same fabric/env/GMU/KV pinning.
+# lil.yaml launch mirrors: block 256, disable_flashinfer_autotune, mm data
+# mode — our text-only flags identical to glm53-flash-nvfp4-lab.
+#
+# Boot markers: "Model loading took ~XX GiB" → "[quantprobe] ..." →
+# "GPU KV cache size" → "Application startup complete".
+#
+# PREREQUISITES: weights on the NAS hub (175 GiB): hf download local-inference-lab/GLM-5.3-Flash-NVFP4-Spark
 
 engine = "vllm"
 name = "GLM-5.3-Flash NVFP4-Spark (vLLM, 2x Spark, MTP-5)"
@@ -63,7 +97,7 @@ args = [
     # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
     # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
     # allocated (weights ~93 + pinned KV 10 + activations).
-    "--gpu-memory-utilization", "0.82",
+    "--gpu-memory-utilization", "0.90",
     # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
     # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
     "--kv-cache-memory-bytes", "10737418240",
@@ -74,19 +108,14 @@ args = [
     # _select_sparse_backend); the sm12x auto-priority list never offers it
     # (FLASHINFER_MLA_SPARSE_SM120 only supports head 576, not GLM's 512).
     "--attention-backend", "B12X",
-    # lab loader: instanttensor (0.1.9 is already in the r12 image) —
-    # ~5 min weight load vs ~16 min from NFS; stays in every bisection run
+    # pure I/O loader (kept across every config; 171 GB in ~5.5 min vs ~16 NFS)
     "--load-format", "instanttensor",
-    # PERF RUN (2026-09-02): full lab execution engine per lil render
-    # (vllm-node-eugr reference): b12x linear+MoE kernels, mamba state
-    # cache align, no enforce-eager (lab runs CUDA graphs; the enforce-eager
-    # quality bug was day-0-build-specific, r12 is the lab-qualified build).
-    "--linear-backend", "b12x",
-    "--moe-backend", "b12x",
-    "--mamba-cache-mode", "align",
+    # [R17 FULL-CLUSTER TEST] lil manifest backends (quality gate: hardmode
+    # must stay ≥88; r12-era this cluster dropped it to 74 — r17 may carry
+    # the fix through — one boot decides).
     "--block-size", "256",
     "--max-model-len", "524288",
-    "--max-num-seqs", "8",
+    "--max-num-seqs", "16",
     "--max-num-batched-tokens", "4096",
     "--enable-chunked-prefill",
     "--async-scheduling",
@@ -95,12 +124,26 @@ args = [
     # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
     "--kernel-config", """
 '{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
-    # lab spec config verbatim (lil render): k=5, moe_backend b12x
-    # (nvfp4->b12x), attention B12X (draft's backend comes from the spec
-    # config, not the global flag — eagle/utils.py _make_eagle_draft_vllm_config)
+    # Built-in MTP draft, 3 tokens (the r12-qualified "mtp:3" mode — their
+    # x86 numbers: 254 tok/s CC1, 2.37 accept; the checkpoint embeds up to
+    # 5 via the model-mtp-0000x shards but 3 is the verified depth). This is
+    # the checkpoint's own MTP head (Glm5NextMTPModel), NOT dspark — dspark
+    # is the DeepSeek-V4 family's mechanism; GLM's other r12 mode is
+    # dflash2:7 with a separate trained draft checkpoint.
+    # The draft model's vllm_config takes its attention backend from the
+    # SPECULATIVE config, not the global flag (eagle/utils.py:
+    # _make_eagle_draft_vllm_config) — B12X here mirrors the main model's.
     "--speculative-config", """
-'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"b12x","attention_backend":"B12X"}'""",
+'{"method":"mtp","num_speculative_tokens":3,"moe_backend":"b12x","attention_backend":"B12X"}'""",
     "--language-model-only",
+    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
+    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
+    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
+    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
+    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
+    # graphs moved BEFORE tool-call-parser; enforce-eager removed for the
+    # r17-graphs test (bisect A2 was 88/100 eager; graphs+MTP fix is public
+    # in r17 refs? verify by bench).
     "--tool-call-parser", "glm47",
     "--enable-auto-tool-choice",
     "--reasoning-parser", "glm45",
@@ -108,29 +151,20 @@ args = [
 
 [env]
 HF_HUB_OFFLINE = "1"
-# lab INSTANTTENSOR loader knobs (lil render: BACKEND/CONCURRENCY/IO_DEPTH
-# + the lil.yaml BUFFER_SIZE/CHUNK)
-INSTANTTENSOR_BACKEND = "BUFFERED"
-INSTANTTENSOR_CONCURRENCY = "1"
-INSTANTTENSOR_IO_DEPTH = "3"
-INSTANTTENSOR_BUFFER_SIZE = "67108864"
-INSTANTTENSOR_CHUNK_SIZE = "8388608"
-# FlashInfer JIT guard: if a kernel is missing from the mounted JIT cache it
-# compiles at boot — 12-way parallel nvcc got OOM-killed (code 137) on GB10's
+# FlashInfer JIT (mxfp8_gemm_cutlass_sm120, ~12 kernels) compiles at first
+# boot with 12 parallel nvcc jobs -> OOM-killed (code 137) on GB10's
 # 121 GB unified while the 93 GB weights + 10 GB KV pin are resident.
+# MAX_JOBS caps ninja parallelism so the JIT fits in the remaining ~20 GB.
 MAX_JOBS = "2"
 # NoPE-MLA rope-pad runtime gate (mod baked into the image)
 VLLM_MLA_NOPE_PAD_ROPE = "1"
 VLLM_ENGINE_READY_TIMEOUT_S = "3600"
-# PERF RUN: lab execution engine (lil render reference)
-VLLM_USE_AOT_COMPILE = "1"
+# Upstream launch-script env (memory + compile knobs)
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
 VLLM_USE_STANDALONE_COMPILE = "1"
-VLLM_USE_V2_MODEL_RUNNER = "1"
 VLLM_USE_FLASHINFER_SAMPLER = "1"
 VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
-# NOTE: no VLLM_USE_MEGA_AOT_ARTIFACT — the prebuilt mega-AOT artifact ships
-# in the lab image, not in ours; the flag would only serialize.
-PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
 TORCH_USE_RTLD_GLOBAL = "1"
 VLLM_NO_USAGE_STATS = "1"
 
@@ -139,17 +173,7 @@ TORCH_CUDA_ARCH_LIST = "12.1a"
 FLASHINFER_CUDA_ARCH_LIST = "12.1f"
 CUTE_DSL_ARCH = "sm_121a"
 CMAKE_CUDA_ARCHITECTURES = "121"
-OMP_NUM_THREADS = "4"
-
-# Pre-baked JIT caches (see header; same pattern as the dsv4 recipe's
-# ~/.cache/dsv4vision-jit mounts). RW on purpose: flashinfer writes lock
-# files into the cache. Missing on a node -> first boot JITs (~15 min at
-# MAX_JOBS=2) and the cache populates for the next boot.
-# NOTE: extra_mounts stay AFTER args (TOML table headers would swallow the
-# args array otherwise).
-[[extra_mounts]]
-host = "~/sm121-cache/glm53-jit-121f/flashinfer"
-container = "/root/.cache/flashinfer"
+OMP_NUM_THREADS = "2"
 
 [[extra_mounts]]
 host = "~/sm121-cache/glm53-jit-121f/b12x"

specs/tech-architecture/glm53-recipe-state.md

@@ -475,3 +475,72 @@ Goal (user): 30 tok/s in --perf. Found the missing pieces:
    deps-src.tar.gz (triton_kernels@v3.5.1, MSA, FlashMLA, FlashKDA,
    flash-attention, qutlass@e74319e, tml-fa4@b206834, cutlass@v4.4.2) +
    the SRC_DIR env escapes each cmake documents. Build 14 compiling.
+
+## r17 (20260902-r17 card) adoption + quality bisect (2026-09-03)
+
+Latest lab card: https://github.com/local-inference-lab/rtx6kpro/blob/master/models/glm-5.3-flash.md
+(names the qualified serving contract per artifact and SHA; TP4 on RTX PRO,
+not Spark — the Spark lil.yaml testdata render remains our TP2 adapter).
+
+1. **Public r17 composition** built from source:
+   `/tmp/vllmr12` = local-inference-lab/vllm `dev/jovian-judgement@9c4dd054`
+   + open PRs **579/580/581/582/586** (branch supersedes the r12 PR set
+   550–557 — same symbols, evolved, see below) + staged-cutlass cmake patch;
+   `/tmp/b12xb17` = local-inference-lab/b12x `master@2bbab479` + PRs
+   **290/284**. Pre-staged as vllm-src.tar.gz / b12x-src.tar.gz on the build
+   node (WAF blocks node→github git). DO NOT re-merge 550-557: creates
+   conflict-marker SyntaxErrors (pr550's pooled_indexer.py conflicts; branch
+   tip already contains the evolved implementation).
+
+2. **Important correction to the 2026-08-28 eager verdict**: on the public
+   r12/r17 refs, graphs+MTP is no longer a quality bug — the lab's private
+   "packed GLM-5.3 package" (card SHAs 47f86087/a02841bc, not public) was the
+   presumed fix carrier, but our public-tree composition measured:
+   - r17 + graphs + 90/100-flags (no b12x/align): **88/100 hardmode (155/176)
+     == lab-quant's own day-0 reference 156/176** (r17graphs01/…perf)
+   - r17 + eager same-flags: 67/100, median turn 11.4 s, 2 scenarios timed
+     out and excluded (r12eager01) — eager is now the WORST mode.
+   So: graphs stay ON; enforce-eager is gone.
+
+3. **Execution-env knobs are quality-clean**: GMU 0.90 (0.95 boot-deaths on
+   GB10 — "free memory 111.96 < desired"), max-num-seqs 16, OMP 2,
+   VLLM_USE_STANDALONE_COMPILE/FLASHINFER_SAMPLER/BREAKABLE envs: 87/100
+   (r17knobsonly). AOT=1+V2 runner: NEUTRAL for quality (47 either way when
+   backends were on) and NEUTRAL for the d0 grid — dropped (AOT=0, no V2).
+
+4. **THE QUALITY TOXIN = the b12x backend-flag cluster** (on the PUBLIC
+   r17/r12 compositions, with our b12x 1.3.0/1.2.6 builds):
+   - k=5 full cluster: 74/100 (r12 img), ~74 → killed-to-16/22 (r12 img)
+   - k=5 full cluster (r17 img): ~70-ish band (r17cluster01, stopped early)
+   - k=3 full cluster: **46/100** (r17clusterk3)
+   - k=3 cluster minus AOT/V2: **47/100** (r17noaotk3)
+   - k=3 cluster minus backends (knobs only): **87/100** (r17knobsonly)
+   The lab's own stack runs these backends qualified — but their composition
+   carries the private packed GLM-5.3 package (RTX-PRO target); the public
+   branch tip alone does NOT make the b12x W4A4/linear/align paths
+   quality-safe. Also: b12x cluster wrecked deep-context speed
+   (d4096 c1 12.6 @k5) while helping d0 (33-34 c1) — nets negative anyway.
+
+5. **SHIP ANCHOR (current recipe)** = r17 image (94ac0882a8b7) + 90/100
+   args (attention B12X, block 256, seqs 16, batched 4096, LM-only,
+   glm47/glm45 parsers, instanttensor load, k=3 spec {mtp,attn B12X} with
+   moe_backend key) + graphs (no eager) + knobs env (GMU 0.90, seqs 16,
+   OMP 2, AOT=0, STANDALONE=1, SAMPLER=1, BREAKABLE=0) + JIT-cache mounts:
+   **87/100 hardmode (153/176; lab-quant ref 156/176), default-suite 88
+   class**; perf d0 c1 24.1 / c2 30.2 / c4 44.7; d4096 c1 23.9; d8192 c1
+   23.8; pp ~1.6-1.8k. Boot 8 min (JIT warm) / ~25 min cold (nvcc re-JIT).
+
+6. Boot-time JIT: changing the rw flashinfer cache invalidates per-config
+   kernel sets; a config's first boot silently nvcc-compiles (MAX_JOBS=2)
+   ~15-25 min inside "No available shared memory" log-silence; later boots
+   of the same config are fast. Expect this on every NEW kernel path.
+
+7. KV-dtype probes: fp8_ds_mla on r12-img = 75/100 default-suite (net zero,
+   TC-45 flipped PASS but others flipped off); graphs here. r17 anchor keeps
+   default dtype. Not a quality lever.
+
+8. Open perf gap: c1 24 vs manifest-cluster 33-34 (which is quality-dead).
+   If the lab ever publishes the packed package (or a public branch commit
+   makes b12x-backends clean), the 30+ c1 target lands with manifest flags.
+   Alternative to probe later: one b12x backend flag at a time (a single
+   toxin member may be free).