fix: image presence check accepts digest refs (docker image ls does not); pin qwen3.8 b12x to the validated nightly-20260913 digest (floating latest 20260925 garbles long contexts, forum 382733 #195-198); min-recipe A/B lane (no MTP, no prefix cache)

recipes/qwen38-flash-next-int4-min/mods/flashnext-int4-b12x/build_views.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""Build, inside the container, the two model folders the b12x vLLM needs -- symlinks + a few small JSON files, nothing
+copied, nothing written outside the container:
+
+  <out>/model      the served model: every snapshot file linked, the 33 PLE-table files (shipped in an unindexed
+                   `ple-table/` subfolder) linked at top level, and
+                     * model.safetensors.index.json = the shipped index + the 128 `ngram_embedding.shard_N.weight` keys
+                       and the table's `ngram_embedding.weight_scale`. ONLY those keys: two of the table files also carry
+                       1,057 tensors of the original checkpoint (layer 1's original experts, shared expert and router
+                       gate). A loader that reads by index key never touches them; one that reads whole files would
+                       silently overwrite V8's healed layer 1 -- see the loader filter in patch_b12x.py.
+                     * config.json = shipped config + `text_config.ple_embedding_dtype` (b12x's PLE storage needs it;
+                       absent -> bf16 assumed) + the `Qwen3_8FlashNextForConditionalGeneration` architecture name this
+                       vLLM registers (it does not know `Qwen4ExpForConditionalGeneration`).
+  <out>/draft-k10  the MTP draft: only the files the speculative head needs (its own tensors, the int4 lm_head, the
+                   embedding shard) + an index of just those keys + a config routing top-k 10 with the head's own
+                   shared-expert width (640; the model's is 1280). Slim on purpose: a draft folder that links every
+                   shard makes the draft load walk the whole model again (2026-09-11 and 2026-09-21 out-of-memory).
+
+  usage: build_views.py <snapshot dir> <output root>          prints one summary line; exit != 0 on any inconsistency
+"""
+import glob, json, os, shutil, struct, sys
+
+snap, root = (os.path.abspath(a) for a in sys.argv[1:3])
+view, draft = os.path.join(root, "model"), os.path.join(root, "draft-k10")
+B12X_ARCH = "Qwen3_8FlashNextForConditionalGeneration"
+
+
+def fail(msg):
+    sys.exit(f"build_views: {msg}")
+
+
+def header(path):
+    with open(path, "rb") as fh:
+        n = struct.unpack("<Q", fh.read(8))[0]
+        h = json.loads(fh.read(n))
+    h.pop("__metadata__", None)
+    return h
+
+
+def fresh(d):
+    if os.path.lexists(d):
+        shutil.rmtree(d)
+    os.makedirs(d)
+
+
+index = json.load(open(os.path.join(snap, "model.safetensors.index.json")))
+wm = dict(index["weight_map"])
+cfg = json.load(open(os.path.join(snap, "config.json")))
+text = cfg.get("text_config", cfg)
+parts = int(text.get("split_ngram_parts", 0)) or fail("config has no split_ngram_parts")
+
+# ---------------------------------------------------------------- served model view
+fresh(view)
+for name in sorted(os.listdir(snap)):
+    if name in ("config.json", "model.safetensors.index.json", "ple-table"):
+        continue
+    os.symlink(os.path.join(snap, name), os.path.join(view, name))
+ple_files = sorted(glob.glob(os.path.join(snap, "ple-table", "*.safetensors")))
+ple_files or fail(f"no ple-table/*.safetensors in {snap}")
+shards, scale_key, table_dtype = set(), None, None
+for f in ple_files:
+    base = os.path.basename(f)
+    if os.path.lexists(os.path.join(view, base)):
+        fail(f"table file name {base} collides with a model file")
+    os.symlink(f, os.path.join(view, base))
+    for key, meta in header(f).items():
+        if ".ple.ple_embedding.ngram_embedding." not in key:
+            continue                                           # foreign tensors stay out of the index
+        if key in wm:
+            fail(f"{key} is already indexed -- unexpected checkpoint layout")
+        wm[key] = base
+        leaf = key.split(".ngram_embedding.", 1)[1]
+        if leaf == "weight_scale":
+            scale_key = key
+        elif leaf.startswith("shard_") and leaf.endswith(".weight"):
+            shards.add(int(leaf[len("shard_"):-len(".weight")]))
+            table_dtype = table_dtype or meta["dtype"]
+if shards != set(range(parts)):
+    fail(f"PLE table incomplete: {len(shards)} of {parts} shards found")
+if table_dtype != "F8_E4M3" or scale_key is None:
+    fail(f"expected an fp8 table with a weight_scale, got {table_dtype} / scale {scale_key}")
+json.dump({"metadata": index.get("metadata", {}), "weight_map": wm},
+          open(os.path.join(view, "model.safetensors.index.json"), "w"))
+text["ple_embedding_dtype"] = "float8_e4m3fn"
+archs = cfg.setdefault("architectures", [])
+if B12X_ARCH not in archs:
+    archs.append(B12X_ARCH)
+json.dump(cfg, open(os.path.join(view, "config.json"), "w"), indent=2)
+
+# ---------------------------------------------------------------- slim MTP draft folder
+fresh(draft)
+keep = {k: v for k, v in wm.items()
+        if k.startswith("mtp.") or k.startswith("lm_head.") or k.endswith("embed_tokens.weight")}
+if not any(k.startswith("mtp.") for k in keep):
+    fail("no mtp.* tensors in the index -- this snapshot has no speculative head")
+for name in sorted(os.listdir(view)):
+    if name.endswith(".safetensors") and name not in set(keep.values()):
+        continue
+    if name in ("config.json", "model.safetensors.index.json"):
+        continue
+    os.symlink(os.path.join(view, name), os.path.join(draft, name))
+json.dump({"metadata": {}, "weight_map": keep}, open(os.path.join(draft, "model.safetensors.index.json"), "w"))
+dcfg = json.loads(json.dumps(cfg))
+dtext = dcfg.get("text_config", dcfg)
+dtext["num_experts_per_tok"] = 10
+head = header(os.path.join(snap, "model_extra_tensors.safetensors"))
+widths = {v["shape"][0] for k, v in head.items() if k.endswith("mlp.shared_expert.gate_proj.weight")}
+if len(widths) == 1:
+    dtext["shared_expert_intermediate_size"] = widths.pop()
+json.dump(dcfg, open(os.path.join(draft, "config.json"), "w"), indent=2)
+
+print(f"model view: {len(os.listdir(view))} entries, +{len(shards)} table shards +1 scale indexed "
+      f"({len(ple_files)} table files) | draft: {len(keep)} keys in {len(set(keep.values()))} files, top-k 10, "
+      f"shared expert {dtext.get('shared_expert_intermediate_size')}")

recipes/qwen38-flash-next-int4-min/mods/flashnext-int4-b12x/patch_b12x.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Patch the vllm-node-b12x image IN THE RUNNING CONTAINER so it can serve the INT4-AutoRound checkpoint. Every edit is
+idempotent (marker comment) and checked (anchor must match exactly once, file must still parse). The image on disk is
+never modified -- a new container starts clean and the mod re-applies this.
+
+  REQUIRED (exit 1 if they cannot be applied -- the launch is aborted rather than serving a broken model):
+    fp8-hybrid   300 side layers of the checkpoint are blockwise fp8 inside a GPTQ checkpoint; stock AutoGPTQConfig would
+                 build them as bf16 and fail to load. Saren's vllm_fp8_hybrid.py (verbatim from the qwen38-flash-dgx
+                 image) + its hook, gated by VLLM_FP8_HYBRID=1.
+    int4 head    the model's ParallelLMHead is built without quant_config, so the int4 AutoRound lm_head cannot load
+                 (the MTP head in this vLLM already passes it).
+    index filter safety net for --load-format safetensors: reads tensors only where the folder's index says they live.
+                 The b12x loader already does this; the stock safetensors loader reads whole files and would load the
+                 original layer-1 weights sitting in two PLE-table files over V8's healed ones.
+  OPTIONAL (a warning, the launch goes on):
+    draft x2     VLLM_MTP_DRAFT_SCALE scales the draft's logits (exact: the verifier decides) -- +2.5pp acceptance.
+    GDN fixes    spark-fla-shmem (GB10 gets the big GDN tiles) and spark-fla-warps (num_warps=2: fla#953 Blackwell race
+                 that corrupts GDN state on the prefix-cache path), same one-liners as the qwen38-flash-dgx image.
+
+  usage: patch_b12x.py <mod dir>
+"""
+import ast, os, re, shutil, sys
+
+MOD = sys.argv[1]
+SP = "/usr/local/lib/python3.12/dist-packages"
+V = f"{SP}/vllm"
+MARK = "flashnext-int4-b12x"
+done, warn, fatal = [], [], []
+
+
+def edit(path, fn, label, required):
+    try:
+        src = open(path).read()
+        if MARK + ":" + label in src:
+            done.append(f"{label} (already)")
+            return
+        new = fn(src)
+        if new is None or new == src:
+            raise ValueError("anchor not found")
+        ast.parse(new)
+        open(path, "w").write(new)
+        done.append(label)
+    except Exception as e:  # noqa: BLE001
+        (fatal if required else warn).append(f"{label}: {e} [{path}]")
+
+
+def once(src, old, new):
+    if src.count(old) != 1:
+        return None
+    return src.replace(old, new)
+
+
+# 1. fp8 hybrid ----------------------------------------------------------------------------------------------------
+shutil.copy(os.path.join(MOD, "vllm_fp8_hybrid.py"), f"{SP}/vllm_fp8_hybrid.py")
+edit(f"{V}/model_executor/layers/quantization/auto_gptq.py",
+     lambda s: s + f"\n\n# --- {MARK}:fp8-hybrid (VLLM_FP8_HYBRID=1) ---\nfrom vllm_fp8_hybrid import apply as _fp8_hybrid_apply\n"
+                   "_fp8_hybrid_apply()\n",
+     "fp8-hybrid", True)
+
+# The model code moved: images up to 2026-09-13 ship it as vllm/models/qwen3_8_flash_next/{model,mtp}.py; from the
+# 2026-09-21 image (vLLM 0.1.dev21473) it is vllm/models/qwen4_exp/nvidia/{model,mtp}.py (qwen3_8_flash_next is only a
+# compatibility alias there). Patch whichever this image has.
+MODEL_DIR = next((d for d in (f"{V}/models/qwen4_exp/nvidia", f"{V}/models/qwen3_8_flash_next")
+                  if os.path.isfile(f"{d}/model.py")), f"{V}/models/qwen3_8_flash_next")
+
+# 2. int4 lm_head ----------------------------------------------------------------------------------------------------
+edit(f"{MODEL_DIR}/model.py",
+     lambda s: once(s, '            prefix=maybe_prefix(prefix, "lm_head"),\n',
+                    f'            quant_config=vllm_config.quant_config,  # {MARK}:int4-head\n'
+                    '            prefix=maybe_prefix(prefix, "lm_head"),\n'),
+     "int4-head", True)
+
+# 3. index filter for the stock safetensors loader ----------------------------------------------------------------------
+FILTER = f'''
+
+# --- {MARK}:index-filter ---
+# Only for folders under /workspace/flashnext: yield a tensor only from the file the folder's index names for it.
+def _flashnext_index_filter():
+    import json as _json
+    from pathlib import Path as _Path
+    _orig = DefaultModelLoader._safetensors_weights_iterator
+    _maps = {{}}
+
+    def _iterate(self, hf_weights_files, source):
+        files = list(hf_weights_files)
+        folders = {{_Path(f).parent for f in files}}
+        if len(folders) != 1 or not str(next(iter(folders))).startswith("/workspace/flashnext/"):
+            yield from _orig(self, files, source)
+            return
+        folder = next(iter(folders))
+        if folder not in _maps:
+            _maps[folder] = _json.load(open(folder / "model.safetensors.index.json"))["weight_map"]
+        weight_map = _maps[folder]
+        for f in files:
+            here = _Path(f).name
+            for name, tensor in _orig(self, [f], source):
+                if weight_map.get(name) == here:
+                    yield name, tensor
+
+    DefaultModelLoader._safetensors_weights_iterator = _iterate
+
+
+_flashnext_index_filter()
+'''
+edit(f"{V}/model_executor/model_loader/default_loader.py", lambda s: s + FILTER, "index-filter", True)
+
+# 3b. the same filter for "mixed" files ------------------------------------------------------------------------------------
+# A file that holds any PLE-table tensor does NOT go through _safetensors_weights_iterator: b12x's vLLM reads it directly in
+# file_backed_safetensors_weights_iterator (every tensor in it, table rows as on-disk references). Two of the table files also
+# carry layer 1's ORIGINAL experts / shared expert / gate -> boot 2 died on "experts has no parameter 'w2_weight'". Wrap that
+# iterator too (default_loader calls it by its module-global name, so rebinding the name there is enough).
+MIXED = f'''
+
+# --- {MARK}:index-filter-mixed ---
+def _flashnext_index_filter_mixed():
+    import json as _json
+    from pathlib import Path as _Path
+    global file_backed_safetensors_weights_iterator
+    _orig = file_backed_safetensors_weights_iterator
+    _maps = {{}}
+
+    def _filtered(hf_weights_files, ordinary_iterator, file_weight_filter, **kwargs):
+        files = list(hf_weights_files)
+        folders = {{_Path(f).parent for f in files}}
+        if len(folders) != 1 or not str(next(iter(folders))).startswith("/workspace/flashnext/"):
+            yield from _orig(files, ordinary_iterator, file_weight_filter, **kwargs)
+            return
+        folder = next(iter(folders))
+        if folder not in _maps:
+            _maps[folder] = _json.load(open(folder / "model.safetensors.index.json"))["weight_map"]
+        weight_map = _maps[folder]
+        for f in files:
+            here = _Path(f).name
+            for name, tensor in _orig([f], ordinary_iterator, file_weight_filter, **kwargs):
+                if weight_map.get(name) == here:
+                    yield name, tensor
+
+    file_backed_safetensors_weights_iterator = _filtered
+
+
+_flashnext_index_filter_mixed()
+'''
+edit(f"{V}/model_executor/model_loader/default_loader.py",
+     lambda s: s + MIXED if "file_backed_safetensors_weights_iterator" in s else None, "index-filter-mixed", True)
+
+# 4. draft x2 --------------------------------------------------------------------------------------------------------
+def _scale(s):
+    pat = re.compile(r"(self\.logits_processor = LogitsProcessor\(\n(\s+)config\.vocab_size,\n)")
+    if len(pat.findall(s)) != 1:
+        return None
+    return pat.sub(lambda m: m.group(1) + m.group(2) +
+                   f'scale=float(__import__("os").environ.get("VLLM_MTP_DRAFT_SCALE", "1.0")),  # {MARK}:draft-scale\n', s)
+edit(f"{MODEL_DIR}/mtp.py", _scale, "draft-scale", False)
+
+# 5. GDN fixes ---------------------------------------------------------------------------------------------------------
+fla = f"{V}/third_party/flash_linear_attention/ops"
+edit(f"{fla}/utils.py", lambda s: once(s, "DEFAULT = 102400", f"DEFAULT = 101376  # {MARK}:fla-shmem GB10 99KiB"),
+     "fla-shmem", False)
+edit(f"{fla}/chunk_delta_h.py",
+     lambda s: once(s, "for num_warps in [2, 4]", f"for num_warps in [2]  # {MARK}:fla-warps fla#953"),
+     "fla-warps", False)
+
+print("patched: " + ", ".join(done) + ("" if not warn else " | WARNING (optional, skipped): " + "; ".join(warn)))
+if fatal:
+    sys.exit("FATAL required patch failed: " + "; ".join(fatal))

recipes/qwen38-flash-next-int4-min/mods/flashnext-int4-b12x/run.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Mod: flashnext-int4-b12x
+# Lets eugr's vllm-node-b12x image serve azampatti/Qwen3.8-Flash-Next-125B-A5B-INT4-AutoRound across two GB10 nodes with
+# the b12x RoCE all-reduce (recipe: recipes/qwen3.8-flash-next-int4-b12x.yaml).
+#
+# The launcher copies this folder into EVERY node's container and runs run.sh before the launch script; a non-zero exit
+# aborts the launch, so a node with a missing or incomplete model never starts serving. Nothing outside the container is
+# written, the Hugging Face cache is read-only here, and nothing is copied: the two folders below are symlinks + small JSON.
+#
+#   1. check the model in the HF cache (what ~/Qwen3.8-Flash-Next-Int4-FAST/setup.sh downloads)
+#   2. build /workspace/flashnext/model     (served model view: PLE table indexed, config fields b12x needs)   build_views.py
+#            /workspace/flashnext/draft-k10 (slim MTP draft folder, top-k 10, the head's own shared-expert width)
+#   3. patch this container's vLLM: fp8 hybrid, int4 lm_head, loader index filter (required); draft x2, GDN fixes (optional)
+#                                                                                                                 patch_b12x.py
+set -euo pipefail
+
+MOD_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO=/root/.cache/huggingface/hub/models--azampatti--Qwen3.8-Flash-Next-125B-A5B-INT4-AutoRound
+OUT=/workspace/flashnext
+die(){ echo "FATAL $(hostname): $*" >&2; exit 1; }
+
+[ -f "$REPO/refs/main" ] || die "model not in the HF cache -- run ./setup.sh in ~/Qwen3.8-Flash-Next-Int4-FAST on this node"
+REV=$(cat "$REPO/refs/main"); SNAP="$REPO/snapshots/$REV"
+[ -d "$SNAP/ple-table" ] || die "snapshot ${REV:0:8} has no ple-table -- run ./setup.sh in ~/Qwen3.8-Flash-Next-Int4-FAST on this node"
+for f in config.json model.safetensors.index.json model-healed-shared-expert.safetensors model_extra_tensors.safetensors \
+         model-lmhead-int4.safetensors medium_chat_template.jinja; do
+  [ -e "$SNAP/$f" ] || die "snapshot ${REV:0:8} is incomplete: $f is missing (download still running?)"
+done
+python3 -c "import b12x" 2>/dev/null || die "this is not the b12x image (no b12x package) -- the recipe needs vllm-node-b12x"
+
+mkdir -p "$OUT"
+VIEWS=$(python3 "$MOD_DIR/build_views.py" "$SNAP" "$OUT") || die "could not build the model folders"
+PATCH=$(python3 "$MOD_DIR/patch_b12x.py" "$MOD_DIR") || die "could not patch vLLM: $PATCH"
+
+echo "$(hostname): snapshot ${REV:0:8} | $VIEWS"
+echo "$(hostname): $PATCH"

recipes/qwen38-flash-next-int4-min/mods/flashnext-int4-b12x/vllm_fp8_hybrid.py

@@ -0,0 +1,125 @@
+"""vllm_fp8_hybrid — int4(GPTQ-Marlin) + blockwise-fp8 hybrid dispatch.
+
+The checkpoint's routed experts + lm_head are GPTQ int4/int8 (served by
+AutoGPTQConfig), while dense side-layers (GDN in_proj/out_proj, QSA q/k/v/o,
+shared experts) are stored as blockwise FP8-e4m3 (`weight` fp8 +
+`weight_scale_inv`, block 128x128). Stock AutoGPTQConfig would init those as
+unquantized bf16 and then fail to load fp8 tensors.
+
+This patch (enabled by VLLM_FP8_HYBRID=1, no-op otherwise):
+  * after ``maybe_update_config``, scans the checkpoint's safetensors metadata
+    for F8_E4M3 weights that have a ``.weight_scale_inv`` sibling and records
+    those layer names;
+  * maps them through the model's hf->vllm mapper alongside
+    ``modules_in_block_to_quantize``;
+  * in ``get_quant_method``, dispatches matching LinearBase layers (including
+    fused qkv_proj / gate_up_proj via ``packed_modules_mapping``) to a shared
+    blockwise ``Fp8Config`` instead of the GPTQ/unquantized path.
+
+Port of Saren's spark-dflash-hybrid-fp8 (vLLM 0.23 INCConfig) onto the
+qwen38next build's AutoGPTQConfig.
+"""
+import logging
+import os
+
+logger = logging.getLogger("vllm.fp8_hybrid")
+
+_SENTINEL = "_fp8_hybrid_patched"
+
+
+def _index_filtered_metadata(model_name, revision=None):
+    """flashnext-int4-b12x change to Saren's module: read each tensor's dtype from the file the INDEX names for it.
+
+    The served-model view links PLE-table files that also carry the original checkpoint's layer-1 tensors, including a bf16
+    shared expert. get_safetensors_params_metadata merges every file in the folder, so whichever file came last in directory
+    order decided that layer's dtype -- different on the two nodes (812 vs 809 fp8 layers, 2026-09-21): the node that saw bf16
+    built layer 1's shared expert unquantized and died loading its fp8 weights. No index (e.g. a hub id) -> stock behaviour.
+    """
+    import json
+    import os
+    import struct
+    index = os.path.join(str(model_name), "model.safetensors.index.json")
+    if not os.path.isfile(index):
+        from vllm.transformers_utils.config import get_safetensors_params_metadata
+        return get_safetensors_params_metadata(model_name, revision=revision)
+    by_file = {}
+    for name, fname in json.load(open(index))["weight_map"].items():
+        by_file.setdefault(fname, set()).add(name)
+    md = {}
+    for fname, names in by_file.items():
+        with open(os.path.join(str(model_name), fname), "rb") as fh:
+            n = struct.unpack("<Q", fh.read(8))[0]
+            header = json.loads(fh.read(n))
+        for name in names:
+            if name in header:
+                md[name] = header[name]
+    return md
+
+
+def apply() -> None:
+    if os.environ.get("VLLM_FP8_HYBRID", "0").lower() not in ("1", "true", "yes"):
+        return
+    from vllm.model_executor.layers.quantization import auto_gptq as m
+
+    cfg_cls = m.AutoGPTQConfig
+    if getattr(cfg_cls, _SENTINEL, False):
+        return
+
+    from vllm.model_executor.layers.linear import LinearBase
+    from vllm.model_executor.layers.quantization.fp8 import Fp8Config
+    from vllm.transformers_utils.config import get_safetensors_params_metadata
+
+    orig_update = cfg_cls.maybe_update_config
+    orig_mapper = cfg_cls.apply_vllm_mapper
+    orig_gqm = cfg_cls.get_quant_method
+
+    def maybe_update_config(self, model_name, hf_config=None, revision=None):
+        orig_update(self, model_name, hf_config=hf_config, revision=revision)
+        md = _index_filtered_metadata(model_name, revision)
+        self.fp8_layers = {
+            name[: -len(".weight")]
+            for name, info in md.items()
+            if name.endswith(".weight")
+            and info.get("dtype") == "F8_E4M3"
+            and name[: -len(".weight")] + ".weight_scale_inv" in md
+        }
+        if self.fp8_layers:
+            logger.info(
+                "fp8 hybrid: %d blockwise-fp8 layers detected", len(self.fp8_layers)
+            )
+
+    def apply_vllm_mapper(self, hf_to_vllm_mapper):
+        orig_mapper(self, hf_to_vllm_mapper)
+        if getattr(self, "fp8_layers", None):
+            self.fp8_layers = set(hf_to_vllm_mapper.apply_list(list(self.fp8_layers)))
+
+    def _is_fp8_layer(self, prefix: str) -> bool:
+        fp8_layers = getattr(self, "fp8_layers", None)
+        if not fp8_layers:
+            return False
+        head, _, proj = prefix.rpartition(".")
+        fused = self.packed_modules_mapping.get(proj)
+        names = [f"{head}.{p}" for p in fused] if fused and head else [prefix]
+        return all(any(l in n for l in fp8_layers) for n in names)
+
+    def get_quant_method(self, layer, prefix):
+        if isinstance(layer, LinearBase) and self._is_fp8_layer(prefix):
+            fp8_cfg = getattr(self, "_fp8_cfg", None)
+            if fp8_cfg is None:
+                fp8_cfg = Fp8Config(
+                    is_checkpoint_fp8_serialized=True,
+                    activation_scheme="dynamic",
+                    weight_block_size=[128, 128],
+                )
+                fp8_cfg.packed_modules_mapping = self.packed_modules_mapping
+                self._fp8_cfg = fp8_cfg
+            logger.debug("fp8 hybrid: %s -> Fp8LinearMethod", prefix)
+            return fp8_cfg.get_quant_method(layer, prefix)
+        return orig_gqm(self, layer, prefix)
+
+    cfg_cls.maybe_update_config = maybe_update_config
+    cfg_cls.apply_vllm_mapper = apply_vllm_mapper
+    cfg_cls._is_fp8_layer = _is_fp8_layer
+    cfg_cls.get_quant_method = get_quant_method
+    setattr(cfg_cls, _SENTINEL, True)
+    logger.info("fp8 hybrid patch applied to AutoGPTQConfig")

recipes/qwen38-flash-next-int4-min/recipe.toml

@@ -0,0 +1,88 @@
+# Qwen3.8-Flash-Next-125B-A5B INT4-AutoRound (azampatti) on a single DGX
+# Spark — the "5 experts per token" cut with the healed shared expert.
+#
+# Source: forum 382733 (posts 167-171, 2026-09-25) + github.com/azampatti/
+# Qwen3.8-Flash-Next-Int4-FAST (recipe repo, public). Numbers: 60-75 tok/s
+# single Spark, ~645K-token KV pool at 20 GB, tool-eval 93/100 (his 69-scen
+# v2.6-era harness; we re-bench on v2.7.0). BENCH NOTE: his quant defaults
+# to MEDIUM thinking (the stock model is xhigh) → the chat template matters
+# for A/B fairness (three switchable templates ship in the model repo; the
+# served view mounts medium_chat_template.jinja — the template the
+# speculative head was trained on).
+#
+# Image: eugr/spark-vllm-b12x (Docker Hub; the b12x stack his 2x recipe uses,
+# ~75 tok/s vs 70 on the classic image). Prebuilt — no docker_image_dir.
+# Digest-pinned (see the main qwen38-flash-next-int4 recipe): floating :latest
+# (nightly c9e22735) intermittently garbles long agent contexts; the pinned
+# build is clean. Forum 382733 #195-198.
+#
+# Model: azampatti/Qwen3.8-Flash-Next-125B-A5B-INT4-AutoRound (130.8 GB)
+# staged at /home/kilork/hf-local on the launch node (dgx5); serve with
+# --hf-cache /home/kilork/hf-local.
+#
+# The mod (mods/flashnext-int4-b12x, vendored from his repo) checks the model
+# in the container's HF cache, builds /workspace/flashnext/{model,draft-k10}
+# (symlink views + small JSON, zero copy) and patches the container's vLLM
+# (fp8 hybrid, int4 lm_head, loader index filter, draft x2, GDN fixes).
+# It must run per node (single node here) before the engine.
+
+engine = "vllm"
+name = "Qwen3.8-Flash-Next-Int4 (b12x, 1x Spark) — MIN: no MTP, no prefix cache"
+docker_image = "eugr/spark-vllm-b12x@sha256:8e7e062186f841453ef0ec6f713043c5b65447decc3835206685128c18e42262"
+# Empty recipe model: the launcher emits no positional model, and args[0] is
+# the mod-built view (vllm serves the view, not the raw repo — the view carries
+# the PLE-table index + the b12x arch rename that the raw snapshot lacks).
+model = ""
+parent_model = "Qwen/Qwen3.8-Flash-Next"
+required_nodes = 1
+
+entrypoint = ["vllm", "serve"]
+
+# vllm serves the mod-built view (not the raw repo): the launcher passes the
+# recipe model to the mod via the HF cache; the mod builds /workspace/...
+# and run.sh validates it. The command targets the view, per his yaml.
+args = [
+    "/workspace/flashnext/model",
+    "--served-model-name", "qwen3.8",
+    "--trust-remote-code",
+    "--enable-expert-parallel",
+    "--mamba-cache-mode", "align",
+    "--linear-backend", "b12x",
+    "--no-enable-prefix-caching",
+    "--enable-chunked-prefill",
+    "--dtype", "bfloat16",
+    "--kv-cache-dtype", "auto",
+    "--block-size", "16",
+    "--load-format", "fastsafetensors",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.01",
+    "--kv-cache-memory-bytes", "20g",
+    "--gdn-decode-kernel", "b12x",
+    "--no-enable-flashinfer-autotune",
+    "--mm-encoder-tp-mode", "data",
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "qwen3_coder",
+    "--reasoning-parser", "qwen3",
+    "--chat-template", "/workspace/flashnext/model/medium_chat_template.jinja",
+]
+
+[mods]
+commands = ["bash /mods/flashnext-int4-b12x/run.sh"]
+
+[env]
+HF_HUB_OFFLINE = "1"
+# b12x stack (his solo yaml, verbatim)
+CUTE_DSL_ARCH = "sm_121a"
+VLLM_WORKER_MULTIPROC_METHOD = "spawn"
+VLLM_SSM_CONV_STATE_LAYOUT = "DS"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+B12X_POLICY_MODE = "auto"
+VLLM_PLE_TABLE_MEMORY = "disk"
+# this checkpoint (the mod patches them into the container's vLLM)
+VLLM_FP8_HYBRID = "1"
+VLLM_USE_DEEP_GEMM = "0"
+VLLM_USE_FLASHINFER_SAMPLER = "1"

recipes/qwen38-flash-next-int4/recipe.toml

@@ -11,7 +11,12 @@
 # speculative head was trained on).
 #
 # Image: eugr/spark-vllm-b12x (Docker Hub; the b12x stack his 2x recipe uses,
-# ~75 tok/s vs 70 on the classic image). Prebuilt — no docker_image_dir.
+# ~75 tok/s vs 70 on the classic image). Pinned to the digest azampatti's repo
+# validates on (config.env B12X_IMAGE_PIN) — NOT floating :latest: the nightly
+# build we pulled on 2026-09-25 (c9e22735) intermittently garbles long
+# agent contexts (empty/degenerate replies, special-token leakage); the pinned
+# build is clean. Forum 382733 #195-198 (0rand: same corruption on unpinned,
+# fixed by the pinned digest). Do not "update" this image casually.
 #
 # Model: azampatti/Qwen3.8-Flash-Next-125B-A5B-INT4-AutoRound (130.8 GB)
 # staged at /home/kilork/hf-local on the launch node (dgx5); serve with
@@ -25,7 +30,7 @@
 
 engine = "vllm"
 name = "Qwen3.8-Flash-Next-Int4-AutoRound (b12x, 1x Spark)"
-docker_image = "eugr/spark-vllm-b12x:latest"
+docker_image = "eugr/spark-vllm-b12x@sha256:8e7e062186f841453ef0ec6f713043c5b65447decc3835206685128c18e42262"
 # Empty recipe model: the launcher emits no positional model, and args[0] is
 # the mod-built view (vllm serves the view, not the raw repo — the view carries
 # the PLE-table index + the b12x arch rename that the raw snapshot lacks).
@@ -64,7 +69,7 @@ args = [
     "--reasoning-parser", "qwen3",
     "--chat-template", "/workspace/flashnext/model/medium_chat_template.jinja",
     "--speculative-config",
-    '{"method":"mtp","num_speculative_tokens":4,"model":"/workspace/flashnext/draft-k10","rejection_sample_method":"block","draft_sample_method":"probabilistic"}',
+    '{"method":"mtp","num_speculative_tokens":3,"model":"/workspace/flashnext/draft-k10","rejection_sample_method":"block","draft_sample_method":"probabilistic"}',
 ]
 
 [mods]

src/docker/sync.rs

@@ -7,7 +7,16 @@ use super::shq;
 
 /// Get the Docker image digest (short ID) from a node.
 async fn get_image_digest(config: &Config, node: &Node, image: &str) -> Result<Option<String>> {
-    let cmd = format!("docker image ls --format '{{{{.ID}}}}' {}", shq(image));
+    // `docker image ls` does not accept digest refs (`repo@sha256:...`) —
+    // use `docker image inspect` for those; it also returns the image ID.
+    let cmd = if image.contains('@') {
+        format!(
+            "docker image inspect --format '{{{{.Id}}}}' {} 2>/dev/null",
+            shq(image)
+        )
+    } else {
+        format!("docker image ls --format '{{{{.ID}}}}' {}", shq(image))
+    };
     let out = run_ssh(config, node, cmd, false).await?;
     let digest = out.trim().to_string();
     if digest.is_empty() {