docker-images: deepseek-v4-flash-eugr

docker-images/deepseek-v4-flash-eugr/Dockerfile

@@ -0,0 +1,23 @@
+# syntax=docker/dockerfile:1.7
+# DeepSeek-V4-Flash-0731 — eugr/spark-vllm-b12x overlay with the
+# instanttensor-hybrid-draft-loader mod baked in.
+#
+# Base: eugr/spark-vllm-b12x — the prebuilt b12x-branch image published in
+# NVIDIA forum comment #28 (https://forums.developer.nvidia.com/t/376220/28).
+# Ubuntu 24.04 + system Python 3.12; vLLM is installed into the system
+# site-packages (the mod's default /usr/local/lib/python3.12/dist-packages,
+# but the path is resolved dynamically below so this is robust to a moved
+# install).
+#
+# eugr's recipe applies `mods/instanttensor-hybrid-draft-loader` at
+# run-recipe.sh time (it patches vLLM's model_loader get_model() to keep
+# InstantTensor for the target while letting an embedded MTP/dspark draft
+# use lazy safetensors). sm121 has no runtime mod-upload step for Docker
+# recipes, so this overlay bakes the patch in at build time instead: the
+# running container already has the hybrid loader, and the recipe carries
+# no [mods] block.
+#
+# Build: cd docker-images/deepseek-v4-flash-eugr && bash build.sh
+# (the sm121 launcher builds on demand and stamps sm121.checksum)
+
+FROM eugr/spark-vllm-b12x:nightly-20260823

docker-images/deepseek-v4-flash-eugr/build.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Build for the deepseek-v4-flash-eugr overlay (eugr/spark-vllm-b12x + baked
+# instanttensor-hybrid-draft-loader mod). No versions.env — this is a FROM
+# overlay on a prebuilt public image, like deepseek-v4-flash-dspark.
+set -euo pipefail
+cd "$(dirname "$0")"
+TAG="${TAG:-eugr/spark-vllm-b12x:sm121}"
+echo "=== Building ${TAG} ==="
+# Persistent buildkit cache (per node, shared across image dirs): the
+# version-independent stages (apt, torch, CUDA base) are reused across
+# rebuilds — only the pinned source stages (vllm/b12x/…) re-download and
+# re-compile. Override the dir with SM121_BUILDKIT_CACHE if desired.
+CACHE_DIR="${SM121_BUILDKIT_CACHE:-$HOME/.cache/sm121-buildkit}"
+mkdir -p "${CACHE_DIR}"
+docker buildx build --cache-from "type=local,src=${CACHE_DIR}" \
+    --cache-to "type=local,dest=${CACHE_DIR},mode=max" \
+    --progress=plain --load -t "${TAG}" . 2>&1
+echo "=== Done: ${TAG} ==="

docker-images/deepseek-v4-flash-eugr/patches/patch_deepseek_v4_effort.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Minimal patch to the eugr image's vendored deepseek_v4 tokenizer to add the
+updated reasoning_effort (low/high/max) support WITHOUT rewriting the whole file.
+
+We deliberately do NOT replace the files with the full 0731 model-card variant.
+Instead this applies the smallest possible edits on top of whatever eugr ships:
+
+  deepseek_v4.py (API layer):
+    - let "low"/"high"/"max" pass through; unknown/garbage -> None (reverts to
+      engine default) instead of coercing everything to "high".
+
+  deepseek_v4_encoding.py:
+    - replace the single REASONING_EFFORT_MAX constant with the per-level
+      REASONING_EFFORT_PROMPTS dict + DEFAULT_REASONING_EFFORT="low"
+      ("high" text == the old "max" text; a stronger "max" added; "low" = "").
+    - in render_message: default None->"low", assert against the dict, and emit
+      the per-level prompt at index 0 in thinking mode.
+
+Everything else in eugr's files (tool handling, `reasoning` key, structure) is
+left intact, so no independent updates are lost.
+
+Usage:  python3 patch_deepseek_v4_effort.py [--check]
+  --check : verify each file is in a CONSISTENT state (either pristine-old or
+            already-patched) and exit 0 if so. Both before and after apply.
+  (apply) : replace old anchors with new (requires exact old-anchor match).
+"""
+import os
+import sys
+
+import vllm
+
+BASE = os.path.join(vllm.__path__[0], "tokenizers")
+F_ENC = os.path.join(BASE, "deepseek_v4_encoding.py")
+F_API = os.path.join(BASE, "deepseek_v4.py")
+
+
+# --- deepseek_v4.py: let low/high/max through; unknown -> None ---
+OLD_API = (
+    "            elif reasoning_effort in (\"max\", \"xhigh\"):\n"
+    "                reasoning_effort = \"max\"\n"
+    "            else:\n"
+    "                reasoning_effort = \"high\""
+)
+NEW_API = (
+    "            elif reasoning_effort in (\"max\", \"xhigh\"):\n"
+    "                reasoning_effort = \"max\"\n"
+    "            elif reasoning_effort not in (\"low\", \"high\", \"max\"):\n"
+    "                reasoning_effort = None"
+)
+
+# --- deepseek_v4_encoding.py: constant -> dict + default ---
+OLD_EFF_CONST = (
+    "REASONING_EFFORT_MAX = (\n"
+    "    \"Reasoning Effort: Absolute maximum with no shortcuts permitted.\\n\"\n"
+    '    "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\\n"\n'
+    '    "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\\n\\n"\n'
+    ")"
+)
+NEW_EFF_CONST = (
+    "REASONING_EFFORT_PROMPTS = {\n"
+    "    \"low\": \"\",\n"
+    "    \"high\": (\n"
+    "        \"Reasoning Effort: Absolute maximum with no shortcuts permitted.\\n\"\n"
+    "        \"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\\n\"\n"
+    "        \"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\\n\\n\"\n"
+    "    ),\n"
+    "    \"max\": (\n"
+    "        \"Reasoning Effort: Beyond maximum \\u2014 exhaustive, relentless, and uncompromising.\\n\"\n"
+    "        \"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\\n\"\n"
+    "        \"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\\n\\n\"\n"
+    "    ),\n"
+    "}\n"
+    'DEFAULT_REASONING_EFFORT = "low"'
+)
+
+# --- deepseek_v4_encoding.py: render_message effort hunk ---
+OLD_EFF_HUNK = (
+    "    # Reasoning effort prefix (only at index 0 in thinking mode with max effort)\n"
+    "    assert reasoning_effort in ['max', None, 'high'], f\"Invalid reasoning effort: {reasoning_effort}\"\n"
+    "    if index == 0 and thinking_mode == \"thinking\" and reasoning_effort == 'max':\n"
+    "        prompt += REASONING_EFFORT_MAX"
+)
+NEW_EFF_HUNK = (
+    "    # Reasoning effort prefix (only at index 0 in thinking mode; \"low\" adds nothing)\n"
+    "    reasoning_effort = reasoning_effort or DEFAULT_REASONING_EFFORT\n"
+    "    assert reasoning_effort in REASONING_EFFORT_PROMPTS, \\\n"
+    "        f\"Invalid reasoning effort: {reasoning_effort}, expected one of {list(REASONING_EFFORT_PROMPTS)}\"\n"
+    "    if index == 0 and thinking_mode == \"thinking\":\n"
+    "        prompt += REASONING_EFFORT_PROMPTS[reasoning_effort]"
+)
+
+# Each entry: (path, [(old_anchor, new_anchor), ...])
+TARGETS = [
+    (F_API, [(OLD_API, NEW_API)]),
+    (F_ENC, [(OLD_EFF_CONST, NEW_EFF_CONST), (OLD_EFF_HUNK, NEW_EFF_HUNK)]),
+]
+
+
+def _consistent(src, old, new):
+    """True if the file holds exactly one of old|new (pristine or patched).
+    Handles overlapping edits where NEW embeds OLD (e.g. an inserted import
+    block that starts with the original lines)."""
+    co, cn = src.count(old), src.count(new)
+    if cn >= 1:
+        # patched: new present once, and old does not appear beyond the copy
+        # embedded inside new itself.
+        return cn == 1 and co <= new.count(old)
+    return co == 1
+
+
+def main() -> None:
+    check = "--check" in sys.argv
+    states = [(path, open(path).read()) for path, _ in TARGETS]
+
+    if check:
+        for path, src in states:
+            for old, new in dict(TARGETS)[path]:
+                if not _consistent(src, old, new):
+                    raise SystemExit(
+                        f"CHECK FAIL: {path} — file is neither pristine-old nor "
+                        "already-patched for an anchor"
+                    )
+        print("check OK: deepseek_v4 reasoning_effort state is consistent")
+        return
+
+    # Apply: require exact old anchors, then replace and verify new.
+    for path, src in states:
+        for old, new in dict(TARGETS)[path]:
+            if src.count(old) != 1:
+                raise SystemExit(
+                    f"refusing: {path} old anchor matched {src.count(old)}x "
+                    "(expected 1); source changed, re-sync patch"
+                )
+        for old, new in dict(TARGETS)[path]:
+            src = src.replace(old, new)
+        for old, new in dict(TARGETS)[path]:
+            if src.count(new) != 1:
+                raise SystemExit(
+                    f"refusing: {path} new anchor present {src.count(new)}x "
+                    "after apply (internal error)"
+                )
+        with open(path, "w") as f:
+            f.write(src)
+        print(f"patched {path}")
+
+
+if __name__ == "__main__":
+    main()

docker-images/deepseek-v4-flash-eugr/patches/patch_deepseek_v4_reasoning_content.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Optional patch to the eugr image's vendored deepseek_v4_encoding.py to read
+the newer `reasoning_content` message key instead of the deprecated `reasoning`
+key.
+
+This is INDEPENDENT of the reasoning_effort patch and is DEFAULT-OFF so the
+pure reasoning_effort (minimal) variant is the first-test behavior. Enable at
+runtime with VLLM_DSV4_REASONING_CONTENT=1 (recipe [env] / -e). It stays
+backward compatible: when enabled it uses `reasoning_content` only when the
+message actually has that key, otherwise falls back to `reasoning`.
+
+Three small edits on top of whatever eugr ships:
+  1. add a module-level env-gated flag (_USE_REASONING_CONTENT_KEY) + import os
+  2. render_message: use reasoning_content (fallback reasoning) when enabled
+  3. drop_thinking: also pop reasoning_content (no-op when key unused)
+
+Everything else in eugr's file is left intact, so no independent updates are
+lost. Idempotent --check (pristine or already-patched) + apply + post-check,
+mirroring patch_deepseek_v4_effort.py.
+
+Usage:  python3 patch_deepseek_v4_reasoning_content.py [--check]
+  --check : verify the file is in a consistent (pristine or patched) state.
+  (apply) : replace old anchors with new (requires exact old-anchor match).
+"""
+import os
+import sys
+
+import vllm
+
+BASE = os.path.join(vllm.__path__[0], "tokenizers")
+F_ENC = os.path.join(BASE, "deepseek_v4_encoding.py")
+
+# 1) import os + env-gated module flag (inserted after the base imports)
+OLD_IMPORTS = (
+    "from typing import Any, Dict, List, Union, Optional\n"
+    "import copy\n"
+    "import json"
+)
+NEW_IMPORTS = (
+    "from typing import Any, Dict, List, Union, Optional\n"
+    "import copy\n"
+    "import json\n"
+    "import os\n"
+    "\n"
+    "# Optional: read the newer `reasoning_content` message key instead of the\n"
+    "# deprecated `reasoning` key. Disabled by default (preserves eugr\n"
+    "# behavior); enable with VLLM_DSV4_REASONING_CONTENT=1.\n"
+    "_USE_REASONING_CONTENT_KEY = bool(\n"
+    "    os.environ.get(\"VLLM_DSV4_REASONING_CONTENT\", \"\").strip().lower()\n"
+    "    in (\"1\", \"true\", \"yes\", \"on\")\n"
+    ")"
+)
+
+# 2) render_message: use reasoning_content when enabled (fallback to reasoning)
+OLD_READ = "    reasoning = msg.get(\"reasoning\")"
+NEW_READ = (
+    "    reasoning = msg.get(\"reasoning\")\n"
+    "    if _USE_REASONING_CONTENT_KEY and \"reasoning_content\" in msg:\n"
+    "        reasoning = msg.get(\"reasoning_content\")"
+)
+
+# 3) drop_thinking: also pop reasoning_content (no-op when key unused)
+OLD_POP = "            msg.pop(\"reasoning\", None)"
+NEW_POP = (
+    "            msg.pop(\"reasoning_content\", None)\n"
+    "            msg.pop(\"reasoning\", None)"
+)
+
+TARGETS = [(F_ENC, [(OLD_IMPORTS, NEW_IMPORTS), (OLD_READ, NEW_READ),
+                    (OLD_POP, NEW_POP)])]
+
+
+def _consistent(src, old, new):
+    """True if the file holds exactly one of old|new (pristine or patched).
+    Handles overlapping edits where NEW embeds OLD (e.g. an inserted import
+    block that starts with the original lines)."""
+    co, cn = src.count(old), src.count(new)
+    if cn >= 1:
+        # patched: new present once, and old does not appear beyond the copy
+        # embedded inside new itself.
+        return cn == 1 and co <= new.count(old)
+    return co == 1
+
+
+def main() -> None:
+    check = "--check" in sys.argv
+    path = TARGETS[0][0]
+    src = open(path).read()
+
+    if check:
+        for old, new in TARGETS[0][1]:
+            if not _consistent(src, old, new):
+                raise SystemExit(
+                    f"CHECK FAIL: {path} — neither pristine-old nor "
+                    "already-patched for an anchor"
+                )
+        print("check OK: deepseek_v4 reasoning_content state is consistent")
+        return
+
+    for old, new in TARGETS[0][1]:
+        if src.count(old) != 1:
+            raise SystemExit(
+                f"refusing: {path} old anchor matched {src.count(old)}x "
+                "(expected 1); source changed, re-sync patch"
+            )
+    for old, new in TARGETS[0][1]:
+        src = src.replace(old, new)
+    for old, new in TARGETS[0][1]:
+        if src.count(new) != 1:
+            raise SystemExit(
+                f"refusing: {path} new anchor present {src.count(new)}x "
+                "after apply (internal error)"
+            )
+    with open(path, "w") as f:
+        f.write(src)
+    print(f"patched {path}")
+
+
+if __name__ == "__main__":
+    main()

docker-images/deepseek-v4-flash-eugr/patches/patch_model_loader.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""Add an opt-in hybrid loader policy for speculative draft models."""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import sys
+from pathlib import Path
+
+MARKER = "# spark-vllm mod: instanttensor-hybrid-draft-loader v1"
+
+GET_MODEL_ANCHOR = """
+def get_model(
+    *,
+    vllm_config: VllmConfig,
+    model_config: ModelConfig | None = None,
+    prefix: str = "",
+    load_config: LoadConfig | None = None,
+) -> nn.Module:
+    loader = get_model_loader(load_config or vllm_config.load_config)
+    if model_config is None:
+        model_config = vllm_config.model_config
+    return loader.load_model(
+        vllm_config=vllm_config, model_config=model_config, prefix=prefix
+    )
+"""
+
+GET_MODEL_REPLACEMENT = f"""
+{MARKER}
+def _instanttensor_draft_load_config(
+    vllm_config: VllmConfig,
+    model_config: ModelConfig,
+    load_config: LoadConfig | None,
+) -> LoadConfig:
+    \"\"\"Resolve the loader for one model without changing the target loader.\"\"\"
+    import os
+
+    from vllm.config import replace
+
+    mode = os.environ.get("INSTANTTENSOR_DRAFT_LOADER", "auto").strip().lower()
+    allowed_modes = ("auto", "safetensors", "instanttensor")
+    if mode not in allowed_modes:
+        raise ValueError(
+            "INSTANTTENSOR_DRAFT_LOADER must be one of "
+            f"{{', '.join(allowed_modes)}}; got {{mode!r}}"
+        )
+
+    effective = load_config or vllm_config.load_config
+    load_format = getattr(effective.load_format, "value", effective.load_format)
+    if mode == "instanttensor" or str(load_format).lower() != "instanttensor":
+        return effective
+
+    speculative_config = getattr(vllm_config, "speculative_config", None)
+    draft_model_config = getattr(speculative_config, "draft_model_config", None)
+    if draft_model_config is None or model_config is not draft_model_config:
+        return effective
+
+    if mode == "auto":
+        target_model_config = (
+            getattr(speculative_config, "target_model_config", None)
+            or vllm_config.model_config
+        )
+        draft_source = (
+            getattr(draft_model_config, "model", None),
+            getattr(draft_model_config, "revision", None),
+        )
+        target_source = (
+            getattr(target_model_config, "model", None),
+            getattr(target_model_config, "revision", None),
+        )
+        if draft_source != target_source:
+            return effective
+
+    logger.info_once(
+        "Hybrid draft loading: using lazy safetensors for speculative draft "
+        "weights while preserving InstantTensor for the target model "
+        "(INSTANTTENSOR_DRAFT_LOADER=%s).",
+        mode,
+    )
+    return replace(
+        effective,
+        load_format="safetensors",
+        safetensors_load_strategy="lazy",
+    )
+
+
+def get_model(
+    *,
+    vllm_config: VllmConfig,
+    model_config: ModelConfig | None = None,
+    prefix: str = "",
+    load_config: LoadConfig | None = None,
+) -> nn.Module:
+    if model_config is None:
+        model_config = vllm_config.model_config
+    resolved_load_config = _instanttensor_draft_load_config(
+        vllm_config, model_config, load_config
+    )
+    loader = get_model_loader(resolved_load_config)
+    return loader.load_model(
+        vllm_config=vllm_config, model_config=model_config, prefix=prefix
+    )
+"""
+
+
+def validate_shape(text: str, *, patched: bool) -> None:
+    tree = ast.parse(text)
+    functions = {
+        node.name
+        for node in tree.body
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+    }
+    required = {"get_model", "get_model_loader"}
+    if patched:
+        required.add("_instanttensor_draft_load_config")
+    missing = sorted(required - functions)
+    if missing:
+        raise ValueError(
+            "unexpected model-loader module; missing " + ", ".join(missing)
+        )
+
+
+def patched_text(text: str) -> str:
+    validate_shape(text, patched=MARKER in text)
+    if MARKER in text:
+        if text.count(MARKER) != 1:
+            raise ValueError("hybrid-loader marker occurs more than once")
+        if "resolved_load_config = _instanttensor_draft_load_config(" not in text:
+            raise ValueError("mod marker exists but get_model does not use the helper")
+        compile(text, "<patched model_loader/__init__.py>", "exec")
+        return text
+
+    count = text.count(GET_MODEL_ANCHOR)
+    if count != 1:
+        raise ValueError(
+            f"expected exactly one supported get_model function; found {count}"
+        )
+    patched = text.replace(GET_MODEL_ANCHOR, GET_MODEL_REPLACEMENT, 1)
+    validate_shape(patched, patched=True)
+    compile(patched, "<patched model_loader/__init__.py>", "exec")
+    return patched
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("target", type=Path)
+    parser.add_argument(
+        "--check", action="store_true", help="validate compatibility without writing"
+    )
+    args = parser.parse_args()
+
+    if not args.target.is_file():
+        print(
+            "[instanttensor-hybrid-draft-loader ERROR] "
+            f"target not found: {args.target}",
+            file=sys.stderr,
+        )
+        return 1
+
+    original = args.target.read_text()
+    try:
+        patched = patched_text(original)
+    except (SyntaxError, ValueError) as exc:
+        print(
+            "[instanttensor-hybrid-draft-loader ERROR] "
+            f"refusing to patch {args.target}: {exc}",
+            file=sys.stderr,
+        )
+        return 1
+
+    if args.check:
+        state = "already patched" if patched == original else "compatible"
+        print(f"[instanttensor-hybrid-draft-loader] {args.target} is {state}.")
+        return 0
+
+    if patched == original:
+        print(
+            "[instanttensor-hybrid-draft-loader] "
+            "Model loader is already patched; skipping."
+        )
+        return 0
+
+    temporary = args.target.with_suffix(
+        args.target.suffix + ".instanttensor-draft-mod.tmp"
+    )
+    temporary.write_text(patched)
+    temporary.replace(args.target)
+    print(f"[instanttensor-hybrid-draft-loader] Patched {args.target}.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())