exl3: two-mount staging (repo at /hub + relative-symlink dir at /model), kv_cap 0; qwen served name qwen3.8

recipes/dsv41-flash-exl3-2x/recipe.toml

@@ -58,13 +58,35 @@ args = [
 
 before_serve = [
   { script = "unlock_display_ram" },
-  # Stable per-node symlinks to the current refs/main snapshots (recreated
-  # every serve; docker resolves the symlink at bind time).
-  "ln -sfn /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/snapshots/$(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/refs/main) /home/kilork/exl3-model",
-  "ln -sfn /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/snapshots/$(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/refs/main) /home/kilork/exl3-draft",
+  # Model + drafter staging: the hub snapshot's internal relative symlinks
+  # (../../../blobs/…) only resolve at the real hub depth. The full repo dir
+  # (blobs/ + snapshots/) is mounted at /hub, and the staging dir at /model
+  # holds RELATIVE symlinks (../hub/...) into it — the kernel lands on the
+  # real correct-depth tree and the inner links resolve.
+  "REPO=/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw && REV=$(cat $REPO/refs/main) && rm -rf /home/kilork/exl3-model && mkdir -p /home/kilork/exl3-model && for f in $REPO/snapshots/$REV/*; do ln -s \"../hub/snapshots/$REV/$(basename $f)\" /home/kilork/exl3-model/$(basename $f); done",
+  "DREPO=/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw && DREV=$(cat $DREPO/refs/main) && rm -rf /home/kilork/exl3-draft && mkdir -p /home/kilork/exl3-draft && for f in $DREPO/snapshots/$DREV/*; do ln -s \"../dh-draft/snapshots/$DREV/$(basename $f)\" /home/kilork/exl3-draft/$(basename $f); done",
+  # The runtime kit (serve.py + conservative.yaml + overlay .so + AOT gemm)
+  # is NOT in the image — it is the pinned release/runtime tree, staged once
+  # per node and bind-mounted as overlays (serving/ -> /opt/ds41-serving,
+  # aot/... -> flashinfer AOT dir). One-time git clone at the pinned commit.
+  "mkdir -p /home/kilork/exl3-cache",
+  "if [ ! -f /home/kilork/exl3-kit/878e0ee/serving/serve.py ]; then rm -rf /home/kilork/exl3-kit/.src-878e0ee /home/kilork/exl3-kit/878e0ee && git clone --quiet https://github.com/coolbho3k/DeepSeek-v4.1-Flash-2x-DGX-Spark.git /home/kilork/exl3-kit/.src-878e0ee && (cd /home/kilork/exl3-kit/.src-878e0ee && git checkout --quiet 878e0eecd893fadc69ad2d58b2df0fabb0fae2ee) && mv /home/kilork/exl3-kit/.src-878e0ee/release/runtime /home/kilork/exl3-kit/878e0ee && rm -rf /home/kilork/exl3-kit/.src-878e0ee; fi && test -f /home/kilork/exl3-kit/878e0ee/serving/serve.py",
 ]
 after_stop = [{ script = "restore_display_ram", on_failure = "warn" }]
 
+# Repo dirs (blobs/ + snapshots/) — the real tree the staging symlinks land
+# in; the internal ../../../blobs links resolve here at full depth.
+[[extra_mounts]]
+host = "/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw"
+container = "/hub"
+read_only = true
+
+[[extra_mounts]]
+host = "/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw"
+container = "/dh-draft"
+read_only = true
+
+# Staging dirs (relative symlinks into /hub and /dh-draft).
 [[extra_mounts]]
 host = "/home/kilork/exl3-model"
 container = "/model"
@@ -75,6 +97,23 @@ host = "/home/kilork/exl3-draft"
 container = "/draft-exl3"
 read_only = true
 
+# Runtime kit overlays (pinned 878e0ee): the image's /opt/ds41-serving is
+# shadowed by the kit's serving tree; the AOT gemm dir is bind-mounted ro.
+[[extra_mounts]]
+host = "/home/kilork/exl3-kit/878e0ee/serving"
+container = "/opt/ds41-serving"
+read_only = true
+
+[[extra_mounts]]
+host = "/home/kilork/exl3-kit/878e0ee/aot/mxfp8_gemm_cutlass_sm120"
+container = "/usr/local/lib/python3.12/dist-packages/flashinfer/data/aot/mxfp8_gemm_cutlass_sm120"
+read_only = true
+
+# Node-local runtime cache (triton/tilelang/torch-ext/cuda flashinfer workspaces).
+[[extra_mounts]]
+host = "/home/kilork/exl3-cache"
+container = "/cache"
+
 [env]
 # DS41 runtime profile (validated against DS41_* envs; kv_cap_mib is
 # env-only, not a CLI flag).
@@ -83,7 +122,7 @@ DS41_MAX_MODEL_LEN = "1048576"
 DS41_MAX_NUM_SEQS = "2"
 DS41_MAX_NUM_BATCHED_TOKENS = "3072"
 DS41_LONG_PREFILL_TOKEN_THRESHOLD = "2816"
-DS41_KV_CAP_MIB = "1536"
+DS41_KV_CAP_MIB = "0" # display-only candidate: all KV from the 2 GB display pool
 # Feature gates (serve.py hard-requires the '1' set; defaults are 1, set
 # explicitly so the guard passes).
 DS41_ENABLE_COOPERATIVE_MOE = "1"
@@ -93,6 +132,15 @@ DS41_ENABLE_SSD_VOCAB = "1"
 DS41_SSD_ENGRAM_SOURCE = "/model/engrams"
 DS41_ENGRAM_CACHE_MIB = "64"
 DS41_DRAFT_EXL3_PATH = "/draft-exl3"
+# Node-local cache directories (/cache mount, persistent between serves).
+VLLM_CACHE_ROOT = "/cache/vllm"
+TILELANG_CACHE_DIR = "/cache/tilelang"
+TRITON_CACHE_DIR = "/cache/triton"
+TORCH_EXTENSIONS_DIR = "/cache/torch-extensions"
+CUDA_CACHE_PATH = "/cache/cuda"
+XDG_CACHE_HOME = "/cache"
+FLASHINFER_WORKSPACE_BASE = "/cache/flashinfer"
+TMPDIR = "/cache/tmp"
 # Engine / offline / threading.
 VLLM_PLUGINS = "ds41"
 VLLM_USE_V2_MODEL_RUNNER = "1"

recipes/qwen38-flash-next-int4/recipe.toml

@@ -40,7 +40,7 @@ entrypoint = ["vllm", "serve"]
 # and run.sh validates it. The command targets the view, per his yaml.
 args = [
     "/workspace/flashnext/model",
-    "--served-model-name", "azampatti/Qwen3.8-Flash-Next-125B-A5B-INT4-AutoRound",
+    "--served-model-name", "qwen3.8",
     "--trust-remote-code",
     "--enable-expert-parallel",
     "--mamba-cache-mode", "align",