|
|
@@ -58,13 +58,35 @@ args = [
|
|
|
|
|
|
before_serve = [
|
|
|
{ script = "unlock_display_ram" },
|
|
|
- # Stable per-node symlinks to the current refs/main snapshots (recreated
|
|
|
- # every serve; docker resolves the symlink at bind time).
|
|
|
- "ln -sfn /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/snapshots/$(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/refs/main) /home/kilork/exl3-model",
|
|
|
- "ln -sfn /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/snapshots/$(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/refs/main) /home/kilork/exl3-draft",
|
|
|
+ # Model + drafter staging: the hub snapshot's internal relative symlinks
|
|
|
+ # (../../../blobs/…) only resolve at the real hub depth. The full repo dir
|
|
|
+ # (blobs/ + snapshots/) is mounted at /hub, and the staging dir at /model
|
|
|
+ # holds RELATIVE symlinks (../hub/...) into it — the kernel lands on the
|
|
|
+ # real correct-depth tree and the inner links resolve.
|
|
|
+ "REPO=/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw && REV=$(cat $REPO/refs/main) && rm -rf /home/kilork/exl3-model && mkdir -p /home/kilork/exl3-model && for f in $REPO/snapshots/$REV/*; do ln -s \"../hub/snapshots/$REV/$(basename $f)\" /home/kilork/exl3-model/$(basename $f); done",
|
|
|
+ "DREPO=/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw && DREV=$(cat $DREPO/refs/main) && rm -rf /home/kilork/exl3-draft && mkdir -p /home/kilork/exl3-draft && for f in $DREPO/snapshots/$DREV/*; do ln -s \"../dh-draft/snapshots/$DREV/$(basename $f)\" /home/kilork/exl3-draft/$(basename $f); done",
|
|
|
+ # The runtime kit (serve.py + conservative.yaml + overlay .so + AOT gemm)
|
|
|
+ # is NOT in the image — it is the pinned release/runtime tree, staged once
|
|
|
+ # per node and bind-mounted as overlays (serving/ -> /opt/ds41-serving,
|
|
|
+ # aot/... -> flashinfer AOT dir). One-time git clone at the pinned commit.
|
|
|
+ "mkdir -p /home/kilork/exl3-cache",
|
|
|
+ "if [ ! -f /home/kilork/exl3-kit/878e0ee/serving/serve.py ]; then rm -rf /home/kilork/exl3-kit/.src-878e0ee /home/kilork/exl3-kit/878e0ee && git clone --quiet https://github.com/coolbho3k/DeepSeek-v4.1-Flash-2x-DGX-Spark.git /home/kilork/exl3-kit/.src-878e0ee && (cd /home/kilork/exl3-kit/.src-878e0ee && git checkout --quiet 878e0eecd893fadc69ad2d58b2df0fabb0fae2ee) && mv /home/kilork/exl3-kit/.src-878e0ee/release/runtime /home/kilork/exl3-kit/878e0ee && rm -rf /home/kilork/exl3-kit/.src-878e0ee; fi && test -f /home/kilork/exl3-kit/878e0ee/serving/serve.py",
|
|
|
]
|
|
|
after_stop = [{ script = "restore_display_ram", on_failure = "warn" }]
|
|
|
|
|
|
+# Repo dirs (blobs/ + snapshots/) — the real tree the staging symlinks land
|
|
|
+# in; the internal ../../../blobs links resolve here at full depth.
|
|
|
+[[extra_mounts]]
|
|
|
+host = "/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw"
|
|
|
+container = "/hub"
|
|
|
+read_only = true
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw"
|
|
|
+container = "/dh-draft"
|
|
|
+read_only = true
|
|
|
+
|
|
|
+# Staging dirs (relative symlinks into /hub and /dh-draft).
|
|
|
[[extra_mounts]]
|
|
|
host = "/home/kilork/exl3-model"
|
|
|
container = "/model"
|
|
|
@@ -75,6 +97,23 @@ host = "/home/kilork/exl3-draft"
|
|
|
container = "/draft-exl3"
|
|
|
read_only = true
|
|
|
|
|
|
+# Runtime kit overlays (pinned 878e0ee): the image's /opt/ds41-serving is
|
|
|
+# shadowed by the kit's serving tree; the AOT gemm dir is bind-mounted ro.
|
|
|
+[[extra_mounts]]
|
|
|
+host = "/home/kilork/exl3-kit/878e0ee/serving"
|
|
|
+container = "/opt/ds41-serving"
|
|
|
+read_only = true
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "/home/kilork/exl3-kit/878e0ee/aot/mxfp8_gemm_cutlass_sm120"
|
|
|
+container = "/usr/local/lib/python3.12/dist-packages/flashinfer/data/aot/mxfp8_gemm_cutlass_sm120"
|
|
|
+read_only = true
|
|
|
+
|
|
|
+# Node-local runtime cache (triton/tilelang/torch-ext/cuda flashinfer workspaces).
|
|
|
+[[extra_mounts]]
|
|
|
+host = "/home/kilork/exl3-cache"
|
|
|
+container = "/cache"
|
|
|
+
|
|
|
[env]
|
|
|
# DS41 runtime profile (validated against DS41_* envs; kv_cap_mib is
|
|
|
# env-only, not a CLI flag).
|
|
|
@@ -83,7 +122,7 @@ DS41_MAX_MODEL_LEN = "1048576"
|
|
|
DS41_MAX_NUM_SEQS = "2"
|
|
|
DS41_MAX_NUM_BATCHED_TOKENS = "3072"
|
|
|
DS41_LONG_PREFILL_TOKEN_THRESHOLD = "2816"
|
|
|
-DS41_KV_CAP_MIB = "1536"
|
|
|
+DS41_KV_CAP_MIB = "0" # display-only candidate: all KV from the 2 GB display pool
|
|
|
# Feature gates (serve.py hard-requires the '1' set; defaults are 1, set
|
|
|
# explicitly so the guard passes).
|
|
|
DS41_ENABLE_COOPERATIVE_MOE = "1"
|
|
|
@@ -93,6 +132,15 @@ DS41_ENABLE_SSD_VOCAB = "1"
|
|
|
DS41_SSD_ENGRAM_SOURCE = "/model/engrams"
|
|
|
DS41_ENGRAM_CACHE_MIB = "64"
|
|
|
DS41_DRAFT_EXL3_PATH = "/draft-exl3"
|
|
|
+# Node-local cache directories (/cache mount, persistent between serves).
|
|
|
+VLLM_CACHE_ROOT = "/cache/vllm"
|
|
|
+TILELANG_CACHE_DIR = "/cache/tilelang"
|
|
|
+TRITON_CACHE_DIR = "/cache/triton"
|
|
|
+TORCH_EXTENSIONS_DIR = "/cache/torch-extensions"
|
|
|
+CUDA_CACHE_PATH = "/cache/cuda"
|
|
|
+XDG_CACHE_HOME = "/cache"
|
|
|
+FLASHINFER_WORKSPACE_BASE = "/cache/flashinfer"
|
|
|
+TMPDIR = "/cache/tmp"
|
|
|
# Engine / offline / threading.
|
|
|
VLLM_PLUGINS = "ds41"
|
|
|
VLLM_USE_V2_MODEL_RUNNER = "1"
|
