launcher/recipes/dsv41-flash-exl3-2x/recipe.toml

# DeepSeek-V4.1-Flash EXL3 3bpw — coolbho3k "2x DGX Spark" production recipe,
# adapted to sm121 (rc2 image, pinned at release commit 878e0ee).
#
# 2x Spark over 200G RoCE (8nodes-switch, first two free ranks):
# TP2/DCP2, DSpark K=3 drafter, EXL3 3bpw experts, FP4 main KV + FP8 SWA,
# 1M context, GMU 0.92, max-num-seqs 2 (C2; the runtime's cooperative
# profile validates 1..6), prefill batch 3072.
#
# 2 GB headless-RAM unlock: the runtime's display-pool KV (1.75 GiB of the
# firmware's 2 GB display reserve per GPU) needs nvidia_drm remoded with
# modeset=1 fbdev=0. That is TEMPORARY by design (technology still in
# approval): before_serve remods per boot, after_stop restores the plain
# module. No UEFI change, no /etc/modprobe.d entry, resets on reboot.
#
# Model + drafter overlay live on the shared NAS (hub layout); the launch
# nodes symlink the current refs/main snapshot and bind it at /model (the
# runtime hard-requires the model at exactly /model) and the drafter overlay
# at /draft-exl3.
#
# Cgroup: the runtime's serving guard hard-requires a 9 GiB / no-swap /
# <=6-CPU cgroup (memory.max==9g, swap 0, cpu quota <=6) — memory_limit +
# cpus below.
engine = "vllm"
docker_image = "ghcr.io/coolbho3k/deepseek-v4.1-flash-exl3-3bpw-2x-dgx-spark:20260917-rc2"
model = ""
name = "DeepSeek-V4.1-Flash EXL3 3bpw (2x Spark, TP2/DCP2, DSpark K=3, 1M)"
required_nodes = 2
cluster = "8nodes-switch"
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "2g"
devices = ["/dev/dri/card0"]
memory_limit = "9g"
cpus = 6

# /model + /draft-exl3 are the exact mounts the runtime validates.
entrypoint = [
  "/opt/ds41-venv/bin/python",
  "/opt/ds41-serving/serve.py",
  "serve",
  "/model",
  "--config",
  "/opt/ds41-serving/conservative.yaml",
  "--worker-cls",
  "combined_worker.CombinedWorker",
  "--disable-custom-all-reduce",
]
args = [
  "--served-model-name",
  "deepseek-v41-flash-exl3",
  "--enable-auto-tool-choice",
  "--tool-call-parser",
  "deepseek_v41",
  "--reasoning-parser",
  "deepseek_v41",
]

before_serve = [
  { script = "unlock_display_ram" },
  # Model + drafter staging: the hub snapshot's internal relative symlinks
  # (../../../blobs/…) only resolve at the real hub depth. The full repo dir
  # (blobs/ + snapshots/) is mounted at /hub, and the staging dir at /model
  # holds RELATIVE symlinks (../hub/...) into it — the kernel lands on the
  # real correct-depth tree and the inner links resolve.
  "REPO=/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw && REV=$(cat $REPO/refs/main) && rm -rf /home/kilork/exl3-model && mkdir -p /home/kilork/exl3-model && for f in $REPO/snapshots/$REV/*; do ln -s \"../hub/snapshots/$REV/$(basename $f)\" /home/kilork/exl3-model/$(basename $f); done",
  "DREPO=/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw && DREV=$(cat $DREPO/refs/main) && rm -rf /home/kilork/exl3-draft && mkdir -p /home/kilork/exl3-draft && for f in $DREPO/snapshots/$DREV/*; do ln -s \"../dh-draft/snapshots/$DREV/$(basename $f)\" /home/kilork/exl3-draft/$(basename $f); done",
  # The runtime kit (serve.py + conservative.yaml + overlay .so + AOT gemm)
  # is NOT in the image — it is the pinned release/runtime tree, staged once
  # per node and bind-mounted as overlays (serving/ -> /opt/ds41-serving,
  # aot/... -> flashinfer AOT dir). One-time git clone at the pinned commit.
  "mkdir -p /home/kilork/exl3-cache",
  "if [ ! -f /home/kilork/exl3-kit/878e0ee/serving/serve.py ]; then rm -rf /home/kilork/exl3-kit/.src-878e0ee /home/kilork/exl3-kit/878e0ee && git clone --quiet https://github.com/coolbho3k/DeepSeek-v4.1-Flash-2x-DGX-Spark.git /home/kilork/exl3-kit/.src-878e0ee && (cd /home/kilork/exl3-kit/.src-878e0ee && git checkout --quiet 878e0eecd893fadc69ad2d58b2df0fabb0fae2ee) && mv /home/kilork/exl3-kit/.src-878e0ee/release/runtime /home/kilork/exl3-kit/878e0ee && rm -rf /home/kilork/exl3-kit/.src-878e0ee; fi && test -f /home/kilork/exl3-kit/878e0ee/serving/serve.py",
]
after_stop = [{ script = "restore_display_ram", on_failure = "warn" }]

# Repo dirs (blobs/ + snapshots/) — the real tree the staging symlinks land
# in; the internal ../../../blobs links resolve here at full depth.
[[extra_mounts]]
host = "/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw"
container = "/hub"
read_only = true

[[extra_mounts]]
host = "/mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw"
container = "/dh-draft"
read_only = true

# Staging dirs (relative symlinks into /hub and /dh-draft).
[[extra_mounts]]
host = "/home/kilork/exl3-model"
container = "/model"
read_only = true

[[extra_mounts]]
host = "/home/kilork/exl3-draft"
container = "/draft-exl3"
read_only = true

# Runtime kit overlays (pinned 878e0ee): the image's /opt/ds41-serving is
# shadowed by the kit's serving tree; the AOT gemm dir is bind-mounted ro.
[[extra_mounts]]
host = "/home/kilork/exl3-kit/878e0ee/serving"
container = "/opt/ds41-serving"
read_only = true

[[extra_mounts]]
host = "/home/kilork/exl3-kit/878e0ee/aot/mxfp8_gemm_cutlass_sm120"
container = "/usr/local/lib/python3.12/dist-packages/flashinfer/data/aot/mxfp8_gemm_cutlass_sm120"
read_only = true

# Node-local runtime cache (triton/tilelang/torch-ext/cuda flashinfer workspaces).
[[extra_mounts]]
host = "/home/kilork/exl3-cache"
container = "/cache"

[env]
# DS41 runtime profile (validated against DS41_* envs; kv_cap_mib is
# env-only, not a CLI flag).
DS41_GPU_MEMORY_UTILIZATION = "0.92"
DS41_MAX_MODEL_LEN = "1048576"
DS41_MAX_NUM_SEQS = "2"
DS41_MAX_NUM_BATCHED_TOKENS = "3072"
DS41_LONG_PREFILL_TOKEN_THRESHOLD = "2816"
DS41_KV_CAP_MIB = "0" # display-only candidate: all KV from the 2 GB display pool
# Feature gates (serve.py hard-requires the '1' set; defaults are 1, set
# explicitly so the guard passes).
DS41_ENABLE_COOPERATIVE_MOE = "1"
DS41_ENABLE_DCP2 = "1"
DS41_ENABLE_DSPARK = "1"
DS41_ENABLE_SSD_VOCAB = "1"
DS41_SSD_ENGRAM_SOURCE = "/model/engrams"
DS41_ENGRAM_CACHE_MIB = "64"
DS41_DRAFT_EXL3_PATH = "/draft-exl3"
# Node-local cache directories (/cache mount, persistent between serves).
VLLM_CACHE_ROOT = "/cache/vllm"
TILELANG_CACHE_DIR = "/cache/tilelang"
TRITON_CACHE_DIR = "/cache/triton"
TORCH_EXTENSIONS_DIR = "/cache/torch-extensions"
CUDA_CACHE_PATH = "/cache/cuda"
XDG_CACHE_HOME = "/cache"
FLASHINFER_WORKSPACE_BASE = "/cache/flashinfer"
TMPDIR = "/cache/tmp"
# Engine / offline / threading.
VLLM_PLUGINS = "ds41"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_NO_USAGE_STATS = "1"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
DSV41_IO_THREADS = "96"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
MAX_JOBS = "1"
MKL_NUM_THREADS = "4"
OPENBLAS_NUM_THREADS = "4"
TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"
MALLOC_ARENA_MAX = "2"
MALLOC_TRIM_THRESHOLD_ = "131072"
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "128"
# NCCL pins (on top of the launcher's cluster NCCL env).
NCCL_NVLS_ENABLE = "0"
NCCL_CROSS_NIC = "0"
NCCL_IB_MERGE_NICS = "0"
NCCL_CUMEM_ENABLE = "0"
NCCL_MAX_CTAS = "8"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_PROTO = "^LL128"
NCCL_MAX_NCHANNELS = "8"
NCCL_IB_ROCE_VERSION_NUM = "2"