launcher/recipes/dsv41-flash-exl3-2x/recipe.toml

# DeepSeek-V4.1-Flash EXL3 3bpw — coolbho3k "2x DGX Spark" production recipe,
# adapted to sm121 (rc2 image, pinned at release commit 878e0ee).
#
# 2x Spark over 200G RoCE (8nodes-switch, first two free ranks):
# TP2/DCP2, DSpark K=3 drafter, EXL3 3bpw experts, FP4 main KV + FP8 SWA,
# 1M context, GMU 0.92, max-num-seqs 2 (C2; the runtime's cooperative
# profile validates 1..6), prefill batch 3072.
#
# 2 GB headless-RAM unlock: the runtime's display-pool KV (1.75 GiB of the
# firmware's 2 GB display reserve per GPU) needs nvidia_drm remoded with
# modeset=1 fbdev=0. That is TEMPORARY by design (technology still in
# approval): before_serve remods per boot, after_stop restores the plain
# module. No UEFI change, no /etc/modprobe.d entry, resets on reboot.
#
# Model + drafter overlay: node-local NVMe mirror of the NAS hub at
# /home/kilork/hf-local/hub (EXL3 streams weights on EVERY decode step — from
# NFS that costs ~2/3 of decode; staged here per node, the read is local).
# Mirror rule: same hub-root-relative layout; refs/main stays pinned to the
# NAS (authoritative). Files not staged locally — engram-page15-v1 (optional
# per its release notes, +1.4% decode) — resolve through the NAS fallback,
# read-only. The launch nodes bind every referenced file per-path at /model
# and /draft-exl3 (the runtime hard-requires the model at exactly /model).
#
# Cgroup: the runtime's serving guard hard-requires a 9 GiB / no-swap /
# <=6-CPU cgroup (memory.max==9g, swap 0, cpu quota <=6) — memory_limit +
# cpus below.
engine = "vllm"
docker_image = "ghcr.io/coolbho3k/deepseek-v4.1-flash-exl3-3bpw-2x-dgx-spark:20260917-rc2"
model = ""
name = "DeepSeek-V4.1-Flash EXL3 3bpw (2x Spark, TP2/DCP2, DSpark K=3, 1M)"
required_nodes = 2
cluster = "8nodes-switch"
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "2g"
devices = ["/dev/dri/card0"]
memory_limit = "9g"
cpus = 6
extra_mounts_manifest = "/tmp/sm121-staging-exl3-manifests.txt"

# /model + /draft-exl3 are the exact mounts the runtime validates.
entrypoint = [
  "/opt/ds41-venv/bin/python",
  "/opt/ds41-serving/serve.py",
  "serve",
  "/model",
  "--config",
  "/opt/ds41-serving/conservative.yaml",
  "--worker-cls",
  "combined_worker.CombinedWorker",
  "--disable-custom-all-reduce",
]
args = [
  "--served-model-name",
  "deepseek-v41-flash-exl3",
  "--enable-auto-tool-choice",
  "--tool-call-parser",
  "deepseek_v41",
  "--reasoning-parser",
  "deepseek_v41",
]

before_serve = [
  { script = "unlock_display_ram" },
  # The runtime kit (serve.py + conservative.yaml + overlay .so + AOT gemm)
  # is NOT in the image — it is the pinned release/runtime tree, staged once
  # per node and bind-mounted as overlays (serving/ -> /opt/ds41-serving,
  # aot/... -> flashinfer AOT dir). One-time git clone at the pinned commit.
  "mkdir -p /home/kilork/exl3-cache",
  "sudo rm -rf /home/kilork/exl3-cache/ds41-backend-attestation /home/kilork/exl3-cache/ds41-combined-ready /home/kilork/exl3-cache/ds41-display-kv",
  "if [ ! -f /home/kilork/exl3-kit/878e0ee/serving/serve.py ]; then rm -rf /home/kilork/exl3-kit/.src-878e0ee /home/kilork/exl3-kit/878e0ee && git clone --quiet https://github.com/coolbho3k/DeepSeek-v4.1-Flash-2x-DGX-Spark.git /home/kilork/exl3-kit/.src-878e0ee && (cd /home/kilork/exl3-kit/.src-878e0ee && git checkout --quiet 878e0eecd893fadc69ad2d58b2df0fabb0fae2ee) && mv /home/kilork/exl3-kit/.src-878e0ee/release/runtime /home/kilork/exl3-kit/878e0ee && rm -rf /home/kilork/exl3-kit/.src-878e0ee; fi && test -f /home/kilork/exl3-kit/878e0ee/serving/serve.py",
]
after_stop = [{ script = "restore_display_ram", on_failure = "warn" }]

# Hub dir (both repos + hub-level blobs/): the repo blobs/ is itself a layer
# of relative symlinks into the sibling hub-level blobs/<xx>/<sha>, so the
# whole hub tree must be in scope for the chain to resolve.
[[extra_mounts]]
host = "/mnt/nas/ai/huggingface/hub"
container = "/hub"
read_only = true

# /model and /draft-exl3: the runtime opens shard files with O_NOFOLLOW
# (symlink final component => ELOOP), so the container must see regular
# files only. The [[staging]] export materializes them node-locally
# (hardlinks onto the mirror blobs); entries the mirror deliberately does
# not hold (the page15 engram table) arrive as per-file NAS binds via the
# export manifest (extra_mounts_manifest).

# Staging (declarative, specs/tech-architecture/staging-feature.md): the
# hub-mirror adopts the node-local copy the manual staging created (the
# 2026-09-26 mirror at /home/kilork/hf-local) — verification accepts
# untracked matching files into the ledger. First serve verifies only.
[[staging]]
kind = "hub-mirror"
id = "exl3"
dst = "/home/kilork/hf-local"
repos = [
  "models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw",
  "models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw",
]
skip = ["engram-page15-v1"]
min_free = "50G"
export = [
  "model:models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw:/model",
  "draft:models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw:/draft-exl3",
]

# /model and /draft-exl3: regular-file views of the staged export trees
# (hardlinks onto mirror blobs). read_only = true is REQUIRED: the runtime
# guards weight shards against a writable filesystem. The skipped page15
# table arrives as per-file binds from the export manifest onto mountpoint
# placeholders that staging creates inside the export trees.
[[extra_mounts]]
host = "/home/kilork/hf-local/model"
container = "/model"
read_only = true

[[extra_mounts]]
host = "/home/kilork/hf-local/draft"
container = "/draft-exl3"
read_only = true

# Runtime kit overlays (pinned 878e0ee): the image's /opt/ds41-serving is
# shadowed by the kit's serving tree; the AOT gemm dir is bind-mounted ro.
[[extra_mounts]]
host = "/home/kilork/exl3-kit/878e0ee/serving"
container = "/opt/ds41-serving"
read_only = true

[[extra_mounts]]
host = "/home/kilork/exl3-kit/878e0ee/aot/mxfp8_gemm_cutlass_sm120"
container = "/usr/local/lib/python3.12/dist-packages/flashinfer/data/aot/mxfp8_gemm_cutlass_sm120"
read_only = true

# Node-local runtime cache (triton/tilelang/torch-ext/cuda flashinfer workspaces).
[[extra_mounts]]
host = "/home/kilork/exl3-cache"
container = "/cache"

[env]
# DS41 runtime profile (validated against DS41_* envs; kv_cap_mib is
# env-only, not a CLI flag).
DS41_GPU_MEMORY_UTILIZATION = "0.92"
DS41_MAX_MODEL_LEN = "1048576"
DS41_MAX_NUM_SEQS = "2"
DS41_MAX_NUM_BATCHED_TOKENS = "3072"
DS41_LONG_PREFILL_TOKEN_THRESHOLD = "2816"
DS41_KV_CAP_MIB = "0" # display-only candidate: all KV from the 2 GB display pool
# Feature gates (serve.py hard-requires the '1' set; defaults are 1, set
# explicitly so the guard passes).
DS41_ENABLE_COOPERATIVE_MOE = "1"
DS41_ENABLE_DCP2 = "1"
DS41_ENABLE_DSPARK = "1"
DS41_ENABLE_SSD_VOCAB = "1"
DS41_SSD_ENGRAM_SOURCE = "/model/engrams"
DS41_ENGRAM_CACHE_MIB = "64"
DS41_DRAFT_EXL3_PATH = "/draft-exl3"
# Node-local cache directories (/cache mount, persistent between serves).
VLLM_CACHE_ROOT = "/cache/vllm"
TILELANG_CACHE_DIR = "/cache/tilelang"
TRITON_CACHE_DIR = "/cache/triton"
TORCH_EXTENSIONS_DIR = "/cache/torch-extensions"
CUDA_CACHE_PATH = "/cache/cuda"
XDG_CACHE_HOME = "/cache"
FLASHINFER_WORKSPACE_BASE = "/cache/flashinfer"
TMPDIR = "/cache/tmp"
# Engine / offline / threading.
VLLM_PLUGINS = "ds41"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_NO_USAGE_STATS = "1"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
DSV41_IO_THREADS = "96"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
MAX_JOBS = "1"
MKL_NUM_THREADS = "4"
OPENBLAS_NUM_THREADS = "4"
TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"
MALLOC_ARENA_MAX = "2"
MALLOC_TRIM_THRESHOLD_ = "131072"
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "128"
# NCCL pins (on top of the launcher's cluster NCCL env).
NCCL_NVLS_ENABLE = "0"
NCCL_CROSS_NIC = "0"
NCCL_IB_MERGE_NICS = "0"
NCCL_CUMEM_ENABLE = "0"
NCCL_MAX_CTAS = "8"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_PROTO = "^LL128"
NCCL_MAX_NCHANNELS = "8"
NCCL_IB_ROCE_VERSION_NUM = "2"