launcher/recipes/dsv41-flash-exl3-2x/recipe.toml

# DeepSeek-V4.1-Flash EXL3 3bpw — coolbho3k "2x DGX Spark" production recipe,
# adapted to sm121 (rc2 image, pinned at release commit 878e0ee).
#
# 2x Spark over 200G RoCE (8nodes-switch, first two free ranks):
# TP2/DCP2, DSpark K=3 drafter, EXL3 3bpw experts, FP4 main KV + FP8 SWA,
# 1M context, GMU 0.92, max-num-seqs 2 (C2; the runtime's cooperative
# profile validates 1..6), prefill batch 3072.
#
# 2 GB headless-RAM unlock: the runtime's display-pool KV (1.75 GiB of the
# firmware's 2 GB display reserve per GPU) needs nvidia_drm remoded with
# modeset=1 fbdev=0. That is TEMPORARY by design (technology still in
# approval): before_serve remods per boot, after_stop restores the plain
# module. No UEFI change, no /etc/modprobe.d entry, resets on reboot.
#
# Model + drafter overlay live on the shared NAS (hub layout); the launch
# nodes symlink the current refs/main snapshot and bind it at /model (the
# runtime hard-requires the model at exactly /model) and the drafter overlay
# at /draft-exl3.
#
# Cgroup: the runtime's serving guard hard-requires a 9 GiB / no-swap /
# <=6-CPU cgroup (memory.max==9g, swap 0, cpu quota <=6) — memory_limit +
# cpus below.
engine = "vllm"
docker_image = "ghcr.io/coolbho3k/deepseek-v4.1-flash-exl3-3bpw-2x-dgx-spark:20260917-rc2"
model = ""
name = "DeepSeek-V4.1-Flash EXL3 3bpw (2x Spark, TP2/DCP2, DSpark K=3, 1M)"
port = 8888
required_nodes = 2
cluster = "8nodes-switch"
tensor_parallel_size = 2
distributed_backend = "mp"
shm_size = "2g"
devices = ["/dev/dri/card0"]
memory_limit = "9g"
cpus = 6

# /model + /draft-exl3 are the exact mounts the runtime validates.
entrypoint = [
  "/opt/ds41-venv/bin/python",
  "/opt/ds41-serving/serve.py",
  "serve",
  "/model",
  "--config",
  "/opt/ds41-serving/conservative.yaml",
  "--worker-cls",
  "combined_worker.CombinedWorker",
  "--disable-custom-all-reduce",
]
args = [
  "--served-model-name",
  "deepseek-v41-flash-exl3",
  "--enable-auto-tool-choice",
  "--tool-call-parser",
  "deepseek_v41",
  "--reasoning-parser",
  "deepseek_v41",
]

before_serve = [
  { script = "unlock_display_ram" },
  # Stable per-node symlinks to the current refs/main snapshots (recreated
  # every serve; docker resolves the symlink at bind time).
  "ln -sfn /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/snapshots/$(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/refs/main) /home/kilork/exl3-model",
  "ln -sfn /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/snapshots/$(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/refs/main) /home/kilork/exl3-draft",
]
after_stop = [{ script = "restore_display_ram", on_failure = "warn" }]

[[extra_mounts]]
host = "/home/kilork/exl3-model"
container = "/model"
read_only = true

[[extra_mounts]]
host = "/home/kilork/exl3-draft"
container = "/draft-exl3"
read_only = true

[env]
# DS41 runtime profile (validated against DS41_* envs; kv_cap_mib is
# env-only, not a CLI flag).
DS41_GPU_MEMORY_UTILIZATION = "0.92"
DS41_MAX_MODEL_LEN = "1048576"
DS41_MAX_NUM_SEQS = "2"
DS41_MAX_NUM_BATCHED_TOKENS = "3072"
DS41_LONG_PREFILL_TOKEN_THRESHOLD = "2816"
DS41_KV_CAP_MIB = "1536"
# Feature gates (serve.py hard-requires the '1' set; defaults are 1, set
# explicitly so the guard passes).
DS41_ENABLE_COOPERATIVE_MOE = "1"
DS41_ENABLE_DCP2 = "1"
DS41_ENABLE_DSPARK = "1"
DS41_ENABLE_SSD_VOCAB = "1"
DS41_SSD_ENGRAM_SOURCE = "/model/engrams"
DS41_ENGRAM_CACHE_MIB = "64"
DS41_DRAFT_EXL3_PATH = "/draft-exl3"
# Engine / offline / threading.
VLLM_PLUGINS = "ds41"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_NO_USAGE_STATS = "1"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
DSV41_IO_THREADS = "96"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
MAX_JOBS = "1"
MKL_NUM_THREADS = "4"
OPENBLAS_NUM_THREADS = "4"
TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"
MALLOC_ARENA_MAX = "2"
MALLOC_TRIM_THRESHOLD_ = "131072"
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "128"
# NCCL pins (on top of the launcher's cluster NCCL env).
NCCL_NVLS_ENABLE = "0"
NCCL_CROSS_NIC = "0"
NCCL_IB_MERGE_NICS = "0"
NCCL_CUMEM_ENABLE = "0"
NCCL_MAX_CTAS = "8"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_PROTO = "^LL128"
NCCL_MAX_NCHANNELS = "8"
NCCL_IB_ROCE_VERSION_NUM = "2"