launcher/recipes/dsv41-flash-exl3-2x/recipe.toml

# DeepSeek-V4.1-Flash EXL3 3bpw — coolbho3k "2x DGX Spark" production recipe,
# adapted to sm121 (rc2 image, pinned at release commit 878e0ee).
#
# 2x Spark over 200G RoCE (8nodes-switch, first two free ranks):
# TP2/DCP2, DSpark K=3 drafter, EXL3 3bpw experts, FP4 main KV + FP8 SWA,
# 1M context, GMU 0.92, max-num-seqs 2 (C2; the runtime's cooperative
# profile validates 1..6), prefill batch 3072.
#
# 2 GB headless-RAM unlock: the runtime's display-pool KV (1.75 GiB of the
# firmware's 2 GB display reserve per GPU) needs nvidia_drm remoded with
# modeset=1 fbdev=0. That is TEMPORARY by design (technology still in
# approval): before_serve remods per boot, after_stop restores the plain
# module. No UEFI change, no /etc/modprobe.d entry, resets on reboot.
#
# Model + drafter overlay: node-local NVMe mirror of the NAS hub at
# /home/kilork/hf-local/hub (EXL3 streams weights on EVERY decode step — from
# NFS that costs ~2/3 of decode; staged here per node, the read is local).
# Mirror rule: same hub-root-relative layout; refs/main stays pinned to the
# NAS (authoritative). Files not staged locally — engram-page15-v1 (optional
# per its release notes, +1.4% decode) — resolve through the NAS fallback,
# read-only. The launch nodes bind every referenced file per-path at /model
# and /draft-exl3 (the runtime hard-requires the model at exactly /model).
#
# Cgroup: the runtime's serving guard hard-requires a 9 GiB / no-swap /
# <=6-CPU cgroup (memory.max==9g, swap 0, cpu quota <=6) — memory_limit +
# cpus below.
engine = "vllm"
docker_image = "ghcr.io/coolbho3k/deepseek-v4.1-flash-exl3-3bpw-2x-dgx-spark:20260917-rc2"
model = ""
name = "DeepSeek-V4.1-Flash EXL3 3bpw (2x Spark, TP2/DCP2, DSpark K=3, 1M)"
required_nodes = 2
cluster = "8nodes-switch"
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "2g"
devices = ["/dev/dri/card0"]
memory_limit = "9g"
cpus = 6
extra_mounts_manifest = "/tmp/sm121-exl3-mounts.txt"

# /model + /draft-exl3 are the exact mounts the runtime validates.
entrypoint = [
  "/opt/ds41-venv/bin/python",
  "/opt/ds41-serving/serve.py",
  "serve",
  "/model",
  "--config",
  "/opt/ds41-serving/conservative.yaml",
  "--worker-cls",
  "combined_worker.CombinedWorker",
  "--disable-custom-all-reduce",
]
args = [
  "--served-model-name",
  "deepseek-v41-flash-exl3",
  "--enable-auto-tool-choice",
  "--tool-call-parser",
  "deepseek_v41",
  "--reasoning-parser",
  "deepseek_v41",
]

before_serve = [
  { script = "unlock_display_ram" },
  # Model + drafter staging: the hub snapshot's internal relative symlinks
  # (../../../blobs/…) only resolve at the real hub depth. The full repo dir
  # (blobs/ + snapshots/) is mounted at /hub, and the staging dir at /model
  # holds RELATIVE symlinks (../hub/...) into it — the kernel lands on the
  # real correct-depth tree and the inner links resolve.
  "python3 - models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw $(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/refs/main) /model models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw $(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/refs/main) /draft-exl3 <<'GENSTAGE'\nimport os, sys\nHUB = '/home/kilork/hf-local/hub'; NAS = '/mnt/nas/ai/huggingface/hub'\ndef fin(p):\n    r = os.path.realpath(p)\n    return r if os.path.exists(r) else r.replace(HUB, NAS, 1)\nMANIFEST = '/tmp/sm121-exl3-mounts.txt'\ndef gen(repo, rev, out):\n    snap = os.path.join(HUB, repo, 'snapshots', rev)\n    lines = []\n    for name in sorted(os.listdir(snap)):\n        src = os.path.join(snap, name)\n        real = fin(src)\n        if os.path.isdir(real):\n            for root, _, files in os.walk(real):\n                for fn in sorted(files):\n                    fp = os.path.join(root, fn)\n                    freal = fin(fp)\n                    sub = os.path.relpath(fp, real)\n                    lines.append(freal + '\\t' + os.path.join(out, name, sub))\n        else:\n            lines.append(real + '\\t' + os.path.join(out, name))\n    ns = set(os.listdir(snap))\n    nasnap = os.path.join(NAS, repo, 'snapshots', rev)\n    for name in sorted(os.listdir(nasnap)):\n        if name in ns:\n            continue\n        src = os.path.join(nasnap, name)\n        if not os.path.isdir(src):\n            continue\n        for root, _, files in os.walk(src):\n            for fn in sorted(files):\n                fp = os.path.join(root, fn)\n                sub = os.path.relpath(fp, src)\n                lines.append(fp + '\\t' + os.path.join(out, name, sub))\n    with open(MANIFEST, 'a') as f:\n        f.write('\\n'.join(lines) + '\\n')\n    print('manifest', MANIFEST, len(lines), 'mounts for', out, flush=True)\nrepo, rev, out, drepo, drev, dout = sys.argv[1:7]\nopen(MANIFEST, 'w').close()  # truncate once before both gens\ngen(repo, rev, out)\ngen(drepo, drev, dout)\nGENSTAGE",
  # The runtime kit (serve.py + conservative.yaml + overlay .so + AOT gemm)
  # is NOT in the image — it is the pinned release/runtime tree, staged once
  # per node and bind-mounted as overlays (serving/ -> /opt/ds41-serving,
  # aot/... -> flashinfer AOT dir). One-time git clone at the pinned commit.
  "mkdir -p /home/kilork/exl3-cache",
  "sudo rm -rf /home/kilork/exl3-cache/ds41-backend-attestation /home/kilork/exl3-cache/ds41-combined-ready /home/kilork/exl3-cache/ds41-display-kv",
  "if [ ! -f /home/kilork/exl3-kit/878e0ee/serving/serve.py ]; then rm -rf /home/kilork/exl3-kit/.src-878e0ee /home/kilork/exl3-kit/878e0ee && git clone --quiet https://github.com/coolbho3k/DeepSeek-v4.1-Flash-2x-DGX-Spark.git /home/kilork/exl3-kit/.src-878e0ee && (cd /home/kilork/exl3-kit/.src-878e0ee && git checkout --quiet 878e0eecd893fadc69ad2d58b2df0fabb0fae2ee) && mv /home/kilork/exl3-kit/.src-878e0ee/release/runtime /home/kilork/exl3-kit/878e0ee && rm -rf /home/kilork/exl3-kit/.src-878e0ee; fi && test -f /home/kilork/exl3-kit/878e0ee/serving/serve.py",
]
after_stop = [{ script = "restore_display_ram", on_failure = "warn" }]

# Hub dir (both repos + hub-level blobs/): the repo blobs/ is itself a layer
# of relative symlinks into the sibling hub-level blobs/<xx>/<sha>, so the
# whole hub tree must be in scope for the chain to resolve.
[[extra_mounts]]
host = "/mnt/nas/ai/huggingface/hub"
container = "/hub"
read_only = true

# /model and /draft-exl3 are NOT dir-mounted: the runtime opens the shard
# files with O_NOFOLLOW (symlink final component => ELOOP), so the
# before-serve hook generates a per-file bind manifest instead
# (extra_mounts_manifest) — the container sees regular files only.

# Runtime kit overlays (pinned 878e0ee): the image's /opt/ds41-serving is
# shadowed by the kit's serving tree; the AOT gemm dir is bind-mounted ro.
[[extra_mounts]]
host = "/home/kilork/exl3-kit/878e0ee/serving"
container = "/opt/ds41-serving"
read_only = true

[[extra_mounts]]
host = "/home/kilork/exl3-kit/878e0ee/aot/mxfp8_gemm_cutlass_sm120"
container = "/usr/local/lib/python3.12/dist-packages/flashinfer/data/aot/mxfp8_gemm_cutlass_sm120"
read_only = true

# Node-local runtime cache (triton/tilelang/torch-ext/cuda flashinfer workspaces).
[[extra_mounts]]
host = "/home/kilork/exl3-cache"
container = "/cache"

[env]
# DS41 runtime profile (validated against DS41_* envs; kv_cap_mib is
# env-only, not a CLI flag).
DS41_GPU_MEMORY_UTILIZATION = "0.92"
DS41_MAX_MODEL_LEN = "1048576"
DS41_MAX_NUM_SEQS = "2"
DS41_MAX_NUM_BATCHED_TOKENS = "3072"
DS41_LONG_PREFILL_TOKEN_THRESHOLD = "2816"
DS41_KV_CAP_MIB = "0" # display-only candidate: all KV from the 2 GB display pool
# Feature gates (serve.py hard-requires the '1' set; defaults are 1, set
# explicitly so the guard passes).
DS41_ENABLE_COOPERATIVE_MOE = "1"
DS41_ENABLE_DCP2 = "1"
DS41_ENABLE_DSPARK = "1"
DS41_ENABLE_SSD_VOCAB = "1"
DS41_SSD_ENGRAM_SOURCE = "/model/engrams"
DS41_ENGRAM_CACHE_MIB = "64"
DS41_DRAFT_EXL3_PATH = "/draft-exl3"
# Node-local cache directories (/cache mount, persistent between serves).
VLLM_CACHE_ROOT = "/cache/vllm"
TILELANG_CACHE_DIR = "/cache/tilelang"
TRITON_CACHE_DIR = "/cache/triton"
TORCH_EXTENSIONS_DIR = "/cache/torch-extensions"
CUDA_CACHE_PATH = "/cache/cuda"
XDG_CACHE_HOME = "/cache"
FLASHINFER_WORKSPACE_BASE = "/cache/flashinfer"
TMPDIR = "/cache/tmp"
# Engine / offline / threading.
VLLM_PLUGINS = "ds41"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_NO_USAGE_STATS = "1"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
DSV41_IO_THREADS = "96"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
MAX_JOBS = "1"
MKL_NUM_THREADS = "4"
OPENBLAS_NUM_THREADS = "4"
TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"
MALLOC_ARENA_MAX = "2"
MALLOC_TRIM_THRESHOLD_ = "131072"
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "128"
# NCCL pins (on top of the launcher's cluster NCCL env).
NCCL_NVLS_ENABLE = "0"
NCCL_CROSS_NIC = "0"
NCCL_IB_MERGE_NICS = "0"
NCCL_CUMEM_ENABLE = "0"
NCCL_MAX_CTAS = "8"
NCCL_BUFFSIZE = "1048576"
NCCL_LL128_BUFFSIZE = "262144"
NCCL_PROTO = "^LL128"
NCCL_MAX_NCHANNELS = "8"
NCCL_IB_ROCE_VERSION_NUM = "2"