exl3 2x: recipe (rc2 image, /model mount, 9g/6cpu cguard, display unlock) + launcher memory_limit/cpus flags

recipes/dsv41-flash-exl3-2x/recipe.toml

@@ -0,0 +1,125 @@
+# DeepSeek-V4.1-Flash EXL3 3bpw — coolbho3k "2x DGX Spark" production recipe,
+# adapted to sm121 (rc2 image, pinned at release commit 878e0ee).
+#
+# 2x Spark over 200G RoCE (8nodes-switch, first two free ranks):
+# TP2/DCP2, DSpark K=3 drafter, EXL3 3bpw experts, FP4 main KV + FP8 SWA,
+# 1M context, GMU 0.92, max-num-seqs 2 (C2; the runtime's cooperative
+# profile validates 1..6), prefill batch 3072.
+#
+# 2 GB headless-RAM unlock: the runtime's display-pool KV (1.75 GiB of the
+# firmware's 2 GB display reserve per GPU) needs nvidia_drm remoded with
+# modeset=1 fbdev=0. That is TEMPORARY by design (technology still in
+# approval): before_serve remods per boot, after_stop restores the plain
+# module. No UEFI change, no /etc/modprobe.d entry, resets on reboot.
+#
+# Model + drafter overlay live on the shared NAS (hub layout); the launch
+# nodes symlink the current refs/main snapshot and bind it at /model (the
+# runtime hard-requires the model at exactly /model) and the drafter overlay
+# at /draft-exl3.
+#
+# Cgroup: the runtime's serving guard hard-requires a 9 GiB / no-swap /
+# <=6-CPU cgroup (memory.max==9g, swap 0, cpu quota <=6) — memory_limit +
+# cpus below.
+engine = "vllm"
+docker_image = "ghcr.io/coolbho3k/deepseek-v4.1-flash-exl3-3bpw-2x-dgx-spark:20260917-rc2"
+model = ""
+name = "DeepSeek-V4.1-Flash EXL3 3bpw (2x Spark, TP2/DCP2, DSpark K=3, 1M)"
+port = 8888
+required_nodes = 2
+cluster = "8nodes-switch"
+tensor_parallel_size = 2
+distributed_backend = "mp"
+shm_size = "2g"
+devices = ["/dev/dri/card0"]
+memory_limit = "9g"
+cpus = 6
+
+# /model + /draft-exl3 are the exact mounts the runtime validates.
+entrypoint = [
+  "/opt/ds41-venv/bin/python",
+  "/opt/ds41-serving/serve.py",
+  "serve",
+  "/model",
+  "--config",
+  "/opt/ds41-serving/conservative.yaml",
+  "--worker-cls",
+  "combined_worker.CombinedWorker",
+  "--disable-custom-all-reduce",
+]
+args = [
+  "--served-model-name",
+  "deepseek-v41-flash-exl3",
+  "--enable-auto-tool-choice",
+  "--tool-call-parser",
+  "deepseek_v41",
+  "--reasoning-parser",
+  "deepseek_v41",
+]
+
+before_serve = [
+  { script = "unlock_display_ram" },
+  # Stable per-node symlinks to the current refs/main snapshots (recreated
+  # every serve; docker resolves the symlink at bind time).
+  "ln -sfn /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/snapshots/$(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-EXL3-3bpw/refs/main) /home/kilork/exl3-model",
+  "ln -sfn /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/snapshots/$(cat /mnt/nas/ai/huggingface/hub/models--coolbho3k--DeepSeek-V4.1-Flash-DSpark-EXL3-3bpw/refs/main) /home/kilork/exl3-draft",
+]
+after_stop = [{ script = "restore_display_ram", on_failure = "warn" }]
+
+[[extra_mounts]]
+host = "/home/kilork/exl3-model"
+container = "/model"
+read_only = true
+
+[[extra_mounts]]
+host = "/home/kilork/exl3-draft"
+container = "/draft-exl3"
+read_only = true
+
+[env]
+# DS41 runtime profile (validated against DS41_* envs; kv_cap_mib is
+# env-only, not a CLI flag).
+DS41_GPU_MEMORY_UTILIZATION = "0.92"
+DS41_MAX_MODEL_LEN = "1048576"
+DS41_MAX_NUM_SEQS = "2"
+DS41_MAX_NUM_BATCHED_TOKENS = "3072"
+DS41_LONG_PREFILL_TOKEN_THRESHOLD = "2816"
+DS41_KV_CAP_MIB = "1536"
+# Feature gates (serve.py hard-requires the '1' set; defaults are 1, set
+# explicitly so the guard passes).
+DS41_ENABLE_COOPERATIVE_MOE = "1"
+DS41_ENABLE_DCP2 = "1"
+DS41_ENABLE_DSPARK = "1"
+DS41_ENABLE_SSD_VOCAB = "1"
+DS41_SSD_ENGRAM_SOURCE = "/model/engrams"
+DS41_ENGRAM_CACHE_MIB = "64"
+DS41_DRAFT_EXL3_PATH = "/draft-exl3"
+# Engine / offline / threading.
+VLLM_PLUGINS = "ds41"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_WORKER_MULTIPROC_METHOD = "spawn"
+VLLM_NO_USAGE_STATS = "1"
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+DSV41_IO_THREADS = "96"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+MAX_JOBS = "1"
+MKL_NUM_THREADS = "4"
+OPENBLAS_NUM_THREADS = "4"
+TORCH_NCCL_ASYNC_ERROR_HANDLING = "1"
+MALLOC_ARENA_MAX = "2"
+MALLOC_TRIM_THRESHOLD_ = "131072"
+VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "128"
+# NCCL pins (on top of the launcher's cluster NCCL env).
+NCCL_NVLS_ENABLE = "0"
+NCCL_CROSS_NIC = "0"
+NCCL_IB_MERGE_NICS = "0"
+NCCL_CUMEM_ENABLE = "0"
+NCCL_MAX_CTAS = "8"
+NCCL_BUFFSIZE = "1048576"
+NCCL_LL128_BUFFSIZE = "262144"
+NCCL_PROTO = "^LL128"
+NCCL_MAX_NCHANNELS = "8"
+NCCL_IB_ROCE_VERSION_NUM = "2"

src/after_stop.rs

@@ -216,6 +216,8 @@ mod tests {
             docker_image_dir: None,
             shm_size: String::new(),
             devices: vec![],
+            memory_limit: None,
+            cpus: None,
             extra_mounts: vec![],
             build: Default::default(),
             composed_model: None,

src/before_serve.rs

@@ -384,6 +384,8 @@ mod tests {
             docker_image_dir: None,
             shm_size: String::new(),
             devices: vec![],
+            memory_limit: None,
+            cpus: None,
             extra_mounts: vec![],
             build: Default::default(),
             composed_model: None,

src/config/recipe.rs

@@ -95,6 +95,15 @@ pub struct Recipe {
     /// display-pool KV runtime). Empty (default) = no passthrough.
     #[serde(default)]
     pub devices: Vec<String>,
+    /// Memory cgroup limit for the container (e.g. `"9g"`); also sets
+    /// `--memory-swap` to the same value (no swap). Some runtimes hard-
+    /// guard on the serving cgroup; leave empty for the default unlimited.
+    #[serde(default)]
+    pub memory_limit: Option<String>,
+    /// CPU count cgroup limit for the container (e.g. `6`); leave empty
+    /// for the default unlimited. Pairs with [`Self::memory_limit`].
+    #[serde(default)]
+    pub cpus: Option<f64>,
     #[serde(default)]
     pub before_serve: Vec<PreServeCommand>,
     /// Commands run per node **after** stopping containers (`sm121 stop`);

src/docker/command.rs

@@ -301,6 +301,15 @@ fn build_docker_flags(ctx: &DockerCommandCtx) -> Vec<String> {
     for device in &ctx.recipe.devices {
         flags.push(format!("--device {}:{}", device, device));
     }
+    if let Some(mem) = &ctx.recipe.memory_limit {
+        flags.push(format!("--memory {}", mem));
+        // Same value for memory-swap = no swap allowed (some runtimes
+        // hard-guard on the 9GiB/no-swap serving cgroup).
+        flags.push(format!("--memory-swap {}", mem));
+    }
+    if let Some(cpus) = ctx.recipe.cpus {
+        flags.push(format!("--cpus {}", cpus));
+    }
     if !ctx.keep {
         flags.push("--rm".to_string());
     }

src/docker/command_tests.rs

@@ -106,6 +106,8 @@ fn base_recipe() -> Recipe {
         mods: Default::default(),
         shm_size: String::new(),
         devices: vec![],
+        memory_limit: None,
+        cpus: None,
         sync_image: true,
         docker_image_dir: None,
         extra_mounts: vec![],
@@ -348,3 +350,21 @@ fn no_devices_no_flag() {
     let cmd = run_with(&base_recipe());
     assert!(!cmd.contains("--device"));
 }
+
+#[test]
+fn memory_and_cpus_cgroup_flags() {
+    let mut r = base_recipe();
+    r.memory_limit = Some("9g".into());
+    r.cpus = Some(6.0);
+    let cmd = run_with(&r);
+    assert!(cmd.contains("--memory 9g"), "in: {cmd}");
+    assert!(cmd.contains("--memory-swap 9g"), "in: {cmd}");
+    assert!(cmd.contains("--cpus 6"), "in: {cmd}");
+}
+
+#[test]
+fn no_memory_limit_no_flags() {
+    let cmd = run_with(&base_recipe());
+    assert!(!cmd.contains("--memory"));
+    assert!(!cmd.contains("--cpus"));
+}

src/docker/mirror.rs

@@ -204,6 +204,8 @@ mod tests {
             mods: Default::default(),
             shm_size: String::new(),
             devices: vec![],
+            memory_limit: None,
+            cpus: None,
             sync_image: true,
             docker_image_dir: None,
             extra_mounts: vec![],

src/docker/sglang_tests.rs

@@ -32,6 +32,8 @@ fn sglang_recipe() -> Recipe {
         mods: Default::default(),
         shm_size: String::new(),
         devices: vec![],
+        memory_limit: None,
+        cpus: None,
         sync_image: true,
         docker_image_dir: None,
         extra_mounts: vec![],

src/docker/tests.rs

@@ -72,6 +72,8 @@ fn trec() -> Recipe {
         mods: Default::default(),
         shm_size: String::new(),
         devices: vec![],
+        memory_limit: None,
+        cpus: None,
         sync_image: true,
         docker_image_dir: None,
         extra_mounts: vec![],

src/doctor.rs

@@ -384,6 +384,8 @@ mod tests {
             docker_image_dir: None,
             shm_size: String::new(),
             devices: vec![],
+            memory_limit: None,
+            cpus: None,
             extra_mounts: vec![],
             build: Default::default(),
             composed_model: None,

src/engine_flags.rs

@@ -136,6 +136,8 @@ mod tests {
             mods: Default::default(),
             shm_size: String::new(),
             devices: vec![],
+            memory_limit: None,
+            cpus: None,
             sync_image: true,
             docker_image_dir: None,
             extra_mounts: vec![],

src/tmux/commands.rs

@@ -170,6 +170,8 @@ mod tests {
             mods: Default::default(),
             shm_size: String::new(),
             devices: vec![],
+            memory_limit: None,
+            cpus: None,
             sync_image: true,
             docker_image_dir: None,
             extra_mounts: vec![],