exl3: temporary 2 GB display-reserve unlock (unlock/restore builtins) + per-recipe device passthrough

host-scripts/restore-display-ram.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# restore-display-ram.sh — restore nvidia_drm to the host default (no
+# modeset). Counterpart to unlock-display-ram.sh (the temporary 2 GB
+# display-reserve unlock): run after the EXL3 2x containers are stopped.
+#
+# Temporary by design: no /etc/modprobe.d entry is ever written; the
+# default modprobe state matches a fresh boot (firmware display reserve
+# stays at its 0.5 GB default until the next unlock).
+set -euo pipefail
+
+if [ ! -d /sys/module/nvidia_drm ]; then
+    echo "nvidia_drm not loaded — nothing to restore"
+    exit 0
+fi
+
+modeset=$(cat /sys/module/nvidia_drm/parameters/modeset)
+if [ "$modeset" != "Y" ]; then
+    echo "nvidia_drm already at default (modeset=$modeset) — nothing to do"
+    exit 0
+fi
+
+if ! sudo rmmod nvidia_drm 2>/dev/null; then
+    echo "nvidia_drm busy — waiting 3 s and retrying once"
+    sleep 3
+    sudo rmmod nvidia_drm
+fi
+sudo modprobe nvidia_drm
+echo "nvidia_drm restored to default (modeset=$(cat /sys/module/nvidia_drm/parameters/modeset))"

host-scripts/unlock-display-ram.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# unlock-display-ram.sh — TEMPORARY 2 GiB display-reserve unlock.
+#
+# Remods nvidia_drm with modeset=1 fbdev=0 so the headless DGX Spark's
+# display framebuffer reserve (2 GB in firmware; 1.75 GiB usable as an
+# external CUDA memory pool) becomes available to the EXL3 2x runtime
+# (libds41_display_kv.so maps /dev/dri/card0 into the KV allocator).
+#
+# Temporary by design: no UEFI/firmware change, no /etc/modprobe.d entry,
+# no initramfs change — the state resets to firmware defaults on reboot.
+# Counterpart: restore-display-ram.sh (plain modprobe back).
+#
+# Requires the GPU to be idle (nvidia_drm in use by a running container
+# will refuse rmmod).
+set -euo pipefail
+
+if [ -d /sys/module/nvidia_drm ]; then
+    modeset=$(cat /sys/module/nvidia_drm/parameters/modeset)
+    fbdev=$(cat /sys/module/nvidia_drm/parameters/fbdev)
+    if [ "$modeset" = "Y" ] && [ "$fbdev" = "N" ]; then
+        if [ -e /dev/dri/card0 ]; then
+            echo "nvidia_drm already modeset=1 fbdev=0, /dev/dri/card0 present"
+            exit 0
+        fi
+        echo "nvidia_drm modeset=1 fbdev=0 but /dev/dri/card0 missing — remod anyway"
+        sudo rmmod nvidia_drm
+    else
+        echo "current nvidia_drm: modeset=$modeset fbdev=$fbdev — remodding"
+        sudo rmmod nvidia_drm
+    fi
+fi
+
+sudo modprobe nvidia_drm modeset=1 fbdev=0
+
+[ "$(cat /sys/module/nvidia_drm/parameters/modeset)" = "Y" ] || {
+    echo "ERROR: modeset not active after modprobe" >&2; exit 1; }
+[ "$(cat /sys/module/nvidia_drm/parameters/fbdev)" = "N" ] || {
+    echo "ERROR: fbdev not disabled after modprobe" >&2; exit 1; }
+if [ ! -e /dev/dri/card0 ]; then
+    sleep 2
+fi
+[ -e /dev/dri/card0 ] || { echo "ERROR: /dev/dri/card0 missing after modprobe" >&2; exit 1; }
+echo "display reserve unlocked: nvidia_drm modeset=1 fbdev=0, /dev/dri/card0 present"

src/after_stop.rs

@@ -215,6 +215,7 @@ mod tests {
             sync_image: true,
             docker_image_dir: None,
             shm_size: String::new(),
+            devices: vec![],
             extra_mounts: vec![],
             build: Default::default(),
             composed_model: None,

src/before_serve.rs

@@ -383,6 +383,7 @@ mod tests {
             sync_image: true,
             docker_image_dir: None,
             shm_size: String::new(),
+            devices: vec![],
             extra_mounts: vec![],
             build: Default::default(),
             composed_model: None,

src/builtin_scripts.rs

@@ -29,6 +29,16 @@ pub static BUILTIN_SCRIPTS: &[BuiltinScript] = &[
         alias: "restore_dgx_services",
         body: include_str!("../host-scripts/restore-dgx-services.sh"),
     },
+    // Temporary (per-boot) 2 GiB display-reserve unlock for the EXL3 2x
+    // runtime (nvidia_drm modeset=1 fbdev=0 remod; no UEFI/boot changes).
+    BuiltinScript {
+        alias: "unlock_display_ram",
+        body: include_str!("../host-scripts/unlock-display-ram.sh"),
+    },
+    BuiltinScript {
+        alias: "restore_display_ram",
+        body: include_str!("../host-scripts/restore-display-ram.sh"),
+    },
 ];
 
 /// Resolve an alias to its script body. Returns `None` (and the caller

src/config/recipe.rs

@@ -90,6 +90,11 @@ pub struct Recipe {
     pub distributed_backend: Option<DistributedBackend>,
     #[serde(default = "default_shm_size")]
     pub shm_size: String,
+    /// Host device nodes to pass through into every container as
+    /// `--device <d>:<d>` (e.g. `["/dev/dri/card0"]` for the EXL3 2x
+    /// display-pool KV runtime). Empty (default) = no passthrough.
+    #[serde(default)]
+    pub devices: Vec<String>,
     #[serde(default)]
     pub before_serve: Vec<PreServeCommand>,
     /// Commands run per node **after** stopping containers (`sm121 stop`);

src/docker/command.rs

@@ -298,6 +298,9 @@ fn build_docker_flags(ctx: &DockerCommandCtx) -> Vec<String> {
     if !ctx.recipe.shm_size.is_empty() {
         flags.push(format!("--shm-size {}", ctx.recipe.shm_size.as_str()));
     }
+    for device in &ctx.recipe.devices {
+        flags.push(format!("--device {}:{}", device, device));
+    }
     if !ctx.keep {
         flags.push("--rm".to_string());
     }

src/docker/command_tests.rs

@@ -105,6 +105,7 @@ fn base_recipe() -> Recipe {
         venv_setup_script: None,
         mods: Default::default(),
         shm_size: String::new(),
+        devices: vec![],
         sync_image: true,
         docker_image_dir: None,
         extra_mounts: vec![],
@@ -330,3 +331,20 @@ fn clear_paths_dedups_and_sorts() {
     assert_eq!(clear_paths(&mounts), vec!["/a/.cache/jit", "/b/.cache/jit"]);
     assert!(clear_paths(&[]).is_empty());
 }
+
+#[test]
+fn devices_pass_through_as_docker_flags() {
+    let mut r = base_recipe();
+    r.devices = vec!["/dev/dri/card0".into()];
+    let cmd = run_with(&r);
+    assert!(
+        cmd.contains("--device /dev/dri/card0:/dev/dri/card0"),
+        "missing --device passthrough in: {cmd}"
+    );
+}
+
+#[test]
+fn no_devices_no_flag() {
+    let cmd = run_with(&base_recipe());
+    assert!(!cmd.contains("--device"));
+}

src/docker/mirror.rs

@@ -203,6 +203,7 @@ mod tests {
             venv_setup_script: None,
             mods: Default::default(),
             shm_size: String::new(),
+            devices: vec![],
             sync_image: true,
             docker_image_dir: None,
             extra_mounts: vec![],

src/docker/sglang_tests.rs

@@ -31,6 +31,7 @@ fn sglang_recipe() -> Recipe {
         venv_setup_script: None,
         mods: Default::default(),
         shm_size: String::new(),
+        devices: vec![],
         sync_image: true,
         docker_image_dir: None,
         extra_mounts: vec![],

src/docker/tests.rs

@@ -71,6 +71,7 @@ fn trec() -> Recipe {
         venv_setup_script: None,
         mods: Default::default(),
         shm_size: String::new(),
+        devices: vec![],
         sync_image: true,
         docker_image_dir: None,
         extra_mounts: vec![],

src/doctor.rs

@@ -383,6 +383,7 @@ mod tests {
             sync_image: true,
             docker_image_dir: None,
             shm_size: String::new(),
+            devices: vec![],
             extra_mounts: vec![],
             build: Default::default(),
             composed_model: None,

src/engine_flags.rs

@@ -135,6 +135,7 @@ mod tests {
             venv_setup_script: None,
             mods: Default::default(),
             shm_size: String::new(),
+            devices: vec![],
             sync_image: true,
             docker_image_dir: None,
             extra_mounts: vec![],

src/tmux/commands.rs

@@ -169,6 +169,7 @@ mod tests {
             venv_setup_script: None,
             mods: Default::default(),
             shm_size: String::new(),
+            devices: vec![],
             sync_image: true,
             docker_image_dir: None,
             extra_mounts: vec![],