feat(recipes): add volumes support and GLM-5.2 Hybrid FP8-MXFP4 recipe

- Add  to Recipe struct for custom
  Docker volume mounts (host_path → container_path)
- Wire volumes into docker run command builder (src/docker/mod.rs)
- Add volumes test in docker/tests.rs
- Add volumes field to test Recipe constructors in before_serve.rs and
  doctor.rs

GLM-5.2 Hybrid FP8-MXFP4 (aidendle94/GLM-5.2-Hybrid-FP8-MXFP4):
- New recipe using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.2
- TP=4, DCP=4, MTP k=4, NVFP4 KV cache with per-token scaling
- 4× DGX Spark / GB10 (sm_121a), 262K context, 688K-token KV pool
- References: NVIDIA forum #377598, model card

Experimental image switch for existing recipes:
- glm52-nf3-hybrid-context/fast: vllm-nf3-hybrid:probe → production-hybrid-1.0
- glm52-int4-int8: sparkrun-vllm-ds4-gb10:glm52 → production-hybrid-1.0

recipes/glm52-hybrid-fp8-mxfp4/recipe.toml

@@ -0,0 +1,126 @@
+# GLM-5.2 Hybrid FP8+NVFP4+MXFP4 — 4× DGX Spark (GB10, sm_121a), TP=4, DCP=4.
+#
+# Model: aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
+#   → https://huggingface.co/aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
+#
+# Mixed-precision GLM-5.2 checkpoint:
+#   - Attention (MLA projections, indexer):  block-FP8
+#   - Dense MLPs & shared experts:           NVFP4 (group-16)
+#   - Routed experts, layers 3–77:           MXFP4 (e2m1 + ue8m0 per-32)
+#   - MTP draft experts (layer 78):          MXFP4
+#   - Norms, gates, lm_head, MTP dense:      BF16
+#
+# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.2
+#   vLLM + b12x GB10 kernels with hybrid_mxfp4_ct quant config,
+#   index-authoritative loader, NVFP4 KV cache with per-token scaling,
+#   sparse-MLA CKV gather, boot-time kernel prewarm.
+#
+# PREREQUISITES:
+#   - Models cached on every node via HF_HOME:
+#       huggingface-cli download aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
+#       (382 GiB, requires free disk on each node)
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/glm-5-2-hybrid-fp8-nvfp4-mxfp4-optimal-runtime-recipe/377598
+#   https://huggingface.co/aidendle94/GLM-5.2-Hybrid-FP8-MXFP4
+
+engine = "vllm"
+name = "GLM-5.2 Hybrid FP8-MXFP4 (vLLM)"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.2"
+model = "aidendle94/GLM-5.2-Hybrid-FP8-MXFP4"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "10gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Model name
+    "--served-model-name", "glm-5.2",
+    # General
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--enable-prefix-caching",
+    # DCP (Decode Context Parallelism)
+    "--decode-context-parallel-size", "4",
+    "--dcp-comm-backend", "ag_rs",
+    # Engine backends
+    "--attention-backend", "B12X_MLA_SPARSE",
+    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
+    "--hf-overrides", """
+'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
+    # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
+    "--kv-cache-dtype", "nvfp4_ds_mla",
+    "--kv-cache-memory-bytes", "5500000000",
+    # Context & batch config
+    "--max-model-len", "262144",
+    "--max-num-batched-tokens", "2048",
+    "--max-num-seqs", "3",
+    "--gpu-memory-utilization", "0.89",
+    # Speculative decode — MTP k=4 with adaptive depths
+    "--speculative-config", """
+'{"model":"aidendle94/GLM-5.2-Hybrid-FP8-MXFP4/mtp-draft","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
+    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
+    # Timeout safety net (cold boot: ranks finish minutes apart)
+    "--distributed-timeout-seconds", "3600",
+    "--cpu-distributed-timeout-seconds", "3600",
+]
+
+[env]
+# ── b12x suite ────────────────────────────────────────────────────────────
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_DCP_SHARD_DRAFT = "1"
+VLLM_DCP_GLOBAL_TOPK = "1"
+VLLM_DSV4_INDEXER_SP = "1"
+
+# ── MLA CKV gather (sparse gather for decode) ─────────────────────────────
+VLLM_B12X_MLA_CKV_GATHER = "1"
+VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
+VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
+VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
+
+# ── Speculative decode ────────────────────────────────────────────────────
+VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
+
+# ── NVFP4 KV cache ────────────────────────────────────────────────────────
+VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
+
+# ── Timeout safety net ────────────────────────────────────────────────────
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+
+# ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ─────────────────
+# The image bakes a value that GLM-5.2's KV coordinator rejects.
+# Empty string = unset at runtime.
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = ""
+
+# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+
+# ── JIT cache ──────────────────────────────────────────────────────────────
+# Uses launcher default: /root/.cache/huggingface
+# Set XDG_CACHE_HOME to override; mount a persistent volume to avoid
+# ~20 min recompilation on each cold boot.
+
+# ── NCCL overrides ─────────────────────────────────────────────────────────
+# Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME
+# Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN
+NCCL_IB_DISABLE = "0"             # enable IB (override launcher default)
+NCCL_MAX_NCHANNELS = "4"
+NCCL_MIN_NCHANNELS = "4"
+NCCL_CUMEM_ENABLE = "0"
+
+# ── Compute ────────────────────────────────────────────────────────────────
+OMP_NUM_THREADS = "4"
+
+# Volumes not needed — model is a Hugging Face Hub ID, downloaded via HF_HOME

recipes/glm52-int4-int8/recipe.toml

@@ -1,10 +1,8 @@
 # GLM-5.2 QuantTrio Int4-Int8Mix — 4× DGX Spark / GB10 (sm_121), TP=4.
 #
-# Image: sparkrun-vllm-ds4-gb10:glm52
-#   Built from docker-images/glm52-int4-int8/Dockerfile, which overlays
-#   sparkrun-vllm-ds4-gb10:fixed with an entrypoint wrapper that unsets
-#   the baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL before calling the
-#   original /opt/nvidia/nvidia_entrypoint.sh.
+# EXPERIMENT: Using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.0
+# instead of locally-built sparkrun-vllm-ds4-gb10:glm52.
+# Pre-built on Docker Hub with CUDA 13.2.1, custom NCCL, b12x suite.
 #
 # Model: QuantTrio/GLM-5.2-Int4-Int8Mix (mixed Int4/Int8, 10% expert-pruned)
 #   → https://huggingface.co/QuantTrio/GLM-5.2-Int4-Int8Mix
@@ -20,15 +18,14 @@
 
 engine = "vllm"
 name = "GLM-5.2 Int4-Int8Mix (vLLM, DS4 image)"
-docker_image = "sparkrun-vllm-ds4-gb10:glm52"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.0"
 model = "QuantTrio/GLM-5.2-Int4-Int8Mix"
-port = 8210
 required_nodes = 4
 tensor_parallel_size = 4
 pipeline_parallel_size = 1
 distributed_backend = "mp"
 shm_size = "64gb"
-docker_image_dir = "glm52-int4-int8"
+# docker_image_dir = "glm52-int4-int8"             # removed for experiment
 
 entrypoint = ["vllm", "serve"]
 

recipes/glm52-nf3-hybrid-context/recipe.toml

@@ -8,10 +8,11 @@
 #   327GB — top-64 experts/layer in NVFP4, remaining 192 in NF3 (3-bit),
 #   non-expert layers in MXFP8.
 #
-# BUILD THE IMAGE FIRST:
-#   cd docker-images/glm52-nf3-hybrid && bash build.sh
-#   cd docker-images/glm52-nf3-hybrid && bash build.sh
-#   Produces image: vllm-nf3-hybrid:probe
+# EXPERIMENT: Using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.0
+# instead of locally-built vllm-nf3-hybrid:probe. Pre-built on Docker Hub.
+# NOTE: production-hybrid-1.0 lacks NF3 hybrid overlay files
+# (hybrid_loader.py, nf3_kernel.py, nvfp4_kernel.py, mxfp8_tier.json).
+# If it fails, revert to vllm-nf3-hybrid:probe.
 #
 # DCP4 lane: KV sharded across all 4 nodes → 877K-token pool.
 #   Every decode step pays ag_rs allreduce on all 78 layers.
@@ -39,13 +40,13 @@
 
 engine = "vllm"
 name = "GLM-5.2 NF3-Hybrid CONTEXT LANE (vLLM)"
-docker_image = "vllm-nf3-hybrid:probe"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.0"
 model = "madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid"
 required_nodes = 4
 tensor_parallel_size = 4
 pipeline_parallel_size = 1
 distributed_backend = "mp"
-docker_image_dir = "glm52-nf3-hybrid"
+# docker_image_dir = "glm52-nf3-hybrid"             # removed for experiment
 
 entrypoint = ["vllm", "serve"]
 
@@ -63,7 +64,7 @@ args = [
     "--max-cudagraph-capture-size", "32",
     "--gpu-memory-utilization", "0.84",
     # Engine backends
-    "--attention-backend", "B12X_MLA_SPARSE",
+    #"--attention-backend", "B12X_MLA_SPARSE",
     "--moe-backend", "b12x",
     # Scheduling
     "--async-scheduling",

recipes/glm52-nf3-hybrid-fast/recipe.toml

@@ -8,10 +8,11 @@
 #   327GB — top-64 experts/layer in NVFP4, remaining 192 in NF3 (3-bit),
 #   non-expert layers in MXFP8.
 #
-# BUILD THE IMAGE FIRST:
-#   cd docker-images/glm52-nf3-hybrid && bash build.sh
-#   cd docker-images/glm52-nf3-hybrid && bash build.sh
-#   Produces image: vllm-nf3-hybrid:probe
+# EXPERIMENT: Using aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.0
+# instead of locally-built vllm-nf3-hybrid:probe. Pre-built on Docker Hub.
+# NOTE: production-hybrid-1.0 lacks NF3 hybrid overlay files
+# (hybrid_loader.py, nf3_kernel.py, nvfp4_kernel.py, mxfp8_tier.json).
+# If it fails, revert to vllm-nf3-hybrid:probe.
 #
 # DCP1 lane: each sequence's KV cache lives entirely on one node → zero
 #   cross-node attention traffic per decode step. ~1/4 the KV pool of DCP4.
@@ -39,14 +40,14 @@
 
 engine = "vllm"
 name = "GLM-5.2 NF3-Hybrid FAST LANE (vLLM)"
-docker_image = "vllm-nf3-hybrid:probe"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.0"
 model = "madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid"
 required_nodes = 4
 tensor_parallel_size = 4
 pipeline_parallel_size = 1
 distributed_backend = "mp"
 shm_size = "64gb"
-docker_image_dir = "glm52-nf3-hybrid"
+# docker_image_dir = "glm52-nf3-hybrid"             # removed for experiment
 
 entrypoint = ["vllm", "serve"]
 

src/before_serve.rs

@@ -252,6 +252,7 @@ mod tests {
             sync_image: true,
             docker_image_dir: None,
             shm_size: String::new(),
+            volumes: HashMap::new(),
         }
     }
 

src/config/types.rs

@@ -138,6 +138,9 @@ pub struct Recipe {
     /// Build directory for docker-images/ (default: derived from recipe name).
     #[serde(default)]
     pub docker_image_dir: Option<String>,
+    /// Docker volume mounts: host_path -> container_path
+    #[serde(default)]
+    pub volumes: HashMap<String, String>,
 }
 
 impl Config {

src/docker/mod.rs

@@ -445,6 +445,9 @@ fn build_docker_command(ctx: &DockerCommandCtx, node: &Node, is_head: bool) -> V
     if !ctx.mods_remote_path.is_empty() && !ctx.recipe.mods.commands.is_empty() {
         cmd.push(format!("-v {}:/mods:ro", ctx.mods_remote_path));
     }
+    for (host_path, container_path) in &ctx.recipe.volumes {
+        cmd.push(format!("-v {}:{}", host_path, container_path));
+    }
     cmd.extend(collect_docker_env(ctx, node, is_head));
     cmd.push(format!("--name sm121_{}_{}", ctx.recipe_name, node.rank));
     cmd.push(format!("--label sm121.recipe={}", ctx.recipe_name));

src/docker/tests.rs

@@ -62,6 +62,7 @@ fn trec() -> Recipe {
         shm_size: String::new(),
         sync_image: true,
         docker_image_dir: None,
+        volumes: HashMap::new(),
     }
 }
 
@@ -245,6 +246,7 @@ fn sglang_recipe() -> Recipe {
         shm_size: String::new(),
         sync_image: true,
         docker_image_dir: None,
+        volumes: HashMap::new(),
     }
 }
 
@@ -317,6 +319,15 @@ fn sglang_entrypoint_model_path_order() {
     assert!(tp < port, "--tp before --port");
 }
 
+#[test]
+fn volumes_are_mounted() {
+    let mut r = trec();
+    r.volumes = HashMap::from([("/models/glm52-hybrid".into(), "/hybridmodel:ro".into())]);
+    let cmd = run(&r, &tcls().nodes[0], true);
+    assert!(cmd.contains("-v /models/glm52-hybrid:/hybridmodel:ro"));
+    assert!(cmd.contains("-v /home/user/.cache/huggingface:/root/.cache/huggingface"));
+}
+
 #[test]
 fn keep_disables_rm() {
     let cmd = run_keep(&trec(), &tcls().nodes[0], true);

src/doctor.rs

@@ -373,6 +373,7 @@ mod tests {
             sync_image: true,
             docker_image_dir: None,
             shm_size: String::new(),
+            volumes: HashMap::new(),
         }
     }