serve: refuse multi-node recipes without distributed_backend — a recipe with required_nodes>1 but no distributed_backend launched N INDEPENDENT single-node engines (no --nnodes/--node-rank/--master-addr/--tensor-parallel-size), each trying to load the FULL checkpoint into one 121 GiB Spark: the actual root cause of the four glm53-flash-fp4-local-2x stalls (b12x routing OOM was the symptom). Guard bails loudly at serve time; the recipe fix (distributed_backend = "mp") restores the dist flags.

src/docker/start.rs

@@ -8,7 +8,7 @@ use super::mirror;
 use super::shq;
 use super::{Config, NcclConfig, Recipe, StartClusterOpts};
 use crate::config::{Cluster, Node};
-use anyhow::Result;
+use anyhow::{Result, bail};
 
 use tokio::time::{Duration, sleep};
 use tracing::info;
@@ -42,6 +42,17 @@ pub(crate) async fn start_cluster(
         }
     };
     let nnodes = nodes.len() as u32;
+    // A multi-node recipe without an explicit distributed backend would launch
+    // N INDEPENDENT single-node engines (no dist flags, each loading the full
+    // checkpoint) — the silent root cause of the glm53-flash-fp4-local-2x
+    // "stalls". Refuse instead.
+    if nnodes > 1 && recipe.distributed_backend.is_none() {
+        bail!(
+            "recipe '{}' runs on {} nodes but sets no distributed_backend — add `distributed_backend = "mp"` (or "ray") so the launcher emits the rendezvous flags; otherwise every node would launch an independent single-node engine and try to load the full checkpoint",
+            recipe_name,
+            nnodes
+        );
+    }
     // `--mirror`: ensure every repo the recipe needs exists in the per-node
     // mirror before launch (copies from the shared HF cache when missing).
     if let Some(ref mirror) = opts.mirror {