feat(nccl): auto-detect NCCL_IB_GID_INDEX per node, opt-out via config pin

The RoCE-v2 IPv4 GID index re-numbers across reboots on DGX Spark, so a
pinned gid can go silently stale. When a cluster [nccl] block omits
ib_gid_index (now Option<u32>, default None), serve resolves it at launch
per node by SSHing in and scanning the sysfs IB GID table for the RoCE-v2
IPv4 GID (mirrors the Aiden serving stack, forum 372268/708), then injects
the value into docker -e / tmux export. Explicit ib_gid_index = <n> opts
out (pins). Dry-run prints the detect command and renders the placeholder.

- src/detect.rs: detect_gid_cmd + resolve_nccl_for_node (per-node, dry-run aware)
- docker/mod.rs: extract per-node launch body into docker/launch_node.rs
  (keeps the file under the 500-line gate), call resolve inside the spawn
- docker/command.rs, tmux/mod.rs: render 'auto' when unresolved
- tmux/serve.rs: resolve per node in mp + ray paths
- config.toml (gitignored): switched to auto-detect (pin removed)

AGENTS.md

@@ -401,7 +401,7 @@ extra_entities = ["switch.network"] # cluster-level HA entities
 [nccl."c"]
 ib_hca = "..."
 socket_ifname = "..."
-ib_gid_index = 3
+ib_gid_index = 3        # optional: omit to auto-detect (RoCE-v2 IPv4 GID, per node)
 ib_disable = 0
 
 [home_assistant]
@@ -426,6 +426,17 @@ commands = ["cmd", { command = "cmd", on_failure = "warn" }]
 - `on`: HA `turn_on` (no SSH, requires HA config)
 - `switch --from <src> --to <tgt>`: disable nodes/extras unique to `from`, enable all in `to`
 
+## NCCL_IB_GID_INDEX auto-detect
+
+The RoCE-v2 IPv4 GID index re-numbers across reboots on DGX Spark, so a pinned
+gid can silently go stale. When a cluster's `[nccl]` block **omits**
+`ib_gid_index`, `serve` resolves it at launch **per node** by SSHing in and
+scanning the sysfs IB GID table for the first RoCE-v2 / IPv4-mapped GID
+(mirrors the Aiden serving stack: forum 372268/708). The resolved value is
+injected into the docker `-e` / tmux export, so both deploy targets benefit.
+Set `ib_gid_index = <n>` explicitly to opt out (pin the value). Dry-run prints
+the detect command per node and renders `NCCL_IB_GID_INDEX=auto`.
+
 ## i18n
 
 `fluent-i18n` crate, auto-detect via `set_locale(None)`. FTL in `locales/`.

src/cluster_state.rs

@@ -186,7 +186,7 @@ mod tests {
         NcclConfig {
             ib_hca: "r".to_string(),
             socket_ifname: "e".to_string(),
-            ib_gid_index: 3,
+            ib_gid_index: Some(3),
             ib_disable: 0,
             gloo_socket_ifname: None,
             ucx_net_devices: None,

src/config/tests.rs

@@ -49,7 +49,7 @@ fn tnccl() -> NcclConfig {
     NcclConfig {
         ib_hca: "rocep1s0f0".to_string(),
         socket_ifname: "enp1s0f0np0".to_string(),
-        ib_gid_index: 3,
+        ib_gid_index: Some(3),
         ib_disable: 0,
         gloo_socket_ifname: None,
         ucx_net_devices: None,

src/config/types.rs

@@ -24,7 +24,11 @@ pub struct Node {
 pub struct NcclConfig {
     pub ib_hca: String,
     pub socket_ifname: String,
-    pub ib_gid_index: u32,
+    /// RoCE GID index. `None` (absent in config) = auto-detect on each node
+    /// at serve time (`NCCL_IB_GID_INDEX` is resolved per node from the
+    /// RoCE-v2 IPv4 GID). `Some(n)` = pin explicitly (opt-out of detection).
+    #[serde(default)]
+    pub ib_gid_index: Option<u32>,
     pub ib_disable: u32,
     #[serde(default)]
     pub gloo_socket_ifname: Option<String>,

src/detect.rs

@@ -0,0 +1,167 @@
+//! Per-node RoCE GID index auto-detection.
+//!
+//! `NCCL_IB_GID_INDEX` re-numbers across reboots on DGX Spark (the RoCE-v2
+//! IPv4 GID's index is not stable), so pinning it in config can go stale.
+//! When the cluster's `[nccl]` block omits `ib_gid_index`, the launcher
+//! resolves it per node at serve time by scanning the node's sysfs IB GID
+//! table for the first index whose type is "RoCE v2" over an IPv4-mapped GID
+//! (mirrors the Aiden/0rand serving stack's `compose.head.yaml` gid auto-detect:
+//! https://forums.developer.nvidia.com/t/372268/708). Explicit
+//! `ib_gid_index = <n>` in config opts out (pins the value).
+
+use crate::config::{Config, NcclConfig, Node};
+use crate::ssh::{run_ssh, shq};
+use anyhow::{Context, Result};
+
+/// Build the shell that scans a node's IB GID table for the RoCE-v2 IPv4 GID
+/// index. Mirrors the reference compose head/worker launch script: for each
+/// HCA in `NCCL_IB_HCA`, walk sysfs `gid_attrs/types` + `gids` up to index 15
+/// and pick the first index whose type is "RoCE v2" and whose GID is the
+/// IPv4-mapped form (`0000:0000:0000:0000:0000:ffff:*`).
+pub(crate) fn detect_gid_cmd(hcas: &str) -> String {
+    // `hcas` is a comma-separated HCA list from config (trusted, but shell
+    // quote it anyway so a crafted value cannot inject).
+    let hcas_q = shq(hcas);
+    format!(
+        "for HCA in $(echo {hcas_q} | tr ',' ' '); do \
+           for i in $(seq 0 15); do \
+             t=$(cat /sys/class/infiniband/$HCA/ports/1/gid_attrs/types/$i 2>/dev/null || true); \
+             g=$(cat /sys/class/infiniband/$HCA/ports/1/gids/$i 2>/dev/null || true); \
+             case \"$t\" in *\"RoCE v2\"*) case \"$g\" in *0000:0000:0000:0000:0000:ffff:*) \
+               echo \"$i\"; exit 0;; esac;; esac; \
+           done; \
+         done; \
+         exit 1"
+    )
+}
+
+/// Auto-detect the RoCE GID index on `node` by SSHing in and reading its
+/// sysfs GID table. On failure returns an error naming the HCA list and
+/// pointing at the explicit-`ib_gid_index` opt-out.
+pub(crate) async fn detect_gid_index_on_node(
+    config: &Config,
+    node: &Node,
+    hcas: &str,
+) -> Result<u32> {
+    let out = run_ssh(config, node, detect_gid_cmd(hcas), false)
+        .await
+        .context("NCCL_IB_GID_INDEX auto-detect failed")?;
+    let idx = out.trim().parse::<u32>().with_context(|| {
+        format!(
+            "Could not determine a RoCE-v2 IPv4 GID index on '{}' (HCA(s) {}). \
+             Set `ib_gid_index = <n>` explicitly in the cluster's [nccl] section \
+             to opt out of auto-detection.",
+            node.name, hcas
+        )
+    })?;
+    if idx > 15 {
+        anyhow::bail!(
+            "Auto-detected NCCL_IB_GID_INDEX={idx} on '{}' is out of range",
+            node.name
+        );
+    }
+    Ok(idx)
+}
+
+/// Resolve a node's NCCL config for launch: if `ib_gid_index` is pinned in
+/// config, return it unchanged; if `None` and `dry_run`, print the detect
+/// command and return a config keeps `ib_gid_index = None` (the docker env
+/// renders a placeholder); if `None` and a real run, SSH-detect per node and
+/// pin the resolved value.
+pub(crate) async fn resolve_nccl_for_node(
+    config: &Config,
+    node: &Node,
+    nccl: Option<&NcclConfig>,
+    dry_run: bool,
+) -> Result<Option<NcclConfig>> {
+    let Some(nccl) = nccl else {
+        // Single-node recipe — no NCCL config at all.
+        return Ok(None);
+    };
+    if nccl.ib_gid_index.is_some() {
+        // Pinned (opt-out) — use as-is.
+        return Ok(Some(nccl.clone()));
+    }
+    if dry_run {
+        // No SSH in dry-run; show the detect command and leave the value
+        // unresolved so the printed `-e NCCL_IB_GID_INDEX=...` is a
+        // self-explanatory placeholder.
+        println!(
+            "  ssh {}@{}: {}",
+            config.ssh_user,
+            node.ssh_ip,
+            detect_gid_cmd(&nccl.ib_hca)
+        );
+        return Ok(Some(nccl.clone()));
+    }
+    let idx = detect_gid_index_on_node(config, node, &nccl.ib_hca).await?;
+    let mut resolved = nccl.clone();
+    resolved.ib_gid_index = Some(idx);
+    Ok(Some(resolved))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::NcclConfig;
+
+    #[test]
+    fn detect_cmd_scans_roce_v2_ipv4() {
+        let cmd = detect_gid_cmd("rocep1s0f1,roceP2p1s0f1");
+        // HCA list is shell-quoted; both HCAs are iterated in order.
+        assert!(cmd.contains("'rocep1s0f1,roceP2p1s0f1'"));
+        assert!(cmd.contains("gid_attrs/types"));
+        assert!(cmd.contains("0000:0000:0000:0000:0000:ffff:"));
+        assert!(cmd.contains("RoCE v2"));
+        assert!(cmd.contains("seq 0 15"));
+        // Must fail loudly when nothing matches.
+        assert!(cmd.ends_with("exit 1"));
+    }
+
+    #[tokio::test]
+    async fn resolve_returns_pinned_config_unchanged() {
+        // A pinned (opt-out) `ib_gid_index` must short-circuit with no SSH —
+        // `detect_gid_index_on_node` is never reached, so this works even
+        // with an unreachable node.
+        let nccl = NcclConfig {
+            ib_hca: "r".into(),
+            socket_ifname: "e".into(),
+            ib_gid_index: Some(3),
+            ib_disable: 0,
+            gloo_socket_ifname: None,
+            ucx_net_devices: None,
+            nccl_cross_nic: None,
+            nccl_proto: None,
+            nccl_p2p_level: None,
+        };
+        let node = crate::config::Node {
+            name: "n1".into(),
+            ssh_ip: "10.0.0.1".into(),
+            ib_ip: Some("10.0.1.1".into()),
+            rank: 0,
+            host_key: None,
+            entity: None,
+        };
+        let config = crate::config::Config {
+            ssh_user: "u".into(),
+            ssh_key_path: std::path::PathBuf::from("/p"),
+            clusters: vec![],
+            nccl: Default::default(),
+            hf_cache_dir: None,
+            init_port: 30000,
+            before_serve: Default::default(),
+            exec_aliases: Default::default(),
+            home_assistant: None,
+            global_mods_dir: None,
+            drop_caches: false,
+            mount_aliases: Default::default(),
+        };
+        for dry in [false, true] {
+            let r = resolve_nccl_for_node(&config, &node, Some(&nccl), dry)
+                .await
+                .unwrap()
+                .unwrap();
+            assert_eq!(r.ib_gid_index, Some(3));
+        }
+    }
+}

src/docker/command.rs

@@ -161,7 +161,11 @@ fn collect_nccl_env<'a>(ctx: &'a DockerCommandCtx, node: &'a Node) -> Vec<(&'a s
             ("NCCL_SOCKET_IFNAME", sock.into()),
             ("GLOO_SOCKET_IFNAME", gloo_if.into()),
             ("TP_SOCKET_IFNAME", gloo_if.into()),
-            ("NCCL_IB_GID_INDEX", nccl.ib_gid_index.to_string().into()),
+            (
+                "NCCL_IB_GID_INDEX",
+                nccl.ib_gid_index
+                    .map_or_else(|| Cow::Borrowed("auto"), |g| g.to_string().into()),
+            ),
             ("MN_IF_NAME", gloo_if.into()),
             ("UCX_NET_DEVICES", ucx_dev.into()),
             ("OMPI_MCA_btl_tcp_if_include", gloo_if.into()),

src/docker/command_tests.rs

@@ -25,7 +25,7 @@ fn tnccl() -> NcclConfig {
     NcclConfig {
         ib_hca: "r".into(),
         socket_ifname: "e".into(),
-        ib_gid_index: 3,
+        ib_gid_index: Some(3),
         ib_disable: 0,
         gloo_socket_ifname: None,
         ucx_net_devices: None,

src/docker/launch_node.rs

@@ -0,0 +1,110 @@
+//! Per-node launch body for [`start_cluster`]: resolves path references and
+//! NCCL GID per node, builds the composed model dir if configured, then
+//! assembles + runs the `docker run` command (or prints it in dry-run).
+//!
+//! Kept as a submodule so `src/docker/mod.rs` stays under the ≤500-line gate;
+//! the per-node work needs SSH (per-node HF snapshots and GID indexes can
+//! differ across nodes) so it must run inside the cluster spawn loop, not
+//! per-cluster.
+
+use crate::config::Node;
+use crate::docker::{command, compose};
+
+/// Per-node inputs for [`launch_node`], grouped so the function stays under
+/// the argument-count lint without scattering fields.
+pub(crate) struct LaunchNodeCtx<'a> {
+    pub node: &'a Node,
+    pub rn: &'a str,
+    pub dry: bool,
+    pub engine: &'a str,
+    pub is_head: bool,
+    pub cleanup: String,
+    pub dm_ctx: &'a command::DockerCommandCtx,
+    pub hf_cache: &'a std::path::Path,
+    pub path_refs: &'a [command::PathRef],
+}
+
+/// Run one node's launch: returns `(node.name, rank, engine, command result)`.
+pub(crate) async fn launch_node(
+    config: &crate::config::Config,
+    ctx: LaunchNodeCtx<'_>,
+) -> (String, u32, String, anyhow::Result<String>) {
+    let LaunchNodeCtx {
+        node,
+        rn,
+        dry,
+        engine,
+        is_head,
+        cleanup,
+        dm_ctx,
+        hf_cache,
+        path_refs,
+    } = ctx;
+    let fail = |e: anyhow::Error| (node.name.clone(), node.rank, engine.to_string(), Err(e));
+    if !dry && let Err(e) = crate::ssh::run_ssh(config, node, cleanup, dry).await {
+        // Ignore cleanup errors — a stale container that won't die must not
+        // abort the launch (the `docker run --name` will surface any real
+        // conflict).
+        tracing::warn!("cleanup on {}: {e}", node.name);
+    }
+    // Resolve model path references per node from that node's HF cache (the
+    // snapshot hash can differ across nodes). Dry-run skips SSH and emits a
+    // descriptive placeholder path instead. The resolved paths are substituted
+    // into the recipe args/env via `{{<name>}}` before building the docker
+    // command — no bind-mount is created for path refs (the HF cache is
+    // already mounted at /root/.cache/huggingface, so symlinks resolve
+    // correctly).
+    let mut resolved_refs: Vec<(String, String)> = Vec::new();
+    for spec in path_refs {
+        let resolved_path = if dry {
+            command::path_ref_placeholder(spec)
+        } else {
+            match crate::docker::path_ref::resolve_path_ref_on_node(config, node, spec, hf_cache)
+                .await
+            {
+                Ok(p) => p,
+                Err(e) => return fail(e),
+            }
+        };
+        resolved_refs.push((spec.name.clone(), resolved_path));
+    }
+    let mut recipe = command::substitute_path_refs(&dm_ctx.recipe, &resolved_refs);
+    let mut extra_mounts = dm_ctx.extra_mounts.clone();
+    // Resolve NCCL_IB_GID_INDEX per node when the cluster config does not pin
+    // `ib_gid_index` (auto-detect: scan this node's sysfs IB GID table for the
+    // RoCE-v2 IPv4 GID — it re-numbers across reboots, mirroring the reference
+    // serving stack). Dry-run skips SSH and renders a placeholder.
+    let nccl =
+        match crate::detect::resolve_nccl_for_node(config, node, dm_ctx.nccl.as_ref(), dry).await {
+            Ok(n) => n,
+            Err(e) => return fail(e),
+        };
+    // Build composed model dir if configured: symlinks into the HF cache +
+    // generated files, bind-mounted at `mount`. `model` is rewritten to the
+    // mount path so vLLM loads from the composed dir.
+    if let Some(ref cm) = recipe.composed_model {
+        match compose::build_composed_dir_on_node(config, node, rn, node.rank, cm, hf_cache, dry)
+            .await
+        {
+            Ok((hp, cp)) => {
+                extra_mounts.push(command::ResolvedMount {
+                    host: hp,
+                    container: cp.clone(),
+                    read_only: false,
+                    env: None,
+                });
+                recipe.model = cp;
+            }
+            Err(e) => return fail(e),
+        }
+    }
+    let ctx = command::DockerCommandCtx {
+        recipe,
+        extra_mounts,
+        nccl,
+        ..dm_ctx.clone()
+    };
+    let cmd = command::build_docker_command(&ctx, node, is_head).join(" ");
+    let result = crate::ssh::run_ssh_with_retry(config, node, cmd, dry, 2).await;
+    (node.name.clone(), node.rank, engine.to_string(), result)
+}

src/docker/mod.rs

@@ -3,6 +3,7 @@ pub mod command;
 pub mod compose;
 pub mod deploy;
 pub mod health;
+pub(crate) mod launch_node;
 pub mod logs;
 pub mod mirror;
 pub mod mods;
@@ -14,7 +15,7 @@ pub mod tail;
 use crate::{
     config::{Cluster, Config, NcclConfig, Node, Recipe, load_recipe},
     doctor::try_repaired_image,
-    ssh::{run_ssh, run_ssh_with_retry},
+    ssh::run_ssh,
 };
 use anyhow::{Context, Result};
 use tokio::time::{Duration, sleep};
@@ -406,62 +407,21 @@ async fn start_cluster(
         let hf_cache = opts.cache_dir.clone();
         let path_refs = resolved.path_refs.clone();
         handles.push(crate::locale::spawn_with_locale(async move {
-            if !dry {
-                let _ = run_ssh(&config, &node, cleanup, dry).await;
-            }
-            // Resolve model path references per node from that node's HF cache
-            // (the snapshot hash can differ across nodes). Dry-run skips SSH
-            // and emits a descriptive placeholder path instead. The resolved
-            // paths are substituted into the recipe args/env via `{{<name>}}`
-            // before building the docker command — no bind-mount is created
-            // for path refs (the HF cache is already mounted at
-            // /root/.cache/huggingface, so symlinks resolve correctly).
-            let mut resolved_refs: Vec<(String, String)> = Vec::new();
-            for spec in &path_refs {
-                let resolved_path = if dry {
-                    command::path_ref_placeholder(spec)
-                } else {
-                    match path_ref::resolve_path_ref_on_node(&config, &node, spec, &hf_cache).await
-                    {
-                        Ok(p) => p,
-                        Err(e) => {
-                            return (node.name, node.rank, engine, Err(e));
-                        }
-                    }
-                };
-                resolved_refs.push((spec.name.clone(), resolved_path));
-            }
-            let mut recipe = command::substitute_path_refs(&dm_ctx.recipe, &resolved_refs);
-            let mut extra_mounts = dm_ctx.extra_mounts.clone();
-            // Build composed model dir if configured: symlinks into the HF
-            // cache + generated files, bind-mounted at `mount`. `model` is
-            // rewritten to the mount path so vLLM loads from the composed dir.
-            if let Some(ref cm) = recipe.composed_model {
-                match compose::build_composed_dir_on_node(
-                    &config, &node, &rn, node.rank, cm, &hf_cache, dry,
-                )
-                .await
-                {
-                    Ok((hp, cp)) => {
-                        extra_mounts.push(command::ResolvedMount {
-                            host: hp,
-                            container: cp.clone(),
-                            read_only: false,
-                            env: None,
-                        });
-                        recipe.model = cp;
-                    }
-                    Err(e) => return (node.name, node.rank, engine, Err(e)),
-                }
-            }
-            let ctx = command::DockerCommandCtx {
-                recipe,
-                extra_mounts,
-                ..dm_ctx.clone()
-            };
-            let cmd = command::build_docker_command(&ctx, &node, is_head).join(" ");
-            let result = run_ssh_with_retry(&config, &node, cmd, dry, 2).await;
-            (node.name, node.rank, engine, result)
+            launch_node::launch_node(
+                &config,
+                launch_node::LaunchNodeCtx {
+                    node: &node,
+                    rn: &rn,
+                    dry,
+                    engine: &engine,
+                    is_head,
+                    cleanup,
+                    dm_ctx: &dm_ctx,
+                    hf_cache: &hf_cache,
+                    path_refs: &path_refs,
+                },
+            )
+            .await
         }));
     }
     // NOTE: all node launches are awaited (join_all) before reporting — a

src/docker/sglang_tests.rs

@@ -0,0 +1,106 @@
+// SGLang serve command tests. Moved out of `docker/tests.rs` so the parent
+// module stays under the ≤500-line gate. Helpers (`tcls`, `make_dm_ctx`,
+// `squash`, ...) are in `super`.
+
+use super::*;
+
+fn sglang_recipe() -> Recipe {
+    Recipe {
+        engine: "sglang".into(),
+        name: "S".into(),
+        docker_image: "sglang-img".into(),
+        model: "org/sgl-model".into(),
+        port: 8000,
+        required_nodes: Some(4),
+        pipeline_parallel_size: None,
+        tensor_parallel_size: 4,
+        args: RecipeArgs::List(vec!["--mem-fraction-static".into(), "0.8".into()]),
+        env: HashMap::from([("TORCH_NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())]),
+        cluster: None,
+        entrypoint: vec!["sglang".into(), "serve".into()],
+        distributed_backend: Some(DistributedBackend::Mp),
+        before_serve: vec![],
+        after_stop: vec![],
+        solo: None,
+        deploy: DeployTarget::Docker,
+        venv: None,
+        venv_setup: VenvSetup::default(),
+        venv_setup_script: None,
+        mods: Default::default(),
+        shm_size: String::new(),
+        sync_image: true,
+        docker_image_dir: None,
+        extra_mounts: vec![],
+        build: Default::default(),
+        composed_model: None,
+    }
+}
+
+fn srun(recipe: &Recipe, node: &Node, head: bool) -> String {
+    let tp = if recipe.required_nodes == Some(recipe.tensor_parallel_size) {
+        4
+    } else {
+        recipe.tensor_parallel_size
+    };
+    let mut ctx = make_dm_ctx(recipe, 4, 4, tp, false);
+    ctx.recipe_name = "sg".to_string();
+    squash(&build_docker_command(&ctx, node, head).join(" "))
+}
+
+#[test]
+fn sglang_host_ip() {
+    let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
+    assert!(cmd.contains("-e SGLANG_WORKER_IP=1"));
+    assert!(!cmd.contains("VLLM_HOST_IP"));
+}
+
+#[test]
+fn sglang_head() {
+    let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
+    assert!(cmd.contains("--model-path"));
+    assert!(cmd.contains("org/sgl-model"));
+    assert!(cmd.contains("--tp 4"));
+    assert!(cmd.contains("--nnodes 4"));
+    assert!(cmd.contains("--node-rank 0"));
+    assert!(cmd.contains("--dist-init-addr 1:25000"));
+    assert!(!cmd.contains("--master-addr"));
+    assert!(!cmd.contains("--tensor-parallel-size"));
+    assert!(!cmd.contains("--pipeline-parallel-size"));
+    assert!(!cmd.contains("--distributed-executor-backend"));
+    assert!(cmd.contains("-e TORCH_NCCL_ASYNC_ERROR_HANDLING=1"));
+    assert!(
+        !cmd.contains("-p "),
+        "host networking: no port mapping expected"
+    );
+    assert!(cmd.contains("--port 8000"));
+    assert!(cmd.contains("-e HF_HOME=/root/.cache/huggingface"));
+    assert!(cmd.contains("-v /home/user/.cache/huggingface:/root/.cache/huggingface"));
+    assert!(
+        cmd.contains("--rm"),
+        "SGLang default --rm should be present"
+    );
+}
+
+#[test]
+fn sglang_worker() {
+    let cmd = srun(&sglang_recipe(), &tcls().nodes[1], false);
+    assert!(!cmd.contains("-p 8000:8000"));
+    assert!(cmd.contains("--node-rank 1"));
+    assert!(cmd.contains("--port 8000"));
+}
+
+#[test]
+fn sglang_entrypoint_model_path_order() {
+    let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
+    let ep = cmd.find("sglang serve").unwrap();
+    let mp = cmd.find("--model-path").unwrap();
+    let mdl = cmd.find("org/sgl-model").unwrap();
+    let tp = cmd.find("--tp 4").unwrap();
+    assert!(
+        ep < mp && mp < mdl,
+        "entrypoint before model-path before model"
+    );
+    assert!(mdl < tp, "model before --tp");
+    let port = cmd.find("--port 8000").unwrap();
+    assert!(tp < port, "--tp before --port");
+}

src/docker/tests.rs

@@ -33,7 +33,7 @@ fn tnccl() -> NcclConfig {
     NcclConfig {
         ib_hca: "r".into(),
         socket_ifname: "e".into(),
-        ib_gid_index: 3,
+        ib_gid_index: Some(3),
         ib_disable: 0,
         gloo_socket_ifname: None,
         ucx_net_devices: None,
@@ -108,6 +108,23 @@ fn run(recipe: &Recipe, node: &Node, head: bool) -> String {
     squash(&build_docker_command(&make_dm_ctx(recipe, 2, 2, tp, false), node, head).join(" "))
 }
 
+#[test]
+fn nccl_gid_unresolved_renders_auto() {
+    // A cluster [nccl] block without `ib_gid_index` (auto-detect) renders
+    // `NCCL_IB_GID_INDEX=auto` — the launcher resolves the real index per
+    // node before the docker run; `auto` only appears on the dry-run/unresolved
+    // path. Confirms the env is emitted and clearly marked, not dropped.
+    let mut nccl = tnccl();
+    nccl.ib_gid_index = None;
+    let mut ctx = make_dm_ctx(&trec(), 2, 2, 2, false);
+    ctx.nccl = Some(nccl);
+    let cmd = squash(&build_docker_command(&ctx, &tcls().nodes[0], true).join(" "));
+    assert!(
+        cmd.contains("-e NCCL_IB_GID_INDEX=auto"),
+        "unresolved GID should render 'auto' placeholder: {cmd}"
+    );
+}
+
 fn run_keep(recipe: &Recipe, node: &Node, head: bool) -> String {
     let tp = if recipe.required_nodes == Some(recipe.tensor_parallel_size) {
         2
@@ -241,109 +258,6 @@ fn entrypoint() {
     );
 }
 
-// ── SGLang engine tests ──────────────────────────────────────
-
-fn sglang_recipe() -> Recipe {
-    Recipe {
-        engine: "sglang".into(),
-        name: "S".into(),
-        docker_image: "sglang-img".into(),
-        model: "org/sgl-model".into(),
-        port: 8000,
-        required_nodes: Some(4),
-        pipeline_parallel_size: None,
-        tensor_parallel_size: 4,
-        args: RecipeArgs::List(vec!["--mem-fraction-static".into(), "0.8".into()]),
-        env: HashMap::from([("TORCH_NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())]),
-        cluster: None,
-        entrypoint: vec!["sglang".into(), "serve".into()],
-        distributed_backend: Some(DistributedBackend::Mp),
-        before_serve: vec![],
-        after_stop: vec![],
-        solo: None,
-        deploy: DeployTarget::Docker,
-        venv: None,
-        venv_setup: VenvSetup::default(),
-        venv_setup_script: None,
-        mods: Default::default(),
-        shm_size: String::new(),
-        sync_image: true,
-        docker_image_dir: None,
-        extra_mounts: vec![],
-        build: Default::default(),
-        composed_model: None,
-    }
-}
-
-fn srun(recipe: &Recipe, node: &Node, head: bool) -> String {
-    let tp = if recipe.required_nodes == Some(recipe.tensor_parallel_size) {
-        4
-    } else {
-        recipe.tensor_parallel_size
-    };
-    let mut ctx = make_dm_ctx(recipe, 4, 4, tp, false);
-    ctx.recipe_name = "sg".to_string();
-    squash(&build_docker_command(&ctx, node, head).join(" "))
-}
-
-#[test]
-fn sglang_host_ip() {
-    let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
-    assert!(cmd.contains("-e SGLANG_WORKER_IP=1"));
-    assert!(!cmd.contains("VLLM_HOST_IP"));
-}
-
-#[test]
-fn sglang_head() {
-    let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
-    assert!(cmd.contains("--model-path"));
-    assert!(cmd.contains("org/sgl-model"));
-    assert!(cmd.contains("--tp 4"));
-    assert!(cmd.contains("--nnodes 4"));
-    assert!(cmd.contains("--node-rank 0"));
-    assert!(cmd.contains("--dist-init-addr 1:25000"));
-    assert!(!cmd.contains("--master-addr"));
-    assert!(!cmd.contains("--tensor-parallel-size"));
-    assert!(!cmd.contains("--pipeline-parallel-size"));
-    assert!(!cmd.contains("--distributed-executor-backend"));
-    assert!(cmd.contains("-e TORCH_NCCL_ASYNC_ERROR_HANDLING=1"));
-    assert!(
-        !cmd.contains("-p "),
-        "host networking: no port mapping expected"
-    );
-    assert!(cmd.contains("--port 8000"));
-    assert!(cmd.contains("-e HF_HOME=/root/.cache/huggingface"));
-    assert!(cmd.contains("-v /home/user/.cache/huggingface:/root/.cache/huggingface"));
-    assert!(
-        cmd.contains("--rm"),
-        "SGLang default --rm should be present"
-    );
-}
-
-#[test]
-fn sglang_worker() {
-    let cmd = srun(&sglang_recipe(), &tcls().nodes[1], false);
-    assert!(!cmd.contains("-p 8000:8000"));
-    assert!(cmd.contains("--node-rank 1"));
-    assert!(cmd.contains("--port 8000"));
-}
-
-#[test]
-fn sglang_entrypoint_model_path_order() {
-    let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
-    let ep = cmd.find("sglang serve").unwrap();
-    let mp = cmd.find("--model-path").unwrap();
-    let mdl = cmd.find("org/sgl-model").unwrap();
-    let tp = cmd.find("--tp 4").unwrap();
-    assert!(
-        ep < mp && mp < mdl,
-        "entrypoint before model-path before model"
-    );
-    assert!(mdl < tp, "model before --tp");
-    let port = cmd.find("--port 8000").unwrap();
-    assert!(tp < port, "--tp before --port");
-}
-
 #[test]
 fn keep_disables_rm() {
     let cmd = run_keep(&trec(), &tcls().nodes[0], true);
@@ -498,3 +412,6 @@ fn entrypoint_script_non_sh_passes_through() {
 
 #[path = "solo_tests.rs"]
 mod solo_tests;
+
+#[path = "sglang_tests.rs"]
+mod sglang_tests;

src/main.rs

@@ -11,6 +11,7 @@ mod builtin_scripts;
 mod cli;
 mod cluster_state;
 mod config;
+mod detect;
 mod docker;
 mod doctor;
 mod ends;

src/orchestrate/tests.rs

@@ -62,7 +62,7 @@ fn test_nccl() -> NcclConfig {
     NcclConfig {
         ib_hca: "r".to_string(),
         socket_ifname: "e".to_string(),
-        ib_gid_index: 3,
+        ib_gid_index: Some(3),
         ib_disable: 0,
         gloo_socket_ifname: None,
         ucx_net_devices: None,

src/tmux/mod.rs

@@ -40,7 +40,13 @@ impl TmuxManager {
             Some(nccl) => vec![
                 format!("export NCCL_IB_HCA=\"{}\"", nccl.ib_hca),
                 format!("export NCCL_SOCKET_IFNAME=\"{}\"", nccl.socket_ifname),
-                format!("export NCCL_IB_GID_INDEX={}", nccl.ib_gid_index),
+                format!(
+                    "export NCCL_IB_GID_INDEX={}",
+                    // Unresolved (dry-run) renders `auto`; real launches
+                    // always have the per-node detected/pinned value.
+                    nccl.ib_gid_index
+                        .map_or_else(|| "auto".to_string(), |g| g.to_string())
+                ),
                 format!("export NCCL_IB_DISABLE={}", nccl.ib_disable),
             ],
             None => Vec::new(),

src/tmux/serve.rs

@@ -113,7 +113,12 @@ pub async fn tmux_serve(
             .find(|n| n.rank == 0)
             .context("No head node (rank 0) in cluster")?;
 
-        let head_cmds = manager.build_session_commands(head_node, &recipe, nccl.as_ref(), &cluster);
+        let head_nccl =
+            crate::detect::resolve_nccl_for_node(config, head_node, nccl.as_ref(), dry_run)
+                .await
+                .context("resolving head node NCCL config")?;
+        let head_cmds =
+            manager.build_session_commands(head_node, &recipe, head_nccl.as_ref(), &cluster);
         manager
             .create_session(config, head_node, &head_cmds, dry_run)
             .await?;
@@ -133,8 +138,10 @@ pub async fn tmux_serve(
             if node.rank == 0 {
                 continue;
             }
+            let node_nccl =
+                crate::detect::resolve_nccl_for_node(config, node, nccl.as_ref(), dry_run).await?;
             let worker_cmds =
-                manager.build_session_commands(node, &recipe, nccl.as_ref(), &cluster);
+                manager.build_session_commands(node, &recipe, node_nccl.as_ref(), &cluster);
             manager
                 .create_session(config, node, &worker_cmds, dry_run)
                 .await?;
@@ -153,7 +160,15 @@ pub async fn tmux_serve(
             let dry = dry_run;
 
             handles.push(crate::locale::spawn_with_locale(async move {
-                let cmds = manager.build_session_commands(&node, &recipe, nccl.as_ref(), &cluster);
+                let node_nccl =
+                    match crate::detect::resolve_nccl_for_node(&config, &node, nccl.as_ref(), dry)
+                        .await
+                    {
+                        Ok(n) => n,
+                        Err(e) => return Err(anyhow::anyhow!(e)),
+                    };
+                let cmds =
+                    manager.build_session_commands(&node, &recipe, node_nccl.as_ref(), &cluster);
                 manager
                     .create_session(&config, &node, &cmds, dry)
                     .await