|
@@ -13,7 +13,9 @@ use std::borrow::Cow;
|
|
|
#[derive(Clone)]
|
|
#[derive(Clone)]
|
|
|
pub(crate) struct DockerCommandCtx {
|
|
pub(crate) struct DockerCommandCtx {
|
|
|
pub cluster: Cluster,
|
|
pub cluster: Cluster,
|
|
|
- pub nccl: NcclConfig,
|
|
|
|
|
|
|
+ /// NCCL config — `None` for single-node recipes (no NCCL env vars
|
|
|
|
|
+ /// are emitted). For multi-node, always `Some`.
|
|
|
|
|
+ pub nccl: Option<NcclConfig>,
|
|
|
pub recipe: Recipe,
|
|
pub recipe: Recipe,
|
|
|
pub recipe_name: String,
|
|
pub recipe_name: String,
|
|
|
pub pp_size: u32,
|
|
pub pp_size: u32,
|
|
@@ -140,38 +142,46 @@ pub(crate) fn path_ref_placeholder(spec: &PathRef) -> String {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/// Collect NCCL cluster env vars for multi-node deployments.
|
|
/// Collect NCCL cluster env vars for multi-node deployments.
|
|
|
|
|
+///
|
|
|
|
|
+/// `ctx.nccl` is `None` for single-node recipes — but this function is
|
|
|
|
|
+/// only called when `ctx.nnodes > 1`, so `None` indicates a configuration
|
|
|
|
|
+/// error (multi-node without NCCL config). In that case, only the worker
|
|
|
|
|
+/// IP env var is emitted (NCCL env vars are skipped), which will cause
|
|
|
|
|
+/// NCCL to fail at runtime with a visible error.
|
|
|
fn collect_nccl_env<'a>(ctx: &'a DockerCommandCtx, node: &'a Node) -> Vec<(&'a str, Cow<'a, str>)> {
|
|
fn collect_nccl_env<'a>(ctx: &'a DockerCommandCtx, node: &'a Node) -> Vec<(&'a str, Cow<'a, str>)> {
|
|
|
let is_sglang = ctx.recipe.engine == "sglang";
|
|
let is_sglang = ctx.recipe.engine == "sglang";
|
|
|
- let sock = ctx.nccl.socket_ifname.as_str();
|
|
|
|
|
- let gloo_if = ctx.nccl.gloo_socket_ifname.as_deref().unwrap_or(sock);
|
|
|
|
|
- let ucx_dev = ctx.nccl.ucx_net_devices.as_deref().unwrap_or(sock);
|
|
|
|
|
- let mut env: Vec<(&str, Cow<str>)> = vec![
|
|
|
|
|
- ("NCCL_IB_DISABLE", format!("{}", ctx.nccl.ib_disable).into()),
|
|
|
|
|
- ("NCCL_IB_HCA", ctx.nccl.ib_hca.as_str().into()),
|
|
|
|
|
- ("NCCL_SOCKET_IFNAME", sock.into()),
|
|
|
|
|
- ("GLOO_SOCKET_IFNAME", gloo_if.into()),
|
|
|
|
|
- ("TP_SOCKET_IFNAME", gloo_if.into()),
|
|
|
|
|
- (
|
|
|
|
|
- "NCCL_IB_GID_INDEX",
|
|
|
|
|
- ctx.nccl.ib_gid_index.to_string().into(),
|
|
|
|
|
- ),
|
|
|
|
|
- ("MN_IF_NAME", gloo_if.into()),
|
|
|
|
|
- ("UCX_NET_DEVICES", ucx_dev.into()),
|
|
|
|
|
- ("OMPI_MCA_btl_tcp_if_include", gloo_if.into()),
|
|
|
|
|
- ("OMP_NUM_THREADS", "4".into()),
|
|
|
|
|
- ("NCCL_IGNORE_CPU_AFFINITY", "1".into()),
|
|
|
|
|
- ("NCCL_NET", "IB".into()),
|
|
|
|
|
- ("NCCL_DEBUG", "WARN".into()),
|
|
|
|
|
- ];
|
|
|
|
|
- if let Some(v) = ctx.nccl.nccl_cross_nic {
|
|
|
|
|
- env.push(("NCCL_CROSS_NIC", v.to_string().into()));
|
|
|
|
|
- }
|
|
|
|
|
- if let Some(ref v) = ctx.nccl.nccl_proto {
|
|
|
|
|
- env.push(("NCCL_PROTO", v.as_str().into()));
|
|
|
|
|
- }
|
|
|
|
|
- if let Some(ref v) = ctx.nccl.nccl_p2p_level {
|
|
|
|
|
- env.push(("NCCL_P2P_LEVEL", v.as_str().into()));
|
|
|
|
|
|
|
+ let mut env: Vec<(&str, Cow<str>)> = Vec::new();
|
|
|
|
|
+ if let Some(nccl) = ctx.nccl.as_ref() {
|
|
|
|
|
+ let sock = nccl.socket_ifname.as_str();
|
|
|
|
|
+ let gloo_if = nccl.gloo_socket_ifname.as_deref().unwrap_or(sock);
|
|
|
|
|
+ let ucx_dev = nccl.ucx_net_devices.as_deref().unwrap_or(sock);
|
|
|
|
|
+ env.extend(vec![
|
|
|
|
|
+ ("NCCL_IB_DISABLE", format!("{}", nccl.ib_disable).into()),
|
|
|
|
|
+ ("NCCL_IB_HCA", nccl.ib_hca.as_str().into()),
|
|
|
|
|
+ ("NCCL_SOCKET_IFNAME", sock.into()),
|
|
|
|
|
+ ("GLOO_SOCKET_IFNAME", gloo_if.into()),
|
|
|
|
|
+ ("TP_SOCKET_IFNAME", gloo_if.into()),
|
|
|
|
|
+ ("NCCL_IB_GID_INDEX", nccl.ib_gid_index.to_string().into()),
|
|
|
|
|
+ ("MN_IF_NAME", gloo_if.into()),
|
|
|
|
|
+ ("UCX_NET_DEVICES", ucx_dev.into()),
|
|
|
|
|
+ ("OMPI_MCA_btl_tcp_if_include", gloo_if.into()),
|
|
|
|
|
+ ("OMP_NUM_THREADS", "4".into()),
|
|
|
|
|
+ ("NCCL_IGNORE_CPU_AFFINITY", "1".into()),
|
|
|
|
|
+ ("NCCL_NET", "IB".into()),
|
|
|
|
|
+ ("NCCL_DEBUG", "WARN".into()),
|
|
|
|
|
+ ]);
|
|
|
|
|
+ if let Some(v) = nccl.nccl_cross_nic {
|
|
|
|
|
+ env.push(("NCCL_CROSS_NIC", v.to_string().into()));
|
|
|
|
|
+ }
|
|
|
|
|
+ if let Some(ref v) = nccl.nccl_proto {
|
|
|
|
|
+ env.push(("NCCL_PROTO", v.as_str().into()));
|
|
|
|
|
+ }
|
|
|
|
|
+ if let Some(ref v) = nccl.nccl_p2p_level {
|
|
|
|
|
+ env.push(("NCCL_P2P_LEVEL", v.as_str().into()));
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
|
|
+ // Worker IP is always set for multi-node (collect_nccl_env is only
|
|
|
|
|
+ // called when nnodes > 1).
|
|
|
let worker_ip = Cow::Borrowed(node.ib_ip.as_str());
|
|
let worker_ip = Cow::Borrowed(node.ib_ip.as_str());
|
|
|
if is_sglang {
|
|
if is_sglang {
|
|
|
env.push(("SGLANG_WORKER_IP", worker_ip));
|
|
env.push(("SGLANG_WORKER_IP", worker_ip));
|