|
|
@@ -182,7 +182,7 @@ fn collect_nccl_env<'a>(ctx: &'a DockerCommandCtx, node: &'a Node) -> Vec<(&'a s
|
|
|
}
|
|
|
// Worker IP is always set for multi-node (collect_nccl_env is only
|
|
|
// called when nnodes > 1).
|
|
|
- let worker_ip = Cow::Borrowed(node.ib_ip.as_str());
|
|
|
+ let worker_ip = Cow::Borrowed(node.effective_ib_ip());
|
|
|
if is_sglang {
|
|
|
env.push(("SGLANG_WORKER_IP", worker_ip));
|
|
|
} else {
|
|
|
@@ -205,11 +205,14 @@ fn collect_docker_env(ctx: &DockerCommandCtx, node: &Node, is_head: bool) -> Vec
|
|
|
if ctx.recipe.distributed_backend == Some(crate::config::DistributedBackend::Ray) {
|
|
|
pairs.push((
|
|
|
"RAY_ADDRESS",
|
|
|
- format!("{}:6379", ctx.cluster.master_ib_ip).into(),
|
|
|
+ format!("{}:6379", ctx.cluster.effective_master_ip()).into(),
|
|
|
));
|
|
|
pairs.push(("RAY_PORT", "6379".into()));
|
|
|
- pairs.push(("RAY_NODE_IP_ADDRESS", node.ib_ip.as_str().into()));
|
|
|
- pairs.push(("RAY_OVERRIDE_NODE_IP_ADDRESS", node.ib_ip.as_str().into()));
|
|
|
+ pairs.push(("RAY_NODE_IP_ADDRESS", node.effective_ib_ip().into()));
|
|
|
+ pairs.push((
|
|
|
+ "RAY_OVERRIDE_NODE_IP_ADDRESS",
|
|
|
+ node.effective_ib_ip().into(),
|
|
|
+ ));
|
|
|
}
|
|
|
// extra_mount `env` defaults (point at the container path) are injected
|
|
|
// before recipe [env] so the recipe can still override them (last wins).
|
|
|
@@ -278,13 +281,14 @@ pub(crate) fn build_docker_command(
|
|
|
if ctx.cluster.nodes.len() == 1 && ctx.cluster.extra_entities.is_empty() {
|
|
|
cmd.push(format!("--label sm121.solo-node={}", node.name));
|
|
|
}
|
|
|
+ let master_ip = ctx.cluster.effective_master_ip();
|
|
|
let tctx = tail::TailCtx {
|
|
|
recipe: &ctx.recipe,
|
|
|
tp_size: ctx.tp_size,
|
|
|
pp_size: ctx.pp_size,
|
|
|
nnodes: ctx.nnodes,
|
|
|
init_port: ctx.init_port,
|
|
|
- master_ib_ip: &ctx.cluster.master_ib_ip,
|
|
|
+ master_ib_ip: &master_ip,
|
|
|
};
|
|
|
let tail = tail::build_tail(&tctx, node);
|
|
|
if ctx.recipe.mods.commands.is_empty() {
|