|
@@ -456,22 +456,28 @@ fn build_docker_command(
|
|
|
"--dist-init-addr".into(),
|
|
"--dist-init-addr".into(),
|
|
|
format!("{}:{}", cluster.master_ib_ip, init_port),
|
|
format!("{}:{}", cluster.master_ib_ip, init_port),
|
|
|
],
|
|
],
|
|
|
- "ray" | "mp" => vec![
|
|
|
|
|
- "--tensor-parallel-size".into(),
|
|
|
|
|
- recipe.tensor_parallel_size.to_string(),
|
|
|
|
|
- "--pipeline-parallel-size".into(),
|
|
|
|
|
- pp_size.to_string(),
|
|
|
|
|
- "--nnodes".into(),
|
|
|
|
|
- nnodes.to_string(),
|
|
|
|
|
- "--node-rank".into(),
|
|
|
|
|
- node.rank.to_string(),
|
|
|
|
|
- "--master-addr".into(),
|
|
|
|
|
- cluster.master_ib_ip.clone(),
|
|
|
|
|
- "--master-port".into(),
|
|
|
|
|
- init_port.to_string(),
|
|
|
|
|
- "--distributed-executor-backend".into(),
|
|
|
|
|
- recipe.distributed_backend.clone(),
|
|
|
|
|
- ],
|
|
|
|
|
|
|
+ "ray" | "mp" => {
|
|
|
|
|
+ let mut fp = vec![
|
|
|
|
|
+ "--tensor-parallel-size".into(),
|
|
|
|
|
+ recipe.tensor_parallel_size.to_string(),
|
|
|
|
|
+ "--pipeline-parallel-size".into(),
|
|
|
|
|
+ pp_size.to_string(),
|
|
|
|
|
+ "--nnodes".into(),
|
|
|
|
|
+ nnodes.to_string(),
|
|
|
|
|
+ "--node-rank".into(),
|
|
|
|
|
+ node.rank.to_string(),
|
|
|
|
|
+ "--master-addr".into(),
|
|
|
|
|
+ cluster.master_ib_ip.clone(),
|
|
|
|
|
+ "--master-port".into(),
|
|
|
|
|
+ init_port.to_string(),
|
|
|
|
|
+ ];
|
|
|
|
|
+ // vLLM auto-detects mp; pass explicitly for ray only.
|
|
|
|
|
+ if recipe.distributed_backend == "ray" {
|
|
|
|
|
+ fp.push("--distributed-executor-backend".into());
|
|
|
|
|
+ fp.push(recipe.distributed_backend.clone());
|
|
|
|
|
+ }
|
|
|
|
|
+ fp
|
|
|
|
|
+ }
|
|
|
_ => vec![],
|
|
_ => vec![],
|
|
|
};
|
|
};
|
|
|
|
|
|
|
@@ -481,8 +487,7 @@ fn build_docker_command(
|
|
|
}
|
|
}
|
|
|
cmd.extend(flags);
|
|
cmd.extend(flags);
|
|
|
|
|
|
|
|
- // Host + port flags — auto-generated for both engines so recipes
|
|
|
|
|
- // don't need to hardcode them.
|
|
|
|
|
|
|
+ // Host + port flags — auto-generated for both engines.
|
|
|
cmd.push("--host".to_string());
|
|
cmd.push("--host".to_string());
|
|
|
cmd.push("0.0.0.0".to_string());
|
|
cmd.push("0.0.0.0".to_string());
|
|
|
cmd.push("--port".to_string());
|
|
cmd.push("--port".to_string());
|