| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173 |
- //! Cluster launch (the `serve` inner half): build/sync the image, compose
- //! the docker run per node, start containers and wait for readiness.
- //! Split from mod.rs for the 500-line cap.
- use super::command;
- use super::launch_node;
- use super::mirror;
- use super::shq;
- use super::{Config, NcclConfig, Recipe, StartClusterOpts};
- use crate::config::{Cluster, Node};
- use anyhow::{Result, bail};
- use tokio::time::{Duration, sleep};
- use tracing::info;
- pub(crate) async fn start_cluster(
- config: &Config,
- cluster: &Cluster,
- nccl: Option<NcclConfig>,
- recipe_name: &str,
- recipe: &Recipe,
- opts: StartClusterOpts,
- ) -> Result<()> {
- let nodes: Vec<&Node> = if let Some(ref nn) = opts.node_name {
- let v: Vec<&Node> = cluster.nodes.iter().filter(|n| n.name == *nn).collect();
- if v.is_empty() {
- anyhow::bail!("Node '{}' not found in cluster '{}'", nn, cluster.name);
- }
- v
- } else {
- // Trim to `required_nodes` when set: a recipe that needs N nodes
- // must launch on exactly N nodes, not on every free node in the
- // cluster. Previously a `required_nodes = 1` recipe on a 4-node
- // cluster launched 4 containers, each told `--nnodes 4 --tp 4` —
- // a broken rendezvous that never coordinated. `prepare_cluster`
- // guarantees >= required_nodes are free, so taking the first N is
- // safe and deterministic.
- let all = cluster.nodes.iter().collect::<Vec<&Node>>();
- match recipe.required_nodes {
- Some(n) if (n as usize) < all.len() => all.into_iter().take(n as usize).collect(),
- _ => all,
- }
- };
- let nnodes = nodes.len() as u32;
- // Renumber to zero-based ranks within the launch set: dist flags
- // (`--node-rank`), head selection and headless-ness must use the
- // serve-local index, not the node's global cluster rank. A subset launch
- // (required_nodes=2 out of a 4-node cluster, dgx3+dgx4 = global ranks
- // 2/3) otherwise sends `--node-rank 2` with `--nnodes 2` — vLLM rejects
- // it ("must be between 0 and 1").
- let launch_nodes: Vec<Node> = nodes
- .iter()
- .enumerate()
- .map(|(i, n)| {
- let mut n = (*n).clone();
- n.rank = i as u32;
- n
- })
- .collect();
- // A multi-node recipe without an explicit distributed backend would launch
- // N INDEPENDENT single-node engines (no dist flags, each loading the full
- // checkpoint) — the silent root cause of the glm53-flash-fp4-local-2x
- // "stalls". Refuse instead.
- if nnodes > 1 && recipe.distributed_backend.is_none() {
- bail!(
- "recipe '{}' runs on {} nodes but sets no distributed_backend — add distributed_backend = \"mp\" (or \"ray\") so the launcher emits the rendezvous flags; otherwise every node launches an independent single-node engine and tries to load the full checkpoint",
- recipe_name,
- nnodes
- );
- }
- // `--mirror`: ensure every repo the recipe needs exists in the per-node
- // mirror before launch (copies from the shared HF cache when missing).
- if let Some(ref mirror) = opts.mirror {
- mirror::ensure_mirrors(
- config,
- &nodes,
- recipe,
- &config.resolved_hf_cache_dir(),
- mirror,
- opts.dry_run,
- )
- .await?;
- }
- let pp_size = if recipe.pipeline_parallel_size.unwrap_or(0) == 0
- || recipe.required_nodes == recipe.pipeline_parallel_size
- {
- nnodes
- } else {
- recipe.pipeline_parallel_size.unwrap()
- };
- let tp_size = if recipe.required_nodes == Some(recipe.tensor_parallel_size) {
- nnodes
- } else {
- recipe.tensor_parallel_size
- };
- // Resolve static mounts once (alias/host + tilde). Model path references
- // (`repo`+`subfolder`) are kept as specs and resolved per node inside the
- // spawn loop, since the HF snapshot hash can differ across nodes.
- let resolved = command::resolve_extra_mounts(config, recipe)?;
- command::clear_extra_mounts(config, &nodes, &resolved, opts.clear, opts.dry_run).await?;
- let mut handles = Vec::new();
- for node in &launch_nodes {
- let node = node.clone();
- let config = config.clone();
- let rn = recipe_name.to_string();
- let dry = opts.dry_run;
- let engine = recipe.engine.clone();
- let is_head = node.rank == 0;
- let cleanup = format!(
- "docker rm -f {} 2>/dev/null; true",
- shq(&format!("sm121_{}_{}", recipe_name, node.rank))
- );
- let dm_ctx = command::DockerCommandCtx {
- cluster: cluster.clone(),
- nccl: nccl.clone(),
- recipe: recipe.clone(),
- recipe_name: rn.clone(),
- pp_size,
- tp_size,
- nnodes,
- cache_dir: opts.cache_dir.to_string_lossy().into_owned(),
- keep: opts.keep,
- init_port: config.init_port,
- mods_remote_path: opts.mods_remote_path.clone(),
- extra_mounts: resolved.static_mounts.clone(),
- };
- let hf_cache = opts.cache_dir.clone();
- let path_refs = resolved.path_refs.clone();
- handles.push(crate::locale::spawn_with_locale(async move {
- launch_node::launch_node(
- &config,
- launch_node::LaunchNodeCtx {
- node: &node,
- rn: &rn,
- dry,
- engine: &engine,
- is_head,
- cleanup,
- dm_ctx: &dm_ctx,
- hf_cache: &hf_cache,
- path_refs: &path_refs,
- },
- )
- .await
- }));
- }
- // NOTE: all node launches are awaited (join_all) before reporting — a
- // failure on one node does not cancel the others, so a partial cluster
- // may be left running. Intentional: reporting every node's result is
- // more useful than aborting mid-launch and leaving the operator to
- // guess which nodes actually started.
- for result in futures::future::join_all(handles).await {
- match result {
- Ok((name, rank, engine, Ok(_))) => info!(
- "{}",
- fluent_i18n::t!("engine-started", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank })
- ),
- Ok((name, rank, engine, Err(e))) => anyhow::bail!(
- "{}",
- fluent_i18n::t!("engine-start-failed", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank, "error" => e.to_string() })
- ),
- Err(e) => anyhow::bail!("Task join error: {}", e),
- }
- }
- if !opts.dry_run {
- sleep(Duration::from_secs(10)).await;
- info!(
- "{}",
- fluent_i18n::t!("cluster-started", { "ip" => cluster.effective_master_ip(), "port" => recipe.port })
- );
- }
- Ok(())
- }
|