start.rs 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173
  1. //! Cluster launch (the `serve` inner half): build/sync the image, compose
  2. //! the docker run per node, start containers and wait for readiness.
  3. //! Split from mod.rs for the 500-line cap.
  4. use super::command;
  5. use super::launch_node;
  6. use super::mirror;
  7. use super::shq;
  8. use super::{Config, NcclConfig, Recipe, StartClusterOpts};
  9. use crate::config::{Cluster, Node};
  10. use anyhow::{Result, bail};
  11. use tokio::time::{Duration, sleep};
  12. use tracing::info;
  13. pub(crate) async fn start_cluster(
  14. config: &Config,
  15. cluster: &Cluster,
  16. nccl: Option<NcclConfig>,
  17. recipe_name: &str,
  18. recipe: &Recipe,
  19. opts: StartClusterOpts,
  20. ) -> Result<()> {
  21. let nodes: Vec<&Node> = if let Some(ref nn) = opts.node_name {
  22. let v: Vec<&Node> = cluster.nodes.iter().filter(|n| n.name == *nn).collect();
  23. if v.is_empty() {
  24. anyhow::bail!("Node '{}' not found in cluster '{}'", nn, cluster.name);
  25. }
  26. v
  27. } else {
  28. // Trim to `required_nodes` when set: a recipe that needs N nodes
  29. // must launch on exactly N nodes, not on every free node in the
  30. // cluster. Previously a `required_nodes = 1` recipe on a 4-node
  31. // cluster launched 4 containers, each told `--nnodes 4 --tp 4` —
  32. // a broken rendezvous that never coordinated. `prepare_cluster`
  33. // guarantees >= required_nodes are free, so taking the first N is
  34. // safe and deterministic.
  35. let all = cluster.nodes.iter().collect::<Vec<&Node>>();
  36. match recipe.required_nodes {
  37. Some(n) if (n as usize) < all.len() => all.into_iter().take(n as usize).collect(),
  38. _ => all,
  39. }
  40. };
  41. let nnodes = nodes.len() as u32;
  42. // Renumber to zero-based ranks within the launch set: dist flags
  43. // (`--node-rank`), head selection and headless-ness must use the
  44. // serve-local index, not the node's global cluster rank. A subset launch
  45. // (required_nodes=2 out of a 4-node cluster, dgx3+dgx4 = global ranks
  46. // 2/3) otherwise sends `--node-rank 2` with `--nnodes 2` — vLLM rejects
  47. // it ("must be between 0 and 1").
  48. let launch_nodes: Vec<Node> = nodes
  49. .iter()
  50. .enumerate()
  51. .map(|(i, n)| {
  52. let mut n = (*n).clone();
  53. n.rank = i as u32;
  54. n
  55. })
  56. .collect();
  57. // A multi-node recipe without an explicit distributed backend would launch
  58. // N INDEPENDENT single-node engines (no dist flags, each loading the full
  59. // checkpoint) — the silent root cause of the glm53-flash-fp4-local-2x
  60. // "stalls". Refuse instead.
  61. if nnodes > 1 && recipe.distributed_backend.is_none() {
  62. bail!(
  63. "recipe '{}' runs on {} nodes but sets no distributed_backend — add distributed_backend = \"mp\" (or \"ray\") so the launcher emits the rendezvous flags; otherwise every node launches an independent single-node engine and tries to load the full checkpoint",
  64. recipe_name,
  65. nnodes
  66. );
  67. }
  68. // `--mirror`: ensure every repo the recipe needs exists in the per-node
  69. // mirror before launch (copies from the shared HF cache when missing).
  70. if let Some(ref mirror) = opts.mirror {
  71. mirror::ensure_mirrors(
  72. config,
  73. &nodes,
  74. recipe,
  75. &config.resolved_hf_cache_dir(),
  76. mirror,
  77. opts.dry_run,
  78. )
  79. .await?;
  80. }
  81. let pp_size = if recipe.pipeline_parallel_size.unwrap_or(0) == 0
  82. || recipe.required_nodes == recipe.pipeline_parallel_size
  83. {
  84. nnodes
  85. } else {
  86. recipe.pipeline_parallel_size.unwrap()
  87. };
  88. let tp_size = if recipe.required_nodes == Some(recipe.tensor_parallel_size) {
  89. nnodes
  90. } else {
  91. recipe.tensor_parallel_size
  92. };
  93. // Resolve static mounts once (alias/host + tilde). Model path references
  94. // (`repo`+`subfolder`) are kept as specs and resolved per node inside the
  95. // spawn loop, since the HF snapshot hash can differ across nodes.
  96. let resolved = command::resolve_extra_mounts(config, recipe)?;
  97. command::clear_extra_mounts(config, &nodes, &resolved, opts.clear, opts.dry_run).await?;
  98. let mut handles = Vec::new();
  99. for node in &launch_nodes {
  100. let node = node.clone();
  101. let config = config.clone();
  102. let rn = recipe_name.to_string();
  103. let dry = opts.dry_run;
  104. let engine = recipe.engine.clone();
  105. let is_head = node.rank == 0;
  106. let cleanup = format!(
  107. "docker rm -f {} 2>/dev/null; true",
  108. shq(&format!("sm121_{}_{}", recipe_name, node.rank))
  109. );
  110. let dm_ctx = command::DockerCommandCtx {
  111. cluster: cluster.clone(),
  112. nccl: nccl.clone(),
  113. recipe: recipe.clone(),
  114. recipe_name: rn.clone(),
  115. pp_size,
  116. tp_size,
  117. nnodes,
  118. cache_dir: opts.cache_dir.to_string_lossy().into_owned(),
  119. keep: opts.keep,
  120. init_port: config.init_port,
  121. mods_remote_path: opts.mods_remote_path.clone(),
  122. extra_mounts: resolved.static_mounts.clone(),
  123. };
  124. let hf_cache = opts.cache_dir.clone();
  125. let path_refs = resolved.path_refs.clone();
  126. handles.push(crate::locale::spawn_with_locale(async move {
  127. launch_node::launch_node(
  128. &config,
  129. launch_node::LaunchNodeCtx {
  130. node: &node,
  131. rn: &rn,
  132. dry,
  133. engine: &engine,
  134. is_head,
  135. cleanup,
  136. dm_ctx: &dm_ctx,
  137. hf_cache: &hf_cache,
  138. path_refs: &path_refs,
  139. },
  140. )
  141. .await
  142. }));
  143. }
  144. // NOTE: all node launches are awaited (join_all) before reporting — a
  145. // failure on one node does not cancel the others, so a partial cluster
  146. // may be left running. Intentional: reporting every node's result is
  147. // more useful than aborting mid-launch and leaving the operator to
  148. // guess which nodes actually started.
  149. for result in futures::future::join_all(handles).await {
  150. match result {
  151. Ok((name, rank, engine, Ok(_))) => info!(
  152. "{}",
  153. fluent_i18n::t!("engine-started", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank })
  154. ),
  155. Ok((name, rank, engine, Err(e))) => anyhow::bail!(
  156. "{}",
  157. fluent_i18n::t!("engine-start-failed", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank, "error" => e.to_string() })
  158. ),
  159. Err(e) => anyhow::bail!("Task join error: {}", e),
  160. }
  161. }
  162. if !opts.dry_run {
  163. sleep(Duration::from_secs(10)).await;
  164. info!(
  165. "{}",
  166. fluent_i18n::t!("cluster-started", { "ip" => cluster.effective_master_ip(), "port" => recipe.port })
  167. );
  168. }
  169. Ok(())
  170. }