|
|
@@ -14,12 +14,14 @@ pub async fn serve(
|
|
|
dry_run: bool,
|
|
|
) -> Result<()> {
|
|
|
let recipe = load_recipe(recipe_name).await?;
|
|
|
+ let rn = recipe.name.clone();
|
|
|
+ let re = recipe.engine.clone();
|
|
|
+ let rb = recipe.distributed_backend.clone();
|
|
|
info!(
|
|
|
- "🚀 Запуск рецепта: {} ({}) [backend: {}]",
|
|
|
- recipe.name, recipe.engine, recipe.distributed_backend
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("recipe-starting", { "name" => rn, "engine" => re, "backend" => rb })
|
|
|
);
|
|
|
let (cluster, nccl) = resolve_cluster(config, &recipe, cluster_name)?;
|
|
|
-
|
|
|
if let Some(required) = recipe.required_nodes {
|
|
|
let actual = cluster.nodes.len() as u32;
|
|
|
if actual != required {
|
|
|
@@ -32,15 +34,12 @@ pub async fn serve(
|
|
|
);
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
crate::before_serve::run(config, &cluster, &recipe, skip_before_serve, dry_run).await?;
|
|
|
-
|
|
|
let image = try_repaired_image(config, &cluster, &recipe).await?;
|
|
|
let mut recipe = recipe;
|
|
|
if image != recipe.docker_image {
|
|
|
recipe.docker_image = image.clone();
|
|
|
}
|
|
|
-
|
|
|
let cache_dir = config.resolved_hf_cache_dir();
|
|
|
start_cluster(
|
|
|
config,
|
|
|
@@ -54,12 +53,11 @@ pub async fn serve(
|
|
|
)
|
|
|
.await?;
|
|
|
if !dry_run {
|
|
|
- info!("\n📜 Последние логи контейнеров:");
|
|
|
+ info!("{}", fluent_i18n::t!("container-log-header"));
|
|
|
show_logs(config, &cluster, recipe_name, false, dry_run).await?;
|
|
|
}
|
|
|
Ok(())
|
|
|
}
|
|
|
-
|
|
|
/// Builds the `docker logs` shell command used to fetch the last `tail`
|
|
|
/// lines of the container for `recipe_name` on a specific `rank`.
|
|
|
pub(crate) fn logs_tail_cmd(recipe_name: &str, rank: u32, tail: usize) -> String {
|
|
|
@@ -68,7 +66,6 @@ pub(crate) fn logs_tail_cmd(recipe_name: &str, rank: u32, tail: usize) -> String
|
|
|
recipe_name, rank, tail
|
|
|
)
|
|
|
}
|
|
|
-
|
|
|
/// Builds the `docker logs -f` shell command used to follow the head
|
|
|
/// container of `recipe_name` in real time.
|
|
|
pub(crate) fn logs_follow_cmd(recipe_name: &str) -> String {
|
|
|
@@ -77,7 +74,6 @@ pub(crate) fn logs_follow_cmd(recipe_name: &str) -> String {
|
|
|
recipe_name
|
|
|
)
|
|
|
}
|
|
|
-
|
|
|
/// Shell-quote `s` (single-quote + escape embedded `'` as `'\''`).
|
|
|
/// Used for entrypoint scripts passed to `sh -c` so bash treats them
|
|
|
/// as a single argv element in the docker run command.
|
|
|
@@ -94,7 +90,6 @@ fn shq(s: &str) -> String {
|
|
|
out.push('\'');
|
|
|
out
|
|
|
}
|
|
|
-
|
|
|
/// Shows Docker logs for a recipe across all nodes (or follows head node).
|
|
|
pub async fn show_logs(
|
|
|
config: &Config,
|
|
|
@@ -105,14 +100,17 @@ pub async fn show_logs(
|
|
|
) -> Result<()> {
|
|
|
if follow {
|
|
|
let head_node = cluster.nodes.first().context("No nodes in cluster")?;
|
|
|
- info!("📜 Following logs for '{}' (head node)...", head_node.name);
|
|
|
+ info!(
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("logs-follow-head", { "name" => &head_node.name })
|
|
|
+ );
|
|
|
let cmd = logs_follow_cmd(recipe_name);
|
|
|
run_ssh_follow(config, head_node, cmd, dry_run).await?;
|
|
|
} else {
|
|
|
+ let cn = cluster.name.clone();
|
|
|
info!(
|
|
|
- "📜 Логи кластера '{}' ({} узлов):",
|
|
|
- cluster.name,
|
|
|
- cluster.nodes.len()
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("logs-header", { "name" => cn, "count" => cluster.nodes.len() })
|
|
|
);
|
|
|
let mut handles = Vec::new();
|
|
|
for node in &cluster.nodes {
|
|
|
@@ -129,14 +127,13 @@ pub async fn show_logs(
|
|
|
if let Ok((name, Ok(output))) = result
|
|
|
&& !output.trim().is_empty()
|
|
|
{
|
|
|
- println!("\n--- {} ---", name);
|
|
|
+ println!("{}", fluent_i18n::t!("logs-section", { "name" => &name }));
|
|
|
println!("{}", output);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
Ok(())
|
|
|
}
|
|
|
-
|
|
|
/// Stops and removes containers for a recipe on all nodes.
|
|
|
pub async fn stop_cluster(
|
|
|
config: &Config,
|
|
|
@@ -144,16 +141,21 @@ pub async fn stop_cluster(
|
|
|
recipe_name: &str,
|
|
|
dry_run: bool,
|
|
|
) -> Result<()> {
|
|
|
- info!("🛑 Останавливаем кластер '{}'...", cluster.name);
|
|
|
+ info!(
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("cluster-stopping", { "name" => &cluster.name })
|
|
|
+ );
|
|
|
let mut failures = Vec::new();
|
|
|
-
|
|
|
for node in &cluster.nodes {
|
|
|
let stop_cmd = format!(
|
|
|
"docker ps -q --filter \"label=sm121.recipe={}\" | xargs -r docker stop",
|
|
|
recipe_name
|
|
|
);
|
|
|
if let Err(e) = run_ssh(config, node, stop_cmd, dry_run).await {
|
|
|
- tracing::warn!("Stop failed on {}: {}", node.name, e);
|
|
|
+ tracing::warn!(
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("stop-failed", { "name" => &node.name, "error" => e.to_string() })
|
|
|
+ );
|
|
|
failures.push((node.name.clone(), "stop".to_string(), e));
|
|
|
}
|
|
|
let rm_cmd = format!(
|
|
|
@@ -161,21 +163,25 @@ pub async fn stop_cluster(
|
|
|
recipe_name
|
|
|
);
|
|
|
if let Err(e) = run_ssh(config, node, rm_cmd, dry_run).await {
|
|
|
- tracing::warn!("Remove failed on {}: {}", node.name, e);
|
|
|
+ tracing::warn!(
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("remove-failed", { "name" => &node.name, "error" => e.to_string() })
|
|
|
+ );
|
|
|
failures.push((node.name.clone(), "rm".to_string(), e));
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
if !failures.is_empty() {
|
|
|
for (name, action, e) in &failures {
|
|
|
- tracing::warn!("Failed to {} on {}: {}", action, name, e);
|
|
|
+ tracing::warn!(
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("failed-to", { "action" => action, "name" => name, "error" => e.to_string() })
|
|
|
+ );
|
|
|
}
|
|
|
anyhow::bail!("Failed to stop containers on {} node(s)", failures.len());
|
|
|
}
|
|
|
- info!("✅ Все контейнеры остановлены.");
|
|
|
+ info!("{}", fluent_i18n::t!("cluster-all-stopped"));
|
|
|
Ok(())
|
|
|
}
|
|
|
-
|
|
|
/// Checks Docker health on a cluster — shows all containers and marks ours.
|
|
|
pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) -> Result<()> {
|
|
|
println!("\n=== Кластер: {} ===", cluster.name);
|
|
|
@@ -211,11 +217,9 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
|
|
|
.get(1)
|
|
|
.map(|s| s.lines().filter(|l| !l.is_empty()).collect())
|
|
|
.unwrap_or_default();
|
|
|
-
|
|
|
if dry_run && all_lines.is_empty() {
|
|
|
continue;
|
|
|
}
|
|
|
-
|
|
|
if output.starts_with("docker:") || output.contains("permission denied") {
|
|
|
println!(" ✗ Docker недоступен");
|
|
|
if let Some(line) = output.lines().next() {
|
|
|
@@ -249,7 +253,6 @@ pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) ->
|
|
|
}
|
|
|
Ok(())
|
|
|
}
|
|
|
-
|
|
|
/// Starts containers on all nodes in parallel.
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
|
async fn start_cluster(
|
|
|
@@ -269,7 +272,6 @@ async fn start_cluster(
|
|
|
recipe.pipeline_parallel_size.unwrap()
|
|
|
};
|
|
|
let mut handles = Vec::new();
|
|
|
-
|
|
|
for node in &cluster.nodes {
|
|
|
let node = node.clone();
|
|
|
let cluster = cluster.clone();
|
|
|
@@ -284,7 +286,6 @@ async fn start_cluster(
|
|
|
"docker rm -f sm121_{}_{} 2>/dev/null; true",
|
|
|
recipe_name, node.rank
|
|
|
);
|
|
|
-
|
|
|
let cd = cache_dir.clone();
|
|
|
let cd_s = cd.to_string_lossy().into_owned();
|
|
|
handles.push(tokio::spawn(async move {
|
|
|
@@ -309,36 +310,28 @@ async fn start_cluster(
|
|
|
(node.name, node.rank, engine, result)
|
|
|
}));
|
|
|
}
|
|
|
-
|
|
|
for result in futures::future::join_all(handles).await {
|
|
|
match result {
|
|
|
Ok((name, rank, engine, Ok(_))) => info!(
|
|
|
- "✅ {} запущен на {} (rank {})",
|
|
|
- engine.to_uppercase(),
|
|
|
- name,
|
|
|
- rank
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("engine-started", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank })
|
|
|
),
|
|
|
Ok((name, rank, engine, Err(e))) => anyhow::bail!(
|
|
|
- "Failed to start {} on {} (rank {}): {}",
|
|
|
- engine.to_uppercase(),
|
|
|
- name,
|
|
|
- rank,
|
|
|
- e
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("engine-start-failed", { "engine" => engine.to_uppercase(), "name" => &name, "rank" => rank, "error" => e.to_string() })
|
|
|
),
|
|
|
Err(e) => anyhow::bail!("Task join error: {}", e),
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
if !dry_run {
|
|
|
sleep(Duration::from_secs(10)).await;
|
|
|
info!(
|
|
|
- "🎉 Кластер успешно запущен! API → http://{}:{}",
|
|
|
- cluster.master_ib_ip, recipe.port
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("cluster-started", { "ip" => &cluster.master_ib_ip, "port" => recipe.port })
|
|
|
);
|
|
|
}
|
|
|
Ok(())
|
|
|
}
|
|
|
-
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
|
fn build_docker_command(
|
|
|
cluster: &Cluster,
|
|
|
@@ -354,7 +347,6 @@ fn build_docker_command(
|
|
|
init_port: u16,
|
|
|
) -> Vec<String> {
|
|
|
let is_sglang = recipe.engine == "sglang";
|
|
|
-
|
|
|
let mut flags: Vec<String> = vec![
|
|
|
"--gpus all".to_string(),
|
|
|
"--ipc=host".to_string(),
|
|
|
@@ -380,7 +372,6 @@ fn build_docker_command(
|
|
|
format!("-e HF_HOME=/root/.cache/huggingface"),
|
|
|
format!("-v {}:/root/.cache/huggingface", cache_dir),
|
|
|
];
|
|
|
-
|
|
|
// Engine-specific host IP env var
|
|
|
if is_sglang {
|
|
|
cmd.push(format!("-e SGLANG_WORKER_IP={}", node.ib_ip));
|
|
|
@@ -394,13 +385,11 @@ fn build_docker_command(
|
|
|
"-e OMPI_MCA_btl_tcp_if_include={}",
|
|
|
nccl.socket_ifname
|
|
|
));
|
|
|
-
|
|
|
// Default cluster env (recipe env below overrides these if duplicated)
|
|
|
cmd.push("-e OMP_NUM_THREADS=4".to_string());
|
|
|
cmd.push("-e NCCL_IGNORE_CPU_AFFINITY=1".to_string());
|
|
|
cmd.push("-e NCCL_NET=IB".to_string());
|
|
|
cmd.push("-e NCCL_DEBUG=WARN".to_string());
|
|
|
-
|
|
|
// Force colored vLLM logs through Docker pipeline
|
|
|
if is_head {
|
|
|
cmd.push("-e VLLM_LOGGING_COLOR=1".to_string());
|
|
|
@@ -434,7 +423,6 @@ fn build_docker_command(
|
|
|
}
|
|
|
cmd.extend(ep);
|
|
|
}
|
|
|
-
|
|
|
// Model argument - engine-specific
|
|
|
if is_sglang {
|
|
|
cmd.push("--model-path".to_string());
|
|
|
@@ -442,9 +430,7 @@ fn build_docker_command(
|
|
|
} else {
|
|
|
cmd.push(recipe.model.clone());
|
|
|
}
|
|
|
-
|
|
|
cmd.extend(recipe.args.clone());
|
|
|
-
|
|
|
// Distributed flags - engine-specific
|
|
|
let mut flags: Vec<String> = match recipe.distributed_backend.as_str() {
|
|
|
"ray" | "mp" if is_sglang => vec![
|
|
|
@@ -481,13 +467,11 @@ fn build_docker_command(
|
|
|
}
|
|
|
_ => vec![],
|
|
|
};
|
|
|
-
|
|
|
// vLLM workers run headless (skip API server) when not rank 0
|
|
|
if !is_sglang && node.rank > 0 && matches!(recipe.distributed_backend.as_str(), "mp" | "ray") {
|
|
|
flags.push("--headless".into());
|
|
|
}
|
|
|
cmd.extend(flags);
|
|
|
-
|
|
|
// Host + port flags — auto-generated for both engines.
|
|
|
cmd.push("--host".to_string());
|
|
|
cmd.push("0.0.0.0".to_string());
|
