|
|
@@ -1,11 +1,13 @@
|
|
|
pub mod deploy;
|
|
|
pub mod health;
|
|
|
+pub mod logs;
|
|
|
pub mod mods;
|
|
|
+pub use logs::show_logs;
|
|
|
pub mod tail;
|
|
|
use crate::{
|
|
|
config::{Cluster, Config, NcclConfig, Node, Recipe, load_recipe, resolve_cluster},
|
|
|
doctor::try_repaired_image,
|
|
|
- ssh::{run_ssh, run_ssh_follow, run_ssh_with_retry},
|
|
|
+ ssh::{run_ssh, run_ssh_with_retry},
|
|
|
};
|
|
|
use anyhow::{Context, Result};
|
|
|
use std::borrow::Cow;
|
|
|
@@ -13,6 +15,20 @@ use std::collections::HashSet;
|
|
|
use tokio::time::{Duration, sleep};
|
|
|
use tracing::info;
|
|
|
|
|
|
+pub(crate) fn shq(s: &str) -> String {
|
|
|
+ let mut out = String::with_capacity(s.len() + 2);
|
|
|
+ out.push('\'');
|
|
|
+ for c in s.chars() {
|
|
|
+ if c == '\'' {
|
|
|
+ out.push_str(r"'\''");
|
|
|
+ } else {
|
|
|
+ out.push(c);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ out.push('\'');
|
|
|
+ out
|
|
|
+}
|
|
|
+
|
|
|
pub async fn serve(
|
|
|
config: &Config,
|
|
|
recipe_name: &str,
|
|
|
@@ -109,75 +125,6 @@ pub async fn serve(
|
|
|
.await?;
|
|
|
Ok(cluster)
|
|
|
}
|
|
|
-pub(crate) fn logs_tail_cmd(recipe_name: &str, rank: u32, tail: usize) -> String {
|
|
|
- format!(
|
|
|
- "docker ps -q --filter \"label=sm121.recipe={}\" --filter \"label=sm121.rank={}\" | xargs -r docker logs --timestamps --tail={} 2>&1",
|
|
|
- recipe_name, rank, tail
|
|
|
- )
|
|
|
-}
|
|
|
-pub(crate) fn logs_follow_cmd(recipe_name: &str) -> String {
|
|
|
- format!(
|
|
|
- "docker ps -q --filter \"label=sm121.recipe={}\" --filter \"label=sm121.rank=0\" | xargs -r docker logs -f --timestamps 2>&1",
|
|
|
- recipe_name
|
|
|
- )
|
|
|
-}
|
|
|
-pub(crate) fn shq(s: &str) -> String {
|
|
|
- let mut out = String::with_capacity(s.len() + 2);
|
|
|
- out.push('\'');
|
|
|
- for c in s.chars() {
|
|
|
- if c == '\'' {
|
|
|
- out.push_str(r"'\''");
|
|
|
- } else {
|
|
|
- out.push(c);
|
|
|
- }
|
|
|
- }
|
|
|
- out.push('\'');
|
|
|
- out
|
|
|
-}
|
|
|
-pub async fn show_logs(
|
|
|
- config: &Config,
|
|
|
- cluster: &Cluster,
|
|
|
- recipe_name: &str,
|
|
|
- follow: bool,
|
|
|
- dry_run: bool,
|
|
|
-) -> Result<()> {
|
|
|
- if follow {
|
|
|
- let head_node = cluster.nodes.first().context("No nodes in cluster")?;
|
|
|
- info!(
|
|
|
- "{}",
|
|
|
- fluent_i18n::t!("logs-follow-head", { "name" => &head_node.name })
|
|
|
- );
|
|
|
- let cmd = logs_follow_cmd(recipe_name);
|
|
|
- run_ssh_follow(config, head_node, cmd, dry_run).await?;
|
|
|
- } else {
|
|
|
- let cn = cluster.name.clone();
|
|
|
- info!(
|
|
|
- "{}",
|
|
|
- fluent_i18n::t!("logs-header", { "name" => cn, "count" => cluster.nodes.len() })
|
|
|
- );
|
|
|
- let mut handles = Vec::new();
|
|
|
- for node in &cluster.nodes {
|
|
|
- let cmd = logs_tail_cmd(recipe_name, node.rank, 100);
|
|
|
- let config = config.clone();
|
|
|
- let node = node.clone();
|
|
|
- let dry = dry_run;
|
|
|
- let name = node.name.clone();
|
|
|
- handles.push(tokio::spawn(async move {
|
|
|
- (name, run_ssh_with_retry(&config, &node, cmd, dry, 1).await)
|
|
|
- }));
|
|
|
- }
|
|
|
- for result in futures::future::join_all(handles).await {
|
|
|
- if let Ok((name, Ok(output))) = result
|
|
|
- && !output.trim().is_empty()
|
|
|
- {
|
|
|
- println!("{}", fluent_i18n::t!("logs-section", { "name" => &name }));
|
|
|
- println!("{}", output);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- Ok(())
|
|
|
-}
|
|
|
-/// Stops containers for a recipe on specific nodes of a cluster.
|
|
|
pub async fn stop_cluster(
|
|
|
config: &Config,
|
|
|
cluster: &Cluster,
|