|
@@ -1,10 +1,9 @@
|
|
|
-use anyhow::{Context, Result};
|
|
|
|
|
-use tokio::time::{Duration, sleep};
|
|
|
|
|
-use tracing::info;
|
|
|
|
|
-
|
|
|
|
|
use crate::config::{Cluster, Config, NcclConfig, Node, Recipe, load_recipe, resolve_cluster};
|
|
use crate::config::{Cluster, Config, NcclConfig, Node, Recipe, load_recipe, resolve_cluster};
|
|
|
use crate::doctor::try_repaired_image;
|
|
use crate::doctor::try_repaired_image;
|
|
|
use crate::ssh::{run_ssh, run_ssh_follow, run_ssh_with_retry};
|
|
use crate::ssh::{run_ssh, run_ssh_follow, run_ssh_with_retry};
|
|
|
|
|
+use anyhow::{Context, Result};
|
|
|
|
|
+use tokio::time::{Duration, sleep};
|
|
|
|
|
+use tracing::info;
|
|
|
/// Launches containers across the cluster for a recipe.
|
|
/// Launches containers across the cluster for a recipe.
|
|
|
pub async fn serve(
|
|
pub async fn serve(
|
|
|
config: &Config,
|
|
config: &Config,
|
|
@@ -261,7 +260,7 @@ async fn start_cluster(
|
|
|
nccl: &NcclConfig,
|
|
nccl: &NcclConfig,
|
|
|
recipe_name: &str,
|
|
recipe_name: &str,
|
|
|
recipe: &Recipe,
|
|
recipe: &Recipe,
|
|
|
- cache_dir: String,
|
|
|
|
|
|
|
+ cache_dir: std::path::PathBuf,
|
|
|
keep: bool,
|
|
keep: bool,
|
|
|
dry_run: bool,
|
|
dry_run: bool,
|
|
|
) -> Result<()> {
|
|
) -> Result<()> {
|
|
@@ -289,6 +288,7 @@ async fn start_cluster(
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
let cd = cache_dir.clone();
|
|
let cd = cache_dir.clone();
|
|
|
|
|
+ let cd_s = cd.to_string_lossy().into_owned();
|
|
|
handles.push(tokio::spawn(async move {
|
|
handles.push(tokio::spawn(async move {
|
|
|
if !dry {
|
|
if !dry {
|
|
|
let _ = run_ssh(&config, &node, cleanup, dry).await;
|
|
let _ = run_ssh(&config, &node, cleanup, dry).await;
|
|
@@ -302,7 +302,7 @@ async fn start_cluster(
|
|
|
is_head,
|
|
is_head,
|
|
|
pp_size,
|
|
pp_size,
|
|
|
nnodes,
|
|
nnodes,
|
|
|
- &cd,
|
|
|
|
|
|
|
+ &cd_s,
|
|
|
keep,
|
|
keep,
|
|
|
config.init_port,
|
|
config.init_port,
|
|
|
)
|
|
)
|