|
@@ -3,7 +3,7 @@ use std::time::Duration;
|
|
|
use tokio::time::sleep;
|
|
use tokio::time::sleep;
|
|
|
use tracing::{info, warn};
|
|
use tracing::{info, warn};
|
|
|
|
|
|
|
|
-use crate::config::{Config, Node, Recipe};
|
|
|
|
|
|
|
+use crate::config::{Config, Recipe};
|
|
|
use crate::tmux::TmuxManager;
|
|
use crate::tmux::TmuxManager;
|
|
|
|
|
|
|
|
/// Launches a cluster via tmux sessions.
|
|
/// Launches a cluster via tmux sessions.
|
|
@@ -274,9 +274,19 @@ async fn prepare_venv(
|
|
|
) -> Result<()> {
|
|
) -> Result<()> {
|
|
|
let venv = match &recipe.venv {
|
|
let venv = match &recipe.venv {
|
|
|
Some(v) => v,
|
|
Some(v) => v,
|
|
|
- None => return Ok(()),
|
|
|
|
|
|
|
+ None => {
|
|
|
|
|
+ info!("prepare_venv: no venv field in recipe — skipping");
|
|
|
|
|
+ return Ok(());
|
|
|
|
|
+ }
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
|
|
+ info!(
|
|
|
|
|
+ "prepare_venv: venv={}, has_venv_setup_script={}, has_venv_setup={}",
|
|
|
|
|
+ venv,
|
|
|
|
|
+ recipe.venv_setup_script.is_some(),
|
|
|
|
|
+ !recipe.venv_setup.to_vec().is_empty()
|
|
|
|
|
+ );
|
|
|
|
|
+
|
|
|
// Resolve setup commands: script file takes precedence over inline.
|
|
// Resolve setup commands: script file takes precedence over inline.
|
|
|
let setup: Vec<String> = if let Some(script_path) = &recipe.venv_setup_script {
|
|
let setup: Vec<String> = if let Some(script_path) = &recipe.venv_setup_script {
|
|
|
let full_path = format!("recipes/{}/{}", recipe_name, script_path);
|
|
let full_path = format!("recipes/{}/{}", recipe_name, script_path);
|
|
@@ -297,104 +307,65 @@ async fn prepare_venv(
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
if setup.is_empty() {
|
|
if setup.is_empty() {
|
|
|
|
|
+ info!("venv setup is empty for {} — skipping", venv);
|
|
|
return Ok(());
|
|
return Ok(());
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- info!("{}", fluent_i18n::t!("venv-preparing", { "venv" => venv }));
|
|
|
|
|
|
|
+ info!(
|
|
|
|
|
+ "{} ({} commands)",
|
|
|
|
|
+ fluent_i18n::t!("venv-preparing", { "venv" => venv }),
|
|
|
|
|
+ setup.len()
|
|
|
|
|
+ );
|
|
|
|
|
|
|
|
- let mut handles = Vec::with_capacity(cluster.nodes.len());
|
|
|
|
|
|
|
+ // Prepare venv on each node sequentially.
|
|
|
for node in &cluster.nodes {
|
|
for node in &cluster.nodes {
|
|
|
- let config = config.clone();
|
|
|
|
|
- let node = node.clone();
|
|
|
|
|
- let venv = venv.clone();
|
|
|
|
|
- let setup = setup.clone();
|
|
|
|
|
- let dry = dry_run;
|
|
|
|
|
-
|
|
|
|
|
- handles.push(crate::locale::spawn_with_locale(async move {
|
|
|
|
|
- (
|
|
|
|
|
- node.name.clone(),
|
|
|
|
|
- prepare_venv_for_node(&config, &node, &venv, &setup, dry).await,
|
|
|
|
|
- )
|
|
|
|
|
- }));
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- let mut any_failed = false;
|
|
|
|
|
- for h in handles {
|
|
|
|
|
- let (name, result) = h.await?;
|
|
|
|
|
- if let Err(e) = result {
|
|
|
|
|
- any_failed = true;
|
|
|
|
|
- warn!(
|
|
|
|
|
- "{}",
|
|
|
|
|
- fluent_i18n::t!("venv-prepare-failed", { "name" => &name, "error" => e.to_string() })
|
|
|
|
|
- );
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- if any_failed {
|
|
|
|
|
- anyhow::bail!("Venv preparation failed on at least one node (see warnings above)");
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ let check_cmd = format!("test -f {}/bin/activate", venv);
|
|
|
|
|
|
|
|
- Ok(())
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
-/// Checks if the venv exists on a single node and creates it if not.
|
|
|
|
|
-async fn prepare_venv_for_node(
|
|
|
|
|
- config: &Config,
|
|
|
|
|
- node: &Node,
|
|
|
|
|
- venv: &str,
|
|
|
|
|
- setup: &[String],
|
|
|
|
|
- dry_run: bool,
|
|
|
|
|
-) -> Result<()> {
|
|
|
|
|
- let check_cmd = format!("test -f {}/bin/activate", venv);
|
|
|
|
|
-
|
|
|
|
|
- if dry_run {
|
|
|
|
|
- info!(
|
|
|
|
|
- " ssh {}@{}: {} (dry-run)",
|
|
|
|
|
- config.ssh_user, node.ssh_ip, check_cmd
|
|
|
|
|
- );
|
|
|
|
|
- for cmd in setup {
|
|
|
|
|
- info!(" {} (dry-run)", cmd);
|
|
|
|
|
- }
|
|
|
|
|
- return Ok(());
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- // Check if venv already exists.
|
|
|
|
|
- match crate::ssh::run_ssh(config, node, check_cmd.clone(), false).await {
|
|
|
|
|
- Ok(_) => {
|
|
|
|
|
- info!(
|
|
|
|
|
- "{}",
|
|
|
|
|
- fluent_i18n::t!("venv-exists", { "name" => &node.name })
|
|
|
|
|
- );
|
|
|
|
|
- return Ok(());
|
|
|
|
|
- }
|
|
|
|
|
- Err(_) => {
|
|
|
|
|
|
|
+ if !dry_run {
|
|
|
|
|
+ // Skip if venv already exists.
|
|
|
|
|
+ if crate::ssh::run_ssh(config, node, check_cmd.clone(), false)
|
|
|
|
|
+ .await
|
|
|
|
|
+ .is_ok()
|
|
|
|
|
+ {
|
|
|
|
|
+ info!(
|
|
|
|
|
+ "{}",
|
|
|
|
|
+ fluent_i18n::t!("venv-exists", { "name" => &node.name })
|
|
|
|
|
+ );
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
info!(
|
|
info!(
|
|
|
"{}",
|
|
"{}",
|
|
|
fluent_i18n::t!("venv-creating", { "name" => &node.name })
|
|
fluent_i18n::t!("venv-creating", { "name" => &node.name })
|
|
|
);
|
|
);
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
|
|
|
|
|
- // Run setup commands in order.
|
|
|
|
|
- for cmd in setup {
|
|
|
|
|
- info!(" ssh {}@{}: {}", config.ssh_user, node.ssh_ip, cmd);
|
|
|
|
|
- crate::ssh::run_ssh(config, node, cmd.clone(), false).await?;
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- // Re-check that the venv was created.
|
|
|
|
|
- match crate::ssh::run_ssh(config, node, check_cmd, false).await {
|
|
|
|
|
- Ok(_) => {
|
|
|
|
|
- info!(
|
|
|
|
|
- "{}",
|
|
|
|
|
- fluent_i18n::t!("venv-created", { "name" => &node.name })
|
|
|
|
|
- );
|
|
|
|
|
- Ok(())
|
|
|
|
|
- }
|
|
|
|
|
- Err(_) => {
|
|
|
|
|
- anyhow::bail!(
|
|
|
|
|
- "{}",
|
|
|
|
|
- fluent_i18n::t!("venv-not-created", { "name" => &node.name })
|
|
|
|
|
|
|
+ // Create venv, then run setup with it activated so that
|
|
|
|
|
+ // uv pip install finds VIRTUAL_ENV. All commands are joined
|
|
|
|
|
+ // with && so the script halts on the first failure.
|
|
|
|
|
+ let joined = setup.join(" && ");
|
|
|
|
|
+ let full_script = format!(
|
|
|
|
|
+ "uv venv {} -p 3.12 && source {}/bin/activate && {}",
|
|
|
|
|
+ venv, venv, joined
|
|
|
);
|
|
);
|
|
|
|
|
+ let script = format!("bash -c '{}'", full_script.replace('\'', "'\\''"));
|
|
|
|
|
+ crate::ssh::run_ssh(config, node, script, false).await?;
|
|
|
|
|
+
|
|
|
|
|
+ // Re-check.
|
|
|
|
|
+ if crate::ssh::run_ssh(config, node, check_cmd, false)
|
|
|
|
|
+ .await
|
|
|
|
|
+ .is_ok()
|
|
|
|
|
+ {
|
|
|
|
|
+ info!(
|
|
|
|
|
+ "{}",
|
|
|
|
|
+ fluent_i18n::t!("venv-created", { "name" => &node.name })
|
|
|
|
|
+ );
|
|
|
|
|
+ } else {
|
|
|
|
|
+ anyhow::bail!(
|
|
|
|
|
+ "{}",
|
|
|
|
|
+ fluent_i18n::t!("venv-not-created", { "name" => &node.name })
|
|
|
|
|
+ );
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
+
|
|
|
|
|
+ Ok(())
|
|
|
}
|
|
}
|