|
@@ -0,0 +1,81 @@
|
|
|
|
|
+use crate::config::{Config, Node};
|
|
|
|
|
+use crate::ssh::run_ssh;
|
|
|
|
|
+use anyhow::Result;
|
|
|
|
|
+use tracing::info;
|
|
|
|
|
+
|
|
|
|
|
+/// Start a Docker image build on a remote node in a tmux session.
|
|
|
|
|
+/// Polls every 30s, streams log output, and returns when the build completes.
|
|
|
|
|
+pub async fn build_image_on_node(
|
|
|
|
|
+ config: &Config,
|
|
|
|
|
+ node: &Node,
|
|
|
|
|
+ local_dir: &std::path::Path,
|
|
|
|
|
+ recipe_name: &str,
|
|
|
|
|
+ image_tag: &str,
|
|
|
|
|
+) -> Result<()> {
|
|
|
|
|
+ let session_name = format!("glm52-build-{}", recipe_name);
|
|
|
|
|
+ let logfile = format!("/tmp/{}-build.log", recipe_name);
|
|
|
|
|
+ let remote_dir = format!("/tmp/{}-build", recipe_name);
|
|
|
|
|
+
|
|
|
|
|
+ info!(
|
|
|
|
|
+ "Building image '{}' on '{}' (tmux: {})...",
|
|
|
|
|
+ image_tag, node.name, session_name
|
|
|
|
|
+ );
|
|
|
|
|
+ crate::ssh::upload_dir(config, node, local_dir, &remote_dir, false).await?;
|
|
|
|
|
+
|
|
|
|
|
+ let tmux_cmd = format!(
|
|
|
|
|
+ "tmux new-session -d -s '{}' -c {} \"TAG={} bash build.sh 2>&1 | tee {}\"",
|
|
|
|
|
+ session_name, remote_dir, image_tag, logfile
|
|
|
|
|
+ );
|
|
|
|
|
+ run_ssh(config, node, tmux_cmd, false).await?;
|
|
|
|
|
+
|
|
|
|
|
+ let start = std::time::Instant::now();
|
|
|
|
|
+ loop {
|
|
|
|
|
+ tokio::time::sleep(std::time::Duration::from_secs(30)).await;
|
|
|
|
|
+ let elapsed = start.elapsed();
|
|
|
|
|
+
|
|
|
|
|
+ let alive = run_ssh(
|
|
|
|
|
+ config,
|
|
|
|
|
+ node,
|
|
|
|
|
+ format!("tmux has-session -t '{}' 2>&1 || echo DEAD", session_name),
|
|
|
|
|
+ false,
|
|
|
|
|
+ )
|
|
|
|
|
+ .await?;
|
|
|
|
|
+
|
|
|
|
|
+ if let Ok(logs) = run_ssh(
|
|
|
|
|
+ config,
|
|
|
|
|
+ node,
|
|
|
|
|
+ format!("tail -10 {} 2>/dev/null || echo '(no log yet)'", logfile),
|
|
|
|
|
+ false,
|
|
|
|
|
+ )
|
|
|
|
|
+ .await
|
|
|
|
|
+ {
|
|
|
|
|
+ for line in logs.lines().filter(|l| !l.is_empty()) {
|
|
|
|
|
+ info!(" [build] {}", line.trim());
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if alive.trim().contains("DEAD") || alive.trim().is_empty() {
|
|
|
|
|
+ let exit_log = run_ssh(config, node, format!("tail -5 {}", logfile), false)
|
|
|
|
|
+ .await
|
|
|
|
|
+ .unwrap_or_default();
|
|
|
|
|
+ if exit_log.contains(&format!("Done: {}", image_tag)) {
|
|
|
|
|
+ info!(
|
|
|
|
|
+ "Image '{}' built on '{}' ({}m)",
|
|
|
|
|
+ image_tag,
|
|
|
|
|
+ node.name,
|
|
|
|
|
+ elapsed.as_secs() / 60
|
|
|
|
|
+ );
|
|
|
|
|
+ } else {
|
|
|
|
|
+ anyhow::bail!(
|
|
|
|
|
+ "Build on '{}' failed after {}m.\n{}",
|
|
|
|
|
+ node.name,
|
|
|
|
|
+ elapsed.as_secs() / 60,
|
|
|
|
|
+ exit_log
|
|
|
|
|
+ );
|
|
|
|
|
+ }
|
|
|
|
|
+ break;
|
|
|
|
|
+ }
|
|
|
|
|
+ info!(" Build running for {}m...", elapsed.as_secs() / 60);
|
|
|
|
|
+ }
|
|
|
|
|
+ Ok(())
|
|
|
|
|
+}
|