|
@@ -107,11 +107,22 @@ pub async fn follow_with_liveness(
|
|
|
tokio::select! {
|
|
tokio::select! {
|
|
|
r = show_logs(config, cluster, recipe_name, true, node_name, dry_run) => {
|
|
r = show_logs(config, cluster, recipe_name, true, node_name, dry_run) => {
|
|
|
r?;
|
|
r?;
|
|
|
- if !crate::readiness::alive(cluster, port).await {
|
|
|
|
|
|
|
+ // The log stream ended. Do NOT decide death from the one-shot API
|
|
|
|
|
+ // probe (`alive`): the API port is closed for the whole (multi-
|
|
|
|
|
+ // minute) weight load, so a false answer here is indistinguishable
|
|
|
|
|
+ // from "still starting" — that mislabelled a slow boot as "engine
|
|
|
|
|
+ // is down — containers died". A real death is a container that has
|
|
|
|
|
+ // EXITED; a still-Up container means the engine is loading or the
|
|
|
|
|
+ // ssh follow simply dropped. Mirror `wait_engine`'s check.
|
|
|
|
|
+ if let Err(e) =
|
|
|
|
|
+ crate::readiness::containers_alive(config, cluster, recipe_name).await
|
|
|
|
|
+ {
|
|
|
let _ = crate::runlock::mark_stop(recipe_name, &config.recipe_sources);
|
|
let _ = crate::runlock::mark_stop(recipe_name, &config.recipe_sources);
|
|
|
- anyhow::bail!("log stream ended and the engine is down — containers died mid-follow");
|
|
|
|
|
|
|
+ anyhow::bail!("log stream ended and a container exited — {e}");
|
|
|
}
|
|
}
|
|
|
- tracing::warn!("log stream ended but the engine is still up (ssh drop?)");
|
|
|
|
|
|
|
+ tracing::warn!(
|
|
|
|
|
+ "log stream ended but the containers are still up (ssh drop, or the engine is still starting)"
|
|
|
|
|
+ );
|
|
|
Ok(())
|
|
Ok(())
|
|
|
}
|
|
}
|
|
|
r = watch => {
|
|
r = watch => {
|