|
|
@@ -118,18 +118,32 @@ pub async fn sync_image_to_nodes(
|
|
|
fluent_i18n::t!("sync-syncing", { "name" => &target.name, "ip" => &target.ssh_ip })
|
|
|
);
|
|
|
// Data path: node-to-node, never through the Mac (mac→node is ~32
|
|
|
- // MB/s vs ~547 MB/s per stream on the 200G fabric). Prefer the
|
|
|
- // target's fabric IP (ib_ip) for the inner hop; nodes without a
|
|
|
- // 200G uplink fall back to the 10G mgmt IP.
|
|
|
- let target_ip = target.ib_ip.clone().unwrap_or_else(|| target.ssh_ip.clone());
|
|
|
+ // MB/s vs ~547 MB/s per stream on the 200G fabric). The inner hop
|
|
|
+ // prefers the target's fabric IP (ib_ip); if that fails (broken
|
|
|
+ // node-to-node ssh over 200.x) it falls back to the 10G mgmt IP with
|
|
|
+ // a warning, so a stale authorized_keys never blocks a serve.
|
|
|
+ let attempts: Vec<String> = match &target.ib_ip {
|
|
|
+ Some(ib) => vec![ib.clone(), target.ssh_ip.clone()],
|
|
|
+ None => vec![target.ssh_ip.clone()],
|
|
|
+ };
|
|
|
// Fetch the target's full public key over a verified russh connection
|
|
|
// (which enforces the pinned fingerprint), then write a per-target
|
|
|
- // known_hosts on the SOURCE node and run the inner ssh with it,
|
|
|
- // StrictHostKeyChecking=yes. A host that doesn't hold the pinned key
|
|
|
- // is rejected before any auth — no key leak, no forgery.
|
|
|
+ // known_hosts on the SOURCE node (one line per IP we may hop to) and
|
|
|
+ // run the inner ssh with it, StrictHostKeyChecking=yes. A host that
|
|
|
+ // doesn't hold the pinned key is rejected before any auth — no key
|
|
|
+ // leak, no forgery.
|
|
|
let pubkey = crate::ssh::fetch_host_pubkey(config, target).await?;
|
|
|
let known_hosts = format!("/tmp/sm121_known_hosts_{}", target.name);
|
|
|
- let entry = format!("{} {}\n", target_ip, pubkey);
|
|
|
+ let mut kh_ips = vec![target.ssh_ip.clone()];
|
|
|
+ if let Some(ib) = &target.ib_ip {
|
|
|
+ kh_ips.push(ib.clone());
|
|
|
+ }
|
|
|
+ let entry = kh_ips
|
|
|
+ .iter()
|
|
|
+ .map(|ip| format!("{} {}", ip, pubkey))
|
|
|
+ .collect::<Vec<_>>()
|
|
|
+ .join("\n")
|
|
|
+ + "\n";
|
|
|
let write_known = format!(
|
|
|
"printf '%s' {} > {} && chmod 600 {}",
|
|
|
shq(&entry),
|
|
|
@@ -138,29 +152,60 @@ pub async fn sync_image_to_nodes(
|
|
|
);
|
|
|
run_ssh(config, src_node, write_known, false).await?;
|
|
|
|
|
|
- let pipe_cmd = format!(
|
|
|
- "docker save {} | ssh -o StrictHostKeyChecking=yes -o HashKnownHosts=no -o UserKnownHostsFile={} -i {} {}@{} docker load; ec=$?; rm -f {}; exit $ec",
|
|
|
- shq(image),
|
|
|
- shq(&known_hosts),
|
|
|
- shq(&config.ssh_key_path.display().to_string()),
|
|
|
- shq(&config.ssh_user),
|
|
|
- shq(&target_ip),
|
|
|
- shq(&known_hosts),
|
|
|
- );
|
|
|
- run_ssh_long_running(config, src_node, pipe_cmd, false).await?;
|
|
|
- let verify = get_image_digest(config, target, image).await?;
|
|
|
- if verify.as_deref() != Some(source_digest) {
|
|
|
+ let pipe_for = |ip: &str| {
|
|
|
+ format!(
|
|
|
+ "docker save {} | ssh -o StrictHostKeyChecking=yes -o HashKnownHosts=no -o UserKnownHostsFile={} -i {} {}@{} docker load; ec=$?; rm -f {}; exit $ec",
|
|
|
+ shq(image),
|
|
|
+ shq(&known_hosts),
|
|
|
+ shq(&config.ssh_key_path.display().to_string()),
|
|
|
+ shq(&config.ssh_user),
|
|
|
+ shq(ip),
|
|
|
+ shq(&known_hosts),
|
|
|
+ )
|
|
|
+ };
|
|
|
+ let mut synced = false;
|
|
|
+ let mut last_err: Option<String> = None;
|
|
|
+ for (idx, ip) in attempts.iter().enumerate() {
|
|
|
+ match run_ssh_long_running(config, src_node, pipe_for(ip), false).await {
|
|
|
+ Ok(_) => {
|
|
|
+ let verify = get_image_digest(config, target, image).await?;
|
|
|
+ if verify.as_deref() == Some(source_digest) {
|
|
|
+ info!(
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("sync-synced", { "name" => &target.name })
|
|
|
+ );
|
|
|
+ synced = true;
|
|
|
+ } else {
|
|
|
+ // a digest mismatch is not a connectivity problem —
|
|
|
+ // the other IP won't help, fail outright
|
|
|
+ last_err = Some(format!(
|
|
|
+ "digest mismatch (source={}, target={:?})",
|
|
|
+ source_digest, verify
|
|
|
+ ));
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Err(e) => {
|
|
|
+ if idx == 0 {
|
|
|
+ tracing::warn!(
|
|
|
+ "{}",
|
|
|
+ fluent_i18n::t!("sync-fabric-fallback", {
|
|
|
+ "name" => &target.name,
|
|
|
+ "err" => e.to_string(),
|
|
|
+ })
|
|
|
+ );
|
|
|
+ }
|
|
|
+ last_err = Some(e.to_string());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if !synced {
|
|
|
anyhow::bail!(
|
|
|
- "Image sync to '{}' failed: digest mismatch (source={}, target={:?})",
|
|
|
+ "Image sync to '{}' failed: {}",
|
|
|
target.name,
|
|
|
- source_digest,
|
|
|
- verify
|
|
|
+ last_err.unwrap_or_default()
|
|
|
);
|
|
|
}
|
|
|
- info!(
|
|
|
- "{}",
|
|
|
- fluent_i18n::t!("sync-synced", { "name" => &target.name })
|
|
|
- );
|
|
|
}
|
|
|
Ok(())
|
|
|
}
|
