docker: image sync falls back 200G→10G with a warning (fabric ssh fix hint)

AGENTS.md

@@ -356,8 +356,10 @@ rm -f` by name, log forensics) — and those get noted in the shift history.
 Image distribution (`sync_image_to_nodes`) is node-to-node, never through
 the Mac: the source node (newest build, or `--build-node`/`--sync-image-source`)
 runs `docker save | ssh <peer> docker load` per peer, with the inner hop
-host-key verified against the pinned fingerprint and routed over the peer's
-`ib_ip` (200G fabric) when it has one, else `ssh_ip` (10G mgmt).
+host-key verified against the pinned fingerprint, and routed over the peer's
+`ib_ip` (200G fabric) when it has one — with a logged fallback to `ssh_ip`
+(10G mgmt) if the fabric hop fails, so a stale node-to-node `authorized_keys`
+warns instead of blocking the serve.
 
 A recipe's `docker_image` is built on demand from `docker-images/<docker_image_dir>/`
 (default `<dir>` = recipe name). `sm121 serve --build-node <n>` (or the

locales/de/main.ftl

@@ -133,6 +133,7 @@ tmux-no-pty = (russh unterstützt kein interaktives PTY. Befehl direkt ausführe
 sync-digest-match = '{ $name }' OK (Digest stimmt überein)
 sync-syncing = { $name } ({ $ip }): synchronisiere...
 sync-synced = { $name } OK
+sync-fabric-fallback = { $name }: 200G-Fabric-Hop fehlgeschlagen ({ $err }) — Fallback auf 10G-Verwaltung; node-to-node-SSH über die Fabric-NICs prüfen (authorized_keys / Host-Keys für 200.x-Adressen)
 health-node-offline =   ✗ { $name } ({ $ip }) — nicht erreichbar
 health-node-idle =   ○ { $name } ({ $ip }){ $net }
 health-node-online =   ● { $name } ({ $ip }){ $net }

locales/en/main.ftl

@@ -133,6 +133,7 @@ tmux-no-pty = (russh does not support interactive PTY. Use the following command
 sync-digest-match = '{ $name }' OK (digest matches)
 sync-syncing = { $name } ({ $ip }): syncing...
 sync-synced = { $name } OK
+sync-fabric-fallback = { $name }: 200G fabric hop failed ({ $err }) — falling back to 10G mgmt; fix node-to-node ssh over the fabric NICs (authorized_keys / host keys for the 200.x addresses)
 health-node-offline = ✗ { $name } ({ $ip }) — offline
 health-node-idle = ○ { $name } ({ $ip }){ $net }
 health-node-online = ● { $name } ({ $ip }){ $net }

locales/es/main.ftl

@@ -133,6 +133,7 @@ tmux-no-pty = (russh no soporta PTY interactivo. Use el siguiente comando direct
 sync-digest-match = '{ $name }' OK (digest coincide)
 sync-syncing = { $name } ({ $ip }): sincronizando...
 sync-synced = { $name } OK
+sync-fabric-fallback = { $name }: el salto por 200G falló ({ $err }) — fallback a mgmt 10G; arreglar ssh nodo-a-nodo por la fabrica (authorized_keys / host keys para las direcciones 200.x)
 health-node-offline =   ✗ { $name } ({ $ip }) — desconectado
 health-node-idle =   ○ { $name } ({ $ip }){ $net }
 health-node-online =   ● { $name } ({ $ip }){ $net }

locales/fr/main.ftl

@@ -133,6 +133,7 @@ tmux-no-pty = (russh ne supporte pas le PTY interactif. Utilisez la commande sui
 sync-digest-match = '{ $name }' OK (digest correspond)
 sync-syncing = { $name } ({ $ip }) : synchronisation...
 sync-synced = { $name } OK
+sync-fabric-fallback = { $name }: saut 200G échoué ({ $err }) — repli sur mgmt 10G ; corriger le ssh inter-nœuds sur le fabric (authorized_keys / host keys pour les adresses 200.x)
 health-node-offline =   ✗ { $name } ({ $ip }) — hors ligne
 health-node-idle =   ○ { $name } ({ $ip }){ $net }
 health-node-online =   ● { $name } ({ $ip }){ $net }

locales/it/main.ftl

@@ -133,6 +133,7 @@ tmux-no-pty = (russh non supporta PTY interattivo. Usa il seguente comando diret
 sync-digest-match = '{ $name }' OK (digest corrisponde)
 sync-syncing = { $name } ({ $ip }): sincronizzazione...
 sync-synced = { $name } OK
+sync-fabric-fallback = { $name }: hop 200G non riuscito ({ $err }) — fallback su mgmt 10G; correggere ssh nodo-nodo sulla fabric (authorized_keys / host key per gli indirizzi 200.x)
 health-node-offline =   ✗ { $name } ({ $ip }) — offline
 health-node-idle =   ○ { $name } ({ $ip }){ $net }
 health-node-online =   ● { $name } ({ $ip }){ $net }

locales/ru/main.ftl

@@ -136,6 +136,7 @@ tmux-no-pty = (russh не поддерживает интерактивный PT
 sync-digest-match = '{ $name }' OK (дайджест совпадает)
 sync-syncing = { $name } ({ $ip }): синхронизация...
 sync-synced = { $name } OK
+sync-fabric-fallback = { $name }: 200G-хоп не удался ({ $err }) — фоллбэк на 10G mgmt; поправьте node-to-node ssh по фабрике (authorized_keys / host keys на 200.x-адресах)
 health-node-offline = ✗ { $name } ({ $ip }) — офлайн
 health-node-idle = ○ { $name } ({ $ip }){ $net }
 health-node-online = ● { $name } ({ $ip }){ $net }

src/docker/sync.rs

@@ -118,18 +118,32 @@ pub async fn sync_image_to_nodes(
             fluent_i18n::t!("sync-syncing", { "name" => &target.name, "ip" => &target.ssh_ip })
         );
         // Data path: node-to-node, never through the Mac (mac→node is ~32
-        // MB/s vs ~547 MB/s per stream on the 200G fabric). Prefer the
-        // target's fabric IP (ib_ip) for the inner hop; nodes without a
-        // 200G uplink fall back to the 10G mgmt IP.
-        let target_ip = target.ib_ip.clone().unwrap_or_else(|| target.ssh_ip.clone());
+        // MB/s vs ~547 MB/s per stream on the 200G fabric). The inner hop
+        // prefers the target's fabric IP (ib_ip); if that fails (broken
+        // node-to-node ssh over 200.x) it falls back to the 10G mgmt IP with
+        // a warning, so a stale authorized_keys never blocks a serve.
+        let attempts: Vec<String> = match &target.ib_ip {
+            Some(ib) => vec![ib.clone(), target.ssh_ip.clone()],
+            None => vec![target.ssh_ip.clone()],
+        };
         // Fetch the target's full public key over a verified russh connection
         // (which enforces the pinned fingerprint), then write a per-target
-        // known_hosts on the SOURCE node and run the inner ssh with it,
-        // StrictHostKeyChecking=yes. A host that doesn't hold the pinned key
-        // is rejected before any auth — no key leak, no forgery.
+        // known_hosts on the SOURCE node (one line per IP we may hop to) and
+        // run the inner ssh with it, StrictHostKeyChecking=yes. A host that
+        // doesn't hold the pinned key is rejected before any auth — no key
+        // leak, no forgery.
         let pubkey = crate::ssh::fetch_host_pubkey(config, target).await?;
         let known_hosts = format!("/tmp/sm121_known_hosts_{}", target.name);
-        let entry = format!("{} {}\n", target_ip, pubkey);
+        let mut kh_ips = vec![target.ssh_ip.clone()];
+        if let Some(ib) = &target.ib_ip {
+            kh_ips.push(ib.clone());
+        }
+        let entry = kh_ips
+            .iter()
+            .map(|ip| format!("{} {}", ip, pubkey))
+            .collect::<Vec<_>>()
+            .join("\n")
+            + "\n";
         let write_known = format!(
             "printf '%s' {} > {} && chmod 600 {}",
             shq(&entry),
@@ -138,29 +152,60 @@ pub async fn sync_image_to_nodes(
         );
         run_ssh(config, src_node, write_known, false).await?;
 
-        let pipe_cmd = format!(
-            "docker save {} | ssh -o StrictHostKeyChecking=yes -o HashKnownHosts=no -o UserKnownHostsFile={} -i {} {}@{} docker load; ec=$?; rm -f {}; exit $ec",
-            shq(image),
-            shq(&known_hosts),
-            shq(&config.ssh_key_path.display().to_string()),
-            shq(&config.ssh_user),
-            shq(&target_ip),
-            shq(&known_hosts),
-        );
-        run_ssh_long_running(config, src_node, pipe_cmd, false).await?;
-        let verify = get_image_digest(config, target, image).await?;
-        if verify.as_deref() != Some(source_digest) {
+        let pipe_for = |ip: &str| {
+            format!(
+                "docker save {} | ssh -o StrictHostKeyChecking=yes -o HashKnownHosts=no -o UserKnownHostsFile={} -i {} {}@{} docker load; ec=$?; rm -f {}; exit $ec",
+                shq(image),
+                shq(&known_hosts),
+                shq(&config.ssh_key_path.display().to_string()),
+                shq(&config.ssh_user),
+                shq(ip),
+                shq(&known_hosts),
+            )
+        };
+        let mut synced = false;
+        let mut last_err: Option<String> = None;
+        for (idx, ip) in attempts.iter().enumerate() {
+            match run_ssh_long_running(config, src_node, pipe_for(ip), false).await {
+                Ok(_) => {
+                    let verify = get_image_digest(config, target, image).await?;
+                    if verify.as_deref() == Some(source_digest) {
+                        info!(
+                            "{}",
+                            fluent_i18n::t!("sync-synced", { "name" => &target.name })
+                        );
+                        synced = true;
+                    } else {
+                        // a digest mismatch is not a connectivity problem —
+                        // the other IP won't help, fail outright
+                        last_err = Some(format!(
+                            "digest mismatch (source={}, target={:?})",
+                            source_digest, verify
+                        ));
+                        break;
+                    }
+                }
+                Err(e) => {
+                    if idx == 0 {
+                        tracing::warn!(
+                            "{}",
+                            fluent_i18n::t!("sync-fabric-fallback", {
+                                "name" => &target.name,
+                                "err" => e.to_string(),
+                            })
+                        );
+                    }
+                    last_err = Some(e.to_string());
+                }
+            }
+        }
+        if !synced {
             anyhow::bail!(
-                "Image sync to '{}' failed: digest mismatch (source={}, target={:?})",
+                "Image sync to '{}' failed: {}",
                 target.name,
-                source_digest,
-                verify
+                last_err.unwrap_or_default()
             );
         }
-        info!(
-            "{}",
-            fluent_i18n::t!("sync-synced", { "name" => &target.name })
-        );
     }
     Ok(())
 }