Эх сурвалжийг харах

feat(host-scripts): flare-swap.sh — the cutover binary swap as one reviewed, idempotent, rank-safe command

Encodes every hard-won rule in one place: swaps BOTH /usr/bin/flare and
~/.local/lib/flare-agent/flare (dpkg owns neither, and only the agent copy runs),
temp + mv so a mapped inode is never overwritten, dated *.orig backups, restarts
only the agent unit, then verifies sha256 AND /proc/<pid>/exe for "(deleted)".

Refuses to touch a node with ranks running unless FORCE=1 is passed explicitly.
Tested today: verified no-op with exit 0 on dgx4 (already swapped), and on dgx8
(a serving rank) it refused with exit 1 leaving shas unchanged, zero tmp-swap
files and the rank untouched.
Alexander Korolev 1 долоо хоног өмнө
parent
commit
a5caf79247

+ 75 - 0
host-scripts/flare-swap.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# flare-swap.sh — move ONE node onto a staged flare binary, safely.
+#
+#   run on the node itself, or:  ssh host 'bash -s' -- /usr/local/share/hyperflare/v0.1.0/flare < flare-swap.sh
+#
+# Rules it encodes (learned the hard way, see specs/tech-architecture/flare-storage-nas.md):
+#   * the unit ExecStarts ~/.local/lib/flare-agent/flare, which dpkg does NOT own:
+#     /usr/bin/flare alone is never "the runtime", so BOTH paths are swapped.
+#   * never `cp` over a mapped executable (SIGBUS) — always temp + `mv` (rename).
+#   * a node with ranks running is only touched with FORCE=1, and only inside a window.
+#   * verification is sha + /proc/<pid>/exe, never `flare --version` (prints 0.1.0 always).
+set -uo pipefail
+
+STAGED=${1:-/usr/local/share/hyperflare/v0.1.0/flare}
+FORCE=${FORCE:-0}
+TS=$(date +%Y%m%d)
+SUDO=""
+[ "$(id -u)" = 0 ] || SUDO="sudo"
+
+die() { echo "REFUSED: $*" >&2; exit 1; }
+need() { command -v "$1" >/dev/null 2>&1 || die "missing command: $1"; }
+
+need sha256sum
+[ -f "$STAGED" ] && [ -x "$STAGED" ] || die "staged binary not found/executable: $STAGED"
+want=$(sha256sum "$STAGED" | awk '{print $1}')
+
+# --- safety gate: never disturb a node that is serving, unless told to ---
+ranks=$(pgrep -fc "[f]lare run" || true)
+ranks=${ranks:-0}
+if [ "$ranks" != 0 ] && [ "$FORCE" != 1 ]; then
+  die "$ranks rank(s) running on $(hostname -s) — pass FORCE=1 only inside a restart window"
+fi
+
+agent=$HOME/.local/lib/flare-agent/flare
+echo "host=$(hostname -s) ranks=$ranks want=${want:0:16}…  (staged: $STAGED)"
+echo "before: /usr/bin=$(sha256sum /usr/bin/flare | cut -c1-16) agent=$(sha256sum "$agent" | cut -c1-16)"
+
+# already there? idempotent — verify and exit without touching anything
+if [ "$(sha256sum /usr/bin/flare | awk '{print $1}')" = "$want" ] \
+&& [ "$(sha256sum "$agent" | awk '{print $1}')" = "$want" ]; then
+  echo "already on the target binary at both paths — nothing to do"
+else
+  # keep one rollback copy per day at both paths, never overwrite an existing backup
+  for f in /usr/bin/flare "$agent"; do
+    b="$f.orig-$TS"
+    if [ ! -e "$b" ]; then $SUDO cp -a "$f" "$b" && echo "backup: $b"; fi
+  done
+  # rename, never cp over the mapped inode
+  $SUDO install -m 0755 "$STAGED" /usr/bin/flare.tmp-swap        && $SUDO mv  /usr/bin/flare.tmp-swap  /usr/bin/flare
+  install -m 0755 "$STAGED" "$agent.tmp-swap"                    && mv         "$agent.tmp-swap"        "$agent"
+  echo "swapped both paths (temp + mv)"
+fi
+
+# the running agent is the only thing that can be stale — restart it (ranks are separate units)
+systemctl --user restart flare-agent.service
+sleep 4
+
+# --- verification: what is installed AND what is actually executing ---
+fail=0
+for f in /usr/bin/flare "$agent"; do
+  got=$(sha256sum "$f" | awk '{print $1}')
+  [ "$got" = "$want" ] && echo "OK   $f ${got:0:16}…" || { echo "BAD  $f ${got:0:16}…"; fail=1; }
+done
+active=$(systemctl --user is-active flare-agent.service || true)
+[ "$active" = active ] && echo "OK   flare-agent.service active" || { echo "BAD  flare-agent.service $active"; fail=1; }
+for pid in $(pgrep -f "[f]lare agent" || true) $(pgrep -f "[f]lare run" || true); do
+  exe=$(readlink "/proc/$pid/exe" 2>/dev/null || echo "?")
+  case "$exe" in
+    *"(deleted)"*) echo "BAD  pid $pid still executing a deleted inode: $exe"; fail=1 ;;
+    *)             echo "OK   pid $pid exe=$exe" ;;
+  esac
+done
+
+[ "$fail" = 0 ] && { echo "RESULT: $(hostname -s) is running the target binary"; exit 0; }
+echo "RESULT: $(hostname -s) verification FAILED — roll back from *.orig-$TS"; exit 1

+ 13 - 2
specs/tech-architecture/flare-storage-nas.md

@@ -251,8 +251,19 @@ previous phase failed.
    `~/.local/lib/flare-agent/flare` were **not touched**, so nothing serving can be
    affected. dgx5–7 are off and still need this.
 
-   The swap in the window is deliberately boring, and note that `/usr/bin/flare` is
-   **not** what runs — the agent executes `~/.local/lib/flare-agent/flare`:
+   **Use `host-scripts/flare-swap.sh <staged-binary>`** — written and tested for this
+   (2026-11-12: verified idempotent no-op on dgx4, and it correctly **refused** on dgx8
+   with a rank running, exit 1, `tmp-swap` files count 0, nothing disturbed). It swaps
+   *both* paths by temp+`mv`, keeps a dated `*.orig-<YYYYMMDD>` backup at each, restarts
+   only the agent unit, and verifies sha256 **and** `/proc/<pid>/exe` for `(deleted)`
+   before exiting 0. Inside the window it is one command per node:
+   ```bash
+   ssh dgxN.lan 'bash -s -- /usr/local/share/hyperflare/v0.1.0/flare' < host-scripts/flare-swap.sh
+   # ranks are already stopped in the window; the script still refuses unless FORCE=1
+   FORCE=1 ssh dgxN.lan 'bash -s -- /usr/local/share/hyperflare/v0.1.0/flare' < host-scripts/flare-swap.sh
+   ```
+   The swap is deliberately boring, and note that `/usr/bin/flare` is **not** what runs —
+   the agent executes `~/.local/lib/flare-agent/flare` (and `dpkg` owns neither copy):
    ```bash
    # per node, in the window (rename, never cp — see the SIGBUS rule)
    sudo install -m 0755 /usr/local/share/hyperflare/v0.1.0/flare /usr/bin/flare.tmp-v010