#!/usr/bin/env bash # flare-swap.sh — move ONE node onto a staged flare binary, safely. # # run on the node itself, or: ssh host 'bash -s' -- /usr/local/share/hyperflare/v0.2.1/flare < flare-swap.sh # # The default staged path tracks the current fleet build (main@6b68b9b4). Keep it in # step when you stage a new one: a stale default here silently DOWNGRADES a node. # # Rules it encodes (learned the hard way, see specs/tech-architecture/flare-storage-nas.md): # * the unit ExecStarts ~/.local/lib/flare-agent/flare, which dpkg does NOT own: # /usr/bin/flare alone is never "the runtime", so BOTH paths are swapped. # * never `cp` over a mapped executable (SIGBUS) — always temp + `mv` (rename). # * a node with ranks running is only touched with FORCE=1, and only inside a window. # * verification is sha + /proc//exe, never `flare --version` (prints 0.1.0 always). set -uo pipefail STAGED=${1:-/usr/local/share/hyperflare/v0.2.1/flare} FORCE=${FORCE:-0} TS=$(date +%Y%m%d) SUDO="" [ "$(id -u)" = 0 ] || SUDO="sudo" die() { echo "REFUSED: $*" >&2; exit 1; } need() { command -v "$1" >/dev/null 2>&1 || die "missing command: $1"; } need sha256sum [ -f "$STAGED" ] && [ -x "$STAGED" ] || die "staged binary not found/executable: $STAGED" want=$(sha256sum "$STAGED" | awk '{print $1}') # --- safety gate: never disturb a node that is serving, unless told to --- ranks=$(pgrep -fc "[f]lare run" || true) ranks=${ranks:-0} if [ "$ranks" != 0 ] && [ "$FORCE" != 1 ]; then die "$ranks rank(s) running on $(hostname -s) — pass FORCE=1 only inside a restart window" fi agent=$HOME/.local/lib/flare-agent/flare echo "host=$(hostname -s) ranks=$ranks want=${want:0:16}… (staged: $STAGED)" echo "before: /usr/bin=$(sha256sum /usr/bin/flare | cut -c1-16) agent=$(sha256sum "$agent" | cut -c1-16)" # already there? idempotent — verify and exit without touching anything if [ "$(sha256sum /usr/bin/flare | awk '{print $1}')" = "$want" ] \ && [ "$(sha256sum "$agent" | awk '{print $1}')" = "$want" ]; then echo "already on the target binary at both paths — nothing to do" else # keep one rollback copy per day at both paths, never overwrite an existing backup for f in /usr/bin/flare "$agent"; do b="$f.orig-$TS" if [ ! -e "$b" ]; then $SUDO cp -a "$f" "$b" && echo "backup: $b"; fi done # rename, never cp over the mapped inode $SUDO install -m 0755 "$STAGED" /usr/bin/flare.tmp-swap && $SUDO mv /usr/bin/flare.tmp-swap /usr/bin/flare install -m 0755 "$STAGED" "$agent.tmp-swap" && mv "$agent.tmp-swap" "$agent" echo "swapped both paths (temp + mv)" fi # the running agent is the only thing that can be stale — restart it (ranks are separate units) systemctl --user restart flare-agent.service sleep 4 # --- verification: what is installed AND what is actually executing --- fail=0 for f in /usr/bin/flare "$agent"; do got=$(sha256sum "$f" | awk '{print $1}') [ "$got" = "$want" ] && echo "OK $f ${got:0:16}…" || { echo "BAD $f ${got:0:16}…"; fail=1; } done active=$(systemctl --user is-active flare-agent.service || true) [ "$active" = active ] && echo "OK flare-agent.service active" || { echo "BAD flare-agent.service $active"; fail=1; } for pid in $(pgrep -f "[f]lare agent" || true) $(pgrep -f "[f]lare run" || true); do exe=$(readlink "/proc/$pid/exe" 2>/dev/null || echo "?") case "$exe" in *"(deleted)"*) echo "BAD pid $pid still executing a deleted inode: $exe"; fail=1 ;; *) echo "OK pid $pid exe=$exe" ;; esac done [ "$fail" = 0 ] && { echo "RESULT: $(hostname -s) is running the target binary"; exit 0; } echo "RESULT: $(hostname -s) verification FAILED — roll back from *.orig-$TS"; exit 1