flare-swap.sh 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778
  1. #!/usr/bin/env bash
  2. # flare-swap.sh — move ONE node onto a staged flare binary, safely.
  3. #
  4. # run on the node itself, or: ssh host 'bash -s' -- /usr/local/share/hyperflare/v0.2.1/flare < flare-swap.sh
  5. #
  6. # The default staged path tracks the current fleet build (main@6b68b9b4). Keep it in
  7. # step when you stage a new one: a stale default here silently DOWNGRADES a node.
  8. #
  9. # Rules it encodes (learned the hard way, see specs/tech-architecture/flare-storage-nas.md):
  10. # * the unit ExecStarts ~/.local/lib/flare-agent/flare, which dpkg does NOT own:
  11. # /usr/bin/flare alone is never "the runtime", so BOTH paths are swapped.
  12. # * never `cp` over a mapped executable (SIGBUS) — always temp + `mv` (rename).
  13. # * a node with ranks running is only touched with FORCE=1, and only inside a window.
  14. # * verification is sha + /proc/<pid>/exe, never `flare --version` (prints 0.1.0 always).
  15. set -uo pipefail
  16. STAGED=${1:-/usr/local/share/hyperflare/v0.2.1/flare}
  17. FORCE=${FORCE:-0}
  18. TS=$(date +%Y%m%d)
  19. SUDO=""
  20. [ "$(id -u)" = 0 ] || SUDO="sudo"
  21. die() { echo "REFUSED: $*" >&2; exit 1; }
  22. need() { command -v "$1" >/dev/null 2>&1 || die "missing command: $1"; }
  23. need sha256sum
  24. [ -f "$STAGED" ] && [ -x "$STAGED" ] || die "staged binary not found/executable: $STAGED"
  25. want=$(sha256sum "$STAGED" | awk '{print $1}')
  26. # --- safety gate: never disturb a node that is serving, unless told to ---
  27. ranks=$(pgrep -fc "[f]lare run" || true)
  28. ranks=${ranks:-0}
  29. if [ "$ranks" != 0 ] && [ "$FORCE" != 1 ]; then
  30. die "$ranks rank(s) running on $(hostname -s) — pass FORCE=1 only inside a restart window"
  31. fi
  32. agent=$HOME/.local/lib/flare-agent/flare
  33. echo "host=$(hostname -s) ranks=$ranks want=${want:0:16}… (staged: $STAGED)"
  34. echo "before: /usr/bin=$(sha256sum /usr/bin/flare | cut -c1-16) agent=$(sha256sum "$agent" | cut -c1-16)"
  35. # already there? idempotent — verify and exit without touching anything
  36. if [ "$(sha256sum /usr/bin/flare | awk '{print $1}')" = "$want" ] \
  37. && [ "$(sha256sum "$agent" | awk '{print $1}')" = "$want" ]; then
  38. echo "already on the target binary at both paths — nothing to do"
  39. else
  40. # keep one rollback copy per day at both paths, never overwrite an existing backup
  41. for f in /usr/bin/flare "$agent"; do
  42. b="$f.orig-$TS"
  43. if [ ! -e "$b" ]; then $SUDO cp -a "$f" "$b" && echo "backup: $b"; fi
  44. done
  45. # rename, never cp over the mapped inode
  46. $SUDO install -m 0755 "$STAGED" /usr/bin/flare.tmp-swap && $SUDO mv /usr/bin/flare.tmp-swap /usr/bin/flare
  47. install -m 0755 "$STAGED" "$agent.tmp-swap" && mv "$agent.tmp-swap" "$agent"
  48. echo "swapped both paths (temp + mv)"
  49. fi
  50. # the running agent is the only thing that can be stale — restart it (ranks are separate units)
  51. systemctl --user restart flare-agent.service
  52. sleep 4
  53. # --- verification: what is installed AND what is actually executing ---
  54. fail=0
  55. for f in /usr/bin/flare "$agent"; do
  56. got=$(sha256sum "$f" | awk '{print $1}')
  57. [ "$got" = "$want" ] && echo "OK $f ${got:0:16}…" || { echo "BAD $f ${got:0:16}…"; fail=1; }
  58. done
  59. active=$(systemctl --user is-active flare-agent.service || true)
  60. [ "$active" = active ] && echo "OK flare-agent.service active" || { echo "BAD flare-agent.service $active"; fail=1; }
  61. for pid in $(pgrep -f "[f]lare agent" || true) $(pgrep -f "[f]lare run" || true); do
  62. exe=$(readlink "/proc/$pid/exe" 2>/dev/null || echo "?")
  63. case "$exe" in
  64. *"(deleted)"*) echo "BAD pid $pid still executing a deleted inode: $exe"; fail=1 ;;
  65. *) echo "OK pid $pid exe=$exe" ;;
  66. esac
  67. done
  68. [ "$fail" = 0 ] && { echo "RESULT: $(hostname -s) is running the target binary"; exit 0; }
  69. echo "RESULT: $(hostname -s) verification FAILED — roll back from *.orig-$TS"; exit 1