flare-fleet.sh 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387
  1. #!/usr/bin/env bash
  2. # flare-fleet.sh — one operator tool for the eight-Spark FLARE fleet.
  3. #
  4. # ./host-scripts/flare-fleet.sh status # read-only snapshot
  5. # ./host-scripts/flare-fleet.sh check # status, exit 1 on drift
  6. # ./host-scripts/flare-fleet.sh build --ref main [--out F] # build on dgx5, print sha
  7. # ./host-scripts/flare-fleet.sh upgrade --binary B --release v0.3.0 --yes
  8. # ./host-scripts/flare-fleet.sh upgrade --binary B --dry-run # print the plan only
  9. # ./host-scripts/flare-fleet.sh rollback --yes # newest .orig-* generation
  10. #
  11. # Order of operations, each step a lesson that cost a window:
  12. # * Both lanes run ONE build: a new CLI cannot handshake old agents. So: stop
  13. # the groups → stage and verify the binary on every node → swap both paths →
  14. # restart the agents → relaunch the groups → verify serving. A partial
  15. # rollout wedges the control plane.
  16. # * Swapping is delegated to host-scripts/flare-swap.sh, which encodes temp+mv
  17. # (never cp over a mapped inode), both paths, one backup generation, and
  18. # sha + /proc/<pid>/exe verification.
  19. # * Groups are persistent (since 2026-11-16), so relaunch is `orbit run --name`.
  20. # * Every wait is bounded; nothing loops forever.
  21. #
  22. # Fleet definition (override by env): FLARE_NODES, FLARE_LANES ("controller:group"),
  23. # FLARE_BUILD_HOST, FLARE_PORT, FLARE_WAIT, FLARE_STAGE_ROOT, FLARE_SRC_DIR.
  24. set -uo pipefail
  25. SELF_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
  26. NODES=${FLARE_NODES:-"dgx1 dgx2 dgx3 dgx4 dgx5 dgx6 dgx7 dgx8"}
  27. LANES=${FLARE_LANES:-"dgx1:deepseek1017 dgx5:ds41-tp4"}
  28. BUILD_HOST=${FLARE_BUILD_HOST:-dgx5}
  29. SRC_DIR=${FLARE_SRC_DIR:-'$HOME/hyperflare-src'}
  30. STAGE_ROOT=${FLARE_STAGE_ROOT:-/usr/local/share/hyperflare}
  31. PORT=${FLARE_PORT:-30000}
  32. WAIT=${FLARE_WAIT:-300}
  33. SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
  34. say() { printf '%s\n' "$*"; }
  35. info() { printf ' %s\n' "$*"; }
  36. warn() { printf 'WARN: %s\n' "$*" >&2; }
  37. die() { printf 'ABORT: %s\n' "$*" >&2; exit 1; }
  38. host_of() { printf '%s.lan' "$1"; }
  39. node_count() { set -- $NODES; printf '%s' "$#"; }
  40. # ssh with a hard timeout so a wedged node cannot hang a window
  41. net() { local node=$1; shift; timeout 90 ssh "${SSH_OPTS[@]}" "$(host_of "$node")" "$@"; }
  42. # ---------------------------------------------------------------- status
  43. cmd_status() {
  44. local strict=${1:-} drift=0 node line
  45. printf '%-6s %-13s %-13s %-11s %-8s %-6s %s\n' NODE /usr/bin AGENT-PATH STAGED AGENT RANKS FREE
  46. for node in $NODES; do
  47. line=$(net "$node" bash -s <<'EOF' 2>/dev/null
  48. a=$(sha256sum /usr/bin/flare 2>/dev/null | cut -c1-12)
  49. b=$(sha256sum "$HOME/.local/lib/flare-agent/flare" 2>/dev/null | cut -c1-12)
  50. s=$(ls /usr/local/share/hyperflare 2>/dev/null | tr '\n' ',' )
  51. agent=$(systemctl --user is-active flare-agent 2>/dev/null || echo inactive)
  52. ranks=$(ps -eo comm,args --no-headers 2>/dev/null | awk '$1=="flare" && / run /' | wc -l)
  53. free=$(df -h / | tail -1 | awk '{print $4}')
  54. printf '%s|%s|%s|%s|%s|%s' "${a:-missing}" "${b:-missing}" "${s:-none}" "$agent" "$ranks" "$free"
  55. EOF
  56. )
  57. IFS='|' read -r usr agent_path staged agent ranks free <<<"$line"
  58. printf '%-6s %-13s %-13s %-11s %-8s %-6s %s\n' \
  59. "$node" "${usr:--}" "${agent_path:--}" "${staged%,}" "${agent:--}" "${ranks:-0}" "${free:--}"
  60. if [ -z "$usr" ] || [ "$usr" != "$agent_path" ] || [ "$agent" != active ]; then
  61. drift=$((drift+1))
  62. fi
  63. done
  64. say ""
  65. say "groups"
  66. local pair ctl group out
  67. for pair in $LANES; do
  68. ctl=${pair%%:*}; group=${pair#*:}
  69. out=$(net "$ctl" "timeout 40 flare orbit ls 2>/dev/null | grep -E '^persistent[[:space:]]+$group'" 2>/dev/null)
  70. [ -n "$out" ] || out="(not listed — is $ctl the controller?)"
  71. say " $group @ $ctl: $out"
  72. done
  73. say ""
  74. say "invariant: one build at both paths on every node, agents active"
  75. if [ "$strict" = strict ] && [ "$drift" -gt 0 ]; then
  76. die "$drift node(s) drift from the invariant"
  77. fi
  78. }
  79. # ---------------------------------------------------------------- build
  80. cmd_build() {
  81. local ref=main out=""
  82. while [ $# -gt 0 ]; do
  83. case $1 in
  84. --ref) ref=$2; shift 2 ;;
  85. --out) out=$2; shift 2 ;;
  86. *) die "build: unknown argument $1" ;;
  87. esac
  88. done
  89. say "building NVIDIA/hyperflare-inference@$ref on $BUILD_HOST"
  90. net "$BUILD_HOST" bash -s -- "$ref" "$SRC_DIR" <<'EOF' 2>&1 | sed 's/^/ /'
  91. set -uo pipefail
  92. ref=$1; src=$2
  93. cd "$src" || { echo "no source tree at $src"; exit 1; }
  94. git fetch --depth 1 origin "$ref" -q || git fetch --depth 1 origin -q
  95. git checkout -q FETCH_HEAD 2>/dev/null || git checkout -q "$ref"
  96. echo "commit: $(git log --oneline -1)"
  97. cargo build --release --locked --manifest-path flare-cli/Cargo.toml --features roce 2>&1 | tail -2
  98. f=flare-cli/target/release/flare
  99. [ -x "$f" ] || { echo "no binary at $f"; exit 1; }
  100. echo "sha256: $(sha256sum "$f" | cut -d' ' -f1)"
  101. echo "size: $(stat -c %s "$f")"
  102. echo "roce: $(ldd "$f" | grep -c ibverbs) libibverbs link(s)"
  103. echo "version: $("$f" --version 2>&1 | head -1) (source builds print 0.0.0 — use the sha)"
  104. EOF
  105. local sha
  106. sha=$(net "$BUILD_HOST" "sha256sum $SRC_DIR/flare-cli/target/release/flare | cut -d' ' -f1")
  107. [ -n "$sha" ] || die "could not read the built binary's sha256"
  108. if [ -n "$out" ]; then
  109. net "$BUILD_HOST" "cat $SRC_DIR/flare-cli/target/release/flare" > "$out" || die "copy failed"
  110. chmod 755 "$out"; info "copied to $out"
  111. fi
  112. say "result: sha256=$sha"
  113. }
  114. # ---------------------------------------------------------------- group helpers
  115. group_pairs() { # "ctl group" lines for groups that are actually running
  116. local pair ctl group n
  117. for pair in $LANES; do
  118. ctl=${pair%%:*}; group=${pair#*:}
  119. n=$(net "$ctl" "timeout 40 flare orbit ls 2>/dev/null | grep -E '^[[:space:]]*$group[[:space:]]' | grep -c running" 2>/dev/null)
  120. [ "${n:-0}" -gt 0 ] && printf '%s %s\n' "$ctl" "$group"
  121. done
  122. return 0
  123. }
  124. stop_groups() {
  125. local ctl group
  126. while read -r ctl group; do
  127. [ -n "${ctl:-}" ] || continue
  128. say " stopping $group on $ctl (drains rank zero first)"
  129. net "$ctl" "timeout 150 flare orbit stop $group" 2>&1 | sed 's/^/ /'
  130. done < <(group_pairs)
  131. }
  132. relaunch_groups() {
  133. local pair ctl group
  134. for pair in $LANES; do
  135. ctl=${pair%%:*}; group=${pair#*:}
  136. say " $group @ $ctl: orbit run --name $group --detach"
  137. net "$ctl" "nohup setsid bash -c 'flare orbit run --name $group --detach > \$HOME/flare-relaunch.log 2>&1' >/dev/null 2>&1 & sleep 5; true" >/dev/null 2>&1
  138. done
  139. }
  140. wait_health() {
  141. local deadline=$((SECONDS + WAIT)) pair ctl code ok
  142. while [ $SECONDS -lt $deadline ]; do
  143. ok=1
  144. for pair in $LANES; do
  145. ctl=${pair%%:*}
  146. code=$(net "$ctl" "curl -s -m 4 -o /dev/null -w '%{http_code}' localhost:$PORT/health" 2>/dev/null)
  147. [ "$code" = 200 ] || ok=0
  148. done
  149. [ $ok -eq 1 ] && return 0
  150. sleep 5
  151. done
  152. return 1
  153. }
  154. verify_serving() {
  155. local pair ctl group code
  156. for pair in $LANES; do
  157. ctl=${pair%%:*}; group=${pair#*:}
  158. code=$(net "$ctl" bash -s -- "$PORT" <<'EOF' 2>/dev/null
  159. port=$1
  160. h=$(curl -s -m 4 -o /dev/null -w '%{http_code}' localhost:$port/health)
  161. m=$(curl -s -m 6 localhost:$port/v1/models | sed -n 's/.*"id":"\([^"]*\)".*/\1/p' | head -1)
  162. r=$(curl -s -m 60 localhost:$port/v1/chat/completions -H 'Content-Type: application/json' \
  163. -d "{\"model\":\"$m\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: OK\"}],\"max_tokens\":16}" \
  164. | sed -n 's/.*"content":"\([^"]*\)".*/\1/p' | head -1)
  165. printf '%s model=%s reply=%s' "$h" "${m:-?}" "${r:-?}"
  166. EOF
  167. )
  168. printf ' %-9s health=%s\n' "$group" "${code:-(no answer)}"
  169. case "$code" in 200*) ;; *) return 1 ;; esac
  170. done
  171. }
  172. prune_backups() {
  173. local node
  174. for node in $NODES; do
  175. net "$node" bash -s <<'EOF' 2>/dev/null | sed "s/^/ $node: /"
  176. keep=$(ls -1 /usr/bin/flare.orig-* 2>/dev/null | sed 's/.*flare\.orig-//' | sort | tail -1)
  177. [ -n "$keep" ] || { echo "no backups"; exit 0; }
  178. for f in /usr/bin/flare.orig-* "$HOME"/.local/lib/flare-agent/flare.orig-*; do
  179. [ -e "$f" ] || continue
  180. case "$f" in *.orig-"$keep") ;; *) sudo rm -f "$f" && echo "removed $(basename "$f")" ;; esac
  181. done
  182. EOF
  183. done
  184. }
  185. # ---------------------------------------------------------------- upgrade
  186. cmd_upgrade() {
  187. local binary="" release="" yes=0 dry=0 relaunch=1 prune=1
  188. while [ $# -gt 0 ]; do
  189. case $1 in
  190. --binary) binary=$2; shift 2 ;;
  191. --release) release=$2; shift 2 ;;
  192. --yes) yes=1; shift ;;
  193. --dry-run) dry=1; shift ;;
  194. --no-relaunch) relaunch=0; shift ;;
  195. --no-prune) prune=0; shift ;;
  196. *) die "upgrade: unknown argument $1" ;;
  197. esac
  198. done
  199. [ -n "$binary" ] || die "upgrade: --binary PATH is required (see 'build')"
  200. [ -f "$binary" ] || die "upgrade: no such binary: $binary"
  201. release=${release:-v$(date +%Y%m%d-%H%M)}
  202. local new_sha; new_sha=$(sha256sum "$binary" | cut -d' ' -f1)
  203. local swap_script=$SELF_DIR/flare-swap.sh
  204. [ -f "$swap_script" ] || die "no flare-swap.sh next to this script"
  205. say "FLARE fleet upgrade -> $release"
  206. info "nodes: $NODES ($(node_count) nodes)"
  207. info "groups: $LANES"
  208. info "binary: $binary"
  209. info "sha256: $new_sha"
  210. say "preflight: reachability and the current build"
  211. local node cur bad=0 sums=""
  212. for node in $NODES; do
  213. cur=$(net "$node" "sha256sum /usr/bin/flare 2>/dev/null | cut -c1-12" 2>/dev/null)
  214. if [ -z "$cur" ]; then warn "$node unreachable"; bad=$((bad+1)); else sums="$sums $cur"; fi
  215. done
  216. [ "$bad" -eq 0 ] || die "$bad node(s) unreachable; fix before opening a window"
  217. local distinct; distinct=$(printf '%s\n' $sums | sort -u | tr '\n' ' ')
  218. info "currently deployed: $distinct"
  219. [ "$(printf '%s\n' $sums | sort -u | wc -l)" -eq 1 ] \
  220. || warn "fleet is ALREADY mixed ($distinct) — this upgrade will fix it, but pairing may be broken now"
  221. case " $distinct " in *" ${new_sha:0:12} "*) warn "target build is already deployed everywhere";; esac
  222. local running; running=$(group_pairs)
  223. [ -n "$running" ] && { say "groups that will stop:"; printf '%s\n' "$running" | sed 's/^/ /'; } \
  224. || say "no groups are running"
  225. if [ "$dry" = 1 ] || [ "$yes" = 0 ]; then
  226. say ""
  227. say "plan (nothing changed)"
  228. info "1. stop the groups above (drain rank zero first)"
  229. info "2. push $release to $STAGE_ROOT/$release on all $(node_count) nodes, sha verified per node"
  230. info "3. flare-swap.sh FORCE=1 per node: temp+mv both paths, backup, agent restart"
  231. info "4. verify one build everywhere (sha at both paths)"
  232. info "5. relaunch: $LANES"
  233. info "6. wait for /health 200 on each controller (max ${WAIT}s) + models + one completion"
  234. [ "$prune" = 1 ] && info "7. prune older .orig-* backups (keep the newest generation)"
  235. say ""
  236. [ "$dry" = 1 ] && { say "dry run: nothing changed"; return 0; }
  237. die "re-run with --yes to execute this window"
  238. fi
  239. local started; started=$(date +%s)
  240. say "step 1/7 stop groups"; stop_groups
  241. say "step 2/7 stage $release"
  242. local got
  243. for node in $NODES; do
  244. net "$node" "cat > /tmp/flare-stage && chmod 755 /tmp/flare-stage" < "$binary" || die "$node: push failed"
  245. got=$(net "$node" "sha256sum /tmp/flare-stage | cut -d' ' -f1")
  246. [ "$got" = "$new_sha" ] || die "$node: staged sha ${got:0:12} != ${new_sha:0:12} — refusing"
  247. net "$node" bash -s <<EOF >/dev/null 2>&1 || die "$node: staging failed"
  248. set -eu
  249. sudo mkdir -p $STAGE_ROOT/$release
  250. sudo install -m 0755 /tmp/flare-stage $STAGE_ROOT/$release/flare
  251. sudo sh -c "cd $STAGE_ROOT/$release && sha256sum flare | cut -d' ' -f1 > binary.sha256"
  252. rm -f /tmp/flare-stage
  253. EOF
  254. for f in LICENSE NOTICE THIRD_PARTY_NOTICES.md; do
  255. net "$BUILD_HOST" "cat $SRC_DIR/$f 2>/dev/null" 2>/dev/null \
  256. | net "$node" "sudo tee $STAGE_ROOT/$release/$f >/dev/null" 2>/dev/null || true
  257. done
  258. info "$node staged"
  259. done
  260. say "step 3/7 swap + restart agents"
  261. for node in $NODES; do
  262. timeout 150 ssh "${SSH_OPTS[@]}" "$(host_of "$node")" "FORCE=1 bash -s -- $STAGE_ROOT/$release/flare" < "$swap_script" 2>&1 \
  263. | grep -E "^RESULT|^BAD|REFUSED" | sed "s/^/ $node: /"
  264. done
  265. say "step 4/7 verify one build everywhere"
  266. local usr agent_path
  267. bad=0
  268. for node in $NODES; do
  269. read -r usr agent_path < <(net "$node" "echo \$(sha256sum /usr/bin/flare | cut -d' ' -f1) \$(sha256sum \$HOME/.local/lib/flare-agent/flare | cut -d' ' -f1)")
  270. if [ "$usr" != "$new_sha" ] || [ "$agent_path" != "$new_sha" ]; then
  271. warn "$node usr=${usr:0:12} agent=${agent_path:0:12} want ${new_sha:0:12}"; bad=$((bad+1))
  272. fi
  273. done
  274. [ "$bad" -eq 0 ] || die "$bad node(s) not on $release — mixed builds cannot pair; run 'rollback --yes'"
  275. info "all $(node_count) nodes run ${new_sha:0:12}"
  276. if [ "$relaunch" = 1 ]; then
  277. say "step 5/7 relaunch groups"; relaunch_groups
  278. say "step 6/7 wait for health"
  279. if wait_health; then info "healthy"; else warn "NOT healthy within ${WAIT}s"; fi
  280. verify_serving || warn "serving verification failed — inspect with 'flare orbit attach GROUP'"
  281. else
  282. say "step 5-6/7 skipped (--no-relaunch)"
  283. fi
  284. if [ "$prune" = 1 ]; then say "step 7/7 prune old backups"; prune_backups; fi
  285. say ""
  286. say "done: $release on $(node_count) nodes in $(( $(date +%s) - started ))s"
  287. }
  288. # ---------------------------------------------------------------- rollback
  289. cmd_rollback() {
  290. local stamp="" yes=0 relaunch=1
  291. while [ $# -gt 0 ]; do
  292. case $1 in
  293. --stamp) stamp=$2; shift 2 ;;
  294. --yes) yes=1; shift ;;
  295. --no-relaunch) relaunch=0; shift ;;
  296. *) die "rollback: unknown argument $1" ;;
  297. esac
  298. done
  299. say "FLARE fleet rollback to a .orig-* generation"
  300. local node avail tmpf one
  301. tmpf=$(mktemp)
  302. for node in $NODES; do
  303. avail=$(net "$node" "ls -1 \$HOME/.local/lib/flare-agent/flare.orig-* 2>/dev/null | sed 's/.*flare\.orig-//' | sort -u")
  304. [ -n "$avail" ] || { rm -f "$tmpf"; die "$node has no agent-path .orig-* backup; fleet-wide rollback impossible"; }
  305. for one in $avail; do printf '%s %s\n' "$node" "$one" >> "$tmpf"; done
  306. info "$node backups: $(printf '%s ' $avail)"
  307. done
  308. # The target must exist on EVERY node: restoring different generations is the
  309. # mixed-build state that rollback exists to escape.
  310. if [ -z "$stamp" ]; then
  311. stamp=$(awk '{print $2}' "$tmpf" | sort | uniq -c \
  312. | awk -v n="$(node_count)" '$1==n{print $2}' | sort | tail -1)
  313. [ -n "$stamp" ] || { rm -f "$tmpf"; die "no backup generation exists on all $(node_count) nodes; pass --stamp"; }
  314. fi
  315. for node in $NODES; do
  316. grep -qx "$node $stamp" "$tmpf" || { rm -f "$tmpf"; die "$node lacks flare.orig-$stamp (would leave a mixed fleet)"; }
  317. done
  318. rm -f "$tmpf"
  319. info "target (present everywhere): flare.orig-$stamp"
  320. if [ "$yes" = 0 ]; then
  321. say "plan: stop groups -> restore flare.orig-$stamp at both paths -> restart agents -> relaunch"
  322. die "re-run with --yes"
  323. fi
  324. stop_groups
  325. local usr
  326. for node in $NODES; do
  327. usr=$(net "$node" bash -s -- "$stamp" <<'EOF' 2>/dev/null | tail -1
  328. set -u
  329. stamp=$1
  330. A=$HOME/.local/lib/flare-agent/flare.orig-$stamp
  331. U=/usr/bin/flare.orig-$stamp
  332. [ -f "$A" ] && [ -f "$U" ] || { echo "missing backup flare.orig-$stamp"; exit 1; }
  333. # rename, never cp over a mapped inode; source copies stay for the next rollback
  334. install -m 0755 "$A" "$HOME/.local/lib/flare-agent/flare.tmp" && mv -f "$HOME/.local/lib/flare-agent/flare.tmp" "$HOME/.local/lib/flare-agent/flare"
  335. sudo install -m 0755 "$U" /usr/bin/flare.tmp && sudo mv -f /usr/bin/flare.tmp /usr/bin/flare
  336. systemctl --user restart flare-agent
  337. sleep 3
  338. echo "$(sha256sum /usr/bin/flare | cut -c1-12)/$(sha256sum "$HOME/.local/lib/flare-agent/flare" | cut -c1-12)"
  339. EOF
  340. )
  341. say " $node: $usr"
  342. [ -n "$usr" ] || die "$node rollback failed"
  343. done
  344. if [ "$relaunch" = 1 ]; then
  345. relaunch_groups
  346. wait_health || warn "not healthy after rollback"
  347. verify_serving || warn "serving verification failed after rollback"
  348. fi
  349. }
  350. usage() { sed -n '2,26p' "$0" | sed 's/^# \{0,1\}//'; }
  351. case ${1:-} in
  352. status) shift; cmd_status ;;
  353. check) shift; cmd_status strict ;;
  354. build) shift; cmd_build "$@" ;;
  355. upgrade) shift; cmd_upgrade "$@" ;;
  356. rollback) shift; cmd_rollback "$@" ;;
  357. ""|help|-h|--help) usage ;;
  358. *) die "unknown command: $1 (try --help)" ;;
  359. esac