#!/usr/bin/env bash # flare-fleet.sh — one operator tool for the eight-Spark FLARE fleet. # # ./host-scripts/flare-fleet.sh status # read-only snapshot # ./host-scripts/flare-fleet.sh check # status, exit 1 on drift # ./host-scripts/flare-fleet.sh build --ref main [--out F] # build on dgx5, print sha # ./host-scripts/flare-fleet.sh upgrade --binary B --release v0.3.0 --yes # ./host-scripts/flare-fleet.sh upgrade --binary B --dry-run # print the plan only # ./host-scripts/flare-fleet.sh rollback --yes # newest .orig-* generation # # Order of operations, each step a lesson that cost a window: # * Both lanes run ONE build: a new CLI cannot handshake old agents. So: stop # the groups → stage and verify the binary on every node → swap both paths → # restart the agents → relaunch the groups → verify serving. A partial # rollout wedges the control plane. # * Swapping is delegated to host-scripts/flare-swap.sh, which encodes temp+mv # (never cp over a mapped inode), both paths, one backup generation, and # sha + /proc//exe verification. # * Groups are persistent (since 2026-11-16), so relaunch is `orbit run --name`. # * Every wait is bounded; nothing loops forever. # # Fleet definition (override by env): FLARE_NODES, FLARE_LANES ("controller:group"), # FLARE_BUILD_HOST, FLARE_PORT, FLARE_WAIT, FLARE_STAGE_ROOT, FLARE_SRC_DIR. set -uo pipefail SELF_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) NODES=${FLARE_NODES:-"dgx1 dgx2 dgx3 dgx4 dgx5 dgx6 dgx7 dgx8"} LANES=${FLARE_LANES:-"dgx1:deepseek1017 dgx5:ds41-tp4"} BUILD_HOST=${FLARE_BUILD_HOST:-dgx5} SRC_DIR=${FLARE_SRC_DIR:-'$HOME/hyperflare-src'} STAGE_ROOT=${FLARE_STAGE_ROOT:-/usr/local/share/hyperflare} PORT=${FLARE_PORT:-30000} WAIT=${FLARE_WAIT:-300} SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new) say() { printf '%s\n' "$*"; } info() { printf ' %s\n' "$*"; } warn() { printf 'WARN: %s\n' "$*" >&2; } die() { printf 'ABORT: %s\n' "$*" >&2; exit 1; } host_of() { printf '%s.lan' "$1"; } node_count() { set -- $NODES; printf '%s' "$#"; } # ssh with a hard timeout so a wedged node cannot hang a window net() { local node=$1; shift; timeout 90 ssh "${SSH_OPTS[@]}" "$(host_of "$node")" "$@"; } # ---------------------------------------------------------------- status cmd_status() { local strict=${1:-} drift=0 node line printf '%-6s %-13s %-13s %-11s %-8s %-6s %s\n' NODE /usr/bin AGENT-PATH STAGED AGENT RANKS FREE for node in $NODES; do line=$(net "$node" bash -s <<'EOF' 2>/dev/null a=$(sha256sum /usr/bin/flare 2>/dev/null | cut -c1-12) b=$(sha256sum "$HOME/.local/lib/flare-agent/flare" 2>/dev/null | cut -c1-12) s=$(ls /usr/local/share/hyperflare 2>/dev/null | tr '\n' ',' ) agent=$(systemctl --user is-active flare-agent 2>/dev/null || echo inactive) ranks=$(ps -eo comm,args --no-headers 2>/dev/null | awk '$1=="flare" && / run /' | wc -l) free=$(df -h / | tail -1 | awk '{print $4}') printf '%s|%s|%s|%s|%s|%s' "${a:-missing}" "${b:-missing}" "${s:-none}" "$agent" "$ranks" "$free" EOF ) IFS='|' read -r usr agent_path staged agent ranks free <<<"$line" printf '%-6s %-13s %-13s %-11s %-8s %-6s %s\n' \ "$node" "${usr:--}" "${agent_path:--}" "${staged%,}" "${agent:--}" "${ranks:-0}" "${free:--}" if [ -z "$usr" ] || [ "$usr" != "$agent_path" ] || [ "$agent" != active ]; then drift=$((drift+1)) fi done say "" say "groups" local pair ctl group out for pair in $LANES; do ctl=${pair%%:*}; group=${pair#*:} out=$(net "$ctl" "timeout 40 flare orbit ls 2>/dev/null | grep -E '^persistent[[:space:]]+$group'" 2>/dev/null) [ -n "$out" ] || out="(not listed — is $ctl the controller?)" say " $group @ $ctl: $out" done say "" say "invariant: one build at both paths on every node, agents active" if [ "$strict" = strict ] && [ "$drift" -gt 0 ]; then die "$drift node(s) drift from the invariant" fi } # ---------------------------------------------------------------- build cmd_build() { local ref=main out="" while [ $# -gt 0 ]; do case $1 in --ref) ref=$2; shift 2 ;; --out) out=$2; shift 2 ;; *) die "build: unknown argument $1" ;; esac done say "building NVIDIA/hyperflare-inference@$ref on $BUILD_HOST" net "$BUILD_HOST" bash -s -- "$ref" "$SRC_DIR" <<'EOF' 2>&1 | sed 's/^/ /' set -uo pipefail ref=$1; src=$2 cd "$src" || { echo "no source tree at $src"; exit 1; } git fetch --depth 1 origin "$ref" -q || git fetch --depth 1 origin -q git checkout -q FETCH_HEAD 2>/dev/null || git checkout -q "$ref" echo "commit: $(git log --oneline -1)" cargo build --release --locked --manifest-path flare-cli/Cargo.toml --features roce 2>&1 | tail -2 f=flare-cli/target/release/flare [ -x "$f" ] || { echo "no binary at $f"; exit 1; } echo "sha256: $(sha256sum "$f" | cut -d' ' -f1)" echo "size: $(stat -c %s "$f")" echo "roce: $(ldd "$f" | grep -c ibverbs) libibverbs link(s)" echo "version: $("$f" --version 2>&1 | head -1) (source builds print 0.0.0 — use the sha)" EOF local sha sha=$(net "$BUILD_HOST" "sha256sum $SRC_DIR/flare-cli/target/release/flare | cut -d' ' -f1") [ -n "$sha" ] || die "could not read the built binary's sha256" if [ -n "$out" ]; then net "$BUILD_HOST" "cat $SRC_DIR/flare-cli/target/release/flare" > "$out" || die "copy failed" chmod 755 "$out"; info "copied to $out" fi say "result: sha256=$sha" } # ---------------------------------------------------------------- group helpers group_pairs() { # "ctl group" lines for groups that are actually running local pair ctl group n for pair in $LANES; do ctl=${pair%%:*}; group=${pair#*:} n=$(net "$ctl" "timeout 40 flare orbit ls 2>/dev/null | grep -E '^[[:space:]]*$group[[:space:]]' | grep -c running" 2>/dev/null) [ "${n:-0}" -gt 0 ] && printf '%s %s\n' "$ctl" "$group" done return 0 } stop_groups() { local ctl group while read -r ctl group; do [ -n "${ctl:-}" ] || continue say " stopping $group on $ctl (drains rank zero first)" net "$ctl" "timeout 150 flare orbit stop $group" 2>&1 | sed 's/^/ /' done < <(group_pairs) } relaunch_groups() { local pair ctl group for pair in $LANES; do ctl=${pair%%:*}; group=${pair#*:} say " $group @ $ctl: orbit run --name $group --detach" net "$ctl" "nohup setsid bash -c 'flare orbit run --name $group --detach > \$HOME/flare-relaunch.log 2>&1' >/dev/null 2>&1 & sleep 5; true" >/dev/null 2>&1 done } wait_health() { local deadline=$((SECONDS + WAIT)) pair ctl code ok while [ $SECONDS -lt $deadline ]; do ok=1 for pair in $LANES; do ctl=${pair%%:*} code=$(net "$ctl" "curl -s -m 4 -o /dev/null -w '%{http_code}' localhost:$PORT/health" 2>/dev/null) [ "$code" = 200 ] || ok=0 done [ $ok -eq 1 ] && return 0 sleep 5 done return 1 } verify_serving() { local pair ctl group code for pair in $LANES; do ctl=${pair%%:*}; group=${pair#*:} code=$(net "$ctl" bash -s -- "$PORT" <<'EOF' 2>/dev/null port=$1 h=$(curl -s -m 4 -o /dev/null -w '%{http_code}' localhost:$port/health) m=$(curl -s -m 6 localhost:$port/v1/models | sed -n 's/.*"id":"\([^"]*\)".*/\1/p' | head -1) r=$(curl -s -m 60 localhost:$port/v1/chat/completions -H 'Content-Type: application/json' \ -d "{\"model\":\"$m\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: OK\"}],\"max_tokens\":16}" \ | sed -n 's/.*"content":"\([^"]*\)".*/\1/p' | head -1) printf '%s model=%s reply=%s' "$h" "${m:-?}" "${r:-?}" EOF ) printf ' %-9s health=%s\n' "$group" "${code:-(no answer)}" case "$code" in 200*) ;; *) return 1 ;; esac done } prune_backups() { local node for node in $NODES; do net "$node" bash -s <<'EOF' 2>/dev/null | sed "s/^/ $node: /" keep=$(ls -1 /usr/bin/flare.orig-* 2>/dev/null | sed 's/.*flare\.orig-//' | sort | tail -1) [ -n "$keep" ] || { echo "no backups"; exit 0; } for f in /usr/bin/flare.orig-* "$HOME"/.local/lib/flare-agent/flare.orig-*; do [ -e "$f" ] || continue case "$f" in *.orig-"$keep") ;; *) sudo rm -f "$f" && echo "removed $(basename "$f")" ;; esac done EOF done } # ---------------------------------------------------------------- upgrade cmd_upgrade() { local binary="" release="" yes=0 dry=0 relaunch=1 prune=1 while [ $# -gt 0 ]; do case $1 in --binary) binary=$2; shift 2 ;; --release) release=$2; shift 2 ;; --yes) yes=1; shift ;; --dry-run) dry=1; shift ;; --no-relaunch) relaunch=0; shift ;; --no-prune) prune=0; shift ;; *) die "upgrade: unknown argument $1" ;; esac done [ -n "$binary" ] || die "upgrade: --binary PATH is required (see 'build')" [ -f "$binary" ] || die "upgrade: no such binary: $binary" release=${release:-v$(date +%Y%m%d-%H%M)} local new_sha; new_sha=$(sha256sum "$binary" | cut -d' ' -f1) local swap_script=$SELF_DIR/flare-swap.sh [ -f "$swap_script" ] || die "no flare-swap.sh next to this script" say "FLARE fleet upgrade -> $release" info "nodes: $NODES ($(node_count) nodes)" info "groups: $LANES" info "binary: $binary" info "sha256: $new_sha" say "preflight: reachability and the current build" local node cur bad=0 sums="" for node in $NODES; do cur=$(net "$node" "sha256sum /usr/bin/flare 2>/dev/null | cut -c1-12" 2>/dev/null) if [ -z "$cur" ]; then warn "$node unreachable"; bad=$((bad+1)); else sums="$sums $cur"; fi done [ "$bad" -eq 0 ] || die "$bad node(s) unreachable; fix before opening a window" local distinct; distinct=$(printf '%s\n' $sums | sort -u | tr '\n' ' ') info "currently deployed: $distinct" [ "$(printf '%s\n' $sums | sort -u | wc -l)" -eq 1 ] \ || warn "fleet is ALREADY mixed ($distinct) — this upgrade will fix it, but pairing may be broken now" case " $distinct " in *" ${new_sha:0:12} "*) warn "target build is already deployed everywhere";; esac local running; running=$(group_pairs) [ -n "$running" ] && { say "groups that will stop:"; printf '%s\n' "$running" | sed 's/^/ /'; } \ || say "no groups are running" if [ "$dry" = 1 ] || [ "$yes" = 0 ]; then say "" say "plan (nothing changed)" info "1. stop the groups above (drain rank zero first)" info "2. push $release to $STAGE_ROOT/$release on all $(node_count) nodes, sha verified per node" info "3. flare-swap.sh FORCE=1 per node: temp+mv both paths, backup, agent restart" info "4. verify one build everywhere (sha at both paths)" info "5. relaunch: $LANES" info "6. wait for /health 200 on each controller (max ${WAIT}s) + models + one completion" [ "$prune" = 1 ] && info "7. prune older .orig-* backups (keep the newest generation)" say "" [ "$dry" = 1 ] && { say "dry run: nothing changed"; return 0; } die "re-run with --yes to execute this window" fi local started; started=$(date +%s) say "step 1/7 stop groups"; stop_groups say "step 2/7 stage $release" local got for node in $NODES; do net "$node" "cat > /tmp/flare-stage && chmod 755 /tmp/flare-stage" < "$binary" || die "$node: push failed" got=$(net "$node" "sha256sum /tmp/flare-stage | cut -d' ' -f1") [ "$got" = "$new_sha" ] || die "$node: staged sha ${got:0:12} != ${new_sha:0:12} — refusing" net "$node" bash -s </dev/null 2>&1 || die "$node: staging failed" set -eu sudo mkdir -p $STAGE_ROOT/$release sudo install -m 0755 /tmp/flare-stage $STAGE_ROOT/$release/flare sudo sh -c "cd $STAGE_ROOT/$release && sha256sum flare | cut -d' ' -f1 > binary.sha256" rm -f /tmp/flare-stage EOF for f in LICENSE NOTICE THIRD_PARTY_NOTICES.md; do net "$BUILD_HOST" "cat $SRC_DIR/$f 2>/dev/null" 2>/dev/null \ | net "$node" "sudo tee $STAGE_ROOT/$release/$f >/dev/null" 2>/dev/null || true done info "$node staged" done say "step 3/7 swap + restart agents" for node in $NODES; do timeout 150 ssh "${SSH_OPTS[@]}" "$(host_of "$node")" "FORCE=1 bash -s -- $STAGE_ROOT/$release/flare" < "$swap_script" 2>&1 \ | grep -E "^RESULT|^BAD|REFUSED" | sed "s/^/ $node: /" done say "step 4/7 verify one build everywhere" local usr agent_path bad=0 for node in $NODES; do read -r usr agent_path < <(net "$node" "echo \$(sha256sum /usr/bin/flare | cut -d' ' -f1) \$(sha256sum \$HOME/.local/lib/flare-agent/flare | cut -d' ' -f1)") if [ "$usr" != "$new_sha" ] || [ "$agent_path" != "$new_sha" ]; then warn "$node usr=${usr:0:12} agent=${agent_path:0:12} want ${new_sha:0:12}"; bad=$((bad+1)) fi done [ "$bad" -eq 0 ] || die "$bad node(s) not on $release — mixed builds cannot pair; run 'rollback --yes'" info "all $(node_count) nodes run ${new_sha:0:12}" if [ "$relaunch" = 1 ]; then say "step 5/7 relaunch groups"; relaunch_groups say "step 6/7 wait for health" if wait_health; then info "healthy"; else warn "NOT healthy within ${WAIT}s"; fi verify_serving || warn "serving verification failed — inspect with 'flare orbit attach GROUP'" else say "step 5-6/7 skipped (--no-relaunch)" fi if [ "$prune" = 1 ]; then say "step 7/7 prune old backups"; prune_backups; fi say "" say "done: $release on $(node_count) nodes in $(( $(date +%s) - started ))s" } # ---------------------------------------------------------------- rollback cmd_rollback() { local stamp="" yes=0 relaunch=1 while [ $# -gt 0 ]; do case $1 in --stamp) stamp=$2; shift 2 ;; --yes) yes=1; shift ;; --no-relaunch) relaunch=0; shift ;; *) die "rollback: unknown argument $1" ;; esac done say "FLARE fleet rollback to a .orig-* generation" local node avail tmpf one tmpf=$(mktemp) for node in $NODES; do avail=$(net "$node" "ls -1 \$HOME/.local/lib/flare-agent/flare.orig-* 2>/dev/null | sed 's/.*flare\.orig-//' | sort -u") [ -n "$avail" ] || { rm -f "$tmpf"; die "$node has no agent-path .orig-* backup; fleet-wide rollback impossible"; } for one in $avail; do printf '%s %s\n' "$node" "$one" >> "$tmpf"; done info "$node backups: $(printf '%s ' $avail)" done # The target must exist on EVERY node: restoring different generations is the # mixed-build state that rollback exists to escape. if [ -z "$stamp" ]; then stamp=$(awk '{print $2}' "$tmpf" | sort | uniq -c \ | awk -v n="$(node_count)" '$1==n{print $2}' | sort | tail -1) [ -n "$stamp" ] || { rm -f "$tmpf"; die "no backup generation exists on all $(node_count) nodes; pass --stamp"; } fi for node in $NODES; do grep -qx "$node $stamp" "$tmpf" || { rm -f "$tmpf"; die "$node lacks flare.orig-$stamp (would leave a mixed fleet)"; } done rm -f "$tmpf" info "target (present everywhere): flare.orig-$stamp" if [ "$yes" = 0 ]; then say "plan: stop groups -> restore flare.orig-$stamp at both paths -> restart agents -> relaunch" die "re-run with --yes" fi stop_groups local usr for node in $NODES; do usr=$(net "$node" bash -s -- "$stamp" <<'EOF' 2>/dev/null | tail -1 set -u stamp=$1 A=$HOME/.local/lib/flare-agent/flare.orig-$stamp U=/usr/bin/flare.orig-$stamp [ -f "$A" ] && [ -f "$U" ] || { echo "missing backup flare.orig-$stamp"; exit 1; } # rename, never cp over a mapped inode; source copies stay for the next rollback install -m 0755 "$A" "$HOME/.local/lib/flare-agent/flare.tmp" && mv -f "$HOME/.local/lib/flare-agent/flare.tmp" "$HOME/.local/lib/flare-agent/flare" sudo install -m 0755 "$U" /usr/bin/flare.tmp && sudo mv -f /usr/bin/flare.tmp /usr/bin/flare systemctl --user restart flare-agent sleep 3 echo "$(sha256sum /usr/bin/flare | cut -c1-12)/$(sha256sum "$HOME/.local/lib/flare-agent/flare" | cut -c1-12)" EOF ) say " $node: $usr" [ -n "$usr" ] || die "$node rollback failed" done if [ "$relaunch" = 1 ]; then relaunch_groups wait_health || warn "not healthy after rollback" verify_serving || warn "serving verification failed after rollback" fi } usage() { sed -n '2,26p' "$0" | sed 's/^# \{0,1\}//'; } case ${1:-} in status) shift; cmd_status ;; check) shift; cmd_status strict ;; build) shift; cmd_build "$@" ;; upgrade) shift; cmd_upgrade "$@" ;; rollback) shift; cmd_rollback "$@" ;; ""|help|-h|--help) usage ;; *) die "unknown command: $1 (try --help)" ;; esac