| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387 |
- #!/usr/bin/env bash
- # flare-fleet.sh — one operator tool for the eight-Spark FLARE fleet.
- #
- # ./host-scripts/flare-fleet.sh status # read-only snapshot
- # ./host-scripts/flare-fleet.sh check # status, exit 1 on drift
- # ./host-scripts/flare-fleet.sh build --ref main [--out F] # build on dgx5, print sha
- # ./host-scripts/flare-fleet.sh upgrade --binary B --release v0.3.0 --yes
- # ./host-scripts/flare-fleet.sh upgrade --binary B --dry-run # print the plan only
- # ./host-scripts/flare-fleet.sh rollback --yes # newest .orig-* generation
- #
- # Order of operations, each step a lesson that cost a window:
- # * Both lanes run ONE build: a new CLI cannot handshake old agents. So: stop
- # the groups → stage and verify the binary on every node → swap both paths →
- # restart the agents → relaunch the groups → verify serving. A partial
- # rollout wedges the control plane.
- # * Swapping is delegated to host-scripts/flare-swap.sh, which encodes temp+mv
- # (never cp over a mapped inode), both paths, one backup generation, and
- # sha + /proc/<pid>/exe verification.
- # * Groups are persistent (since 2026-11-16), so relaunch is `orbit run --name`.
- # * Every wait is bounded; nothing loops forever.
- #
- # Fleet definition (override by env): FLARE_NODES, FLARE_LANES ("controller:group"),
- # FLARE_BUILD_HOST, FLARE_PORT, FLARE_WAIT, FLARE_STAGE_ROOT, FLARE_SRC_DIR.
- set -uo pipefail
- SELF_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
- NODES=${FLARE_NODES:-"dgx1 dgx2 dgx3 dgx4 dgx5 dgx6 dgx7 dgx8"}
- LANES=${FLARE_LANES:-"dgx1:deepseek1017 dgx5:ds41-tp4"}
- BUILD_HOST=${FLARE_BUILD_HOST:-dgx5}
- SRC_DIR=${FLARE_SRC_DIR:-'$HOME/hyperflare-src'}
- STAGE_ROOT=${FLARE_STAGE_ROOT:-/usr/local/share/hyperflare}
- PORT=${FLARE_PORT:-30000}
- WAIT=${FLARE_WAIT:-300}
- SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
- say() { printf '%s\n' "$*"; }
- info() { printf ' %s\n' "$*"; }
- warn() { printf 'WARN: %s\n' "$*" >&2; }
- die() { printf 'ABORT: %s\n' "$*" >&2; exit 1; }
- host_of() { printf '%s.lan' "$1"; }
- node_count() { set -- $NODES; printf '%s' "$#"; }
- # ssh with a hard timeout so a wedged node cannot hang a window
- net() { local node=$1; shift; timeout 90 ssh "${SSH_OPTS[@]}" "$(host_of "$node")" "$@"; }
- # ---------------------------------------------------------------- status
- cmd_status() {
- local strict=${1:-} drift=0 node line
- printf '%-6s %-13s %-13s %-11s %-8s %-6s %s\n' NODE /usr/bin AGENT-PATH STAGED AGENT RANKS FREE
- for node in $NODES; do
- line=$(net "$node" bash -s <<'EOF' 2>/dev/null
- a=$(sha256sum /usr/bin/flare 2>/dev/null | cut -c1-12)
- b=$(sha256sum "$HOME/.local/lib/flare-agent/flare" 2>/dev/null | cut -c1-12)
- s=$(ls /usr/local/share/hyperflare 2>/dev/null | tr '\n' ',' )
- agent=$(systemctl --user is-active flare-agent 2>/dev/null || echo inactive)
- ranks=$(ps -eo comm,args --no-headers 2>/dev/null | awk '$1=="flare" && / run /' | wc -l)
- free=$(df -h / | tail -1 | awk '{print $4}')
- printf '%s|%s|%s|%s|%s|%s' "${a:-missing}" "${b:-missing}" "${s:-none}" "$agent" "$ranks" "$free"
- EOF
- )
- IFS='|' read -r usr agent_path staged agent ranks free <<<"$line"
- printf '%-6s %-13s %-13s %-11s %-8s %-6s %s\n' \
- "$node" "${usr:--}" "${agent_path:--}" "${staged%,}" "${agent:--}" "${ranks:-0}" "${free:--}"
- if [ -z "$usr" ] || [ "$usr" != "$agent_path" ] || [ "$agent" != active ]; then
- drift=$((drift+1))
- fi
- done
- say ""
- say "groups"
- local pair ctl group out
- for pair in $LANES; do
- ctl=${pair%%:*}; group=${pair#*:}
- out=$(net "$ctl" "timeout 40 flare orbit ls 2>/dev/null | grep -E '^persistent[[:space:]]+$group'" 2>/dev/null)
- [ -n "$out" ] || out="(not listed — is $ctl the controller?)"
- say " $group @ $ctl: $out"
- done
- say ""
- say "invariant: one build at both paths on every node, agents active"
- if [ "$strict" = strict ] && [ "$drift" -gt 0 ]; then
- die "$drift node(s) drift from the invariant"
- fi
- }
- # ---------------------------------------------------------------- build
- cmd_build() {
- local ref=main out=""
- while [ $# -gt 0 ]; do
- case $1 in
- --ref) ref=$2; shift 2 ;;
- --out) out=$2; shift 2 ;;
- *) die "build: unknown argument $1" ;;
- esac
- done
- say "building NVIDIA/hyperflare-inference@$ref on $BUILD_HOST"
- net "$BUILD_HOST" bash -s -- "$ref" "$SRC_DIR" <<'EOF' 2>&1 | sed 's/^/ /'
- set -uo pipefail
- ref=$1; src=$2
- cd "$src" || { echo "no source tree at $src"; exit 1; }
- git fetch --depth 1 origin "$ref" -q || git fetch --depth 1 origin -q
- git checkout -q FETCH_HEAD 2>/dev/null || git checkout -q "$ref"
- echo "commit: $(git log --oneline -1)"
- cargo build --release --locked --manifest-path flare-cli/Cargo.toml --features roce 2>&1 | tail -2
- f=flare-cli/target/release/flare
- [ -x "$f" ] || { echo "no binary at $f"; exit 1; }
- echo "sha256: $(sha256sum "$f" | cut -d' ' -f1)"
- echo "size: $(stat -c %s "$f")"
- echo "roce: $(ldd "$f" | grep -c ibverbs) libibverbs link(s)"
- echo "version: $("$f" --version 2>&1 | head -1) (source builds print 0.0.0 — use the sha)"
- EOF
- local sha
- sha=$(net "$BUILD_HOST" "sha256sum $SRC_DIR/flare-cli/target/release/flare | cut -d' ' -f1")
- [ -n "$sha" ] || die "could not read the built binary's sha256"
- if [ -n "$out" ]; then
- net "$BUILD_HOST" "cat $SRC_DIR/flare-cli/target/release/flare" > "$out" || die "copy failed"
- chmod 755 "$out"; info "copied to $out"
- fi
- say "result: sha256=$sha"
- }
- # ---------------------------------------------------------------- group helpers
- group_pairs() { # "ctl group" lines for groups that are actually running
- local pair ctl group n
- for pair in $LANES; do
- ctl=${pair%%:*}; group=${pair#*:}
- n=$(net "$ctl" "timeout 40 flare orbit ls 2>/dev/null | grep -E '^[[:space:]]*$group[[:space:]]' | grep -c running" 2>/dev/null)
- [ "${n:-0}" -gt 0 ] && printf '%s %s\n' "$ctl" "$group"
- done
- return 0
- }
- stop_groups() {
- local ctl group
- while read -r ctl group; do
- [ -n "${ctl:-}" ] || continue
- say " stopping $group on $ctl (drains rank zero first)"
- net "$ctl" "timeout 150 flare orbit stop $group" 2>&1 | sed 's/^/ /'
- done < <(group_pairs)
- }
- relaunch_groups() {
- local pair ctl group
- for pair in $LANES; do
- ctl=${pair%%:*}; group=${pair#*:}
- say " $group @ $ctl: orbit run --name $group --detach"
- net "$ctl" "nohup setsid bash -c 'flare orbit run --name $group --detach > \$HOME/flare-relaunch.log 2>&1' >/dev/null 2>&1 & sleep 5; true" >/dev/null 2>&1
- done
- }
- wait_health() {
- local deadline=$((SECONDS + WAIT)) pair ctl code ok
- while [ $SECONDS -lt $deadline ]; do
- ok=1
- for pair in $LANES; do
- ctl=${pair%%:*}
- code=$(net "$ctl" "curl -s -m 4 -o /dev/null -w '%{http_code}' localhost:$PORT/health" 2>/dev/null)
- [ "$code" = 200 ] || ok=0
- done
- [ $ok -eq 1 ] && return 0
- sleep 5
- done
- return 1
- }
- verify_serving() {
- local pair ctl group code
- for pair in $LANES; do
- ctl=${pair%%:*}; group=${pair#*:}
- code=$(net "$ctl" bash -s -- "$PORT" <<'EOF' 2>/dev/null
- port=$1
- h=$(curl -s -m 4 -o /dev/null -w '%{http_code}' localhost:$port/health)
- m=$(curl -s -m 6 localhost:$port/v1/models | sed -n 's/.*"id":"\([^"]*\)".*/\1/p' | head -1)
- r=$(curl -s -m 60 localhost:$port/v1/chat/completions -H 'Content-Type: application/json' \
- -d "{\"model\":\"$m\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: OK\"}],\"max_tokens\":16}" \
- | sed -n 's/.*"content":"\([^"]*\)".*/\1/p' | head -1)
- printf '%s model=%s reply=%s' "$h" "${m:-?}" "${r:-?}"
- EOF
- )
- printf ' %-9s health=%s\n' "$group" "${code:-(no answer)}"
- case "$code" in 200*) ;; *) return 1 ;; esac
- done
- }
- prune_backups() {
- local node
- for node in $NODES; do
- net "$node" bash -s <<'EOF' 2>/dev/null | sed "s/^/ $node: /"
- keep=$(ls -1 /usr/bin/flare.orig-* 2>/dev/null | sed 's/.*flare\.orig-//' | sort | tail -1)
- [ -n "$keep" ] || { echo "no backups"; exit 0; }
- for f in /usr/bin/flare.orig-* "$HOME"/.local/lib/flare-agent/flare.orig-*; do
- [ -e "$f" ] || continue
- case "$f" in *.orig-"$keep") ;; *) sudo rm -f "$f" && echo "removed $(basename "$f")" ;; esac
- done
- EOF
- done
- }
- # ---------------------------------------------------------------- upgrade
- cmd_upgrade() {
- local binary="" release="" yes=0 dry=0 relaunch=1 prune=1
- while [ $# -gt 0 ]; do
- case $1 in
- --binary) binary=$2; shift 2 ;;
- --release) release=$2; shift 2 ;;
- --yes) yes=1; shift ;;
- --dry-run) dry=1; shift ;;
- --no-relaunch) relaunch=0; shift ;;
- --no-prune) prune=0; shift ;;
- *) die "upgrade: unknown argument $1" ;;
- esac
- done
- [ -n "$binary" ] || die "upgrade: --binary PATH is required (see 'build')"
- [ -f "$binary" ] || die "upgrade: no such binary: $binary"
- release=${release:-v$(date +%Y%m%d-%H%M)}
- local new_sha; new_sha=$(sha256sum "$binary" | cut -d' ' -f1)
- local swap_script=$SELF_DIR/flare-swap.sh
- [ -f "$swap_script" ] || die "no flare-swap.sh next to this script"
- say "FLARE fleet upgrade -> $release"
- info "nodes: $NODES ($(node_count) nodes)"
- info "groups: $LANES"
- info "binary: $binary"
- info "sha256: $new_sha"
- say "preflight: reachability and the current build"
- local node cur bad=0 sums=""
- for node in $NODES; do
- cur=$(net "$node" "sha256sum /usr/bin/flare 2>/dev/null | cut -c1-12" 2>/dev/null)
- if [ -z "$cur" ]; then warn "$node unreachable"; bad=$((bad+1)); else sums="$sums $cur"; fi
- done
- [ "$bad" -eq 0 ] || die "$bad node(s) unreachable; fix before opening a window"
- local distinct; distinct=$(printf '%s\n' $sums | sort -u | tr '\n' ' ')
- info "currently deployed: $distinct"
- [ "$(printf '%s\n' $sums | sort -u | wc -l)" -eq 1 ] \
- || warn "fleet is ALREADY mixed ($distinct) — this upgrade will fix it, but pairing may be broken now"
- case " $distinct " in *" ${new_sha:0:12} "*) warn "target build is already deployed everywhere";; esac
- local running; running=$(group_pairs)
- [ -n "$running" ] && { say "groups that will stop:"; printf '%s\n' "$running" | sed 's/^/ /'; } \
- || say "no groups are running"
- if [ "$dry" = 1 ] || [ "$yes" = 0 ]; then
- say ""
- say "plan (nothing changed)"
- info "1. stop the groups above (drain rank zero first)"
- info "2. push $release to $STAGE_ROOT/$release on all $(node_count) nodes, sha verified per node"
- info "3. flare-swap.sh FORCE=1 per node: temp+mv both paths, backup, agent restart"
- info "4. verify one build everywhere (sha at both paths)"
- info "5. relaunch: $LANES"
- info "6. wait for /health 200 on each controller (max ${WAIT}s) + models + one completion"
- [ "$prune" = 1 ] && info "7. prune older .orig-* backups (keep the newest generation)"
- say ""
- [ "$dry" = 1 ] && { say "dry run: nothing changed"; return 0; }
- die "re-run with --yes to execute this window"
- fi
- local started; started=$(date +%s)
- say "step 1/7 stop groups"; stop_groups
- say "step 2/7 stage $release"
- local got
- for node in $NODES; do
- net "$node" "cat > /tmp/flare-stage && chmod 755 /tmp/flare-stage" < "$binary" || die "$node: push failed"
- got=$(net "$node" "sha256sum /tmp/flare-stage | cut -d' ' -f1")
- [ "$got" = "$new_sha" ] || die "$node: staged sha ${got:0:12} != ${new_sha:0:12} — refusing"
- net "$node" bash -s <<EOF >/dev/null 2>&1 || die "$node: staging failed"
- set -eu
- sudo mkdir -p $STAGE_ROOT/$release
- sudo install -m 0755 /tmp/flare-stage $STAGE_ROOT/$release/flare
- sudo sh -c "cd $STAGE_ROOT/$release && sha256sum flare | cut -d' ' -f1 > binary.sha256"
- rm -f /tmp/flare-stage
- EOF
- for f in LICENSE NOTICE THIRD_PARTY_NOTICES.md; do
- net "$BUILD_HOST" "cat $SRC_DIR/$f 2>/dev/null" 2>/dev/null \
- | net "$node" "sudo tee $STAGE_ROOT/$release/$f >/dev/null" 2>/dev/null || true
- done
- info "$node staged"
- done
- say "step 3/7 swap + restart agents"
- for node in $NODES; do
- timeout 150 ssh "${SSH_OPTS[@]}" "$(host_of "$node")" "FORCE=1 bash -s -- $STAGE_ROOT/$release/flare" < "$swap_script" 2>&1 \
- | grep -E "^RESULT|^BAD|REFUSED" | sed "s/^/ $node: /"
- done
- say "step 4/7 verify one build everywhere"
- local usr agent_path
- bad=0
- for node in $NODES; do
- read -r usr agent_path < <(net "$node" "echo \$(sha256sum /usr/bin/flare | cut -d' ' -f1) \$(sha256sum \$HOME/.local/lib/flare-agent/flare | cut -d' ' -f1)")
- if [ "$usr" != "$new_sha" ] || [ "$agent_path" != "$new_sha" ]; then
- warn "$node usr=${usr:0:12} agent=${agent_path:0:12} want ${new_sha:0:12}"; bad=$((bad+1))
- fi
- done
- [ "$bad" -eq 0 ] || die "$bad node(s) not on $release — mixed builds cannot pair; run 'rollback --yes'"
- info "all $(node_count) nodes run ${new_sha:0:12}"
- if [ "$relaunch" = 1 ]; then
- say "step 5/7 relaunch groups"; relaunch_groups
- say "step 6/7 wait for health"
- if wait_health; then info "healthy"; else warn "NOT healthy within ${WAIT}s"; fi
- verify_serving || warn "serving verification failed — inspect with 'flare orbit attach GROUP'"
- else
- say "step 5-6/7 skipped (--no-relaunch)"
- fi
- if [ "$prune" = 1 ]; then say "step 7/7 prune old backups"; prune_backups; fi
- say ""
- say "done: $release on $(node_count) nodes in $(( $(date +%s) - started ))s"
- }
- # ---------------------------------------------------------------- rollback
- cmd_rollback() {
- local stamp="" yes=0 relaunch=1
- while [ $# -gt 0 ]; do
- case $1 in
- --stamp) stamp=$2; shift 2 ;;
- --yes) yes=1; shift ;;
- --no-relaunch) relaunch=0; shift ;;
- *) die "rollback: unknown argument $1" ;;
- esac
- done
- say "FLARE fleet rollback to a .orig-* generation"
- local node avail tmpf one
- tmpf=$(mktemp)
- for node in $NODES; do
- avail=$(net "$node" "ls -1 \$HOME/.local/lib/flare-agent/flare.orig-* 2>/dev/null | sed 's/.*flare\.orig-//' | sort -u")
- [ -n "$avail" ] || { rm -f "$tmpf"; die "$node has no agent-path .orig-* backup; fleet-wide rollback impossible"; }
- for one in $avail; do printf '%s %s\n' "$node" "$one" >> "$tmpf"; done
- info "$node backups: $(printf '%s ' $avail)"
- done
- # The target must exist on EVERY node: restoring different generations is the
- # mixed-build state that rollback exists to escape.
- if [ -z "$stamp" ]; then
- stamp=$(awk '{print $2}' "$tmpf" | sort | uniq -c \
- | awk -v n="$(node_count)" '$1==n{print $2}' | sort | tail -1)
- [ -n "$stamp" ] || { rm -f "$tmpf"; die "no backup generation exists on all $(node_count) nodes; pass --stamp"; }
- fi
- for node in $NODES; do
- grep -qx "$node $stamp" "$tmpf" || { rm -f "$tmpf"; die "$node lacks flare.orig-$stamp (would leave a mixed fleet)"; }
- done
- rm -f "$tmpf"
- info "target (present everywhere): flare.orig-$stamp"
- if [ "$yes" = 0 ]; then
- say "plan: stop groups -> restore flare.orig-$stamp at both paths -> restart agents -> relaunch"
- die "re-run with --yes"
- fi
- stop_groups
- local usr
- for node in $NODES; do
- usr=$(net "$node" bash -s -- "$stamp" <<'EOF' 2>/dev/null | tail -1
- set -u
- stamp=$1
- A=$HOME/.local/lib/flare-agent/flare.orig-$stamp
- U=/usr/bin/flare.orig-$stamp
- [ -f "$A" ] && [ -f "$U" ] || { echo "missing backup flare.orig-$stamp"; exit 1; }
- # rename, never cp over a mapped inode; source copies stay for the next rollback
- install -m 0755 "$A" "$HOME/.local/lib/flare-agent/flare.tmp" && mv -f "$HOME/.local/lib/flare-agent/flare.tmp" "$HOME/.local/lib/flare-agent/flare"
- sudo install -m 0755 "$U" /usr/bin/flare.tmp && sudo mv -f /usr/bin/flare.tmp /usr/bin/flare
- systemctl --user restart flare-agent
- sleep 3
- echo "$(sha256sum /usr/bin/flare | cut -c1-12)/$(sha256sum "$HOME/.local/lib/flare-agent/flare" | cut -c1-12)"
- EOF
- )
- say " $node: $usr"
- [ -n "$usr" ] || die "$node rollback failed"
- done
- if [ "$relaunch" = 1 ]; then
- relaunch_groups
- wait_health || warn "not healthy after rollback"
- verify_serving || warn "serving verification failed after rollback"
- fi
- }
- usage() { sed -n '2,26p' "$0" | sed 's/^# \{0,1\}//'; }
- case ${1:-} in
- status) shift; cmd_status ;;
- check) shift; cmd_status strict ;;
- build) shift; cmd_build "$@" ;;
- upgrade) shift; cmd_upgrade "$@" ;;
- rollback) shift; cmd_rollback "$@" ;;
- ""|help|-h|--help) usage ;;
- *) die "unknown command: $1 (try --help)" ;;
- esac
|