|
@@ -0,0 +1,387 @@
|
|
|
|
|
+#!/usr/bin/env bash
|
|
|
|
|
+# flare-fleet.sh — one operator tool for the eight-Spark FLARE fleet.
|
|
|
|
|
+#
|
|
|
|
|
+# ./host-scripts/flare-fleet.sh status # read-only snapshot
|
|
|
|
|
+# ./host-scripts/flare-fleet.sh check # status, exit 1 on drift
|
|
|
|
|
+# ./host-scripts/flare-fleet.sh build --ref main [--out F] # build on dgx5, print sha
|
|
|
|
|
+# ./host-scripts/flare-fleet.sh upgrade --binary B --release v0.3.0 --yes
|
|
|
|
|
+# ./host-scripts/flare-fleet.sh upgrade --binary B --dry-run # print the plan only
|
|
|
|
|
+# ./host-scripts/flare-fleet.sh rollback --yes # newest .orig-* generation
|
|
|
|
|
+#
|
|
|
|
|
+# Order of operations, each step a lesson that cost a window:
|
|
|
|
|
+# * Both lanes run ONE build: a new CLI cannot handshake old agents. So: stop
|
|
|
|
|
+# the groups → stage and verify the binary on every node → swap both paths →
|
|
|
|
|
+# restart the agents → relaunch the groups → verify serving. A partial
|
|
|
|
|
+# rollout wedges the control plane.
|
|
|
|
|
+# * Swapping is delegated to host-scripts/flare-swap.sh, which encodes temp+mv
|
|
|
|
|
+# (never cp over a mapped inode), both paths, one backup generation, and
|
|
|
|
|
+# sha + /proc/<pid>/exe verification.
|
|
|
|
|
+# * Groups are persistent (since 2026-11-16), so relaunch is `orbit run --name`.
|
|
|
|
|
+# * Every wait is bounded; nothing loops forever.
|
|
|
|
|
+#
|
|
|
|
|
+# Fleet definition (override by env): FLARE_NODES, FLARE_LANES ("controller:group"),
|
|
|
|
|
+# FLARE_BUILD_HOST, FLARE_PORT, FLARE_WAIT, FLARE_STAGE_ROOT, FLARE_SRC_DIR.
|
|
|
|
|
+set -uo pipefail
|
|
|
|
|
+
|
|
|
|
|
+SELF_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
|
|
|
|
+NODES=${FLARE_NODES:-"dgx1 dgx2 dgx3 dgx4 dgx5 dgx6 dgx7 dgx8"}
|
|
|
|
|
+LANES=${FLARE_LANES:-"dgx1:deepseek1017 dgx5:ds41-tp4"}
|
|
|
|
|
+BUILD_HOST=${FLARE_BUILD_HOST:-dgx5}
|
|
|
|
|
+SRC_DIR=${FLARE_SRC_DIR:-'$HOME/hyperflare-src'}
|
|
|
|
|
+STAGE_ROOT=${FLARE_STAGE_ROOT:-/usr/local/share/hyperflare}
|
|
|
|
|
+PORT=${FLARE_PORT:-30000}
|
|
|
|
|
+WAIT=${FLARE_WAIT:-300}
|
|
|
|
|
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
|
|
|
|
+
|
|
|
|
|
+say() { printf '%s\n' "$*"; }
|
|
|
|
|
+info() { printf ' %s\n' "$*"; }
|
|
|
|
|
+warn() { printf 'WARN: %s\n' "$*" >&2; }
|
|
|
|
|
+die() { printf 'ABORT: %s\n' "$*" >&2; exit 1; }
|
|
|
|
|
+host_of() { printf '%s.lan' "$1"; }
|
|
|
|
|
+node_count() { set -- $NODES; printf '%s' "$#"; }
|
|
|
|
|
+
|
|
|
|
|
+# ssh with a hard timeout so a wedged node cannot hang a window
|
|
|
|
|
+net() { local node=$1; shift; timeout 90 ssh "${SSH_OPTS[@]}" "$(host_of "$node")" "$@"; }
|
|
|
|
|
+
|
|
|
|
|
+# ---------------------------------------------------------------- status
|
|
|
|
|
+cmd_status() {
|
|
|
|
|
+ local strict=${1:-} drift=0 node line
|
|
|
|
|
+ printf '%-6s %-13s %-13s %-11s %-8s %-6s %s\n' NODE /usr/bin AGENT-PATH STAGED AGENT RANKS FREE
|
|
|
|
|
+ for node in $NODES; do
|
|
|
|
|
+ line=$(net "$node" bash -s <<'EOF' 2>/dev/null
|
|
|
|
|
+a=$(sha256sum /usr/bin/flare 2>/dev/null | cut -c1-12)
|
|
|
|
|
+b=$(sha256sum "$HOME/.local/lib/flare-agent/flare" 2>/dev/null | cut -c1-12)
|
|
|
|
|
+s=$(ls /usr/local/share/hyperflare 2>/dev/null | tr '\n' ',' )
|
|
|
|
|
+agent=$(systemctl --user is-active flare-agent 2>/dev/null || echo inactive)
|
|
|
|
|
+ranks=$(ps -eo comm,args --no-headers 2>/dev/null | awk '$1=="flare" && / run /' | wc -l)
|
|
|
|
|
+free=$(df -h / | tail -1 | awk '{print $4}')
|
|
|
|
|
+printf '%s|%s|%s|%s|%s|%s' "${a:-missing}" "${b:-missing}" "${s:-none}" "$agent" "$ranks" "$free"
|
|
|
|
|
+EOF
|
|
|
|
|
+)
|
|
|
|
|
+ IFS='|' read -r usr agent_path staged agent ranks free <<<"$line"
|
|
|
|
|
+ printf '%-6s %-13s %-13s %-11s %-8s %-6s %s\n' \
|
|
|
|
|
+ "$node" "${usr:--}" "${agent_path:--}" "${staged%,}" "${agent:--}" "${ranks:-0}" "${free:--}"
|
|
|
|
|
+ if [ -z "$usr" ] || [ "$usr" != "$agent_path" ] || [ "$agent" != active ]; then
|
|
|
|
|
+ drift=$((drift+1))
|
|
|
|
|
+ fi
|
|
|
|
|
+ done
|
|
|
|
|
+ say ""
|
|
|
|
|
+ say "groups"
|
|
|
|
|
+ local pair ctl group out
|
|
|
|
|
+ for pair in $LANES; do
|
|
|
|
|
+ ctl=${pair%%:*}; group=${pair#*:}
|
|
|
|
|
+ out=$(net "$ctl" "timeout 40 flare orbit ls 2>/dev/null | grep -E '^persistent[[:space:]]+$group'" 2>/dev/null)
|
|
|
|
|
+ [ -n "$out" ] || out="(not listed — is $ctl the controller?)"
|
|
|
|
|
+ say " $group @ $ctl: $out"
|
|
|
|
|
+ done
|
|
|
|
|
+ say ""
|
|
|
|
|
+ say "invariant: one build at both paths on every node, agents active"
|
|
|
|
|
+ if [ "$strict" = strict ] && [ "$drift" -gt 0 ]; then
|
|
|
|
|
+ die "$drift node(s) drift from the invariant"
|
|
|
|
|
+ fi
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+# ---------------------------------------------------------------- build
|
|
|
|
|
+cmd_build() {
|
|
|
|
|
+ local ref=main out=""
|
|
|
|
|
+ while [ $# -gt 0 ]; do
|
|
|
|
|
+ case $1 in
|
|
|
|
|
+ --ref) ref=$2; shift 2 ;;
|
|
|
|
|
+ --out) out=$2; shift 2 ;;
|
|
|
|
|
+ *) die "build: unknown argument $1" ;;
|
|
|
|
|
+ esac
|
|
|
|
|
+ done
|
|
|
|
|
+ say "building NVIDIA/hyperflare-inference@$ref on $BUILD_HOST"
|
|
|
|
|
+ net "$BUILD_HOST" bash -s -- "$ref" "$SRC_DIR" <<'EOF' 2>&1 | sed 's/^/ /'
|
|
|
|
|
+set -uo pipefail
|
|
|
|
|
+ref=$1; src=$2
|
|
|
|
|
+cd "$src" || { echo "no source tree at $src"; exit 1; }
|
|
|
|
|
+git fetch --depth 1 origin "$ref" -q || git fetch --depth 1 origin -q
|
|
|
|
|
+git checkout -q FETCH_HEAD 2>/dev/null || git checkout -q "$ref"
|
|
|
|
|
+echo "commit: $(git log --oneline -1)"
|
|
|
|
|
+cargo build --release --locked --manifest-path flare-cli/Cargo.toml --features roce 2>&1 | tail -2
|
|
|
|
|
+f=flare-cli/target/release/flare
|
|
|
|
|
+[ -x "$f" ] || { echo "no binary at $f"; exit 1; }
|
|
|
|
|
+echo "sha256: $(sha256sum "$f" | cut -d' ' -f1)"
|
|
|
|
|
+echo "size: $(stat -c %s "$f")"
|
|
|
|
|
+echo "roce: $(ldd "$f" | grep -c ibverbs) libibverbs link(s)"
|
|
|
|
|
+echo "version: $("$f" --version 2>&1 | head -1) (source builds print 0.0.0 — use the sha)"
|
|
|
|
|
+EOF
|
|
|
|
|
+ local sha
|
|
|
|
|
+ sha=$(net "$BUILD_HOST" "sha256sum $SRC_DIR/flare-cli/target/release/flare | cut -d' ' -f1")
|
|
|
|
|
+ [ -n "$sha" ] || die "could not read the built binary's sha256"
|
|
|
|
|
+ if [ -n "$out" ]; then
|
|
|
|
|
+ net "$BUILD_HOST" "cat $SRC_DIR/flare-cli/target/release/flare" > "$out" || die "copy failed"
|
|
|
|
|
+ chmod 755 "$out"; info "copied to $out"
|
|
|
|
|
+ fi
|
|
|
|
|
+ say "result: sha256=$sha"
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+# ---------------------------------------------------------------- group helpers
|
|
|
|
|
+group_pairs() { # "ctl group" lines for groups that are actually running
|
|
|
|
|
+ local pair ctl group n
|
|
|
|
|
+ for pair in $LANES; do
|
|
|
|
|
+ ctl=${pair%%:*}; group=${pair#*:}
|
|
|
|
|
+ n=$(net "$ctl" "timeout 40 flare orbit ls 2>/dev/null | grep -E '^[[:space:]]*$group[[:space:]]' | grep -c running" 2>/dev/null)
|
|
|
|
|
+ [ "${n:-0}" -gt 0 ] && printf '%s %s\n' "$ctl" "$group"
|
|
|
|
|
+ done
|
|
|
|
|
+ return 0
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+stop_groups() {
|
|
|
|
|
+ local ctl group
|
|
|
|
|
+ while read -r ctl group; do
|
|
|
|
|
+ [ -n "${ctl:-}" ] || continue
|
|
|
|
|
+ say " stopping $group on $ctl (drains rank zero first)"
|
|
|
|
|
+ net "$ctl" "timeout 150 flare orbit stop $group" 2>&1 | sed 's/^/ /'
|
|
|
|
|
+ done < <(group_pairs)
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+relaunch_groups() {
|
|
|
|
|
+ local pair ctl group
|
|
|
|
|
+ for pair in $LANES; do
|
|
|
|
|
+ ctl=${pair%%:*}; group=${pair#*:}
|
|
|
|
|
+ say " $group @ $ctl: orbit run --name $group --detach"
|
|
|
|
|
+ net "$ctl" "nohup setsid bash -c 'flare orbit run --name $group --detach > \$HOME/flare-relaunch.log 2>&1' >/dev/null 2>&1 & sleep 5; true" >/dev/null 2>&1
|
|
|
|
|
+ done
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+wait_health() {
|
|
|
|
|
+ local deadline=$((SECONDS + WAIT)) pair ctl code ok
|
|
|
|
|
+ while [ $SECONDS -lt $deadline ]; do
|
|
|
|
|
+ ok=1
|
|
|
|
|
+ for pair in $LANES; do
|
|
|
|
|
+ ctl=${pair%%:*}
|
|
|
|
|
+ code=$(net "$ctl" "curl -s -m 4 -o /dev/null -w '%{http_code}' localhost:$PORT/health" 2>/dev/null)
|
|
|
|
|
+ [ "$code" = 200 ] || ok=0
|
|
|
|
|
+ done
|
|
|
|
|
+ [ $ok -eq 1 ] && return 0
|
|
|
|
|
+ sleep 5
|
|
|
|
|
+ done
|
|
|
|
|
+ return 1
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+verify_serving() {
|
|
|
|
|
+ local pair ctl group code
|
|
|
|
|
+ for pair in $LANES; do
|
|
|
|
|
+ ctl=${pair%%:*}; group=${pair#*:}
|
|
|
|
|
+ code=$(net "$ctl" bash -s -- "$PORT" <<'EOF' 2>/dev/null
|
|
|
|
|
+port=$1
|
|
|
|
|
+h=$(curl -s -m 4 -o /dev/null -w '%{http_code}' localhost:$port/health)
|
|
|
|
|
+m=$(curl -s -m 6 localhost:$port/v1/models | sed -n 's/.*"id":"\([^"]*\)".*/\1/p' | head -1)
|
|
|
|
|
+r=$(curl -s -m 60 localhost:$port/v1/chat/completions -H 'Content-Type: application/json' \
|
|
|
|
|
+ -d "{\"model\":\"$m\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: OK\"}],\"max_tokens\":16}" \
|
|
|
|
|
+ | sed -n 's/.*"content":"\([^"]*\)".*/\1/p' | head -1)
|
|
|
|
|
+printf '%s model=%s reply=%s' "$h" "${m:-?}" "${r:-?}"
|
|
|
|
|
+EOF
|
|
|
|
|
+)
|
|
|
|
|
+ printf ' %-9s health=%s\n' "$group" "${code:-(no answer)}"
|
|
|
|
|
+ case "$code" in 200*) ;; *) return 1 ;; esac
|
|
|
|
|
+ done
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+prune_backups() {
|
|
|
|
|
+ local node
|
|
|
|
|
+ for node in $NODES; do
|
|
|
|
|
+ net "$node" bash -s <<'EOF' 2>/dev/null | sed "s/^/ $node: /"
|
|
|
|
|
+keep=$(ls -1 /usr/bin/flare.orig-* 2>/dev/null | sed 's/.*flare\.orig-//' | sort | tail -1)
|
|
|
|
|
+[ -n "$keep" ] || { echo "no backups"; exit 0; }
|
|
|
|
|
+for f in /usr/bin/flare.orig-* "$HOME"/.local/lib/flare-agent/flare.orig-*; do
|
|
|
|
|
+ [ -e "$f" ] || continue
|
|
|
|
|
+ case "$f" in *.orig-"$keep") ;; *) sudo rm -f "$f" && echo "removed $(basename "$f")" ;; esac
|
|
|
|
|
+done
|
|
|
|
|
+EOF
|
|
|
|
|
+ done
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+# ---------------------------------------------------------------- upgrade
|
|
|
|
|
+cmd_upgrade() {
|
|
|
|
|
+ local binary="" release="" yes=0 dry=0 relaunch=1 prune=1
|
|
|
|
|
+ while [ $# -gt 0 ]; do
|
|
|
|
|
+ case $1 in
|
|
|
|
|
+ --binary) binary=$2; shift 2 ;;
|
|
|
|
|
+ --release) release=$2; shift 2 ;;
|
|
|
|
|
+ --yes) yes=1; shift ;;
|
|
|
|
|
+ --dry-run) dry=1; shift ;;
|
|
|
|
|
+ --no-relaunch) relaunch=0; shift ;;
|
|
|
|
|
+ --no-prune) prune=0; shift ;;
|
|
|
|
|
+ *) die "upgrade: unknown argument $1" ;;
|
|
|
|
|
+ esac
|
|
|
|
|
+ done
|
|
|
|
|
+ [ -n "$binary" ] || die "upgrade: --binary PATH is required (see 'build')"
|
|
|
|
|
+ [ -f "$binary" ] || die "upgrade: no such binary: $binary"
|
|
|
|
|
+ release=${release:-v$(date +%Y%m%d-%H%M)}
|
|
|
|
|
+ local new_sha; new_sha=$(sha256sum "$binary" | cut -d' ' -f1)
|
|
|
|
|
+ local swap_script=$SELF_DIR/flare-swap.sh
|
|
|
|
|
+ [ -f "$swap_script" ] || die "no flare-swap.sh next to this script"
|
|
|
|
|
+
|
|
|
|
|
+ say "FLARE fleet upgrade -> $release"
|
|
|
|
|
+ info "nodes: $NODES ($(node_count) nodes)"
|
|
|
|
|
+ info "groups: $LANES"
|
|
|
|
|
+ info "binary: $binary"
|
|
|
|
|
+ info "sha256: $new_sha"
|
|
|
|
|
+
|
|
|
|
|
+ say "preflight: reachability and the current build"
|
|
|
|
|
+ local node cur bad=0 sums=""
|
|
|
|
|
+ for node in $NODES; do
|
|
|
|
|
+ cur=$(net "$node" "sha256sum /usr/bin/flare 2>/dev/null | cut -c1-12" 2>/dev/null)
|
|
|
|
|
+ if [ -z "$cur" ]; then warn "$node unreachable"; bad=$((bad+1)); else sums="$sums $cur"; fi
|
|
|
|
|
+ done
|
|
|
|
|
+ [ "$bad" -eq 0 ] || die "$bad node(s) unreachable; fix before opening a window"
|
|
|
|
|
+ local distinct; distinct=$(printf '%s\n' $sums | sort -u | tr '\n' ' ')
|
|
|
|
|
+ info "currently deployed: $distinct"
|
|
|
|
|
+ [ "$(printf '%s\n' $sums | sort -u | wc -l)" -eq 1 ] \
|
|
|
|
|
+ || warn "fleet is ALREADY mixed ($distinct) — this upgrade will fix it, but pairing may be broken now"
|
|
|
|
|
+ case " $distinct " in *" ${new_sha:0:12} "*) warn "target build is already deployed everywhere";; esac
|
|
|
|
|
+
|
|
|
|
|
+ local running; running=$(group_pairs)
|
|
|
|
|
+ [ -n "$running" ] && { say "groups that will stop:"; printf '%s\n' "$running" | sed 's/^/ /'; } \
|
|
|
|
|
+ || say "no groups are running"
|
|
|
|
|
+
|
|
|
|
|
+ if [ "$dry" = 1 ] || [ "$yes" = 0 ]; then
|
|
|
|
|
+ say ""
|
|
|
|
|
+ say "plan (nothing changed)"
|
|
|
|
|
+ info "1. stop the groups above (drain rank zero first)"
|
|
|
|
|
+ info "2. push $release to $STAGE_ROOT/$release on all $(node_count) nodes, sha verified per node"
|
|
|
|
|
+ info "3. flare-swap.sh FORCE=1 per node: temp+mv both paths, backup, agent restart"
|
|
|
|
|
+ info "4. verify one build everywhere (sha at both paths)"
|
|
|
|
|
+ info "5. relaunch: $LANES"
|
|
|
|
|
+ info "6. wait for /health 200 on each controller (max ${WAIT}s) + models + one completion"
|
|
|
|
|
+ [ "$prune" = 1 ] && info "7. prune older .orig-* backups (keep the newest generation)"
|
|
|
|
|
+ say ""
|
|
|
|
|
+ [ "$dry" = 1 ] && { say "dry run: nothing changed"; return 0; }
|
|
|
|
|
+ die "re-run with --yes to execute this window"
|
|
|
|
|
+ fi
|
|
|
|
|
+
|
|
|
|
|
+ local started; started=$(date +%s)
|
|
|
|
|
+ say "step 1/7 stop groups"; stop_groups
|
|
|
|
|
+
|
|
|
|
|
+ say "step 2/7 stage $release"
|
|
|
|
|
+ local got
|
|
|
|
|
+ for node in $NODES; do
|
|
|
|
|
+ net "$node" "cat > /tmp/flare-stage && chmod 755 /tmp/flare-stage" < "$binary" || die "$node: push failed"
|
|
|
|
|
+ got=$(net "$node" "sha256sum /tmp/flare-stage | cut -d' ' -f1")
|
|
|
|
|
+ [ "$got" = "$new_sha" ] || die "$node: staged sha ${got:0:12} != ${new_sha:0:12} — refusing"
|
|
|
|
|
+ net "$node" bash -s <<EOF >/dev/null 2>&1 || die "$node: staging failed"
|
|
|
|
|
+set -eu
|
|
|
|
|
+sudo mkdir -p $STAGE_ROOT/$release
|
|
|
|
|
+sudo install -m 0755 /tmp/flare-stage $STAGE_ROOT/$release/flare
|
|
|
|
|
+sudo sh -c "cd $STAGE_ROOT/$release && sha256sum flare | cut -d' ' -f1 > binary.sha256"
|
|
|
|
|
+rm -f /tmp/flare-stage
|
|
|
|
|
+EOF
|
|
|
|
|
+ for f in LICENSE NOTICE THIRD_PARTY_NOTICES.md; do
|
|
|
|
|
+ net "$BUILD_HOST" "cat $SRC_DIR/$f 2>/dev/null" 2>/dev/null \
|
|
|
|
|
+ | net "$node" "sudo tee $STAGE_ROOT/$release/$f >/dev/null" 2>/dev/null || true
|
|
|
|
|
+ done
|
|
|
|
|
+ info "$node staged"
|
|
|
|
|
+ done
|
|
|
|
|
+
|
|
|
|
|
+ say "step 3/7 swap + restart agents"
|
|
|
|
|
+ for node in $NODES; do
|
|
|
|
|
+ timeout 150 ssh "${SSH_OPTS[@]}" "$(host_of "$node")" "FORCE=1 bash -s -- $STAGE_ROOT/$release/flare" < "$swap_script" 2>&1 \
|
|
|
|
|
+ | grep -E "^RESULT|^BAD|REFUSED" | sed "s/^/ $node: /"
|
|
|
|
|
+ done
|
|
|
|
|
+
|
|
|
|
|
+ say "step 4/7 verify one build everywhere"
|
|
|
|
|
+ local usr agent_path
|
|
|
|
|
+ bad=0
|
|
|
|
|
+ for node in $NODES; do
|
|
|
|
|
+ read -r usr agent_path < <(net "$node" "echo \$(sha256sum /usr/bin/flare | cut -d' ' -f1) \$(sha256sum \$HOME/.local/lib/flare-agent/flare | cut -d' ' -f1)")
|
|
|
|
|
+ if [ "$usr" != "$new_sha" ] || [ "$agent_path" != "$new_sha" ]; then
|
|
|
|
|
+ warn "$node usr=${usr:0:12} agent=${agent_path:0:12} want ${new_sha:0:12}"; bad=$((bad+1))
|
|
|
|
|
+ fi
|
|
|
|
|
+ done
|
|
|
|
|
+ [ "$bad" -eq 0 ] || die "$bad node(s) not on $release — mixed builds cannot pair; run 'rollback --yes'"
|
|
|
|
|
+ info "all $(node_count) nodes run ${new_sha:0:12}"
|
|
|
|
|
+
|
|
|
|
|
+ if [ "$relaunch" = 1 ]; then
|
|
|
|
|
+ say "step 5/7 relaunch groups"; relaunch_groups
|
|
|
|
|
+ say "step 6/7 wait for health"
|
|
|
|
|
+ if wait_health; then info "healthy"; else warn "NOT healthy within ${WAIT}s"; fi
|
|
|
|
|
+ verify_serving || warn "serving verification failed — inspect with 'flare orbit attach GROUP'"
|
|
|
|
|
+ else
|
|
|
|
|
+ say "step 5-6/7 skipped (--no-relaunch)"
|
|
|
|
|
+ fi
|
|
|
|
|
+
|
|
|
|
|
+ if [ "$prune" = 1 ]; then say "step 7/7 prune old backups"; prune_backups; fi
|
|
|
|
|
+
|
|
|
|
|
+ say ""
|
|
|
|
|
+ say "done: $release on $(node_count) nodes in $(( $(date +%s) - started ))s"
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+# ---------------------------------------------------------------- rollback
|
|
|
|
|
+cmd_rollback() {
|
|
|
|
|
+ local stamp="" yes=0 relaunch=1
|
|
|
|
|
+ while [ $# -gt 0 ]; do
|
|
|
|
|
+ case $1 in
|
|
|
|
|
+ --stamp) stamp=$2; shift 2 ;;
|
|
|
|
|
+ --yes) yes=1; shift ;;
|
|
|
|
|
+ --no-relaunch) relaunch=0; shift ;;
|
|
|
|
|
+ *) die "rollback: unknown argument $1" ;;
|
|
|
|
|
+ esac
|
|
|
|
|
+ done
|
|
|
|
|
+ say "FLARE fleet rollback to a .orig-* generation"
|
|
|
|
|
+ local node avail tmpf one
|
|
|
|
|
+ tmpf=$(mktemp)
|
|
|
|
|
+ for node in $NODES; do
|
|
|
|
|
+ avail=$(net "$node" "ls -1 \$HOME/.local/lib/flare-agent/flare.orig-* 2>/dev/null | sed 's/.*flare\.orig-//' | sort -u")
|
|
|
|
|
+ [ -n "$avail" ] || { rm -f "$tmpf"; die "$node has no agent-path .orig-* backup; fleet-wide rollback impossible"; }
|
|
|
|
|
+ for one in $avail; do printf '%s %s\n' "$node" "$one" >> "$tmpf"; done
|
|
|
|
|
+ info "$node backups: $(printf '%s ' $avail)"
|
|
|
|
|
+ done
|
|
|
|
|
+ # The target must exist on EVERY node: restoring different generations is the
|
|
|
|
|
+ # mixed-build state that rollback exists to escape.
|
|
|
|
|
+ if [ -z "$stamp" ]; then
|
|
|
|
|
+ stamp=$(awk '{print $2}' "$tmpf" | sort | uniq -c \
|
|
|
|
|
+ | awk -v n="$(node_count)" '$1==n{print $2}' | sort | tail -1)
|
|
|
|
|
+ [ -n "$stamp" ] || { rm -f "$tmpf"; die "no backup generation exists on all $(node_count) nodes; pass --stamp"; }
|
|
|
|
|
+ fi
|
|
|
|
|
+ for node in $NODES; do
|
|
|
|
|
+ grep -qx "$node $stamp" "$tmpf" || { rm -f "$tmpf"; die "$node lacks flare.orig-$stamp (would leave a mixed fleet)"; }
|
|
|
|
|
+ done
|
|
|
|
|
+ rm -f "$tmpf"
|
|
|
|
|
+ info "target (present everywhere): flare.orig-$stamp"
|
|
|
|
|
+
|
|
|
|
|
+ if [ "$yes" = 0 ]; then
|
|
|
|
|
+ say "plan: stop groups -> restore flare.orig-$stamp at both paths -> restart agents -> relaunch"
|
|
|
|
|
+ die "re-run with --yes"
|
|
|
|
|
+ fi
|
|
|
|
|
+
|
|
|
|
|
+ stop_groups
|
|
|
|
|
+ local usr
|
|
|
|
|
+ for node in $NODES; do
|
|
|
|
|
+ usr=$(net "$node" bash -s -- "$stamp" <<'EOF' 2>/dev/null | tail -1
|
|
|
|
|
+set -u
|
|
|
|
|
+stamp=$1
|
|
|
|
|
+A=$HOME/.local/lib/flare-agent/flare.orig-$stamp
|
|
|
|
|
+U=/usr/bin/flare.orig-$stamp
|
|
|
|
|
+[ -f "$A" ] && [ -f "$U" ] || { echo "missing backup flare.orig-$stamp"; exit 1; }
|
|
|
|
|
+# rename, never cp over a mapped inode; source copies stay for the next rollback
|
|
|
|
|
+install -m 0755 "$A" "$HOME/.local/lib/flare-agent/flare.tmp" && mv -f "$HOME/.local/lib/flare-agent/flare.tmp" "$HOME/.local/lib/flare-agent/flare"
|
|
|
|
|
+sudo install -m 0755 "$U" /usr/bin/flare.tmp && sudo mv -f /usr/bin/flare.tmp /usr/bin/flare
|
|
|
|
|
+systemctl --user restart flare-agent
|
|
|
|
|
+sleep 3
|
|
|
|
|
+echo "$(sha256sum /usr/bin/flare | cut -c1-12)/$(sha256sum "$HOME/.local/lib/flare-agent/flare" | cut -c1-12)"
|
|
|
|
|
+EOF
|
|
|
|
|
+)
|
|
|
|
|
+ say " $node: $usr"
|
|
|
|
|
+ [ -n "$usr" ] || die "$node rollback failed"
|
|
|
|
|
+ done
|
|
|
|
|
+ if [ "$relaunch" = 1 ]; then
|
|
|
|
|
+ relaunch_groups
|
|
|
|
|
+ wait_health || warn "not healthy after rollback"
|
|
|
|
|
+ verify_serving || warn "serving verification failed after rollback"
|
|
|
|
|
+ fi
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+usage() { sed -n '2,26p' "$0" | sed 's/^# \{0,1\}//'; }
|
|
|
|
|
+
|
|
|
|
|
+case ${1:-} in
|
|
|
|
|
+ status) shift; cmd_status ;;
|
|
|
|
|
+ check) shift; cmd_status strict ;;
|
|
|
|
|
+ build) shift; cmd_build "$@" ;;
|
|
|
|
|
+ upgrade) shift; cmd_upgrade "$@" ;;
|
|
|
|
|
+ rollback) shift; cmd_rollback "$@" ;;
|
|
|
|
|
+ ""|help|-h|--help) usage ;;
|
|
|
|
|
+ *) die "unknown command: $1 (try --help)" ;;
|
|
|
|
|
+esac
|