|
|
@@ -3,9 +3,10 @@
|
|
|
|
|
|
use super::HaClient;
|
|
|
use super::mikro;
|
|
|
-use crate::config::{Cluster, MikrotikConfig, Node};
|
|
|
+use crate::config::{Cluster, Config, MikrotikConfig, Node};
|
|
|
use crate::home_assistant::split_entity;
|
|
|
use anyhow::Result;
|
|
|
+use std::collections::{HashMap, HashSet};
|
|
|
use tracing::{info, warn};
|
|
|
|
|
|
/// Entity ids of a cluster's `extra_entities` (both forms).
|
|
|
@@ -17,17 +18,57 @@ pub(crate) fn extra_ids(cluster: &Cluster) -> Vec<String> {
|
|
|
.collect()
|
|
|
}
|
|
|
|
|
|
-/// Cluster-level extras to toggle OFF for a power off: the full list only for
|
|
|
-/// a whole-cluster off (no `--node`); empty for a subset. The fabric switch
|
|
|
-/// and fan are shared by every node, so powering off one node must not take
|
|
|
-/// the cluster's fabric down for the nodes that stay up (it wedges live
|
|
|
-/// multi-node serves — NCCL over the switch dies).
|
|
|
-pub(crate) fn off_extras(cluster: &Cluster, node_names: &[String]) -> Vec<String> {
|
|
|
- if node_names.is_empty() {
|
|
|
- extra_ids(cluster)
|
|
|
- } else {
|
|
|
- Vec::new()
|
|
|
+/// Extra-entity ids that must stay ON because **another** cluster (≠
|
|
|
+/// `target`) has an open run-ledger entry that references them. The fabric
|
|
|
+/// switch and cooling fan are shared across clusters (e.g. `8nodes-switch`
|
|
|
+/// and `dgx58-switch` both list `switch.sw6_mt`), so powering one cluster
|
|
|
+/// down must not cut the fabric a live serve on a *different* cluster depends
|
|
|
+/// on — that wedges its NCCL and kills whatever it is serving. Maps each
|
|
|
+/// entity to the active other-cluster names that hold it. The run ledger is
|
|
|
+/// the offline source of truth (the single-launcher serialization point); a
|
|
|
+/// stale open run over-reports and keeps the entity on — the safe direction
|
|
|
+/// (never cut a shared resource that might still be in use).
|
|
|
+pub(crate) fn extras_held_by_other_clusters(config: &Config, target: &Cluster) -> HashMap<String, Vec<String>> {
|
|
|
+ let active: HashSet<String> = crate::runlock::open_runs(config)
|
|
|
+ .into_iter()
|
|
|
+ .filter_map(|(_, _, cluster)| cluster)
|
|
|
+ .filter(|c| c != &target.name)
|
|
|
+ .collect();
|
|
|
+ let mut held: HashMap<String, Vec<String>> = HashMap::new();
|
|
|
+ for c in &config.clusters {
|
|
|
+ if !active.contains(&c.name) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ for e in &c.extra_entities {
|
|
|
+ held.entry(e.entity().to_string()).or_default().push(c.name.clone());
|
|
|
+ }
|
|
|
}
|
|
|
+ held
|
|
|
+}
|
|
|
+
|
|
|
+/// Cluster-level extras to toggle OFF for a power off. Two guards keep a
|
|
|
+/// shared resource (fabric switch, fan) from being cut out from under a live
|
|
|
+/// serve:
|
|
|
+/// 1. A SUBSET power off (`--node`) never touches extras — the nodes of this
|
|
|
+/// cluster that stay up still need the fabric (NCCL over the switch dies
|
|
|
+/// otherwise).
|
|
|
+/// 2. A WHOLE-cluster power off still keeps any extra that another cluster
|
|
|
+/// with an open run references (`held`, from
|
|
|
+/// [`extras_held_by_other_clusters`]) — the fabric is shared across
|
|
|
+/// clusters, so powering `8nodes-switch` down must not kill a serve
|
|
|
+/// running on `dgx58-switch` over the same switch.
|
|
|
+pub(crate) fn off_extras(
|
|
|
+ cluster: &Cluster,
|
|
|
+ node_names: &[String],
|
|
|
+ held: &HashSet<String>,
|
|
|
+) -> Vec<String> {
|
|
|
+ if !node_names.is_empty() {
|
|
|
+ return Vec::new();
|
|
|
+ }
|
|
|
+ extra_ids(cluster)
|
|
|
+ .into_iter()
|
|
|
+ .filter(|e| !held.contains(e))
|
|
|
+ .collect()
|
|
|
}
|
|
|
|
|
|
/// Handle one entity id for a power toggle: dispatch `[mikrotik]`
|
