|
@@ -79,6 +79,22 @@ pub fn created_epoch(s: &str) -> Option<u64> {
|
|
|
/// reason the docker path reads `CreatedAt`).
|
|
/// reason the docker path reads `CreatedAt`).
|
|
|
pub fn parse_orbit_groups(stdout: &str) -> Vec<(String, Option<u64>)> {
|
|
pub fn parse_orbit_groups(stdout: &str) -> Vec<(String, Option<u64>)> {
|
|
|
let mut out: Vec<(String, Option<u64>)> = Vec::new();
|
|
let mut out: Vec<(String, Option<u64>)> = Vec::new();
|
|
|
|
|
+ // The GROUPS column reads `<group> rank <n>`; the start time only appears
|
|
|
|
|
+ // in the `held by <group>-<stamp>-<hex>` detail line, so collect names
|
|
|
|
|
+ // first and attach a timestamp when one is found.
|
|
|
|
|
+ for line in stdout.lines() {
|
|
|
|
|
+ if let Some((before, _)) = line.split_once(" rank ")
|
|
|
|
|
+ && let Some(name) = before.split_whitespace().last()
|
|
|
|
|
+ && !name.starts_with("gx10")
|
|
|
|
|
+ && !name.contains("GiB")
|
|
|
|
|
+ && name
|
|
|
|
|
+ .chars()
|
|
|
|
|
+ .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'))
|
|
|
|
|
+ && !out.iter().any(|(n, _)| n == name)
|
|
|
|
|
+ {
|
|
|
|
|
+ out.push((name.to_string(), None));
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
for tok in stdout.split(|c: char| c.is_whitespace() || c == ',') {
|
|
for tok in stdout.split(|c: char| c.is_whitespace() || c == ',') {
|
|
|
let t = tok.trim_matches(|c: char| c == '`' || c == '"' || c == '\'');
|
|
let t = tok.trim_matches(|c: char| c == '`' || c == '"' || c == '\'');
|
|
|
let parts: Vec<&str> = t.split('-').collect();
|
|
let parts: Vec<&str> = t.split('-').collect();
|
|
@@ -117,14 +133,28 @@ pub fn parse_orbit_groups(stdout: &str) -> Vec<(String, Option<u64>)> {
|
|
|
let doy = (153 * mp + 2) / 5 + da - 1;
|
|
let doy = (153 * mp + 2) / 5 + da - 1;
|
|
|
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
|
|
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
|
|
|
let secs = (era * 146_097 + doe - 719_468) * 86_400 + h * 3600 + mi * 60 + se;
|
|
let secs = (era * 146_097 + doe - 719_468) * 86_400 + h * 3600 + mi * 60 + se;
|
|
|
- let entry = (base.to_string(), u64::try_from(secs).ok());
|
|
|
|
|
- if !out.iter().any(|(n, _)| n == &entry.0) {
|
|
|
|
|
- out.push(entry);
|
|
|
|
|
|
|
+ let created = u64::try_from(secs).ok();
|
|
|
|
|
+ match out.iter_mut().find(|(n, _)| n == &base) {
|
|
|
|
|
+ Some(slot) => slot.1 = created,
|
|
|
|
|
+ None => out.push((base.to_string(), created)),
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
out
|
|
out
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+/// The ssh_ip of a node by its name (the probe keys its map on node names).
|
|
|
|
|
+fn node_ip_of<'a>(config: &'a crate::config::Config, name: &str) -> &'a String {
|
|
|
|
|
+ config
|
|
|
|
|
+ .clusters
|
|
|
|
|
+ .iter()
|
|
|
|
|
+ .flat_map(|c| c.nodes.iter())
|
|
|
|
|
+ .find(|n| n.name == *name)
|
|
|
|
|
+ .map(|n| &n.ssh_ip)
|
|
|
|
|
+ .unwrap_or(&EMPTY_IP)
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+static EMPTY_IP: std::sync::LazyLock<String> = std::sync::LazyLock::new(String::new);
|
|
|
|
|
+
|
|
|
/// Per-node state with containers.
|
|
/// Per-node state with containers.
|
|
|
#[derive(Clone, Debug, Default)]
|
|
#[derive(Clone, Debug, Default)]
|
|
|
pub struct WebNodeState {
|
|
pub struct WebNodeState {
|
|
@@ -147,12 +177,7 @@ pub async fn flare_groups(config: &crate::config::Config) -> Vec<(String, Option
|
|
|
.orbit_cluster
|
|
.orbit_cluster
|
|
|
.clone()
|
|
.clone()
|
|
|
.unwrap_or_else(|| "sparks".to_string());
|
|
.unwrap_or_else(|| "sparks".to_string());
|
|
|
- let Some(controller) = config
|
|
|
|
|
- .clusters
|
|
|
|
|
- .iter()
|
|
|
|
|
- .flat_map(|c| c.nodes.iter())
|
|
|
|
|
- .find(|n| n.rank == 0)
|
|
|
|
|
- else {
|
|
|
|
|
|
|
+ let Some(controller) = config.flake_controller_node() else {
|
|
|
return Vec::new();
|
|
return Vec::new();
|
|
|
};
|
|
};
|
|
|
let cmd = format!("{binary} orbit status --cluster {orbit_cluster} 2>/dev/null");
|
|
let cmd = format!("{binary} orbit status --cluster {orbit_cluster} 2>/dev/null");
|
|
@@ -256,7 +281,19 @@ pub async fn probe(config: &Config) -> Fleet {
|
|
|
let mut out: HashMap<String, WebNodeState> = HashMap::new();
|
|
let mut out: HashMap<String, WebNodeState> = HashMap::new();
|
|
|
// FLARE serves out of `flare orbit`, not docker: its groups are folded in
|
|
// FLARE serves out of `flare orbit`, not docker: its groups are folded in
|
|
|
// as pseudo-containers so the dashboard shows them like any other lane.
|
|
// as pseudo-containers so the dashboard shows them like any other lane.
|
|
|
|
|
+ // They carry the controller's cluster label — the snapshot builder drops
|
|
|
|
|
+ // containers that belong to no known cluster.
|
|
|
let flare = flare_groups(config).await;
|
|
let flare = flare_groups(config).await;
|
|
|
|
|
+ let controller_cluster = config
|
|
|
|
|
+ .clusters
|
|
|
|
|
+ .iter()
|
|
|
|
|
+ .find(|c| {
|
|
|
|
|
+ config
|
|
|
|
|
+ .flake_controller_node()
|
|
|
|
|
+ .is_some_and(|n| c.nodes.iter().any(|x| x.name == n.name))
|
|
|
|
|
+ })
|
|
|
|
|
+ .map(|c| c.name.clone())
|
|
|
|
|
+ .unwrap_or_default();
|
|
|
for h in futures::future::join_all(handles).await {
|
|
for h in futures::future::join_all(handles).await {
|
|
|
match h {
|
|
match h {
|
|
|
Ok((name, Ok(output))) => {
|
|
Ok((name, Ok(output))) => {
|
|
@@ -275,11 +312,17 @@ pub async fn probe(config: &Config) -> Fleet {
|
|
|
});
|
|
});
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
- for (group, created) in &flare {
|
|
|
|
|
|
|
+ // Only the controller node hosts the orbit groups (the fold
|
|
|
|
|
+ // below runs per node — without this guard every node would
|
|
|
|
|
+ // report the same group).
|
|
|
|
|
+ let is_controller = config
|
|
|
|
|
+ .flake_controller_node()
|
|
|
|
|
+ .is_some_and(|c| c.ssh_ip == *node_ip_of(config, &name));
|
|
|
|
|
+ for (group, created) in flare.iter().filter(|_| is_controller) {
|
|
|
containers.push(ContainerSnap {
|
|
containers.push(ContainerSnap {
|
|
|
name: group.clone(),
|
|
name: group.clone(),
|
|
|
status: "Up (flare)".into(),
|
|
status: "Up (flare)".into(),
|
|
|
- cluster: String::new(),
|
|
|
|
|
|
|
+ cluster: controller_cluster.clone(),
|
|
|
recipe: group.clone(),
|
|
recipe: group.clone(),
|
|
|
created: *created,
|
|
created: *created,
|
|
|
});
|
|
});
|