|
|
@@ -71,6 +71,60 @@ pub fn created_epoch(s: &str) -> Option<u64> {
|
|
|
u64::try_from(secs).ok()
|
|
|
}
|
|
|
|
|
|
+/// Parse `flare orbit status --cluster <c>` into (group, created-epoch) pairs.
|
|
|
+///
|
|
|
+/// The status table lists `… GROUPS` per node plus a `held by
|
|
|
+/// <name>-<YYYYMMDDTHHMMSSZ>-<hex>` detail line; a group name carries its
|
|
|
+/// start time, so the dashboard gets an uptime without a run ledger (the same
|
|
|
+/// reason the docker path reads `CreatedAt`).
|
|
|
+pub fn parse_orbit_groups(stdout: &str) -> Vec<(String, Option<u64>)> {
|
|
|
+ let mut out: Vec<(String, Option<u64>)> = Vec::new();
|
|
|
+ for tok in stdout.split(|c: char| c.is_whitespace() || c == ',') {
|
|
|
+ let t = tok.trim_matches(|c: char| c == '`' || c == '"' || c == '\'');
|
|
|
+ let parts: Vec<&str> = t.split('-').collect();
|
|
|
+ if parts.len() < 3 {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ let base = parts[..parts.len() - 2].join("-");
|
|
|
+ let stamp = parts[parts.len() - 2];
|
|
|
+ let Some(stamp) = stamp.strip_suffix('Z') else {
|
|
|
+ continue;
|
|
|
+ };
|
|
|
+ let Some((d, hms)) = stamp.split_once('T') else {
|
|
|
+ continue;
|
|
|
+ };
|
|
|
+ if d.len() != 8
|
|
|
+ || hms.len() != 6
|
|
|
+ || base.is_empty()
|
|
|
+ || !base
|
|
|
+ .chars()
|
|
|
+ .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.'))
|
|
|
+ {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ let num = |s: &str| s.parse::<i64>().ok();
|
|
|
+ let (Some(y), Some(mo), Some(da)) = (num(&d[0..4]), num(&d[4..6]), num(&d[6..8])) else {
|
|
|
+ continue;
|
|
|
+ };
|
|
|
+ let (Some(h), Some(mi), Some(se)) = (num(&hms[0..2]), num(&hms[2..4]), num(&hms[4..6]))
|
|
|
+ else {
|
|
|
+ continue;
|
|
|
+ };
|
|
|
+ let y2 = y - if mo <= 2 { 1 } else { 0 };
|
|
|
+ let era = if y2 >= 0 { y2 } else { y2 - 399 } / 400;
|
|
|
+ let yoe = y2 - era * 400;
|
|
|
+ let mp = (mo + 9) % 12;
|
|
|
+ let doy = (153 * mp + 2) / 5 + da - 1;
|
|
|
+ let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
|
|
|
+ let secs = (era * 146_097 + doe - 719_468) * 86_400 + h * 3600 + mi * 60 + se;
|
|
|
+ let entry = (base.to_string(), u64::try_from(secs).ok());
|
|
|
+ if !out.iter().any(|(n, _)| n == &entry.0) {
|
|
|
+ out.push(entry);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ out
|
|
|
+}
|
|
|
+
|
|
|
/// Per-node state with containers.
|
|
|
#[derive(Clone, Debug, Default)]
|
|
|
pub struct WebNodeState {
|
|
|
@@ -80,6 +134,36 @@ pub struct WebNodeState {
|
|
|
}
|
|
|
|
|
|
/// Fleet-wide probe result: per-node state + per-cluster HA switch state.
|
|
|
+/// `flare orbit status` groups → pseudo-containers on the controller node, so
|
|
|
+/// a FLARE serve shows up in the dashboard exactly like a docker container
|
|
|
+/// (green "up (fleet)" + uptime). The recipe column carries the orbit group's
|
|
|
+/// base name (`[flare].name`), which the report maps back to a recipe.
|
|
|
+pub async fn flare_groups(config: &crate::config::Config) -> Vec<(String, Option<u64>)> {
|
|
|
+ let Some(f) = config.flare.as_ref() else {
|
|
|
+ return Vec::new();
|
|
|
+ };
|
|
|
+ let binary = f.binary.clone().unwrap_or_else(|| "flare".to_string());
|
|
|
+ let orbit_cluster = f
|
|
|
+ .orbit_cluster
|
|
|
+ .clone()
|
|
|
+ .unwrap_or_else(|| "sparks".to_string());
|
|
|
+ let Some(controller) = config
|
|
|
+ .clusters
|
|
|
+ .iter()
|
|
|
+ .flat_map(|c| c.nodes.iter())
|
|
|
+ .find(|n| n.rank == 0)
|
|
|
+ else {
|
|
|
+ return Vec::new();
|
|
|
+ };
|
|
|
+ let cmd = format!("{binary} orbit status --cluster {orbit_cluster} 2>/dev/null");
|
|
|
+ match crate::ssh::run_ssh(config, controller, cmd, false).await {
|
|
|
+ Ok(out) => parse_orbit_groups(&out),
|
|
|
+ Err(e) => {
|
|
|
+ tracing::debug!("flare orbit status failed: {e}");
|
|
|
+ Vec::new()
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
pub struct Fleet {
|
|
|
pub nodes: HashMap<String, WebNodeState>,
|
|
|
/// cluster name → a blocking `extra_entity` reports anything but "on".
|
|
|
@@ -170,6 +254,9 @@ pub async fn probe(config: &Config) -> Fleet {
|
|
|
));
|
|
|
}
|
|
|
let mut out: HashMap<String, WebNodeState> = HashMap::new();
|
|
|
+ // FLARE serves out of `flare orbit`, not docker: its groups are folded in
|
|
|
+ // as pseudo-containers so the dashboard shows them like any other lane.
|
|
|
+ let flare = flare_groups(config).await;
|
|
|
for h in futures::future::join_all(handles).await {
|
|
|
match h {
|
|
|
Ok((name, Ok(output))) => {
|
|
|
@@ -188,6 +275,15 @@ pub async fn probe(config: &Config) -> Fleet {
|
|
|
});
|
|
|
}
|
|
|
}
|
|
|
+ for (group, created) in &flare {
|
|
|
+ containers.push(ContainerSnap {
|
|
|
+ name: group.clone(),
|
|
|
+ status: "Up (flare)".into(),
|
|
|
+ cluster: String::new(),
|
|
|
+ recipe: group.clone(),
|
|
|
+ created: *created,
|
|
|
+ });
|
|
|
+ }
|
|
|
LAST_WARNED.lock().await.remove(&name);
|
|
|
out.insert(
|
|
|
name,
|
|
|
@@ -254,3 +350,28 @@ mod created_tests {
|
|
|
assert_eq!(created_epoch("yesterday"), None);
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+#[cfg(test)]
|
|
|
+mod orbit_tests {
|
|
|
+ use super::parse_orbit_groups;
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn orbit_status_yields_group_and_start_time() {
|
|
|
+ let out = "cluster sparks\n\
|
|
|
+ NODE FINGERPRINT FREE GiB REVISION GROUPS\n\
|
|
|
+ gx10-dgx5 e3d68d8a3deb 86.2 1 deepseek1017 rank 0\n\
|
|
|
+ gpu 0 NVIDIA GB10 (gb10) 40.7/121.6 GiB free, held by deepseek1017-20261024T062147Z-e2ea\n";
|
|
|
+ let groups = parse_orbit_groups(out);
|
|
|
+ assert!(
|
|
|
+ groups.iter().any(|(n, _)| n == "deepseek1017"),
|
|
|
+ "{groups:?}"
|
|
|
+ );
|
|
|
+ let ts = groups.iter().find(|(n, _)| n == "deepseek1017").unwrap().1;
|
|
|
+ assert!(ts.is_some(), "group start time parsed: {groups:?}");
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn plain_status_has_no_groups() {
|
|
|
+ assert!(parse_orbit_groups("cluster sparks\nNODE FINGERPRINT FREE GiB REVISION GROUPS\ngx10-dgx5 ab12 562.3 1 -\n").is_empty());
|
|
|
+ }
|
|
|
+}
|