|
|
@@ -14,6 +14,21 @@ use anyhow::Result;
|
|
|
use serde::{Deserialize, Serialize};
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
|
|
+/// `sm121 runlock` subcommands — offline .lock ledger management. `sm121
|
|
|
+/// stop` only closes the ledger after verifying every node of the cluster,
|
|
|
+/// so a dead node (or a killed serve) leaves the run open forever; these
|
|
|
+/// fix that without touching the nodes.
|
|
|
+#[derive(clap::Subcommand)]
|
|
|
+pub enum RunlockCommands {
|
|
|
+ /// Close the open run of a recipe's .lock ledger (offline)
|
|
|
+ Close {
|
|
|
+ /// Recipe name (its directory under the recipe sources)
|
|
|
+ recipe: String,
|
|
|
+ },
|
|
|
+ /// List every recipe with an open (running) ledger entry
|
|
|
+ List,
|
|
|
+}
|
|
|
+
|
|
|
/// One launch: unix-epoch seconds (UTC), open-ended while running.
|
|
|
#[derive(Serialize, Deserialize, Clone)]
|
|
|
pub(crate) struct RunEntry {
|
|
|
@@ -110,6 +125,73 @@ pub(crate) fn mark_stop(recipe: &str, sources: &[String]) -> Result<()> {
|
|
|
l.save(&dir)
|
|
|
}
|
|
|
|
|
|
+/// Every recipe with an open (running) ledger entry: (name, start, cluster).
|
|
|
+/// Same dir precedence as collect: first dir wins on name collisions.
|
|
|
+pub(crate) fn open_runs(config: &crate::config::Config) -> Vec<(String, i64, Option<String>)> {
|
|
|
+ let mut out = Vec::new();
|
|
|
+ let mut seen = std::collections::HashSet::new();
|
|
|
+ for dir in crate::config::recipe_dirs(config) {
|
|
|
+ let Ok(entries) = std::fs::read_dir(&dir) else {
|
|
|
+ continue;
|
|
|
+ };
|
|
|
+ for entry in entries.flatten() {
|
|
|
+ let path = entry.path();
|
|
|
+ let Some(name) = entry.file_name().to_str().map(String::from) else {
|
|
|
+ continue;
|
|
|
+ };
|
|
|
+ if !seen.insert(name.clone()) || !path.join("recipe.toml").is_file() {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if let Some(run) = Lock::load(&path).running {
|
|
|
+ out.push((name, run.start, run.cluster));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ out.sort_by(|a, b| a.0.cmp(&b.0));
|
|
|
+ out
|
|
|
+}
|
|
|
+
|
|
|
+/// `sm121 runlock` — offline ledger management: no node verification
|
|
|
+/// (unlike `sm121 stop`, which refuses to close the ledger when it cannot
|
|
|
+/// reach every node of the cluster). Used after a node died mid-run or the
|
|
|
+/// serve process was killed.
|
|
|
+pub(crate) fn run(config: &crate::config::Config, cmd: &RunlockCommands) -> Result<()> {
|
|
|
+ match cmd {
|
|
|
+ RunlockCommands::Close { recipe } => {
|
|
|
+ let open = open_runs(config)
|
|
|
+ .into_iter()
|
|
|
+ .find(|(n, _, _)| n == recipe)
|
|
|
+ .map(|(_, start, cluster)| (start, cluster));
|
|
|
+ match open {
|
|
|
+ None => println!("no open run for '{recipe}' (nothing to close)"),
|
|
|
+ Some((start, cluster)) => {
|
|
|
+ mark_stop(recipe, &config.recipe_sources)?;
|
|
|
+ println!(
|
|
|
+ "closed open run of '{recipe}' (started {} UTC, cluster {})",
|
|
|
+ crate::stamp::fmt_ts(start),
|
|
|
+ cluster.as_deref().unwrap_or("tmux"),
|
|
|
+ );
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ RunlockCommands::List => {
|
|
|
+ let open = open_runs(config);
|
|
|
+ if open.is_empty() {
|
|
|
+ println!("no open runs (all ledgers closed)");
|
|
|
+ return Ok(());
|
|
|
+ }
|
|
|
+ for (name, start, cluster) in open {
|
|
|
+ println!(
|
|
|
+ "{name}\tstarted {} UTC\tcluster {}",
|
|
|
+ crate::stamp::fmt_ts(start),
|
|
|
+ cluster.as_deref().unwrap_or("tmux"),
|
|
|
+ );
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ Ok(())
|
|
|
+}
|
|
|
+
|
|
|
/// Flip the recipe's `archived` flag in its own toml (comment-preserving).
|
|
|
pub(crate) fn set_archived(recipe: &str, sources: &[String], archived: bool) -> Result<()> {
|
|
|
let file = crate::config::recipe_file(recipe, sources)
|
|
|
@@ -198,6 +280,32 @@ mod tests {
|
|
|
std::fs::remove_dir_all(&d).ok();
|
|
|
}
|
|
|
|
|
|
+ #[test]
|
|
|
+ fn open_runs_lists_only_open_ledgers() {
|
|
|
+ let d = dir("open");
|
|
|
+ let src = sources(&d);
|
|
|
+ mark_start("r-lock", &src, Some("c1")).unwrap();
|
|
|
+ std::fs::create_dir_all(d.join("other")).unwrap();
|
|
|
+ std::fs::write(
|
|
|
+ d.join("other/recipe.toml"),
|
|
|
+ "engine = \"vllm\"\nname = \"O\"\n",
|
|
|
+ )
|
|
|
+ .unwrap();
|
|
|
+ mark_start("other", &src, Some("c1")).unwrap();
|
|
|
+ mark_stop("other", &src).unwrap();
|
|
|
+ let cfg: crate::config::Config =
|
|
|
+ toml::from_str(&format!("recipe_sources = [\"{}\"]\n", d.display())).unwrap();
|
|
|
+ let open = open_runs(&cfg);
|
|
|
+ let mine = open.iter().find(|(n, _, _)| n == "r-lock");
|
|
|
+ assert!(mine.is_some(), "open run not listed: {open:?}");
|
|
|
+ assert_eq!(mine.unwrap().2.as_deref(), Some("c1"));
|
|
|
+ assert!(
|
|
|
+ open.iter().all(|(n, _, _)| n != "other"),
|
|
|
+ "closed run must not be listed: {open:?}",
|
|
|
+ );
|
|
|
+ std::fs::remove_dir_all(&d).ok();
|
|
|
+ }
|
|
|
+
|
|
|
#[test]
|
|
|
fn archived_flag_flips_round_trip() {
|
|
|
let d = dir("arch");
|
