//! Versioned run package for `sm121 bench`. //! //! Every bench run lands next to its reports in the recipe's `reports/` //! dir (same stem as the report triplet): //! //! - `.recipe.toml` — a byte-exact copy of the recipe file the //! serve was launched from, so the run's recipe is pinned even after //! the recipe (or its image pins) move on. //! - `.manifest.json` — provenance: recipe + config SHA-256, the //! launcher's git commit, the bench tool's version, the served model's //! resolved `refs/main` snapshot (the model reference floats, so the //! snapshot pins the run), and the served image's identity per node //! (docker id + `sm121.checksum` build label, captured over SSH at bench //! time). //! //! `sm121 report` reads the manifests back (see [`crate::report`]) and //! shows each run's provenance — and flags a run whose recipe has since //! changed (drift). use std::fs; use std::path::Path; use anyhow::Result; use sha2::{Digest, Sha256}; use crate::config::{Cluster, Config, Recipe}; use crate::ssh::run_ssh; /// SHA-256 of a byte slice as a 64-char lowercase hex string. pub fn sha256_hex(bytes: &[u8]) -> String { let digest = Sha256::digest(bytes); digest.iter().map(|b| format!("{b:02x}")).collect() } /// First 12 chars — the display width for hashes in the report UI. pub fn short12(s: &str) -> String { s.chars().take(12).collect() } /// UTC "YYYY-MM-DDTHH:MM:SSZ" for the manifest's capture time. fn utc_now_iso() -> String { let secs = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); let (y, m, d) = civil_date((secs / 86400) as i64); let rem = (secs % 86400) as u32; format!( "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", y, m, d, rem / 3600, rem % 3600 / 60, rem % 60 ) } /// Days-since-civil conversion (Hansen/Clayton), same as `bench::civil_date`. fn civil_date(z: i64) -> (u32, u32, u32) { let z = z + 719_468; let era = if z >= 0 { z } else { z - 146_096 } / 146_097; let doe = z - era * 146_097; let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); let mp = (5 * doy + 2) / 153; let day = doy - (153 * mp + 2) / 5 + 1; let month = if mp < 10 { mp + 3 } else { mp - 9 }; let year = era * 400 + yoe + if month <= 2 { 1 } else { 0 }; (year as u32, month as u32, day as u32) } #[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone)] pub struct ManifestRecipe { pub name: String, /// Path of the recipe file as resolved by the launcher. pub file: String, pub sha256: String, } #[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone)] pub struct ManifestCluster { pub name: String, /// ssh_ip per node (node order from the cluster definition). pub nodes: Vec, } #[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone)] pub struct ManifestConfig { pub file: String, pub sha256: String, } #[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone)] pub struct ManifestLauncher { /// Canonical path of the `sm121` binary that ran the bench. pub bin: String, /// `git rev-parse HEAD` of the repo the binary was built in (cwd), /// null when the cwd is not a git checkout. pub git_sha: Option, } #[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone)] pub struct ManifestBench { pub tool: String, /// `tool_eval_bench_version` from the run's JSON report. pub version: Option, /// The exact `tool-eval-bench` arguments (base-url, model, seed, …). pub args: Vec, pub final_score: Option, /// `total/max` points, e.g. "163/176". pub points: Option, } /// Provenance of the served model. The recipe's `model` reference floats /// (the engine follows the HF cache's `refs/main` → latest snapshot), so the /// snapshot it actually resolved to at bench time is what makes a run /// reproducible. `snapshot` is null when `model` is a local path (no /// `org/name` id to resolve) or the repo isn't cached on the head node. #[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone)] pub struct ManifestModel { /// The recipe's `model` field (an `org/name` repo id or a local path). pub id: String, /// The `refs/main` snapshot hash the id resolved to on the head node. pub snapshot: Option, } /// One per bench run; the top-level shape of `.manifest.json`. #[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone)] pub struct Manifest { pub schema_version: u32, /// `_