//! Run-package provenance for the report table: one bench run's recipe //! copy + served-image identity, parsed from `.manifest.json` //! (written by `sm121 bench`, see [`crate::bench_artifacts`]). //! //! The report also flags **drift**: the manifest's recipe SHA-256 vs the //! current `recipe.toml` — a run whose recipe has since changed shows ⚠, //! so "what did this bench actually test?" is always answerable. use crate::bench_artifacts::{Manifest, short12}; use crate::report::ReportRow; use serde::{Deserialize, Serialize}; /// Run package provenance for one bench run. Empty fields for runs that /// predate the run package. #[derive(Serialize, Deserialize, Default, Debug, Clone)] pub struct Provenance { /// 12-char SHA-256 of the recipe copy captured at bench time. pub recipe_sha: String, /// True when the current recipe.toml's SHA-256 differs from the one /// captured at bench time — the report may no longer represent the /// recipe as it stands now. pub recipe_drift: bool, /// Docker image name from the manifest. pub image: String, /// `sm121.checksum` build label of the image (short hex). pub image_sum: String, /// Docker image id (first 12 hex chars) as seen on a launch node. pub image_id: String, /// tool-eval-bench version string from the run's JSON. pub tool: String, /// Launcher git commit (first 12 hex chars) that ran the bench. pub launcher: String, } /// Parse a `.manifest.json` body into provenance; `None` when the /// body is not a valid manifest (the entry then falls back to the empty /// shape, as for runs without a package). pub fn from_manifest(content: &str, current_recipe_sha: &str) -> Option { let m: Manifest = serde_json::from_str(content).ok()?; let (image, image_sum, image_id) = m .images .iter() .next() .map(|(name, val)| { let sum = val .get("build_checksum") .and_then(|v| v.as_str()) .unwrap_or_default(); let id = val .get("nodes") .and_then(|v| v.as_object()) .and_then(|n| n.values().next()) .and_then(|n| n.get("id")) .and_then(|v| v.as_str()) .map(|id| short12(id.strip_prefix("sha256:").unwrap_or(id))) .unwrap_or_default(); (name.clone(), sum.to_string(), id) }) .unwrap_or_default(); Some(Provenance { recipe_sha: short12(&m.recipe.sha256), recipe_drift: m.recipe.sha256 != current_recipe_sha, image, image_sum, image_id, tool: m.bench.version.unwrap_or_default(), launcher: m .launcher .git_sha .as_deref() .map(short12) .unwrap_or_default(), }) } /// Latest report entry carrying a run package (newest first); the row's /// provenance cell mirrors what the run's chip shows in the web UI. pub fn latest(r: &ReportRow) -> Option<&Provenance> { r.reports .iter() .rev() .find(|e| !e.provenance.recipe_sha.is_empty() || !e.provenance.image_sum.is_empty()) .map(|e| &e.provenance) } /// `img@·rcp@` (+ ⚠ on drift) for the markdown table. pub fn cell(r: &ReportRow) -> String { latest(r) .map(|p| { let sum = p.image_sum.chars().take(12).collect::(); let mut s = format!("img@{sum}·rcp@{}", p.recipe_sha); if p.recipe_drift { s.push_str(" ⚠"); } s }) .unwrap_or_else(|| "—".into()) } #[cfg(test)] mod tests { use super::*; use crate::config::Config; use crate::report::collect; use crate::report::tests::fixture; use std::fs; #[test] fn manifest_parses_and_drifts() { use crate::bench_artifacts::sha256_hex; let dir = fixture(); let recipe_path = dir.join("test-recipe-nvfp4/recipe.toml"); let recipe = fs::read_to_string(&recipe_path).unwrap(); // run package for the roce-run stem: manifest + recipe copy let stem = "2026-01-04_roce-run"; fs::write( dir.join(format!("test-recipe-nvfp4/reports/{stem}.manifest.json")), format!( "{{\"schema_version\":1,\"run\":\"{stem}\",\"created_utc\":\"x\",\"recipe\":{{\"name\":\"Test Recipe\",\"file\":\"x\",\"sha256\":\"{}\"}},\"cluster\":{{\"name\":\"c\",\"nodes\":[]}},\"launcher\":{{\"bin\":\"b\",\"git_sha\":\"deadbeefcafe42\"}},\"bench\":{{\"tool\":\"tool-eval-bench\",\"version\":\"2.6.1.dev99\",\"args\":[]}},\"images\":{{\"img:tag\":{{\"build_checksum\":\"8bb87348\",\"nodes\":{{\"dgx1\":{{\"id\":\"sha256:abcdef1234567890\",\"sm121_checksum\":\"8bb87348\"}}}}}}}}}}", sha256_hex(recipe.as_bytes()) ), ) .unwrap(); fs::write( dir.join(format!("test-recipe-nvfp4/reports/{stem}.recipe.toml")), &recipe, ) .unwrap(); let mk_config = || { toml::from_str::(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())) .unwrap() }; // no drift: manifest sha == current recipe sha let rows = collect(&mk_config(), false); let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap(); let e = row.reports.iter().find(|e| e.label == "roce-run").unwrap(); assert_eq!(e.provenance.image_sum, "8bb87348"); assert_eq!(e.provenance.image, "img:tag"); assert_eq!(e.provenance.image_id, "abcdef123456"); assert_eq!(e.provenance.tool, "2.6.1.dev99"); assert_eq!(e.provenance.launcher, "deadbeefcafe"); assert_eq!( e.provenance.recipe_sha, &sha256_hex(recipe.as_bytes())[..12] ); assert!(!e.provenance.recipe_drift); // run-package files ride on the run's stem (links in the UI) assert!(e.files.contains(&format!("{stem}.manifest.json"))); assert!(e.files.contains(&format!("{stem}.recipe.toml"))); // the score still comes from the REPORT json, not the manifest assert_eq!(e.pct, Some(92.0)); // the markdown row carries the provenance cell assert!(cell(row).starts_with("img@8bb87348·rcp@")); // drift: the recipe changed since the bench fs::write(&recipe_path, recipe + "port = 30001\n").unwrap(); let rows = collect(&mk_config(), false); let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap(); let e = row.reports.iter().find(|e| e.label == "roce-run").unwrap(); assert!(e.provenance.recipe_drift, "recipe changed after the run"); assert!(cell(row).ends_with('⚠')); fs::remove_dir_all(&dir).ok(); } #[test] fn bad_manifest_yields_empty_provenance() { assert!(from_manifest("not json", "x").is_none()); // complete manifest, empty images + drifted sha let m = from_manifest( r#"{"schema_version":1,"run":"r","created_utc":"x","recipe":{"name":"n","file":"f","sha256":"a"},"cluster":{"name":"c","nodes":[]},"launcher":{"bin":"b"},"bench":{"tool":"t","args":[]},"images":{}}"#, "b", ) .unwrap(); assert!(m.recipe_drift, "a != b"); assert!(m.image.is_empty()); } }