//! Post-run report repair. tool-eval-bench's `--json-file` flag implies //! `--json`, and that dispatch path (v2.7.0) drops the llama-benchy perf //! section from the scenario report md (the samples are computed and printed //! to the bench log, but never passed to the report writer). The launcher //! re-attaches the table from the log so every campaign report carries its //! perf numbers like a `--perf-only` run would. use std::fs; use anyhow::Result; pub struct PerfRow { pub test: String, pub concurrency: u32, pub pp: String, pub tg: String, pub ttft: String, pub total: String, pub tokens: String, } /// Append a `## Throughput Metrics` section to `md` when it lacks one, /// taking the rows from the llama-benchy box in `log`. Returns true when /// the md was modified. pub fn ensure_throughput_section(md: &std::path::Path, log: &std::path::Path) -> Result { let body = fs::read_to_string(md)?; if body.contains("## Throughput Metrics") || body.contains("## Results") { return Ok(false); } let rows = parse_perf_rows(&fs::read_to_string(log)?) .ok_or_else(|| anyhow::anyhow!("no llama-benchy perf rows in {}", log.display()))?; let mut out = body; if !out.ends_with('\n') { out.push('\n'); } out.push_str("\n## Throughput Metrics\n\n"); out.push_str("> Appended by `sm121 bench`: the tool's `--json-file` path drops the\n"); out.push_str("> perf section from this report; the rows come from the llama-benchy table\n"); out.push_str("> in the bench log (same run).\n\n"); out.push_str("| Test | pp t/s | tg t/s | TTFT (ms) | Total (ms) | Tokens |\n"); out.push_str("|---|---:|---:|---:|---:|---:|\n"); for r in &rows { let conc = if r.concurrency > 1 { format!(" c{}", r.concurrency) } else { String::new() }; out.push_str(&format!( "| {}{} | {} | {} | {} | {} | {} |\n", r.test, conc, r.pp, r.tg, r.ttft, r.total, r.tokens )); } fs::write(md, out)?; Ok(true) } /// Extract the llama-benchy result rows from a bench log. Box rows look like /// `│ pp2048 tg128 @ d0 │ c1 │ 154 │ 19.7 │ 12,620 │ 18,810 │ 2048+128`. fn parse_perf_rows(log: &str) -> Option> { let mut rows = Vec::new(); for line in log.lines() { let line = line.trim(); if !line.starts_with('│') { continue; } let parts: Vec<&str> = line .split('│') .map(|s| s.trim()) .filter(|s| !s.is_empty()) .collect(); if parts.len() != 7 || !parts[0].starts_with("pp") { continue; } let concurrency = parts .get(1) .and_then(|c| c.strip_prefix('c')) .and_then(|n| n.parse().ok()) .unwrap_or(1); rows.push(PerfRow { test: parts[0].to_string(), concurrency, pp: parts[2].to_string(), tg: parts[3].to_string(), ttft: parts[4].to_string(), total: parts[5].to_string(), tokens: parts[6].to_string(), }); } (!rows.is_empty()).then_some(rows) } #[cfg(test)] mod tests { use super::*; const LOG: &str = "\ ╭──────────────────── ⚡ llama-benchy Throughput Benchmark ────────────────────╮ ┃ Test ┃ c ┃ pp t/s ┃ tg t/s ┃ TTFT ┃ Total ┃ ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━ │ pp2048 tg128 @ d0 │ c1 │ 154 │ 19.7 │ 12,620 │ 18,810 │ 2048+128 │ pp2048 tg128 @ d0 │ c2 │ 177 │ 21.8 │ 20,470 │ 30,788 │ 2048+128 └──────────────────────┴────┴─────────┴─────────┴──────────┴──────────┴───────── "; #[test] fn appends_section_from_log_box() { let tmp = std::env::temp_dir().join(format!("sm121-bench-report-test-{}", std::process::id())); fs::create_dir_all(&tmp).unwrap(); let md = tmp.join("r.md"); let log = tmp.join("r.log"); fs::write(&md, "# Report\n\n## Scenario Results\n\n| ok |\n").unwrap(); fs::write(&log, LOG).unwrap(); assert!(ensure_throughput_section(&md, &log).unwrap()); let body = fs::read_to_string(&md).unwrap(); assert!(body.contains("## Throughput Metrics")); assert!(body.contains("| pp2048 tg128 @ d0 | 154 | 19.7 | 12,620 | 18,810 | 2048+128 |")); assert!( body.contains("| pp2048 tg128 @ d0 c2 | 177 | 21.8 | 20,470 | 30,788 | 2048+128 |") ); // idempotent: an existing section means no rewrite assert!(!ensure_throughput_section(&md, &log).unwrap()); let _ = fs::remove_dir_all(&tmp); } #[test] fn no_rows_is_an_error() { let tmp = std::env::temp_dir().join(format!("sm121-bench-report-test2-{}", std::process::id())); fs::create_dir_all(&tmp).unwrap(); let md = tmp.join("r.md"); let log = tmp.join("r.log"); fs::write(&md, "# Report\n").unwrap(); fs::write(&log, "no box here\n").unwrap(); assert!(ensure_throughput_section(&md, &log).is_err()); let _ = fs::remove_dir_all(&tmp); } }