launcher/src/bench_report.rs

//! Post-run report repair. tool-eval-bench's `--json-file` flag implies
//! `--json`, and that dispatch path (v2.7.0) drops the llama-benchy perf
//! section from the scenario report md (the samples are computed and printed
//! to the bench log, but never passed to the report writer). The launcher
//! re-attaches the table from the log so every campaign report carries its
//! perf numbers like a `--perf-only` run would.

use std::fs;

use anyhow::Result;

pub struct PerfRow {
    pub test: String,
    pub concurrency: u32,
    pub pp: String,
    pub tg: String,
    pub ttft: String,
    pub total: String,
    pub tokens: String,
}

/// Append a `## Throughput Metrics` section to `md` when it lacks one,
/// taking the rows from the llama-benchy box in `log`. Returns true when
/// the md was modified.
pub fn ensure_throughput_section(md: &std::path::Path, log: &std::path::Path) -> Result<bool> {
    let body = fs::read_to_string(md)?;
    if body.contains("## Throughput Metrics") || body.contains("## Results") {
        return Ok(false);
    }
    let rows = parse_perf_rows(&fs::read_to_string(log)?)
        .ok_or_else(|| anyhow::anyhow!("no llama-benchy perf rows in {}", log.display()))?;
    let mut out = body;
    if !out.ends_with('\n') {
        out.push('\n');
    }
    out.push_str("\n## Throughput Metrics\n\n");
    out.push_str("> Appended by `sm121 bench`: the tool's `--json-file` path drops the\n");
    out.push_str("> perf section from this report; the rows come from the llama-benchy table\n");
    out.push_str("> in the bench log (same run).\n\n");
    out.push_str("| Test | pp t/s | tg t/s | TTFT (ms) | Total (ms) | Tokens |\n");
    out.push_str("|---|---:|---:|---:|---:|---:|\n");
    for r in &rows {
        let conc = if r.concurrency > 1 {
            format!(" c{}", r.concurrency)
        } else {
            String::new()
        };
        out.push_str(&format!(
            "| {}{} | {} | {} | {} | {} | {} |\n",
            r.test, conc, r.pp, r.tg, r.ttft, r.total, r.tokens
        ));
    }
    fs::write(md, out)?;
    Ok(true)
}

/// Extract the llama-benchy result rows from a bench log. Box rows look like
/// `│ pp2048 tg128 @ d0 │ c1 │ 154 │ 19.7 │ 12,620 │ 18,810 │ 2048+128`.
fn parse_perf_rows(log: &str) -> Option<Vec<PerfRow>> {
    let mut rows = Vec::new();
    for line in log.lines() {
        let line = line.trim();
        if !line.starts_with('│') {
            continue;
        }
        let parts: Vec<&str> = line
            .split('│')
            .map(|s| s.trim())
            .filter(|s| !s.is_empty())
            .collect();
        if parts.len() != 7 || !parts[0].starts_with("pp") {
            continue;
        }
        let concurrency = parts
            .get(1)
            .and_then(|c| c.strip_prefix('c'))
            .and_then(|n| n.parse().ok())
            .unwrap_or(1);
        rows.push(PerfRow {
            test: parts[0].to_string(),
            concurrency,
            pp: parts[2].to_string(),
            tg: parts[3].to_string(),
            ttft: parts[4].to_string(),
            total: parts[5].to_string(),
            tokens: parts[6].to_string(),
        });
    }
    (!rows.is_empty()).then_some(rows)
}

#[cfg(test)]
mod tests {
    use super::*;

    const LOG: &str = "\
╭──────────────────── ⚡ llama-benchy Throughput Benchmark ────────────────────╮
┃ Test                 ┃ c  ┃  pp t/s ┃  tg t/s ┃     TTFT ┃    Total ┃
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━
│ pp2048 tg128 @ d0    │ c1 │     154 │    19.7 │   12,620 │   18,810 │ 2048+128
│ pp2048 tg128 @ d0    │ c2 │     177 │    21.8 │   20,470 │   30,788 │ 2048+128
└──────────────────────┴────┴─────────┴─────────┴──────────┴──────────┴─────────
";

    #[test]
    fn appends_section_from_log_box() {
        let tmp =
            std::env::temp_dir().join(format!("sm121-bench-report-test-{}", std::process::id()));
        fs::create_dir_all(&tmp).unwrap();
        let md = tmp.join("r.md");
        let log = tmp.join("r.log");
        fs::write(&md, "# Report\n\n## Scenario Results\n\n| ok |\n").unwrap();
        fs::write(&log, LOG).unwrap();

        assert!(ensure_throughput_section(&md, &log).unwrap());
        let body = fs::read_to_string(&md).unwrap();
        assert!(body.contains("## Throughput Metrics"));
        assert!(body.contains("| pp2048 tg128 @ d0 | 154 | 19.7 | 12,620 | 18,810 | 2048+128 |"));
        assert!(
            body.contains("| pp2048 tg128 @ d0 c2 | 177 | 21.8 | 20,470 | 30,788 | 2048+128 |")
        );

        // idempotent: an existing section means no rewrite
        assert!(!ensure_throughput_section(&md, &log).unwrap());
        let _ = fs::remove_dir_all(&tmp);
    }

    #[test]
    fn no_rows_is_an_error() {
        let tmp =
            std::env::temp_dir().join(format!("sm121-bench-report-test2-{}", std::process::id()));
        fs::create_dir_all(&tmp).unwrap();
        let md = tmp.join("r.md");
        let log = tmp.join("r.log");
        fs::write(&md, "# Report\n").unwrap();
        fs::write(&log, "no box here\n").unwrap();
        assert!(ensure_throughput_section(&md, &log).is_err());
        let _ = fs::remove_dir_all(&tmp);
    }
}