launcher/src/report/parse.rs

//! Parsers for benchmark report files: the tool-eval-bench `--json-file`
//! output (authoritative score) and the llama-benchy perf tables found in
//! the `.md`/`.log` reports (generation speed).

use serde::{Deserialize, Serialize};

/// One llama-benchy perf cell: decode speed at a context depth and
/// concurrency.
#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Debug)]
pub(crate) struct PerfCell {
    pub(crate) d: u32,  // context depth (0, 4096, 8192)
    pub(crate) c: u32,  // concurrency (1, 2, …)
    pub(crate) tg: f64, // decode tokens/s
}

/// Parse the tool-eval-bench `--json-file` output (authoritative for
/// `sm121 bench` runs): `scores.final_score` (0–100), `scores.total_points`
/// / `max_points` (hardmode). Returns `(pct, points, ratio)`; `points` is
/// suppressed when max is 100 (already normalized, same rule as the text
/// parsers).
pub(crate) fn parse_bench_json(content: &str) -> Option<(f64, Option<String>, Option<f64>)> {
    let v: serde_json::Value = serde_json::from_str(content).ok()?;
    let s = v.get("scores")?;
    let pct = s.get("final_score").and_then(|x| x.as_f64())?;
    let tp = s.get("total_points").and_then(|x| x.as_u64());
    let mp = s.get("max_points").and_then(|x| x.as_u64());
    let (points, ratio) = match (tp, mp) {
        (Some(p), Some(m)) if m > 0 && m != 100 => {
            (Some(format!("{p}/{m}")), Some(p as f64 / m as f64))
        }
        _ => (None, None),
    };
    Some((pct, points, ratio))
}

/// Did this bench run to the end? Decided from the JSON's own metadata,
/// never from a fixed pool size: the tool stamps `status` (absent on
/// legacy files) and `total_scenarios` beside `scenario_results`, so
/// coverage is self-describing for every harness generation (88×2=176 in
/// the dev-era, 92×2=184 since v2.7.0, whatever comes next). A run
/// interrupted early either says so in `status` or leaves the results
/// map shorter than the scenario count.
pub(crate) fn bench_complete(content: &str) -> bool {
    let Ok(v) = serde_json::from_str::<serde_json::Value>(content) else {
        return false;
    };
    let scores = match v.get("scores") {
        Some(s) if s.is_object() => s,
        _ => return false,
    };
    let covered = match (
        v.get("total_scenarios").and_then(|x| x.as_u64()),
        scores
            .get("scenario_results")
            .and_then(|x| x.as_object())
            .map(|m| m.len() as u64),
    ) {
        (Some(total), Some(ran)) => ran >= total,
        _ => true, // no coverage metadata to contradict
    };
    match v.get("status").and_then(|x| x.as_str()) {
        Some("completed") => covered,
        Some(_) => false,
        // Legacy files carry no status — a printed final score was
        // written only by a finished run.
        None => scores.get("final_score").and_then(|x| x.as_f64()).is_some() && covered,
    }
}

/// First non-empty perf box among the given contents (md first, then log —
/// older reports carry the box in the .md, `sm121 bench` runs in the .log).
pub(crate) fn perf_from(md: Option<&str>, log: Option<&str>) -> Vec<PerfCell> {
    for c in [md, log] {
        let v = c.map(parse_perf_box).unwrap_or_default();
        if !v.is_empty() {
            return v;
        }
    }
    Vec::new()
}

fn split_tokens(line: &str) -> impl Iterator<Item = &str> + '_ {
    line.split(|ch: char| ch.is_whitespace() || ch == '|' || ch == '│')
}

/// Parse llama-benchy perf table rows into `(depth, concurrency, tg t/s)`
/// cells. Two known shapes:
///   box: `│ pp2048 tg128 @ d0    │ c1 │ 1,500 │ 27.4 │ 1,398 │ 5,959 │ 2048+128`
///   md:  `| pp2048 tg128 @ d0 c2 | 1,728 | 36.0 | 2,097 | 8,802 | 2048+128`
/// The `c` token (concurrency column or label suffix) is optional — absent
/// means c1. Header/box-chrome lines have no `@ d<depth>` marker and are
/// skipped.
pub(crate) fn parse_perf_box(content: &str) -> Vec<PerfCell> {
    let mut out = Vec::new();
    for line in content.lines() {
        // Perf rows are table rows — box/md chrome with ≥3 column
        // separators. llama-benchy progress/spinner frames embed the same
        // `pp2048 tg128 @ d0` labels with no columns (they parsed as
        // d=0/c=1/tg=0.0 and shadowed the real cells in lookups).
        if line.matches(['|', '│']).count() < 3 {
            continue;
        }
        let Some(at) = line.find("@ d") else {
            continue;
        };
        let rest = &line[at + 3..];
        let dd = rest
            .find(|ch: char| !ch.is_ascii_digit())
            .unwrap_or(rest.len());
        let d: u32 = match rest[..dd].parse() {
            Ok(x) => x,
            Err(_) => continue,
        };
        let c: u32 = split_tokens(line)
            .find_map(|tok| tok.strip_prefix("c").and_then(|n| n.parse::<u32>().ok()))
            .unwrap_or(1);
        // numbers from the label onward: pp t/s, tg t/s, TTFT, total, 2048, 128
        // (llama-benchy marks caveat/cold cells with a trailing `*`, e.g. `675*`)
        let nums: Vec<f64> = split_tokens(&line[at..])
            .filter_map(|tok| {
                tok.trim_end_matches('*')
                    .replace(',', "")
                    .parse::<f64>()
                    .ok()
            })
            .collect();
        if nums.len() >= 4 && nums[1] > 0.0 {
            out.push(PerfCell { d, c, tg: nums[1] });
        }
    }
    out
}

/// Quantization tokens, longest first so `nvfp4kv` wins over `nvfp4`, etc.
pub(crate) const QUANTS: &[&str] = &[
    "nvfp4kv", "nvfp4", "mxfp8", "mxfp4", "w4a16", "dcp4", "exl3", "int8", "int4", "nf4", "fp8",
    "awq", "gptq", "bf16", "fp16",
];

pub(crate) fn detect_quant(name: &str, model: &str) -> String {
    let hay = format!("{name} {model}").to_lowercase();
    // ponytail: token-scan heuristic over dir+model names; per-recipe `quant`
    // field if recipes ever grow one.
    for q in QUANTS {
        if hay.contains(q) {
            return q.to_uppercase();
        }
    }
    "—".to_string()
}

/// Parse the score headers used by campaign reports and raw bench logs:
/// `## Score: 159/176 (90.4)`, `- **Final Score**: **81** / 100`, and the
/// box-drawn log summary `│    Score:  88 / 100  │`. Lines must start with
/// a score prefix (after stripping markdown bullets / box-drawing chars),
/// so prose and embedded tool transcripts can't produce false positives.
/// Returns `(points, max, normalized_pct)`.
pub(crate) fn parse_score(content: &str) -> Option<(u32, u32, Option<f64>)> {
    for line in content.lines() {
        let l = line.trim_start_matches(|c: char| {
            matches!(c, '#' | '-' | '•' | '|' | '│' | '┃') || c.is_whitespace()
        });
        let is_score = l.starts_with("**Final Score**") || l.starts_with("Score:");
        if is_score && let Some((p, m)) = slash_pair(l) {
            let pct = paren_pct(l)
                // `Final Score` / log box are already out of 100
                .or_else(|| (m == 100).then_some(p as f64))
                .or_else(|| Some(p as f64 / m as f64 * 100.0));
            return Some((p, m, pct));
        }
    }
    None
}

/// First parenthesized number, e.g. `(90.4)` in `## Score: 159/176 (90.4)`.
pub(crate) fn paren_pct(s: &str) -> Option<f64> {
    let open = s.find('(')?;
    let close = open + s[open..].find(')')?;
    s[open + 1..close].trim().parse().ok()
}

/// First `YYYY-MM-DDTHH:MM` (bench `**Date**` line) or `YYYY-MM-DDTHH-MM`
/// (Run ID / full-report path) timestamp; returns `HH:MM`.
pub(crate) fn extract_time(content: &str) -> Option<String> {
    for line in content.lines() {
        let b = line.as_bytes();
        for i in 0..b.len().saturating_sub(15) {
            let d = |j: usize| b[i + j].is_ascii_digit();
            if d(0)
                && d(1)
                && d(2)
                && d(3)
                && b[i + 4] == b'-'
                && d(5)
                && d(6)
                && b[i + 7] == b'-'
                && d(8)
                && d(9)
                && b[i + 10] == b'T'
                && d(11)
                && d(12)
                && (b[i + 13] == b':' || b[i + 13] == b'-')
                && d(14)
                && d(15)
            {
                let mut t = line[i + 11..i + 16].to_string();
                if t.as_bytes()[2] == b'-' {
                    t.replace_range(2..3, ":");
                }
                return Some(t);
            }
        }
    }
    None
}

/// First `N…/M` pair in `s`, tolerating `*` and spaces around both sides
/// (bold markdown). Plain digit scan — no regex dependency.
pub(crate) fn slash_pair(s: &str) -> Option<(u32, u32)> {
    let b = s.as_bytes();
    let mut i = 0;
    while i < b.len() {
        if !b[i].is_ascii_digit() {
            i += 1;
            continue;
        }
        let start = i;
        while i < b.len() && b[i].is_ascii_digit() {
            i += 1;
        }
        let mut j = i;
        while j < b.len() && matches!(b[j], b' ' | b'*') {
            j += 1;
        }
        if j < b.len() && b[j] == b'/' {
            let mut k = j + 1;
            while k < b.len() && matches!(b[k], b' ' | b'*') {
                k += 1;
            }
            let dstart = k;
            while k < b.len() && b[k].is_ascii_digit() {
                k += 1;
            }
            if let (Ok(n), Ok(m)) = (s[start..i].parse(), s[dstart..k].parse()) {
                return Some((n, m));
            }
        }
    }
    None
}