//! Parsers for benchmark report files: the tool-eval-bench `--json-file` //! output (authoritative score) and the llama-benchy perf tables found in //! the `.md`/`.log` reports (generation speed). use serde::{Deserialize, Serialize}; /// One llama-benchy perf cell: decode speed at a context depth and /// concurrency. #[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Debug)] pub(crate) struct PerfCell { pub(crate) d: u32, // context depth (0, 4096, 8192) pub(crate) c: u32, // concurrency (1, 2, …) pub(crate) tg: f64, // decode tokens/s } /// Parse the tool-eval-bench `--json-file` output (authoritative for /// `sm121 bench` runs): `scores.final_score` (0–100), `scores.total_points` /// / `max_points` (hardmode). Returns `(pct, points, ratio)`; `points` is /// suppressed when max is 100 (already normalized, same rule as the text /// parsers). pub(crate) fn parse_bench_json(content: &str) -> Option<(f64, Option, Option)> { let v: serde_json::Value = serde_json::from_str(content).ok()?; let s = v.get("scores")?; let pct = s.get("final_score").and_then(|x| x.as_f64())?; let tp = s.get("total_points").and_then(|x| x.as_u64()); let mp = s.get("max_points").and_then(|x| x.as_u64()); let (points, ratio) = match (tp, mp) { (Some(p), Some(m)) if m > 0 && m != 100 => { (Some(format!("{p}/{m}")), Some(p as f64 / m as f64)) } _ => (None, None), }; Some((pct, points, ratio)) } /// Did this bench run to the end? Decided from the JSON's own metadata, /// never from a fixed pool size: the tool stamps `status` (absent on /// legacy files) and `total_scenarios` beside `scenario_results`, so /// coverage is self-describing for every harness generation (88×2=176 in /// the dev-era, 92×2=184 since v2.7.0, whatever comes next). A run /// interrupted early either says so in `status` or leaves the results /// map shorter than the scenario count. pub(crate) fn bench_complete(content: &str) -> bool { let Ok(v) = serde_json::from_str::(content) else { return false; }; let scores = match v.get("scores") { Some(s) if s.is_object() => s, _ => return false, }; let covered = match ( v.get("total_scenarios").and_then(|x| x.as_u64()), scores .get("scenario_results") .and_then(|x| x.as_object()) .map(|m| m.len() as u64), ) { (Some(total), Some(ran)) => ran >= total, _ => true, // no coverage metadata to contradict }; match v.get("status").and_then(|x| x.as_str()) { Some("completed") => covered, Some(_) => false, // Legacy files carry no status — a printed final score was // written only by a finished run. None => scores.get("final_score").and_then(|x| x.as_f64()).is_some() && covered, } } /// First non-empty perf box among the given contents (md first, then log — /// older reports carry the box in the .md, `sm121 bench` runs in the .log). pub(crate) fn perf_from(md: Option<&str>, log: Option<&str>) -> Vec { for c in [md, log] { let v = c.map(parse_perf_box).unwrap_or_default(); if !v.is_empty() { return v; } } Vec::new() } fn split_tokens(line: &str) -> impl Iterator + '_ { line.split(|ch: char| ch.is_whitespace() || ch == '|' || ch == '│') } /// Parse llama-benchy perf table rows into `(depth, concurrency, tg t/s)` /// cells. Two known shapes: /// box: `│ pp2048 tg128 @ d0 │ c1 │ 1,500 │ 27.4 │ 1,398 │ 5,959 │ 2048+128` /// md: `| pp2048 tg128 @ d0 c2 | 1,728 | 36.0 | 2,097 | 8,802 | 2048+128` /// The `c` token (concurrency column or label suffix) is optional — absent /// means c1. Header/box-chrome lines have no `@ d` marker and are /// skipped. pub(crate) fn parse_perf_box(content: &str) -> Vec { let mut out = Vec::new(); for line in content.lines() { // Perf rows are table rows — box/md chrome with ≥3 column // separators. llama-benchy progress/spinner frames embed the same // `pp2048 tg128 @ d0` labels with no columns (they parsed as // d=0/c=1/tg=0.0 and shadowed the real cells in lookups). if line.matches(['|', '│']).count() < 3 { continue; } let Some(at) = line.find("@ d") else { continue; }; let rest = &line[at + 3..]; let dd = rest .find(|ch: char| !ch.is_ascii_digit()) .unwrap_or(rest.len()); let d: u32 = match rest[..dd].parse() { Ok(x) => x, Err(_) => continue, }; let c: u32 = split_tokens(line) .find_map(|tok| tok.strip_prefix("c").and_then(|n| n.parse::().ok())) .unwrap_or(1); // numbers from the label onward: pp t/s, tg t/s, TTFT, total, 2048, 128 // (llama-benchy marks caveat/cold cells with a trailing `*`, e.g. `675*`) let nums: Vec = split_tokens(&line[at..]) .filter_map(|tok| { tok.trim_end_matches('*') .replace(',', "") .parse::() .ok() }) .collect(); if nums.len() >= 4 && nums[1] > 0.0 { out.push(PerfCell { d, c, tg: nums[1] }); } } out } /// Quantization tokens, longest first so `nvfp4kv` wins over `nvfp4`, etc. pub(crate) const QUANTS: &[&str] = &[ "nvfp4kv", "nvfp4", "mxfp8", "mxfp4", "w4a16", "dcp4", "exl3", "int8", "int4", "nf4", "fp8", "awq", "gptq", "bf16", "fp16", ]; pub(crate) fn detect_quant(name: &str, model: &str) -> String { let hay = format!("{name} {model}").to_lowercase(); // ponytail: token-scan heuristic over dir+model names; per-recipe `quant` // field if recipes ever grow one. for q in QUANTS { if hay.contains(q) { return q.to_uppercase(); } } "—".to_string() } /// Parse the score headers used by campaign reports and raw bench logs: /// `## Score: 159/176 (90.4)`, `- **Final Score**: **81** / 100`, and the /// box-drawn log summary `│ Score: 88 / 100 │`. Lines must start with /// a score prefix (after stripping markdown bullets / box-drawing chars), /// so prose and embedded tool transcripts can't produce false positives. /// Returns `(points, max, normalized_pct)`. pub(crate) fn parse_score(content: &str) -> Option<(u32, u32, Option)> { for line in content.lines() { let l = line.trim_start_matches(|c: char| { matches!(c, '#' | '-' | '•' | '|' | '│' | '┃') || c.is_whitespace() }); let is_score = l.starts_with("**Final Score**") || l.starts_with("Score:"); if is_score && let Some((p, m)) = slash_pair(l) { let pct = paren_pct(l) // `Final Score` / log box are already out of 100 .or_else(|| (m == 100).then_some(p as f64)) .or_else(|| Some(p as f64 / m as f64 * 100.0)); return Some((p, m, pct)); } } None } /// First parenthesized number, e.g. `(90.4)` in `## Score: 159/176 (90.4)`. pub(crate) fn paren_pct(s: &str) -> Option { let open = s.find('(')?; let close = open + s[open..].find(')')?; s[open + 1..close].trim().parse().ok() } /// First `YYYY-MM-DDTHH:MM` (bench `**Date**` line) or `YYYY-MM-DDTHH-MM` /// (Run ID / full-report path) timestamp; returns `HH:MM`. pub(crate) fn extract_time(content: &str) -> Option { for line in content.lines() { let b = line.as_bytes(); for i in 0..b.len().saturating_sub(15) { let d = |j: usize| b[i + j].is_ascii_digit(); if d(0) && d(1) && d(2) && d(3) && b[i + 4] == b'-' && d(5) && d(6) && b[i + 7] == b'-' && d(8) && d(9) && b[i + 10] == b'T' && d(11) && d(12) && (b[i + 13] == b':' || b[i + 13] == b'-') && d(14) && d(15) { let mut t = line[i + 11..i + 16].to_string(); if t.as_bytes()[2] == b'-' { t.replace_range(2..3, ":"); } return Some(t); } } } None } /// First `N…/M` pair in `s`, tolerating `*` and spaces around both sides /// (bold markdown). Plain digit scan — no regex dependency. pub(crate) fn slash_pair(s: &str) -> Option<(u32, u32)> { let b = s.as_bytes(); let mut i = 0; while i < b.len() { if !b[i].is_ascii_digit() { i += 1; continue; } let start = i; while i < b.len() && b[i].is_ascii_digit() { i += 1; } let mut j = i; while j < b.len() && matches!(b[j], b' ' | b'*') { j += 1; } if j < b.len() && b[j] == b'/' { let mut k = j + 1; while k < b.len() && matches!(b[k], b' ' | b'*') { k += 1; } let dstart = k; while k < b.len() && b[k].is_ascii_digit() { k += 1; } if let (Ok(n), Ok(m)) = (s[start..i].parse(), s[dstart..k].parse()) { return Some((n, m)); } } } None }