bench: re-attach the perf table the --json-file path drops; fix aiden recipe toml

recipes/dsv41-flash-4x-sglang-aiden16/recipe.toml

@@ -26,7 +26,7 @@
 # layer. Computed per node by aiden4x-launch.sh (next to this recipe) from
 # $NODE_RANK + the checkpoint's safetensors headers (floor/ceil tiling; the
 # launcher exports NODE_RANK per rank). A full range on every rank would
-double-count row contributions in the reduce — the range is mandatory.
+# double-count row contributions in the reduce — the range is mandatory.
 #
 # B12X_ROCE_GID_INDEX reuses the launcher's per-node NCCL GID auto-detect
 # (the wrapper exports it), so no pinned GID anywhere.

recipes/dsv41-flash-4x-sglang-aiden16/reports/2026-09-26_015216_aiden16-sglang-4x.md

@@ -4742,3 +4742,18 @@ final_answer=01928374655647382910
 verdict=fail
 summary=Returned extra text or a value that was not exactly 20 digits.
 ```
+
+## Throughput Metrics
+
+> Appended by `sm121 bench`: the tool's `--json-file` path drops the
+> perf section from this report; the rows come from the llama-benchy table
+> in the bench log (same run).
+
+| Test | pp t/s | tg t/s | TTFT (ms) | Total (ms) | Tokens |
+|---|---:|---:|---:|---:|---:|
+| pp2048 tg128 @ d0 | 154 | 19.7 | 12,620 | 18,810 | 2048+128 |
+| pp2048 tg128 @ d0 c2 | 177 | 21.8 | 20,470 | 30,788 | 2048+128 |
+| pp2048 tg128 @ d4096 | 286 | 22.9 | 22,183 | 27,467 | 2048+128 |
+| pp2048 tg128 @ d4096 c2 | 309 | 13.1 | 33,447 | 45,173 | 2048+128 |
+| pp2048 tg128 @ d8192 | 268 | 20.3 | 35,731 | 41,724 | 2048+128 |
+| pp2048 tg128 @ d8192 c2 | 325 | 8.6 | 51,541 | 67,162 | 2048+128 |

recipes/qwen38-flash-next-int4/reports/2026-09-26_022931_int4-auto-1x-first.md

@@ -6586,3 +6586,18 @@ verdict=pass
 summary=Returned three distinct values satisfying every visible cross-turn constraint.
 note=Reasoning transport is a separate diagnostic; number occurrence does not prove private verification.
 ```
+
+## Throughput Metrics
+
+> Appended by `sm121 bench`: the tool's `--json-file` path drops the
+> perf section from this report; the rows come from the llama-benchy table
+> in the bench log (same run).
+
+| Test | pp t/s | tg t/s | TTFT (ms) | Total (ms) | Tokens |
+|---|---:|---:|---:|---:|---:|
+| pp2048 tg128 @ d0 | 1,886 | 66.3 | 1,103 | 2,945 | 2048+128 |
+| pp2048 tg128 @ d0 c2 | 1,946 | 100.3 | 1,928 | 4,339 | 2048+128 |
+| pp2048 tg128 @ d4096 | 2,103 | 66.7 | 2,741 | 4,572 | 2048+128 |
+| pp2048 tg128 @ d4096 c2 | 2,070 | 98.7 | 5,383 | 7,706 | 2048+128 |
+| pp2048 tg128 @ d8192 | 2,125 | 63.2 | 4,495 | 6,431 | 2048+128 |
+| pp2048 tg128 @ d8192 c2 | 2,074 | 72.4 | 8,431 | 11,213 | 2048+128 |

src/bench.rs

@@ -274,6 +274,12 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
     let md_dest = reports_dir.join(format!("{stamp}_{label}.md"));
     fs::copy(&md_src, &md_dest)?;
 
+    // The `--json-file` dispatch path drops the perf section from the md —
+    // re-attach it from the llama-benchy table in the log.
+    if crate::bench_report::ensure_throughput_section(&md_dest, &log_path).is_ok_and(|m| m) {
+        eprintln!("bench: appended the perf table to the report md");
+    }
+
     // Summary: final score from the JSON + the perf table from the log.
     // Perf-only runs have no quality score and the tool writes no JSON —
     // the run package is skipped for them (the campaign run is the record).

src/bench_report.rs

@@ -0,0 +1,140 @@
+//! Post-run report repair. tool-eval-bench's `--json-file` flag implies
+//! `--json`, and that dispatch path (v2.7.0) drops the llama-benchy perf
+//! section from the scenario report md (the samples are computed and printed
+//! to the bench log, but never passed to the report writer). The launcher
+//! re-attaches the table from the log so every campaign report carries its
+//! perf numbers like a `--perf-only` run would.
+
+use std::fs;
+
+use anyhow::Result;
+
+pub struct PerfRow {
+    pub test: String,
+    pub concurrency: u32,
+    pub pp: String,
+    pub tg: String,
+    pub ttft: String,
+    pub total: String,
+    pub tokens: String,
+}
+
+/// Append a `## Throughput Metrics` section to `md` when it lacks one,
+/// taking the rows from the llama-benchy box in `log`. Returns true when
+/// the md was modified.
+pub fn ensure_throughput_section(md: &std::path::Path, log: &std::path::Path) -> Result<bool> {
+    let body = fs::read_to_string(md)?;
+    if body.contains("## Throughput Metrics") || body.contains("## Results") {
+        return Ok(false);
+    }
+    let rows = parse_perf_rows(&fs::read_to_string(log)?)
+        .ok_or_else(|| anyhow::anyhow!("no llama-benchy perf rows in {}", log.display()))?;
+    let mut out = body;
+    if !out.ends_with('\n') {
+        out.push('\n');
+    }
+    out.push_str("\n## Throughput Metrics\n\n");
+    out.push_str("> Appended by `sm121 bench`: the tool's `--json-file` path drops the\n");
+    out.push_str("> perf section from this report; the rows come from the llama-benchy table\n");
+    out.push_str("> in the bench log (same run).\n\n");
+    out.push_str("| Test | pp t/s | tg t/s | TTFT (ms) | Total (ms) | Tokens |\n");
+    out.push_str("|---|---:|---:|---:|---:|---:|\n");
+    for r in &rows {
+        let conc = if r.concurrency > 1 {
+            format!(" c{}", r.concurrency)
+        } else {
+            String::new()
+        };
+        out.push_str(&format!(
+            "| {}{} | {} | {} | {} | {} | {} |\n",
+            r.test, conc, r.pp, r.tg, r.ttft, r.total, r.tokens
+        ));
+    }
+    fs::write(md, out)?;
+    Ok(true)
+}
+
+/// Extract the llama-benchy result rows from a bench log. Box rows look like
+/// `│ pp2048 tg128 @ d0 │ c1 │ 154 │ 19.7 │ 12,620 │ 18,810 │ 2048+128`.
+fn parse_perf_rows(log: &str) -> Option<Vec<PerfRow>> {
+    let mut rows = Vec::new();
+    for line in log.lines() {
+        let line = line.trim();
+        if !line.starts_with('│') {
+            continue;
+        }
+        let parts: Vec<&str> = line
+            .split('│')
+            .map(|s| s.trim())
+            .filter(|s| !s.is_empty())
+            .collect();
+        if parts.len() != 7 || !parts[0].starts_with("pp") {
+            continue;
+        }
+        let concurrency = parts
+            .get(1)
+            .and_then(|c| c.strip_prefix('c'))
+            .and_then(|n| n.parse().ok())
+            .unwrap_or(1);
+        rows.push(PerfRow {
+            test: parts[0].to_string(),
+            concurrency,
+            pp: parts[2].to_string(),
+            tg: parts[3].to_string(),
+            ttft: parts[4].to_string(),
+            total: parts[5].to_string(),
+            tokens: parts[6].to_string(),
+        });
+    }
+    (!rows.is_empty()).then_some(rows)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const LOG: &str = "\
+╭──────────────────── ⚡ llama-benchy Throughput Benchmark ────────────────────╮
+┃ Test                 ┃ c  ┃  pp t/s ┃  tg t/s ┃     TTFT ┃    Total ┃
+┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━
+│ pp2048 tg128 @ d0    │ c1 │     154 │    19.7 │   12,620 │   18,810 │ 2048+128
+│ pp2048 tg128 @ d0    │ c2 │     177 │    21.8 │   20,470 │   30,788 │ 2048+128
+└──────────────────────┴────┴─────────┴─────────┴──────────┴──────────┴─────────
+";
+
+    #[test]
+    fn appends_section_from_log_box() {
+        let tmp =
+            std::env::temp_dir().join(format!("sm121-bench-report-test-{}", std::process::id()));
+        fs::create_dir_all(&tmp).unwrap();
+        let md = tmp.join("r.md");
+        let log = tmp.join("r.log");
+        fs::write(&md, "# Report\n\n## Scenario Results\n\n| ok |\n").unwrap();
+        fs::write(&log, LOG).unwrap();
+
+        assert!(ensure_throughput_section(&md, &log).unwrap());
+        let body = fs::read_to_string(&md).unwrap();
+        assert!(body.contains("## Throughput Metrics"));
+        assert!(body.contains("| pp2048 tg128 @ d0 | 154 | 19.7 | 12,620 | 18,810 | 2048+128 |"));
+        assert!(
+            body.contains("| pp2048 tg128 @ d0 c2 | 177 | 21.8 | 20,470 | 30,788 | 2048+128 |")
+        );
+
+        // idempotent: an existing section means no rewrite
+        assert!(!ensure_throughput_section(&md, &log).unwrap());
+        let _ = fs::remove_dir_all(&tmp);
+    }
+
+    #[test]
+    fn no_rows_is_an_error() {
+        let tmp =
+            std::env::temp_dir().join(format!("sm121-bench-report-test2-{}", std::process::id()));
+        fs::create_dir_all(&tmp).unwrap();
+        let md = tmp.join("r.md");
+        let log = tmp.join("r.log");
+        fs::write(&md, "# Report\n").unwrap();
+        fs::write(&log, "no box here\n").unwrap();
+        assert!(ensure_throughput_section(&md, &log).is_err());
+        let _ = fs::remove_dir_all(&tmp);
+    }
+}

src/main.rs

@@ -10,6 +10,7 @@ mod ansi;
 mod before_serve;
 mod bench;
 mod bench_artifacts;
+mod bench_report;
 mod builtin_scripts;
 mod cli;
 mod cluster_state;