bench: depth-suite phase (16K/64K/128K/256K @ c2, 2 repeats) appended to the campaign log; report/tests.rs split (perf_box_tests) for the line cap

src/bench.rs

@@ -105,6 +105,10 @@ fn pump<R: Read>(mut reader: R, to_console: bool, file: Arc<Mutex<fs::File>>) {
 /// Run the bench tool in `cwd`, teeing stdout+stderr to the console and the
 /// log file; returns the exit code.
 ///
+/// `append_log = false` truncates the log (main campaign phase); `true`
+/// appends (the depth-suite phase shares one log so its rows land in the
+/// report's Throughput table).
+///
 /// The child runs under a pty (`script -q /dev/null`) instead of raw pipes:
 /// tool-eval-bench's rich Live display deadlocks on a non-tty stdout pipe
 /// right after the perf phase (deterministic on the 2x lane; 4x runs got
@@ -121,8 +125,21 @@ fn pump<R: Read>(mut reader: R, to_console: bool, file: Arc<Mutex<fs::File>>) {
 /// keeps the deadlock workaround intact. `NO_COLOR=1` covers non-rich
 /// output. Verified locally under `script`: 1.3 KB/8 CR of soup → 141
 /// bytes/1 static final frame.
-fn run_tool(bin: &str, args: &[String], cwd: &Path, log_path: &Path) -> Result<i32> {
-    let file = Arc::new(Mutex::new(fs::File::create(log_path)?));
+fn run_tool(
+    bin: &str,
+    args: &[String],
+    cwd: &Path,
+    log_path: &Path,
+    append_log: bool,
+) -> Result<i32> {
+    let file = Arc::new(Mutex::new(if append_log {
+        fs::OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(log_path)?
+    } else {
+        fs::File::create(log_path)?
+    }));
     let mut cmd = if std::process::Command::new("/usr/bin/script")
         .arg("-q")
         .arg("/dev/null")
@@ -231,6 +248,9 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
         .join("bench")
         .join(format!("{stamp}_{label}"));
     let bin = "tool-eval-bench".to_string();
+    let depth_scratch = scratch.with_file_name(format!("{stem}_depth"));
+    let depth_args =
+        crate::bench_depth::depth_args(&base_url, &model, a.seed, a.tokenizer.as_ref());
 
     let mut args = vec![
         "--base-url".into(),
@@ -261,6 +281,7 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
     if dry_run {
         println!("bench: cwd {}", scratch.display());
         println!("bench: {} {}", bin, args.join(" "));
+        println!("bench: depth suite: {} {}", bin, depth_args.join(" "));
         println!("bench: reports -> {}", reports_dir.display());
         println!(
             "bench: run package -> {}.recipe.toml + {}.manifest.json",
@@ -276,7 +297,7 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
     crate::readiness::wait_ready(&base_url, crate::readiness::BENCH_WAIT_SECS).await?;
     eprintln!("bench: API ready — running {} (seed {})", bin, a.seed);
 
-    let code = run_tool(&bin, &args, &scratch, &log_path)?;
+    let code = run_tool(&bin, &args, &scratch, &log_path, false)?;
     if code != 0 {
         anyhow::bail!(
             "bench: tool exited with {code} (log: {})",
@@ -284,6 +305,29 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
         );
     }
 
+    // Phase 2 — context-depth suite (16K..256K @ c2, 2 repeats): appended
+    // to the same log so `parse_perf_rows` picks its rows up too. A depth
+    // failure must not cost the finished quality run — warn, keep going.
+    eprintln!(
+        "bench: depth suite: {} @ c{}, {} repeats{}",
+        crate::bench_depth::DEPTHS,
+        crate::bench_depth::CONCURRENCY,
+        crate::bench_depth::RUNS,
+        if a.tokenizer.is_some() {
+            " (+tokenizer)"
+        } else {
+            ""
+        }
+    );
+    fs::create_dir_all(&depth_scratch)?;
+    let dcode = run_tool(&bin, &depth_args, &depth_scratch, &log_path, true)?;
+    if dcode != 0 {
+        eprintln!(
+            "bench: WARNING depth suite exited with {dcode} — main report unaffected ({})",
+            log_path.display()
+        );
+    }
+
     let md_src = latest_md(&scratch)
         .ok_or_else(|| anyhow::anyhow!("no report .md under {}", scratch.join("runs").display()))?;
     let md_dest = reports_dir.join(format!("{stamp}_{label}.md"));
@@ -404,6 +448,7 @@ mod tests {
             ],
             &tmp,
             &log,
+            false,
         )
         .unwrap();
         let body = fs::read_to_string(&log).unwrap();
@@ -414,6 +459,28 @@ mod tests {
         assert!(!body.contains('\r'), "pty ONLCR leaked CRLF: {body:?}");
     }
 
+    /// The append mode must not truncate the main phase's log — the depth
+    /// suite shares it so one report carries both tables.
+    #[test]
+    fn run_tool_append_keeps_prior_output() {
+        let tmp =
+            std::env::temp_dir().join(format!("sm121-bench-append-test-{}", std::process::id()));
+        fs::create_dir_all(&tmp).unwrap();
+        let log = tmp.join("append.log");
+        let run =
+            |a: &str, append| run_tool("sh", &["-c".into(), a.into()], &tmp, &log, append).unwrap();
+        assert_eq!(run("echo main", false), 0);
+        assert!(fs::read_to_string(&log).unwrap().contains("main\n"));
+        assert_eq!(run("echo depth", true), 0);
+        let body = fs::read_to_string(&log).unwrap();
+        let (i1, i2) = (body.find("main\n"), body.find("depth\n"));
+        assert!(
+            i1.is_some() && i2.is_some_and(|d| i1.unwrap() < d),
+            "append truncated the log: {body:?}"
+        );
+        let _ = fs::remove_dir_all(&tmp);
+    }
+
     #[test]
     fn served_model_parsed_from_args() {
         let content = fs::read_to_string("recipes/glm53-flash-r27-spark/recipe.toml")

src/bench_depth.rs

@@ -0,0 +1,93 @@
+//! Depth-suite phase of `sm121 bench`: a second, scenario-free perf pass
+//! that maps decode throughput across context depths.
+//!
+//! The campaign perf phase covers 0/8K (with the quality run); long-context
+//! decode is a lane-defining characteristic (vLLM vs sglang flattening the
+//! 256K cliff) and used to need a manual slope sweep to observe. The suite
+//! runs it at fixed 2 streams with 2 repeats per point — operator convention
+//! 2026-09-29: "16K/64K/128K/256K at c2, 2 runs, keep it short".
+
+/// Context depths of the suite (0/8K excluded — covered by the main run).
+pub const DEPTHS: &str = "16384,65536,131072,262144";
+/// Fixed stream count (matches the campaign concurrency's second entry).
+pub const CONCURRENCY: &str = "2";
+/// Repeats per depth point (kept at 2 to bound the wall time).
+pub const RUNS: &str = "2";
+
+/// `tool-eval-bench` args for the depth sweep (perf-only, no scenarios).
+///
+/// pp/tg match the campaign convention so the rows land under the same
+/// `pp2048 tg128 @ dNN` names the report's Throughput table already parses.
+pub fn depth_args(
+    base_url: &str,
+    model: &str,
+    seed: u64,
+    tokenizer: Option<&String>,
+) -> Vec<String> {
+    let mut args = vec![
+        "--base-url".into(),
+        base_url.to_string(),
+        "--model".into(),
+        model.to_string(),
+        "--seed".into(),
+        seed.to_string(),
+        "--perf-only".into(),
+        "--skip-tool-eval".into(),
+        "--pp".into(),
+        "2048".into(),
+        "--tg".into(),
+        "128".into(),
+        "--depth".into(),
+        DEPTHS.to_string(),
+        "--concurrency".into(),
+        CONCURRENCY.to_string(),
+        "--benchy-runs".into(),
+        RUNS.to_string(),
+    ];
+    if let Some(tok) = tokenizer {
+        args.push("--tokenizer".into());
+        args.push(tok.clone());
+    }
+    args
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// The suite must be scenario-free and pinned to the campaign pp/tg —
+    /// rows named differently would not match the report's perf parser.
+    #[test]
+    fn depth_args_are_perf_only_with_campaign_pp_tg() {
+        let a = depth_args("http://h:3000/v1", "m", 42, None);
+        let joined = a.join(" ");
+        assert!(joined.contains("--perf-only"), "{joined}");
+        assert!(joined.contains("--skip-tool-eval"), "{joined}");
+        assert!(joined.contains("--pp 2048"), "{joined}");
+        assert!(joined.contains("--tg 128"), "{joined}");
+        assert!(joined.contains(&format!("--depth {DEPTHS}")), "{joined}");
+        assert!(
+            joined.contains(&format!("--concurrency {CONCURRENCY}")),
+            "{joined}"
+        );
+        assert!(
+            joined.contains(&format!("--benchy-runs {RUNS}")),
+            "{joined}"
+        );
+        assert!(!joined.contains("--tokenizer"), "{joined}");
+    }
+
+    #[test]
+    fn depths_exclude_the_covered_points() {
+        let depths: Vec<u32> = DEPTHS.split(',').map(|d| d.parse().unwrap()).collect();
+        assert_eq!(depths, vec![16384, 65536, 131072, 262144]);
+        assert!(!depths.contains(&0) && !depths.contains(&8192));
+    }
+
+    #[test]
+    fn tokenizer_forwarded_when_given() {
+        let a = depth_args("u", "m", 42, Some(&"/tmp/tok".to_string()));
+        assert!(a.contains(&"--tokenizer".to_string()));
+        assert!(a.contains(&"/tmp/tok".to_string()));
+    }
+}

src/main.rs

@@ -10,6 +10,7 @@ mod ansi;
 mod before_serve;
 mod bench;
 mod bench_artifacts;
+mod bench_depth;
 mod bench_report;
 mod builtin_scripts;
 mod cli;

src/report/mod.rs

@@ -456,5 +456,8 @@ mod stale;
 #[cfg(test)]
 mod tests;
 
+#[cfg(test)]
+mod perf_box_tests;
+
 #[cfg(test)]
 mod ui_tests;

src/report/perf_box_tests.rs

@@ -0,0 +1,78 @@
+//! `parse_perf_box` shape/soup regression tests — extracted from `tests.rs`
+//! for the 500-line cap (same `super::` scope, no other imports needed).
+
+#[test]
+fn parse_perf_box_reads_both_table_shapes() {
+    // modern box rows (concurrency as its own column)
+    let box_log = "\u{2502} pp2048 tg128 @ d0    \u{2502} c1 \u{2502}   1,500 \u{2502}    27.4 \u{2502}    1,398 \u{2502}    5,959 \u{2502} 2048+128\n\
+\u{2502} pp2048 tg128 @ d4096 \u{2502} c2 \u{2502}   1,642 \u{2502}    30.6 \u{2502}    6,040 \u{2502}   12,822 \u{2502} 2048+128\n";
+    assert_eq!(
+        super::parse::parse_perf_box(box_log),
+        vec![
+            super::PerfCell {
+                d: 0,
+                c: 1,
+                tg: 27.4
+            },
+            super::PerfCell {
+                d: 4096,
+                c: 2,
+                tg: 30.6
+            },
+        ]
+    );
+    // legacy markdown rows (concurrency in the label, absent = c1)
+    let md = "| pp2048 tg128 @ d0 | 1,861 | 24.5 | 1,236 | 6,228 | 2048+128 |\n\
+| pp2048 tg128 @ d0 c2 | 1,728 | 36.0 | 2,097 | 8,802 | 2048+128 |\n\
+| Test | pp t/s | tg t/s | TTFT (ms) | Total (ms) | Tokens |\n";
+    assert_eq!(
+        super::parse::parse_perf_box(md),
+        vec![
+            super::PerfCell {
+                d: 0,
+                c: 1,
+                tg: 24.5
+            },
+            super::PerfCell {
+                d: 0,
+                c: 2,
+                tg: 36.0
+            },
+        ]
+    );
+    // header chrome and prose must not produce cells
+    assert!(super::parse::parse_perf_box("pp=[2048] tg=[128] depth=[0, 4096]").is_empty());
+    assert!(super::parse::parse_perf_box("no perf here").is_empty());
+}
+
+#[test]
+fn parse_perf_box_ignores_progress_spinner_soup() {
+    // llama-benchy redraws its progress bar on one \r-overwritten line that
+    // embeds the same `pp2048 tg128 @ d0` labels with no table columns —
+    // it used to parse as a bogus {d:0, c:1, tg:0.0} cell that shadowed the
+    // real one in first-match lookups (glm53-flash-eugr-2x).
+    let soup = "⠋ Initializing… ━━━━━━━━━━━━━━━━━━━━━━  0/18 0:00:00⠙ Warming… pp2048 tg128 @ d0  0/18 0:00:00⠹ ━━━━ 0/18\n\
+│ pp2048 tg128 @ d0    │ c1 │   1,198 │    29.0 │    2,113 │    6,437 │ 2048+128\n\
+│ pp2048 tg128 @ d0    │ c2 │   1,287 │    35.1 │    2,148 │    8,054 │ 2048+128\n";
+    let cells = super::parse::parse_perf_box(soup);
+    assert_eq!(
+        cells,
+        vec![
+            super::PerfCell {
+                d: 0,
+                c: 1,
+                tg: 29.0
+            },
+            super::PerfCell {
+                d: 0,
+                c: 2,
+                tg: 35.1
+            },
+        ]
+    );
+    // first-match lookup must see the real speed, not 0
+    assert_eq!(
+        cells.iter().find(|p| p.d == 0 && p.c == 1).map(|p| p.tg),
+        Some(29.0)
+    );
+}

src/report/tests.rs

@@ -382,82 +382,6 @@ fn parse_bench_json_reads_scores_block() {
     assert_eq!(super::parse_bench_json(r#"{"scores": {}}"#), None);
 }
 
-#[test]
-fn parse_perf_box_reads_both_table_shapes() {
-    // modern box rows (concurrency as its own column)
-    let box_log = "\u{2502} pp2048 tg128 @ d0    \u{2502} c1 \u{2502}   1,500 \u{2502}    27.4 \u{2502}    1,398 \u{2502}    5,959 \u{2502} 2048+128\n\
-\u{2502} pp2048 tg128 @ d4096 \u{2502} c2 \u{2502}   1,642 \u{2502}    30.6 \u{2502}    6,040 \u{2502}   12,822 \u{2502} 2048+128\n";
-    assert_eq!(
-        super::parse::parse_perf_box(box_log),
-        vec![
-            super::PerfCell {
-                d: 0,
-                c: 1,
-                tg: 27.4
-            },
-            super::PerfCell {
-                d: 4096,
-                c: 2,
-                tg: 30.6
-            },
-        ]
-    );
-    // legacy markdown rows (concurrency in the label, absent = c1)
-    let md = "| pp2048 tg128 @ d0 | 1,861 | 24.5 | 1,236 | 6,228 | 2048+128 |\n\
-| pp2048 tg128 @ d0 c2 | 1,728 | 36.0 | 2,097 | 8,802 | 2048+128 |\n\
-| Test | pp t/s | tg t/s | TTFT (ms) | Total (ms) | Tokens |\n";
-    assert_eq!(
-        super::parse::parse_perf_box(md),
-        vec![
-            super::PerfCell {
-                d: 0,
-                c: 1,
-                tg: 24.5
-            },
-            super::PerfCell {
-                d: 0,
-                c: 2,
-                tg: 36.0
-            },
-        ]
-    );
-    // header chrome and prose must not produce cells
-    assert!(super::parse::parse_perf_box("pp=[2048] tg=[128] depth=[0, 4096]").is_empty());
-    assert!(super::parse::parse_perf_box("no perf here").is_empty());
-}
-
-#[test]
-fn parse_perf_box_ignores_progress_spinner_soup() {
-    // llama-benchy redraws its progress bar on one \r-overwritten line that
-    // embeds the same `pp2048 tg128 @ d0` labels with no table columns —
-    // it used to parse as a bogus {d:0, c:1, tg:0.0} cell that shadowed the
-    // real one in first-match lookups (glm53-flash-eugr-2x).
-    let soup = "⠋ Initializing… ━━━━━━━━━━━━━━━━━━━━━━  0/18 0:00:00⠙ Warming… pp2048 tg128 @ d0  0/18 0:00:00⠹ ━━━━ 0/18\n\
-│ pp2048 tg128 @ d0    │ c1 │   1,198 │    29.0 │    2,113 │    6,437 │ 2048+128\n\
-│ pp2048 tg128 @ d0    │ c2 │   1,287 │    35.1 │    2,148 │    8,054 │ 2048+128\n";
-    let cells = super::parse::parse_perf_box(soup);
-    assert_eq!(
-        cells,
-        vec![
-            super::PerfCell {
-                d: 0,
-                c: 1,
-                tg: 29.0
-            },
-            super::PerfCell {
-                d: 0,
-                c: 2,
-                tg: 35.1
-            },
-        ]
-    );
-    // first-match lookup must see the real speed, not 0
-    assert_eq!(
-        cells.iter().find(|p| p.d == 0 && p.c == 1).map(|p| p.tg),
-        Some(29.0)
-    );
-}
-
 #[test]
 fn html_page_keeps_view_state_in_the_url() {
     // the full view (filter, grouping, sort, columns, paging) must be