launcher/src/bench_depth.rs

//! Depth-suite phase of `sm121 bench`: a second, scenario-free perf pass
//! that maps decode throughput across context depths.
//!
//! The campaign perf phase covers 0/8K (with the quality run); long-context
//! decode is a lane-defining characteristic (vLLM vs sglang flattening the
//! 256K cliff) and used to need a manual slope sweep to observe. The suite
//! runs it at both stream counts with 2 repeats per point — operator
//! convention 2026-09-29: "16K/64K/128K/256K at c2, 2 runs"; extended the
//! same day after the first suites came back c2-only: c1 is the reference
//! (stays flat) and shows what the c2 collapse costs.

/// Context depths of the suite (0/8K excluded — covered by the main run).
pub const DEPTHS: &str = "16384,65536,131072,262144";
/// Stream counts: the same pair the campaign perf phase records (c1 baseline
/// + c2 — the long-context collapse only shows at c2, so both are needed).
pub const CONCURRENCY: &str = "1,2";
/// Repeats per depth point (kept at 2 to bound the wall time).
pub const RUNS: &str = "2";

/// `tool-eval-bench` args for the depth sweep (perf-only, no scenarios).
///
/// pp/tg match the campaign convention so the rows land under the same
/// `pp2048 tg128 @ dNN` names the report's Throughput table already parses.
pub fn depth_args(
    base_url: &str,
    model: &str,
    seed: u64,
    tokenizer: Option<&String>,
) -> Vec<String> {
    let mut args = vec![
        "--base-url".into(),
        base_url.to_string(),
        "--model".into(),
        model.to_string(),
        "--seed".into(),
        seed.to_string(),
        "--perf-only".into(),
        "--skip-tool-eval".into(),
        "--pp".into(),
        "2048".into(),
        "--tg".into(),
        "128".into(),
        "--depth".into(),
        DEPTHS.to_string(),
        "--concurrency".into(),
        CONCURRENCY.to_string(),
        "--benchy-runs".into(),
        RUNS.to_string(),
    ];
    if let Some(tok) = tokenizer {
        args.push("--tokenizer".into());
        args.push(tok.clone());
    }
    args
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The suite must be scenario-free and pinned to the campaign pp/tg —
    /// rows named differently would not match the report's perf parser.
    #[test]
    fn depth_args_are_perf_only_with_campaign_pp_tg() {
        let a = depth_args("http://h:3000/v1", "m", 42, None);
        let joined = a.join(" ");
        assert!(joined.contains("--perf-only"), "{joined}");
        assert!(joined.contains("--skip-tool-eval"), "{joined}");
        assert!(joined.contains("--pp 2048"), "{joined}");
        assert!(joined.contains("--tg 128"), "{joined}");
        assert!(joined.contains(&format!("--depth {DEPTHS}")), "{joined}");
        assert!(
            joined.contains(&format!("--concurrency {CONCURRENCY}")),
            "{joined}"
        );
        assert!(
            joined.contains(&format!("--benchy-runs {RUNS}")),
            "{joined}"
        );
        assert!(!joined.contains("--tokenizer"), "{joined}");
    }

    #[test]
    fn depths_exclude_the_covered_points() {
        let depths: Vec<u32> = DEPTHS.split(',').map(|d| d.parse().unwrap()).collect();
        assert_eq!(depths, vec![16384, 65536, 131072, 262144]);
        assert!(!depths.contains(&0) && !depths.contains(&8192));
    }

    #[test]
    fn tokenizer_forwarded_when_given() {
        let a = depth_args("u", "m", 42, Some(&"/tmp/tok".to_string()));
        assert!(a.contains(&"--tokenizer".to_string()));
        assert!(a.contains(&"/tmp/tok".to_string()));
    }
}