//! Depth-suite phase of `sm121 bench`: a second, scenario-free perf pass //! that maps decode throughput across context depths. //! //! The campaign perf phase covers 0/8K (with the quality run); long-context //! decode is a lane-defining characteristic (vLLM vs sglang flattening the //! 256K cliff) and used to need a manual slope sweep to observe. The suite //! runs it at both stream counts with 2 repeats per point — operator //! convention 2026-09-29: "16K/64K/128K/256K at c2, 2 runs"; extended the //! same day after the first suites came back c2-only: c1 is the reference //! (stays flat) and shows what the c2 collapse costs. /// Context depths of the suite (0/8K excluded — covered by the main run). pub const DEPTHS: &str = "16384,65536,131072,262144"; /// Stream counts: the same pair the campaign perf phase records (c1 baseline /// + c2 — the long-context collapse only shows at c2, so both are needed). pub const CONCURRENCY: &str = "1,2"; /// Repeats per depth point (kept at 2 to bound the wall time). pub const RUNS: &str = "2"; /// `tool-eval-bench` args for the depth sweep (perf-only, no scenarios). /// /// pp/tg match the campaign convention so the rows land under the same /// `pp2048 tg128 @ dNN` names the report's Throughput table already parses. pub fn depth_args( base_url: &str, model: &str, seed: u64, tokenizer: Option<&String>, ) -> Vec { let mut args = vec![ "--base-url".into(), base_url.to_string(), "--model".into(), model.to_string(), "--seed".into(), seed.to_string(), "--perf-only".into(), "--skip-tool-eval".into(), "--pp".into(), "2048".into(), "--tg".into(), "128".into(), "--depth".into(), DEPTHS.to_string(), "--concurrency".into(), CONCURRENCY.to_string(), "--benchy-runs".into(), RUNS.to_string(), ]; if let Some(tok) = tokenizer { args.push("--tokenizer".into()); args.push(tok.clone()); } args } #[cfg(test)] mod tests { use super::*; /// The suite must be scenario-free and pinned to the campaign pp/tg — /// rows named differently would not match the report's perf parser. #[test] fn depth_args_are_perf_only_with_campaign_pp_tg() { let a = depth_args("http://h:3000/v1", "m", 42, None); let joined = a.join(" "); assert!(joined.contains("--perf-only"), "{joined}"); assert!(joined.contains("--skip-tool-eval"), "{joined}"); assert!(joined.contains("--pp 2048"), "{joined}"); assert!(joined.contains("--tg 128"), "{joined}"); assert!(joined.contains(&format!("--depth {DEPTHS}")), "{joined}"); assert!( joined.contains(&format!("--concurrency {CONCURRENCY}")), "{joined}" ); assert!( joined.contains(&format!("--benchy-runs {RUNS}")), "{joined}" ); assert!(!joined.contains("--tokenizer"), "{joined}"); } #[test] fn depths_exclude_the_covered_points() { let depths: Vec = DEPTHS.split(',').map(|d| d.parse().unwrap()).collect(); assert_eq!(depths, vec![16384, 65536, 131072, 262144]); assert!(!depths.contains(&0) && !depths.contains(&8192)); } #[test] fn tokenizer_forwarded_when_given() { let a = depth_args("u", "m", 42, Some(&"/tmp/tok".to_string())); assert!(a.contains(&"--tokenizer".to_string())); assert!(a.contains(&"/tmp/tok".to_string())); } }