bench.rs 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487
  1. //! `sm121 bench` — run `tool-eval-bench` against a live serve and land the
  2. //! reports in `recipes/<lane>/reports/`. The tool needs a scratch cwd (it
  3. //! writes `runs/<date>/<run>.md` relative to cwd), an absolute `--json-file`
  4. //! path, and the `.md` copied into the lane — this command does the
  5. //! babysitting. Campaign convention (seed 42, hardmode, perf, c1,2) is the
  6. //! default; `sm121 bench -d` prints the exact command.
  7. use std::fs;
  8. use std::path::PathBuf;
  9. use anyhow::Result;
  10. use crate::bench_target::{pick_head, run_tool, sanitize, served_model};
  11. use crate::config::{Config, load_recipe, select_cluster_by_name};
  12. /// Fields of the `sm121 bench` subcommand (mirrored by the clap variant in
  13. /// `cli.rs`). Served name/port come from the recipe; the tool is on PATH.
  14. #[derive(Clone)]
  15. pub struct Args {
  16. pub recipe: String,
  17. pub cluster: String,
  18. pub label: Option<String>,
  19. pub concurrency: String,
  20. pub seed: u64,
  21. /// Scenario streams run concurrently (tool `--parallel`; None = serial).
  22. pub parallel: Option<u32>,
  23. /// Skip the scenario phase (tool `--perf-only`).
  24. pub perf_only: bool,
  25. /// Run the context-depth suite (16K..256K) after the main pass — off by
  26. /// default (a full sweep takes ~45 min on a 2x lane).
  27. pub depth: bool,
  28. /// Local tokenizer source for llama-benchy (tool `--tokenizer`): a
  29. /// tokenizer.json or a dir holding one. Needed when the served model
  30. /// id is not resolvable in the local HF cache (custom export lanes).
  31. pub tokenizer: Option<String>,
  32. /// When set, the tool-eval-bench invocation runs on this host over SSH
  33. /// (the local machine only orchestrates), keeping the bench client's own
  34. /// network (e.g. a WiFi laptop) off the measurement path. The tokenizer
  35. /// is uploaded to the host; the report + run package land locally. The
  36. /// host must have `tool-eval-bench` on PATH and reach the serve.
  37. pub ssh_host: Option<String>,
  38. /// Bench target node within the cluster (default: rank 0). Use when the
  39. /// serve was launched with `serve --node <n>` — a pinned/solo run may
  40. /// live on a non-rank-0 node, and benching the wrong head measures a
  41. /// different (or no) engine.
  42. pub node: Option<String>,
  43. /// Disable thinking/reasoning in the harness (tool `--no-think`, sets
  44. /// `enable_thinking=false`). Needed for thinking-by-default models (Ling,
  45. /// GLM-4.5-style) whose long default reasoning blows the per-turn timeout
  46. /// and excludes most scenarios.
  47. pub no_think: bool,
  48. /// Per-turn request timeout seconds forwarded to the harness (tool
  49. /// `--timeout`). Raise above the 120 s default for slow/thinking models.
  50. pub timeout: Option<u64>,
  51. /// Job cancellation flag (web bench window). `run_tool` polls it and
  52. /// kills the harness child; None for the CLI.
  53. pub cancel: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
  54. }
  55. /// Served model name from the recipe's `--served-model-name` arg.
  56. /// Run the bench and land `recipes/<lane>/reports/<stamp>_<label>.{md,json,log}`.
  57. pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
  58. let recipe = load_recipe(&a.recipe, &config.recipe_sources).await?;
  59. let file = crate::config::recipe_file(&a.recipe, &config.recipe_sources)
  60. .ok_or_else(|| anyhow::anyhow!("recipe file not found for '{}'", a.recipe))?
  61. .canonicalize()?;
  62. let cluster = select_cluster_by_name(config, &a.cluster)?;
  63. let head = pick_head(&cluster.nodes, &cluster.name, a.node.as_deref())?;
  64. // FLARE serves on its own fixed API port (`[flare].http_port`), not the
  65. // cluster default the docker lanes use.
  66. let port = if recipe.engine == "flare" {
  67. crate::native::flare::http_port(&recipe)
  68. } else {
  69. recipe.port
  70. };
  71. let model = served_model(&recipe).ok_or_else(|| {
  72. anyhow::anyhow!(
  73. "recipe '{}' has no --served-model-name in args — add it to the recipe",
  74. a.recipe
  75. )
  76. })?;
  77. let base_url = format!("http://{}:{}/v1", head.ssh_ip, port);
  78. let stamp = crate::stamp::utc_stamp();
  79. // Label: required for quality runs, optional for perf-only (default `perf`).
  80. let label = match (&a.label, a.perf_only) {
  81. (Some(l), _) => sanitize(l),
  82. (None, true) => String::from("perf"),
  83. (None, false) => {
  84. anyhow::bail!("--label is required for a quality run (the campaign record)")
  85. }
  86. };
  87. let stem = format!("{stamp}_{label}");
  88. let reports_dir = file
  89. .parent()
  90. .ok_or_else(|| anyhow::anyhow!("no parent dir for {}", file.display()))?
  91. .join("reports");
  92. let log_path = reports_dir.join(format!("{stamp}_{label}.log"));
  93. let ssh_host = a.ssh_host.as_deref();
  94. // The local scratch dir (the report reads the md + json from here). For
  95. // the remote host the tool writes to a remote scratch dir; its md/json are
  96. // downloaded here after the run so the local report generation is unchanged.
  97. let scratch = dirs::home_dir()
  98. .ok_or_else(|| anyhow::anyhow!("no home dir"))?
  99. .join(".cache/sm121/bench")
  100. .join(&stem);
  101. // The scratch dir the tool writes to (local by default, or the remote host
  102. // when `--ssh-host` is set — the bench client's network then stays off the
  103. // measurement path).
  104. let tool_scratch = if ssh_host.is_some() {
  105. PathBuf::from(format!("/tmp/sm121-bench-{stem}"))
  106. } else {
  107. scratch.clone()
  108. };
  109. // The local json path (the report reads it).
  110. let json_path = reports_dir.join(format!("{stamp}_{label}.json"));
  111. // The json path the tool writes to (local by default, or the remote host).
  112. let tool_json = if ssh_host.is_some() {
  113. tool_scratch.join(format!("{stem}.json"))
  114. } else {
  115. json_path.clone()
  116. };
  117. let bin = "tool-eval-bench".to_string();
  118. // The tokenizer path the tool uses (local by default, or the remote
  119. // scratch dir when `--ssh-host` is set — the tokenizer is uploaded there).
  120. let tool_tokenizer = if ssh_host.is_some() {
  121. a.tokenizer.as_ref().map(|tok| {
  122. let name = tok.rsplit('/').next().unwrap_or("tokenizer");
  123. tool_scratch.join(name).to_string_lossy().to_string()
  124. })
  125. } else {
  126. a.tokenizer.clone()
  127. };
  128. let depth_scratch = tool_scratch.with_file_name(format!("{stem}_depth"));
  129. let depth_args =
  130. crate::bench_depth::depth_args(&base_url, &model, a.seed, tool_tokenizer.as_ref());
  131. let mut args = vec![
  132. "--base-url".into(),
  133. base_url.clone(),
  134. "--model".into(),
  135. model.clone(),
  136. "--seed".into(),
  137. a.seed.to_string(),
  138. "--hardmode".into(),
  139. "--perf".into(),
  140. // Warm-prefix decode: without it llama-benchy's default branch
  141. // serializes dual 67K prefills (c2 collapses to 13.7 t/s — an
  142. // admission artifact, not the engine; 2026-09-29 diagnosis).
  143. "--benchy-args=--enable-prefix-caching".into(),
  144. "--concurrency".into(),
  145. a.concurrency.clone(),
  146. "--json-file".into(),
  147. tool_json.to_string_lossy().to_string(),
  148. ];
  149. if let Some(p) = a.parallel {
  150. args.push("--parallel".into());
  151. args.push(p.to_string());
  152. }
  153. if a.perf_only {
  154. args.push("--perf-only".into());
  155. }
  156. if let Some(tok) = &tool_tokenizer {
  157. args.push("--tokenizer".into());
  158. args.push(tok.clone());
  159. }
  160. if a.no_think {
  161. args.push("--no-think".into());
  162. }
  163. if let Some(t) = a.timeout {
  164. args.push("--timeout".into());
  165. args.push(t.to_string());
  166. }
  167. if dry_run {
  168. println!("bench: cwd {}", tool_scratch.display());
  169. println!("bench: {} {}", bin, args.join(" "));
  170. println!("bench: depth suite: {} {}", bin, depth_args.join(" "));
  171. println!(
  172. "bench: reports -> {} (run package: {}.recipe.toml + {}.manifest.json)",
  173. reports_dir.display(),
  174. stem,
  175. stem
  176. );
  177. return Ok(());
  178. }
  179. fs::create_dir_all(&reports_dir)?;
  180. // One truncate per run (run_tool appends, so its output and the notes
  181. // below interleave in file order instead of clobbering each other).
  182. fs::write(&log_path, "")?;
  183. let n = |m: String| crate::bench_io::note(&log_path, &m);
  184. n(format!(
  185. "bench: {} · seed {} · concurrency {} · model {} · target {}",
  186. stem, a.seed, a.concurrency, model, base_url
  187. ));
  188. if let Some(host) = ssh_host {
  189. n(format!(
  190. "bench: harness host {host} ({})",
  191. tool_scratch.display()
  192. ));
  193. crate::bench_remote::prepare(host, &tool_scratch, &a.tokenizer)?;
  194. } else {
  195. n(format!(
  196. "bench: harness host (local) {bin} in {}",
  197. tool_scratch.display()
  198. ));
  199. fs::create_dir_all(&scratch)?;
  200. }
  201. n(format!("bench: waiting for {base_url} ..."));
  202. crate::readiness::wait_ready(&base_url, crate::readiness::BENCH_WAIT_SECS).await?;
  203. n(format!(
  204. "bench: API ready — running {bin} (seed {})",
  205. a.seed
  206. ));
  207. let code = run_tool(
  208. &bin,
  209. &args,
  210. &tool_scratch,
  211. &log_path,
  212. ssh_host,
  213. a.cancel.as_ref(),
  214. )?;
  215. if code != 0 {
  216. if a.cancel
  217. .as_ref()
  218. .is_some_and(|c| c.load(std::sync::atomic::Ordering::Relaxed))
  219. {
  220. anyhow::bail!("bench: cancelled");
  221. }
  222. anyhow::bail!(
  223. "bench: tool exited with {code} (log: {})",
  224. log_path.display()
  225. );
  226. }
  227. // Phase 2 — context-depth suite (16K..256K @ c2, 2 repeats): appended
  228. // to the same log so `parse_perf_rows` picks its rows up too. A depth
  229. // failure must not cost the finished quality run — warn, keep going.
  230. // Opt-in (`--depth`): a full sweep takes ~45 min on a 2x lane.
  231. if a.depth {
  232. n(format!(
  233. "bench: depth suite: {} @ c{}, {} repeats{}",
  234. crate::bench_depth::DEPTHS,
  235. crate::bench_depth::CONCURRENCY,
  236. crate::bench_depth::RUNS,
  237. if a.tokenizer.is_some() {
  238. " (+tokenizer)"
  239. } else {
  240. ""
  241. }
  242. ));
  243. fs::create_dir_all(&depth_scratch)?;
  244. let dcode = run_tool(
  245. &bin,
  246. &depth_args,
  247. &depth_scratch,
  248. &log_path,
  249. ssh_host,
  250. a.cancel.as_ref(),
  251. )?;
  252. if dcode != 0 {
  253. if a.cancel
  254. .as_ref()
  255. .is_some_and(|c| c.load(std::sync::atomic::Ordering::Relaxed))
  256. {
  257. anyhow::bail!("bench: cancelled in depth suite");
  258. }
  259. eprintln!(
  260. "bench: WARNING depth suite exited with {dcode} — main report unaffected ({})",
  261. log_path.display()
  262. );
  263. }
  264. } else {
  265. n(String::from("bench: depth suite skipped (opt-in: --depth)"));
  266. }
  267. // Remote host: download the json + the md from the remote scratch dir so
  268. // the local report generation is unchanged.
  269. if let Some(host) = ssh_host {
  270. crate::bench_remote::collect(
  271. host,
  272. &tool_json,
  273. &json_path,
  274. &tool_scratch,
  275. &scratch,
  276. !a.perf_only,
  277. )?;
  278. }
  279. let md_src = crate::bench_io::latest_md(&scratch)
  280. .ok_or_else(|| anyhow::anyhow!("no report .md under {}", scratch.join("runs").display()))?;
  281. let md_dest = reports_dir.join(format!("{stamp}_{label}.md"));
  282. fs::copy(&md_src, &md_dest)?;
  283. // The `--json-file` dispatch path drops the perf section from the md —
  284. // re-attach it from the llama-benchy table in the log.
  285. if crate::bench_report::ensure_throughput_section(&md_dest, &log_path).is_ok_and(|m| m) {
  286. eprintln!("bench: appended the perf table to the report md");
  287. }
  288. // Summary: final score from the JSON + the perf table from the log.
  289. // Perf-only runs have no quality score and the tool writes no JSON —
  290. // the run package is skipped for them (the campaign run is the record).
  291. let j = match a.perf_only {
  292. true => {
  293. eprintln!("bench: perf-only run (no quality score, no run package)");
  294. None
  295. }
  296. false => {
  297. let j: serde_json::Value = serde_json::from_str(&fs::read_to_string(&json_path)?)?;
  298. let sc = &j["scores"];
  299. let final_score = sc["final_score"].as_u64();
  300. if let Some(score) = final_score {
  301. let completion = sc["completion_rate"]
  302. .as_f64()
  303. .map(|v| v.to_string())
  304. .unwrap_or_default();
  305. eprintln!(
  306. "bench: final score {score} / {} ({} points), completion {completion}",
  307. sc["max_points"], sc["total_points"]
  308. );
  309. } else {
  310. eprintln!("bench: perf-only run (no quality score)");
  311. }
  312. Some(j)
  313. }
  314. };
  315. for line in fs::read_to_string(&log_path)?.lines() {
  316. if line.contains("pp2048") && line.contains("tg128") {
  317. println!("{line}");
  318. }
  319. }
  320. if let Some(j) = &j {
  321. crate::bench_artifacts::write_run_package(&crate::bench_artifacts::RunPackage {
  322. config,
  323. cluster: &cluster,
  324. recipe: &recipe,
  325. recipe_file: &file,
  326. stem: stem.as_str(),
  327. reports_dir: &reports_dir,
  328. bench_args: &args,
  329. bench_json: j,
  330. })
  331. .await?;
  332. }
  333. n(format!(
  334. "bench: reports landed in {}",
  335. reports_dir.display()
  336. ));
  337. Ok(())
  338. }
  339. #[cfg(test)]
  340. mod tests {
  341. use super::*;
  342. use crate::config::{Node, Recipe};
  343. use std::process::{Command, Stdio};
  344. use std::sync::{Arc, Mutex};
  345. fn node(name: &str, rank: u32) -> Node {
  346. Node {
  347. name: name.to_string(),
  348. ssh_ip: "192.0.2.1".into(),
  349. ib_ip: None,
  350. rank,
  351. host_key: None,
  352. entity: None,
  353. }
  354. }
  355. #[test]
  356. fn pick_head_defaults_to_rank0() {
  357. let nodes = vec![node("a.lan", 2), node("b.lan", 0), node("c.lan", 1)];
  358. assert_eq!(pick_head(&nodes, "c", None).unwrap().name, "b.lan");
  359. }
  360. #[test]
  361. fn pick_head_node_override_tolerates_lan_suffix() {
  362. let nodes = vec![node("a.lan", 2), node("b.lan", 0)];
  363. assert_eq!(pick_head(&nodes, "c", Some("a")).unwrap().name, "a.lan");
  364. assert_eq!(pick_head(&nodes, "c", Some("b.lan")).unwrap().name, "b.lan");
  365. assert!(pick_head(&nodes, "c", Some("zzz")).is_err());
  366. }
  367. /// Regression: the pump must not treat a not-yet-written pipe as EOF.
  368. /// `read()` into a zero-length buffer returns 0, which the old pump
  369. /// treated as EOF — closing the pipe and killing the child on its first
  370. /// stderr write with EPIPE (silent exit 120 in CPython).
  371. #[test]
  372. fn pump_keeps_pipe_open_until_child_exits() {
  373. let tmp =
  374. std::env::temp_dir().join(format!("sm121-bench-pump-test-{}", std::process::id()));
  375. fs::create_dir_all(&tmp).unwrap();
  376. let log = tmp.join("pump.log");
  377. let mut child = Command::new("sh")
  378. .args(["-c", "sleep 0.3; echo late-output >&2"])
  379. .stdout(Stdio::piped())
  380. .stderr(Stdio::piped())
  381. .spawn()
  382. .unwrap();
  383. let out = child.stdout.take().unwrap();
  384. let err = child.stderr.take().unwrap();
  385. let f = Arc::new(Mutex::new(fs::File::create(&log).unwrap()));
  386. let f1 = f.clone();
  387. let t1 = std::thread::spawn(move || crate::bench_io::pump(out, false, f1));
  388. let t2 = std::thread::spawn(move || crate::bench_io::pump(err, false, f));
  389. let status = child.wait().unwrap();
  390. t1.join().unwrap();
  391. t2.join().unwrap();
  392. let body = fs::read_to_string(&log).unwrap();
  393. assert_eq!(status.code(), Some(0), "child must not hit EPIPE: {body}");
  394. assert!(body.contains("late-output"), "piped stderr lost: {body:?}");
  395. let _ = fs::remove_dir_all(&tmp);
  396. }
  397. #[test]
  398. fn sanitize_replaces_bad_chars() {
  399. assert_eq!(sanitize("arm B: 2x"), "arm-B--2x");
  400. assert_eq!(sanitize("ok_name-2.v3"), "ok_name-2.v3");
  401. }
  402. /// The non-interactive env must reach the child through the `script`
  403. /// pty wrapper — rich reads TTY_COMPATIBLE from the child's env. Also:
  404. /// `stty -onlcr` must hold, so the saved log is LF-only (CRLF made the
  405. /// report viewer render every line twice).
  406. #[test]
  407. fn run_tool_passes_non_interactive_env() {
  408. let tmp = std::env::temp_dir().join(format!("sm121-bench-env-test-{}", std::process::id()));
  409. fs::create_dir_all(&tmp).unwrap();
  410. let log = tmp.join("env.log");
  411. let code = run_tool(
  412. "sh",
  413. &[
  414. "-c".to_string(),
  415. "echo TTY=$TTY_COMPATIBLE NO=$NO_COLOR".to_string(),
  416. ],
  417. &tmp,
  418. &log,
  419. None,
  420. None,
  421. )
  422. .unwrap();
  423. let body = fs::read_to_string(&log).unwrap();
  424. let _ = fs::remove_dir_all(&tmp);
  425. assert_eq!(code, 0, "exit code, log: {body:?}");
  426. assert!(body.contains("TTY=0"), "TTY_COMPATIBLE lost: {body:?}");
  427. assert!(body.contains("NO=1"), "NO_COLOR lost: {body:?}");
  428. assert!(!body.contains('\r'), "pty ONLCR leaked CRLF: {body:?}");
  429. }
  430. /// The depth suite shares the main phase's log (one report, both tables):
  431. /// appending keeps the main rows instead of replacing them.
  432. #[test]
  433. fn run_tool_append_keeps_prior_output() {
  434. let tmp =
  435. std::env::temp_dir().join(format!("sm121-bench-append-test-{}", std::process::id()));
  436. fs::create_dir_all(&tmp).unwrap();
  437. let log = tmp.join("append.log");
  438. // run_tool always appends; the caller owns the one truncate.
  439. fs::write(&log, "").unwrap();
  440. let run =
  441. |a: &str| run_tool("sh", &["-c".into(), a.into()], &tmp, &log, None, None).unwrap();
  442. assert_eq!(run("echo main"), 0);
  443. assert!(fs::read_to_string(&log).unwrap().contains("main\n"));
  444. assert_eq!(run("echo depth"), 0);
  445. let body = fs::read_to_string(&log).unwrap();
  446. let (i1, i2) = (body.find("main\n"), body.find("depth\n"));
  447. assert!(
  448. i1.is_some() && i2.is_some_and(|d| i1.unwrap() < d),
  449. "append truncated the log: {body:?}"
  450. );
  451. let _ = fs::remove_dir_all(&tmp);
  452. }
  453. #[test]
  454. fn served_model_parsed_from_args() {
  455. let content = "name = \"t\"\ndocker_image = \"x:1\"\nmodel = \"org/m\"\nargs = [\"--served-model-name\", \"glm53\"]\n";
  456. let r: Recipe = toml::from_str(content).expect("recipe toml");
  457. assert_eq!(served_model(&r).as_deref(), Some("glm53"));
  458. let mut r2 = r.clone();
  459. r2.args = crate::pre_serve::RecipeArgs::List(vec!["--trust-remote-code".into()]);
  460. assert!(served_model(&r2).is_none());
  461. }
  462. }