use super::parse::bench_complete; use super::quality::{Medals, award_medals, incomplete}; use super::render::fmt_pct; use super::serve::load_report; use super::{ReportRow, collect, detect_quant, extract_time, html_page, parse_score, slash_pair}; use crate::config::Config; use std::fs; use std::path::PathBuf; #[test] fn slash_pair_tolerates_bold_and_spaces() { assert_eq!(slash_pair("**81** / 100"), Some((81, 100))); assert_eq!( slash_pair("Score: 159/176 (90.4) — 74 pass"), Some((159, 176)) ); assert_eq!(slash_pair("no pair here"), None); } #[test] fn parse_score_handles_both_header_styles() { let bench = "- **Final Score**: **81** / 100\n- more\n"; assert_eq!(parse_score(bench), Some((81, 100, Some(81.0)))); let campaign = "intro\n## Score: 159/176 (90.4) — 74 pass / 11 partial\n"; assert_eq!(parse_score(campaign), Some((159, 176, Some(90.4)))); assert_eq!(parse_score("# no score in here\n"), None); } #[test] fn parse_score_reads_log_summary_box() { // raw tool-eval-bench log: the score box sits deep in the file let log = "│ Quality: 88/100 │\n\ │ Score: 88 / 100 │\n\ │ • Final score: (total points / max points) × 100 │\n"; assert_eq!(parse_score(log), Some((88, 100, Some(88.0)))); } #[test] fn parse_score_ignores_transcript_noise() { // embedded tool transcripts contain "score" but no leading score header let noise = "tool_result={\"relevance_score\": 0.95, \"url\": \"x/1\"}\n\ Note: my initial search returned no matches, would you like more?\n\ │ Deployability: 73/100 (α=0.7) │\n\ │ Responsiveness: 39/100 (median turn: 4.1s) │\n"; assert_eq!(parse_score(noise), None); } #[test] fn fmt_pct_whole_and_fractional() { assert_eq!(fmt_pct(81.0), "81"); assert_eq!(fmt_pct(90.4), "90.4"); } #[test] fn extract_time_reads_both_timestamp_shapes() { let md = "- **Run ID**: `2026-09-06T01-10-29.307431Z_edbd57fa`\n\ - **Date**: `2026-09-06T01:58:31.440817+00:00`\n"; assert_eq!(extract_time(md), Some("01:10".into())); assert_eq!( extract_time("x 2026-09-07T23:05:00 y\n"), Some("23:05".into()) ); assert_eq!(extract_time("# no time here\n"), None); assert_eq!(extract_time("2026-09-07 at 12:30 o'clock\n"), None); assert_eq!(extract_time("2026-09-07T12:3 short\n"), None); } #[test] fn detect_quant_prefers_longest_token() { assert_eq!(detect_quant("glm52-hybrid-fp8-mxfp4", "x"), "MXFP4"); assert_eq!(detect_quant("x", "unsloth/Qwen3.8-27B-NVFP4"), "NVFP4"); assert_eq!(detect_quant("glm53-flash-r18-4x-nvfp4kv", ""), "NVFP4KV"); assert_eq!(detect_quant("qwen38-27b-int4", ""), "INT4"); assert_eq!(detect_quant("deepseek-v4-flash-dspark38", "m"), "—"); } pub(crate) fn fixture() -> PathBuf { use std::sync::atomic::{AtomicU32, Ordering}; static N: AtomicU32 = AtomicU32::new(0); // unique per call — tests run in parallel and clean up after themselves let dir = std::env::temp_dir().join(format!( "sm121-report-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed) )); let r = dir.join("test-recipe-nvfp4"); fs::create_dir_all(r.join("reports")).unwrap(); fs::write( r.join("recipe.toml"), "engine = \"vllm\"\nname = \"Test Recipe\"\nmodel = \"org/Test-Model-NVFP4\"\nrequired_nodes = 2\n", ) .unwrap(); fs::write( r.join("reports/2026-01-02_demo-run.md"), "# demo\n## Score: 159/176 (90.4)\n", ) .unwrap(); fs::write( r.join("reports/2026-01-01_raw.log"), "noise\nrun 2026-01-01T08-30-00Z\n│ Score: 88 / 100 │\n", ) .unwrap(); fs::write( r.join("reports/2026-01-03_hardmode.md"), "# hm\n## Score: 159/176 (90.4) — 74 pass / 11 partial\n", ) .unwrap(); // a full `sm121 bench` run triplet: one bench run must yield ONE entry fs::write( r.join("reports/2026-01-04_roce-run.json"), r#"{"scores": {"final_score": 92, "total_points": 162, "max_points": 176}}"#, ) .unwrap(); fs::write( r.join("reports/2026-01-04_roce-run.md"), "- **Run ID**: `2026-01-04T11:08:03.183685Z_ef01052a`\n- **Final Score**: **92** / 100\n", ) .unwrap(); fs::write( r.join("reports/2026-01-04_roce-run.log"), "noise\n│ Score: 92 / 100 │\n\n\ │ pp2048 tg128 @ d0 │ c1 │ 1,500 │ 27.4 │ 1,398 │ 5,959 │ 2048+128\n\ │ pp2048 tg128 @ d0 │ c2 │ 1,346 │ 33.3 │ 1,999 │ 8,653 │ 2048+128\n", ) .unwrap(); dir } #[test] fn collect_carries_file_and_time() { let dir = fixture(); let config: Config = toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap(); let rows = collect(&config, false); fs::remove_dir_all(&dir).ok(); let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap(); assert_eq!(row.reports[2].file, "2026-01-03_hardmode.md"); assert_eq!(row.reports[2].time, None, "fixture .md has no timestamp"); assert_eq!( row.reports[0].time, Some("08:30".into()), "from Run ID in .log" ); } fn entry(pct: f64, done: bool) -> super::ReportEntry { serde_json::from_value(serde_json::json!({ "date": "2026-01-02", "time": null, "label": "l", "file": "f.md", "files": ["f.md"], "pct": pct, "points": null, "ratio": null, "done": done, "perf": [], })) .unwrap() } fn qrow(name: &str, parent: &str, model: &str, tp: u32, pct: Option) -> ReportRow { let reports = match pct { Some(p) => serde_json::json!([ {"file": "f.md", "files": ["f.md"], "date": "2026-01-02", "label": "l", "pct": p, "done": true} ]), None => serde_json::json!([]), }; serde_json::from_value(serde_json::json!({ "name": name, "title": name, "model": model, "parent": parent, "quant": "NVFP4", "engine": "vllm", "image": "img", "nodes": tp, "tp": tp, "deploy": "docker", "port": 30000, "reports": reports, })) .unwrap() } #[test] fn bench_complete_requires_full_pool() { // Legacy dev-era file (176-point pool, no status marker). assert!(bench_complete( r#"{"scores": {"final_score": 92, "total_points": 162, "max_points": 176, "scenario_results": {}}}"# )); // v2.7.0 pool (184 pts) — completed by coverage, no hardcode. let mut results = serde_json::Map::new(); for i in 0..92 { results.insert(format!("TC-{i}"), serde_json::json!({"pass": true})); } assert!(bench_complete( &serde_json::json!({"status": "completed", "total_scenarios": 92, "scores": {"final_score": 91, "total_points": 168, "max_points": 184, "scenario_results": results}}) .to_string() )); // Interrupted early: fewer scenarios ran than the tool declares. let partial = serde_json::Map::from_iter((0..44).map(|i| (format!("TC-{i}"), serde_json::json!({})))); assert!(!bench_complete( &serde_json::json!({"status": "completed", "total_scenarios": 92, "scores": {"final_score": 45, "total_points": 88, "max_points": 184, "scenario_results": partial}}) .to_string() )); // Explicit non-completed status always fails, coverage or not. assert!(!bench_complete( r#"{"status": "interrupted", "total_scenarios": 92, "scores": {"final_score": 90, "scenario_results": {}}}"# )); assert!(!bench_complete("not json")); } #[test] fn incomplete_flags_pool_not_finished() { assert!(!incomplete(&[])); assert!(incomplete(&[entry(0.0, false)])); assert!(!incomplete(&[entry(0.0, false), entry(50.0, true)])); } #[test] fn medals_rank_family_model_and_tp_tiers() { let mut rows = vec![ qrow("a", "M", "m1", 2, Some(90.0)), qrow("b", "M", "m1", 2, Some(94.0)), // family gold, model gold, tp(2) gold qrow("c", "M", "m1", 2, Some(88.0)), // family bronze, model bronze, tp(2) bronze qrow("d", "M", "m2", 4, Some(91.0)), // family silver, model-m2 gold, tp(4) gold qrow("e", "M", "m1", 2, None), // no score — no medal anywhere qrow("f", "N", "n1", 1, Some(50.0)), // own family/model/tp golds ]; award_medals(&mut rows); let got = |n: &str| rows.iter().find(|r| r.name == n).unwrap().medals; assert_eq!( got("b"), Medals { p: Some(1), m: Some(1), t: Some(1) } ); assert_eq!( got("a"), Medals { p: Some(3), m: Some(2), t: Some(2) } ); // family rank 4 (d=91 outranks) — no family medal, but model/tp medals hold assert_eq!( got("c"), Medals { p: None, m: Some(3), t: Some(3) } ); assert_eq!( got("d"), Medals { p: Some(2), m: Some(1), t: Some(1) }, "family silver; tp(4) own group gold" ); assert_eq!(got("e"), Medals::default()); assert_eq!( got("f"), Medals { p: Some(1), m: Some(1), t: Some(1) } ); } #[test] fn load_report_resolves_and_rejects_traversal() { let dir = fixture(); let config: Config = toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap(); let ok = load_report(&config, "test-recipe-nvfp4", "2026-01-03_hardmode.md"); fs::remove_dir_all(&dir).ok(); assert!(ok.unwrap().contains("159/176")); let dir = fixture(); let config: Config = toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap(); let toml = load_report(&config, "test-recipe-nvfp4", "recipe.toml"); fs::remove_dir_all(&dir).ok(); assert!( toml.unwrap().contains("engine = \"vllm\""), "recipe.toml served" ); let dir = fixture(); let config: Config = toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap(); fs::remove_dir_all(&dir).ok(); assert_eq!( load_report(&config, "test-recipe-nvfp4", "../../etc/passwd"), None ); assert_eq!(load_report(&config, "..", "x.md"), None); assert_eq!( load_report(&config, "test-recipe-nvfp4", "missing.md"), None ); } #[test] fn collect_reads_recipe_reports_and_dedups() { let dir = fixture(); let config: Config = toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap(); let rows = collect(&config, false); fs::remove_dir_all(&dir).ok(); let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap(); assert_eq!(row.title, "Test Recipe"); assert_eq!(row.quant, "NVFP4"); assert_eq!(row.nodes, Some(2)); assert_eq!(row.reports.len(), 4); assert_eq!(row.reports[0].label, "raw"); assert_eq!(row.reports[0].pct, Some(88.0)); assert_eq!(row.reports[0].points, None); assert_eq!(row.reports[1].label, "demo-run"); assert_eq!(row.reports[1].pct, Some(90.4)); assert_eq!(row.reports[1].points.as_deref(), Some("159/176")); assert_eq!(row.reports[1].ratio, Some(159.0 / 176.0)); assert_eq!(row.reports[2].label, "hardmode"); assert_eq!(row.reports[2].points.as_deref(), Some("159/176")); assert_eq!(row.reports[2].pct, Some(90.4)); assert_eq!(row.reports[2].ratio, Some(159.0 / 176.0)); } #[test] fn bench_triplet_groups_into_one_entry_with_json_authority() { let dir = fixture(); let config: Config = toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap(); let rows = collect(&config, false); fs::remove_dir_all(&dir).ok(); let row = rows.iter().find(|r| r.name == "test-recipe-nvfp4").unwrap(); let e = &row.reports[3]; assert_eq!(e.label, "roce-run"); // the JSON score wins over the md/log text (same value here — 92) assert_eq!(e.pct, Some(92.0)); assert_eq!(e.points.as_deref(), Some("162/176")); assert_eq!(e.ratio, Some(162.0 / 176.0)); // primary file = the human .md; all three files are listed assert_eq!(e.file, "2026-01-04_roce-run.md"); assert_eq!( e.files, vec![ "2026-01-04_roce-run.json", "2026-01-04_roce-run.log", "2026-01-04_roce-run.md" ] ); // time comes from the md Run ID (11:08) assert_eq!(e.time, Some("11:08".into())); // decode speeds parsed from the log perf box assert_eq!( e.perf, vec![ super::PerfCell { d: 0, c: 1, tg: 27.4 }, super::PerfCell { d: 0, c: 2, tg: 33.3 }, ] ); } #[test] fn parse_bench_json_reads_scores_block() { let v = r#"{"scores": {"final_score": 88, "total_points": 155, "max_points": 176}}"#; assert_eq!( super::parse_bench_json(v), Some((88.0, Some("155/176".into()), Some(155.0 / 176.0))) ); // max_points 100 → points suppressed (already normalized), pct kept let v = r#"{"scores": {"final_score": 81.0, "total_points": 81, "max_points": 100}}"#; assert_eq!(super::parse_bench_json(v), Some((81.0, None, None))); // missing block / garbage assert_eq!(super::parse_bench_json("{}"), None); assert_eq!(super::parse_bench_json("not json"), None); assert_eq!(super::parse_bench_json(r#"{"scores": {}}"#), None); } #[test] fn parse_perf_box_reads_both_table_shapes() { // modern box rows (concurrency as its own column) let box_log = "\u{2502} pp2048 tg128 @ d0 \u{2502} c1 \u{2502} 1,500 \u{2502} 27.4 \u{2502} 1,398 \u{2502} 5,959 \u{2502} 2048+128\n\ \u{2502} pp2048 tg128 @ d4096 \u{2502} c2 \u{2502} 1,642 \u{2502} 30.6 \u{2502} 6,040 \u{2502} 12,822 \u{2502} 2048+128\n"; assert_eq!( super::parse::parse_perf_box(box_log), vec![ super::PerfCell { d: 0, c: 1, tg: 27.4 }, super::PerfCell { d: 4096, c: 2, tg: 30.6 }, ] ); // legacy markdown rows (concurrency in the label, absent = c1) let md = "| pp2048 tg128 @ d0 | 1,861 | 24.5 | 1,236 | 6,228 | 2048+128 |\n\ | pp2048 tg128 @ d0 c2 | 1,728 | 36.0 | 2,097 | 8,802 | 2048+128 |\n\ | Test | pp t/s | tg t/s | TTFT (ms) | Total (ms) | Tokens |\n"; assert_eq!( super::parse::parse_perf_box(md), vec![ super::PerfCell { d: 0, c: 1, tg: 24.5 }, super::PerfCell { d: 0, c: 2, tg: 36.0 }, ] ); // header chrome and prose must not produce cells assert!(super::parse::parse_perf_box("pp=[2048] tg=[128] depth=[0, 4096]").is_empty()); assert!(super::parse::parse_perf_box("no perf here").is_empty()); } #[test] fn parse_perf_box_ignores_progress_spinner_soup() { // llama-benchy redraws its progress bar on one \r-overwritten line that // embeds the same `pp2048 tg128 @ d0` labels with no table columns — // it used to parse as a bogus {d:0, c:1, tg:0.0} cell that shadowed the // real one in first-match lookups (glm53-flash-eugr-2x). let soup = "⠋ Initializing… ━━━━━━━━━━━━━━━━━━━━━━ 0/18 0:00:00⠙ Warming… pp2048 tg128 @ d0 0/18 0:00:00⠹ ━━━━ 0/18\n\ │ pp2048 tg128 @ d0 │ c1 │ 1,198 │ 29.0 │ 2,113 │ 6,437 │ 2048+128\n\ │ pp2048 tg128 @ d0 │ c2 │ 1,287 │ 35.1 │ 2,148 │ 8,054 │ 2048+128\n"; let cells = super::parse::parse_perf_box(soup); assert_eq!( cells, vec![ super::PerfCell { d: 0, c: 1, tg: 29.0 }, super::PerfCell { d: 0, c: 2, tg: 35.1 }, ] ); // first-match lookup must see the real speed, not 0 assert_eq!( cells.iter().find(|p| p.d == 0 && p.c == 1).map(|p| p.tg), Some(29.0) ); } #[test] fn html_page_keeps_view_state_in_the_url() { // the full view (filter, grouping, sort, columns, paging) must be // deep-link-backed: parse from the query string on load, and // replaceState it on every render let page = html_page(&[]); assert!(page.contains("new URLSearchParams(location.search)")); assert!(page.contains("history.replaceState")); assert!(page.contains("syncUrl();")); } #[test] fn report_keys_fall_back_outside_en() { let _ = fluent_i18n::set_locale(Some("ru")); let s = fluent_i18n::t!("report-title", { "count" => 3 }); let _ = fluent_i18n::set_locale(None); assert!(s.contains('3'), "fallback broken, got: {s}"); assert_ne!(s, "report-title", "missing fluent fallback, got: {s}"); } #[tokio::test] async fn refresher_publishes_on_change_only() { use super::serve::refresh_loop; use std::time::Duration; let dir = fixture(); let config: Config = toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap(); let cfg = std::sync::Arc::new(config); let (tx, mut rx) = tokio::sync::watch::channel(String::new()); // keepalive 100 ms: if the file watcher fires, the update lands via the // event path; if not, the keepalive picks it up — either way within 3 s let h = tokio::spawn(refresh_loop(cfg, tx, Duration::from_millis(100), false)); // the first collect runs immediately: the fixture's report is visible tokio::time::timeout( Duration::from_secs(3), rx.wait_for(|j| j.contains("demo-run")), ) .await .expect("initial collect not published") .expect("channel closed"); // a new report file appears → the next tick publishes the fresh JSON fs::write( dir.join("test-recipe-nvfp4/reports/2026-01-05_live-update.md"), "# lu\n## Score: 159/176 (90.4)\n", ) .unwrap(); tokio::time::timeout( Duration::from_secs(3), rx.wait_for(|j| j.contains("live-update")), ) .await .expect("change not published within 3 s") .expect("channel closed"); // stability: with no further changes the JSON stays put (no re-publish // storm on a quiet tree — the SSE side only wakes on watch changes) let stable = rx.borrow().clone(); tokio::time::sleep(Duration::from_millis(150)).await; assert_eq!(*rx.borrow(), stable, "unchanged tree must not re-publish"); h.abort(); fs::remove_dir_all(&dir).ok(); }