ui_tests.rs 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300
  1. // Report page UI tests: the live-update bell (SSE → diff → badge → panel).
  2. // Split from tests.rs at the 500-line cap (same split as web: tests + ui);
  3. // shared helpers (fixture) live in tests.rs.
  4. use super::tests::fixture;
  5. use super::{ReportRow, collect, html_page, markdown, serve};
  6. use crate::config::Config;
  7. use std::fs;
  8. #[test]
  9. fn page_has_bell_and_notif_panel() {
  10. let page = html_page(&[]);
  11. for needle in [
  12. "id=\"bell\"",
  13. "id=\"bellcount\"",
  14. "id=\"notifbox\"",
  15. "notifDiff(DATA, rows)",
  16. "auth(\"/events\")",
  17. "NOTIFDIFF:START",
  18. ] {
  19. assert!(page.contains(needle), "page missing {needle}");
  20. }
  21. }
  22. /// The stale branch of the state cell: an open ledger run the live probe
  23. /// contradicts renders orange with the `sm121 runlock close` hint.
  24. #[test]
  25. fn page_renders_stale_state() {
  26. let page = html_page(&[]);
  27. for needle in ["r.stale", "sm121 runlock close", "stale"] {
  28. assert!(page.contains(needle), "page missing {needle}");
  29. }
  30. }
  31. /// The `notifDiff` JS function (extracted between markers from the live
  32. /// page) must classify every update kind and order newest first — probed
  33. /// under node, like the web dashboard's UI probes.
  34. #[test]
  35. fn page_notif_diff_js() {
  36. if std::process::Command::new("node")
  37. .arg("--version")
  38. .output()
  39. .is_err()
  40. {
  41. eprintln!("SKIP page_notif_diff_js: node not available");
  42. return;
  43. }
  44. let page = html_page(&[]);
  45. let s = page.find("/*NOTIFDIFF:START*/").unwrap();
  46. let e = page.find("/*NOTIFDIFF:END*/").unwrap();
  47. let fnjs = &page[s + "/*NOTIFDIFF:START*/".len()..e];
  48. let probe = format!(
  49. "{fn}\n\
  50. const rep = (n, l, date, pct) => ({{ date: date, time: null, label: l, \
  51. file: date + \"_\" + l + \".md\", files: [date + \"_\" + l + \".md\"], \
  52. pct: pct, points: null, ratio: null, perf: [] }});
  53. const row = (n, model, image, reports) => ({{ name: n, title: n, model: model, \
  54. quant: \"NVFP4\", engine: \"vllm\", image: image, nodes: 1, tp: 1, \
  55. deploy: \"docker\", port: 30000, reports: reports }});
  56. const a0 = row(\"r-a\", \"m/A\", \"img:1\", [rep(\"r-a\", \"old\", \"2026-01-01\", 90.0)]);
  57. // 1: identical snapshots → empty
  58. if (JSON.stringify(notifDiff([a0], [a0])) !== \"[]\")
  59. throw new Error(\"identical must diff empty: \" + JSON.stringify(notifDiff([a0], [a0])));
  60. // 2: image change + new bench + new recipe
  61. const a1 = row(\"r-a\", \"m/A\", \"img:2\", [rep(\"r-a\", \"old\", \"2026-01-01\", 90.0), rep(\"r-a\", \"new\", \"2026-01-02\", 92.0)]);
  62. const b0 = row(\"r-b\", \"m/B\", \"img:3\", []);
  63. const d = notifDiff([a0], [a1, b0]);
  64. if (d.map(x => x.t).sort().join(\",\") !== \"bench,config,recipe\")
  65. throw new Error(\"kinds: \" + JSON.stringify(d));
  66. const bi = d.find(x => x.t === \"bench\");
  67. if (bi.name !== \"r-a\" || bi.label !== \"new\" || bi.score !== 92.0)
  68. throw new Error(\"bench item: \" + JSON.stringify(bi));
  69. if (bi.files[0] !== \"2026-01-02_new.md\") throw new Error(\"bench files: \" + JSON.stringify(bi));
  70. if (d.find(x => x.t === \"config\").name !== \"r-a\") throw new Error(\"config item\");
  71. const ri = d.find(x => x.t === \"recipe\");
  72. if (ri.name !== \"r-b\" || ri.label !== \"m/B\") throw new Error(\"recipe item: \" + JSON.stringify(ri));
  73. // 3: entry value change → change
  74. const a2 = row(\"r-a\", \"m/A\", \"img:1\", [rep(\"r-a\", \"old\", \"2026-01-01\", 95.0)]);
  75. const d2 = notifDiff([a0], [a2]);
  76. if (d2.length !== 1 || d2[0].t !== \"change\")
  77. throw new Error(\"expected one change: \" + JSON.stringify(d2));
  78. // 4: dropped row → removed
  79. const d3 = notifDiff([a1, b0], [a1]);
  80. if (d3.length !== 1 || d3[0].t !== \"removed\" || d3[0].name !== \"r-b\")
  81. throw new Error(\"expected removed r-b: \" + JSON.stringify(d3));
  82. // 5: newest first — the new bench must precede the config note
  83. if (d.findIndex(x => x.t === \"bench\") > d.findIndex(x => x.t === \"config\"))
  84. throw new Error(\"newest-first violated\");
  85. console.log(\"ok\");", fn = fnjs
  86. );
  87. let js = std::env::temp_dir().join(format!("sm121-report-notif-{}.js", std::process::id()));
  88. fs::write(&js, &probe).unwrap();
  89. let out = std::process::Command::new("node")
  90. .arg(&js)
  91. .output()
  92. .unwrap();
  93. let _ = fs::remove_file(&js);
  94. assert!(
  95. out.status.success(),
  96. "notifDiff probe failed: {}",
  97. String::from_utf8_lossy(&out.stderr)
  98. );
  99. }
  100. /// Click-through checks for the live-update bell: boots the real report
  101. /// router on an ephemeral port (fixture recipe dir), drives page.html with
  102. /// Playwright (deno, npm-free), drops a new report file into the watched
  103. /// dir and expects the badge + panel to react. Skipped without deno/browser.
  104. #[tokio::test]
  105. async fn report_playwright_checks() {
  106. if std::process::Command::new("deno")
  107. .arg("--version")
  108. .output()
  109. .is_err()
  110. {
  111. eprintln!("SKIP report_playwright_checks: deno not available");
  112. return;
  113. }
  114. let dir = fixture();
  115. let config: Config =
  116. toml::from_str(&format!("recipe_sources = [\"{}\"]", dir.to_str().unwrap())).unwrap();
  117. let cfg = std::sync::Arc::new(config);
  118. let rows = collect(&cfg, false);
  119. let page = html_page(&rows);
  120. let initial = serde_json::to_string(&rows).unwrap();
  121. let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
  122. let addr = listener.local_addr().unwrap();
  123. let app = serve::router(cfg, page, initial, false, "", None);
  124. let _h = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
  125. let reports = dir.join("test-recipe-nvfp4/reports");
  126. // Serialize with the web dashboard's Playwright suite (shared browser).
  127. // The lock + deno run live in spawn_blocking: the std guard must not
  128. // be held across an await, and std::process::output must not block
  129. // this runtime's only thread (the axum server runs on it).
  130. let url = format!("http://{addr}");
  131. let fixture_path = reports.to_str().unwrap().to_string();
  132. let out = tokio::task::spawn_blocking(move || {
  133. let _pw = crate::ui_test_lock::playwright_session();
  134. std::process::Command::new("deno")
  135. .arg("run")
  136. .arg("-A")
  137. .arg("src/report/report.playwright.mjs")
  138. .env("SM121_REPORT_URL", url)
  139. .env("SM121_REPORT_FIXTURE", fixture_path)
  140. .output()
  141. })
  142. .await
  143. .unwrap()
  144. .unwrap();
  145. let stdout = String::from_utf8_lossy(&out.stdout);
  146. let stderr = String::from_utf8_lossy(&out.stderr);
  147. let skip = stdout.contains("SKIP");
  148. fs::remove_dir_all(&dir).ok();
  149. if skip {
  150. eprintln!("SKIP report_playwright_checks:\n{stdout}");
  151. return;
  152. }
  153. assert!(
  154. out.status.success(),
  155. "report playwright checks failed:\n{stdout}{stderr}"
  156. );
  157. eprintln!("{stdout}");
  158. }
  159. // ----- rendering (moved from tests.rs at the 500-line cap) -----
  160. fn row(name: &str, model: &str, quant: &str, score: Option<serde_json::Value>) -> ReportRow {
  161. let reports = score.unwrap_or(
  162. serde_json::json!([{"file": "f.md", "files": ["f.md"], "perf": [], "date": "", "label": ""}]),
  163. );
  164. serde_json::from_value(serde_json::json!({
  165. "name": name, "title": name, "model": model, "quant": quant,
  166. "engine": "vllm", "image": "img:1", "nodes": 1, "tp": 1,
  167. "deploy": "docker", "port": 30000, "reports": reports,
  168. }))
  169. .unwrap()
  170. }
  171. #[test]
  172. fn markdown_lists_rows_and_scores() {
  173. let rows = vec![row(
  174. "r-a",
  175. "m/one",
  176. "NVFP4",
  177. Some(serde_json::json!([
  178. {"file": "f.md", "files": ["f.md"], "perf": [{"d": 0, "c": 1, "tg": 27.4}, {"d": 0, "c": 2, "tg": 33.3}], "date": "2026-01-02", "label": "l", "pct": 90.4, "points": "159/176", "ratio": 0.9034},
  179. ])),
  180. )];
  181. let md = markdown(&rows);
  182. assert!(md.contains(
  183. "| m/one | NVFP4 | r-a | — | vllm | 1×1 | 1 | 2026-01-02 | — | 0 | 159/176 | 90.4 | 27.4 | 33.3 |"
  184. ));
  185. let empty = markdown(&[]);
  186. assert!(empty.contains(fluent_i18n::t!("report-title", { "count" => 0 }).as_str()));
  187. }
  188. /// Mounted variant (the dashboard nests the panel at /recipes): the page's
  189. /// own absolute paths must move with the prefix or its SSE stream and
  190. /// viewer links would hit the dashboard's routes.
  191. #[test]
  192. fn html_page_at_rewrites_own_paths() {
  193. let page = super::html_page_at(&[], "/recipes");
  194. assert!(
  195. page.contains("auth(\"/recipes/events\")"),
  196. "SSE path not rewritten"
  197. );
  198. assert!(!page.contains("auth(\"/events\")"));
  199. assert!(
  200. page.contains("auth(\"/recipes/r/"),
  201. "viewer link not rewritten"
  202. );
  203. assert!(!page.contains("href=\"/r/"));
  204. assert!(
  205. !super::html_page(&[]).contains("/recipes"),
  206. "unprefixed page must stay clean"
  207. );
  208. // hosted copy is marked so it can drop its own header/live chrome
  209. assert!(
  210. page.contains("<body class=\"embed\">"),
  211. "embed marker missing"
  212. );
  213. assert!(
  214. super::html_page(&[]).contains("<body>"),
  215. "standalone page keeps its unmarked body"
  216. );
  217. // viewer links carry the auth token (the token gate rejects bare /r/)
  218. assert!(
  219. page.contains("auth(\"/recipes/r/\""),
  220. "viewer link not auth-wrapped"
  221. );
  222. assert!(
  223. !page.contains("href=\"/r/\""),
  224. "unwrapped viewer link remains"
  225. );
  226. }
  227. #[test]
  228. fn html_page_embeds_data_escaped() {
  229. let rows = vec![row("r-a", "m/one", "NVFP4", None)];
  230. let page = html_page(&rows);
  231. assert!(page.contains("r-a"));
  232. assert!(!page.contains("{{DATA}}"));
  233. // a model name containing </script> cannot break out of the JSON island
  234. let evil = vec![row("x", "a</script>b", "NVFP4", None)];
  235. assert!(!html_page(&evil).contains("</script>b"));
  236. }
  237. // ----- log rendering (serve.rs clean_log), moved at the 500-line cap -----
  238. #[test]
  239. fn clean_log_collapses_spinner_soup_and_dups() {
  240. use super::serve::clean_log;
  241. // one llama-benchy phase = many rich redraw frames (spinner glyph,
  242. // elapsed clock and run counter differ, the test name does not); old
  243. // pty-captured logs keep every frame as its own line, plus pty ^D/⌫
  244. // artifacts and duplicated summary lines
  245. let soup = "^D\u{8}\u{8}{\"event\": \"backend_detected\"}\n\n\
  246. \u{280b} Initializing\u{2026} \u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501} 0/18 0:00:00\n\
  247. \u{2819} Initializing\u{2026} \u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501} 0/18 0:00:00\n\
  248. \u{2839} pp2048 tg128 @ d0 c1 run 0 \u{2501}\u{2501}\u{2501}\u{2501}\u{257a}\u{2501} 1/18 0:00:02\n\
  249. \u{2838} pp2048 tg128 @ d0 c1 run 0 \u{2501}\u{2501}\u{2501}\u{2501}\u{257a}\u{2501} 1/18 0:00:02\n\
  250. \u{283c} pp2048 tg128 @ d0 c1 run 1 \u{2501}\u{2501}\u{2501}\u{2501}\u{257a}\u{2501} 2/18 0:00:05\n\
  251. \u{2713} Complete \u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501} 18/18 0:01:48\n\
  252. \u{2713} Complete \u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501}\u{2501} 18/18 0:01:48\n\
  253. \u{2502} pp2048 tg128 @ d0 \u{2502} c1 \u{2502} 1,230 \u{2502} 59.2 \u{2502} 2,630 \u{2502} 4,641 \u{2502} 2048+128\n";
  254. let out = clean_log(soup);
  255. let lines: Vec<&str> = out.lines().collect();
  256. // pty artifact gone; each spinner phase keeps its LAST frame; one
  257. // ✓ Complete; table row intact
  258. assert_eq!(
  259. lines.len(),
  260. 6,
  261. "event, blank, init, run, check, table: {}",
  262. out
  263. );
  264. assert!(lines[0].starts_with("{\"event\""), "no ^D: {}", lines[0]);
  265. assert!(
  266. lines[2].starts_with("\u{2819} Initializing"),
  267. "init last frame: {}",
  268. out
  269. );
  270. assert!(
  271. lines[3].starts_with("\u{283c} pp2048"),
  272. "run-0 frames dropped: {}",
  273. out
  274. );
  275. assert!(lines[3].contains("run 1"), "keeps the LAST frame: {}", out);
  276. assert!(
  277. lines[4].trim().starts_with("\u{2713} Complete"),
  278. "one check: {}",
  279. out
  280. );
  281. assert!(
  282. lines[5].starts_with("\u{2502} pp2048"),
  283. "table row intact: {}",
  284. out
  285. );
  286. // idempotent on already-clean logs
  287. assert_eq!(clean_log(&out), out);
  288. }