Преглед на файлове

feat: sm121 bench --perf-only (skip scenarios, re-baseline the perf table after a cold first row)

Alexander Korolev преди 1 месец
родител
ревизия
314812116c
променени са 3 файла, в които са добавени 25 реда и са изтрити 9 реда
  1. 19 9
      src/bench.rs
  2. 4 0
      src/cli.rs
  3. 2 0
      src/main.rs

+ 19 - 9
src/bench.rs

@@ -31,6 +31,8 @@ pub struct Args {
     pub label: String,
     pub concurrency: String,
     pub seed: u64,
+    /// Skip the scenario phase (tool `--perf-only`).
+    pub perf_only: bool,
 }
 
 /// Served model name from the recipe's `--served-model-name` arg.
@@ -223,7 +225,7 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
         .join(format!("{stamp}_{label}"));
     let bin = "tool-eval-bench".to_string();
 
-    let args = vec![
+    let mut args = vec![
         "--base-url".into(),
         base_url.clone(),
         "--model".into(),
@@ -237,6 +239,9 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
         "--json-file".into(),
         json_path.to_string_lossy().to_string(),
     ];
+    if a.perf_only {
+        args.push("--perf-only".into());
+    }
 
     if dry_run {
         println!("bench: cwd {}", scratch.display());
@@ -272,14 +277,19 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
     // Summary: final score from the JSON + the perf table from the log.
     let j: serde_json::Value = serde_json::from_str(&fs::read_to_string(&json_path)?)?;
     let sc = &j["scores"];
-    let completion = sc["completion_rate"]
-        .as_f64()
-        .map(|v| v.to_string())
-        .unwrap_or_default();
-    eprintln!(
-        "bench: final score {} / {} ({} points), completion {completion}",
-        sc["final_score"], sc["max_points"], sc["total_points"]
-    );
+    let final_score = sc["final_score"].as_u64();
+    if let Some(score) = final_score {
+        let completion = sc["completion_rate"]
+            .as_f64()
+            .map(|v| v.to_string())
+            .unwrap_or_default();
+        eprintln!(
+            "bench: final score {score} / {} ({} points), completion {completion}",
+            sc["max_points"], sc["total_points"]
+        );
+    } else {
+        eprintln!("bench: perf-only run (no quality score)");
+    }
     for line in fs::read_to_string(&log_path)?.lines() {
         if line.contains("pp2048") && line.contains("tg128") {
             println!("{line}");

+ 4 - 0
src/cli.rs

@@ -154,6 +154,10 @@ pub enum Commands {
         concurrency: String,
         #[arg(long, default_value_t = 42)]
         seed: u64,
+        /// Skip the scenario phase — run only the perf micro-benchmarks
+        /// (use to re-baseline the perf table after a cold first row).
+        #[arg(long)]
+        perf_only: bool,
     },
 
     /// Check cluster health

+ 2 - 0
src/main.rs

@@ -171,6 +171,7 @@ async fn main() -> Result<()> {
             label,
             concurrency,
             seed,
+            perf_only,
         } => {
             let args = crate::bench::Args {
                 recipe,
@@ -178,6 +179,7 @@ async fn main() -> Result<()> {
                 label,
                 concurrency,
                 seed,
+                perf_only,
             };
             crate::bench::run(&config, &args, dry_run).await?;
         }