|
|
@@ -31,6 +31,8 @@ pub struct Args {
|
|
|
pub label: String,
|
|
|
pub concurrency: String,
|
|
|
pub seed: u64,
|
|
|
+ /// Skip the scenario phase (tool `--perf-only`).
|
|
|
+ pub perf_only: bool,
|
|
|
}
|
|
|
|
|
|
/// Served model name from the recipe's `--served-model-name` arg.
|
|
|
@@ -223,7 +225,7 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
|
|
|
.join(format!("{stamp}_{label}"));
|
|
|
let bin = "tool-eval-bench".to_string();
|
|
|
|
|
|
- let args = vec![
|
|
|
+ let mut args = vec![
|
|
|
"--base-url".into(),
|
|
|
base_url.clone(),
|
|
|
"--model".into(),
|
|
|
@@ -237,6 +239,9 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
|
|
|
"--json-file".into(),
|
|
|
json_path.to_string_lossy().to_string(),
|
|
|
];
|
|
|
+ if a.perf_only {
|
|
|
+ args.push("--perf-only".into());
|
|
|
+ }
|
|
|
|
|
|
if dry_run {
|
|
|
println!("bench: cwd {}", scratch.display());
|
|
|
@@ -272,14 +277,19 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
|
|
|
// Summary: final score from the JSON + the perf table from the log.
|
|
|
let j: serde_json::Value = serde_json::from_str(&fs::read_to_string(&json_path)?)?;
|
|
|
let sc = &j["scores"];
|
|
|
- let completion = sc["completion_rate"]
|
|
|
- .as_f64()
|
|
|
- .map(|v| v.to_string())
|
|
|
- .unwrap_or_default();
|
|
|
- eprintln!(
|
|
|
- "bench: final score {} / {} ({} points), completion {completion}",
|
|
|
- sc["final_score"], sc["max_points"], sc["total_points"]
|
|
|
- );
|
|
|
+ let final_score = sc["final_score"].as_u64();
|
|
|
+ if let Some(score) = final_score {
|
|
|
+ let completion = sc["completion_rate"]
|
|
|
+ .as_f64()
|
|
|
+ .map(|v| v.to_string())
|
|
|
+ .unwrap_or_default();
|
|
|
+ eprintln!(
|
|
|
+ "bench: final score {score} / {} ({} points), completion {completion}",
|
|
|
+ sc["max_points"], sc["total_points"]
|
|
|
+ );
|
|
|
+ } else {
|
|
|
+ eprintln!("bench: perf-only run (no quality score)");
|
|
|
+ }
|
|
|
for line in fs::read_to_string(&log_path)?.lines() {
|
|
|
if line.contains("pp2048") && line.contains("tg128") {
|
|
|
println!("{line}");
|