فهرست منبع

bench: add --no-think and --timeout passthrough to tool-eval-bench — unblocks valid quality benches for thinking-by-default models (Ling, GLM-4.5-style) whose default reasoning blows the 120s/turn timeout and excludes most scenarios (the 10-16 fp8 Ling run scored 67/100 at 3.1% completion for exactly this reason)

Alexander Korolev 6 روز پیش
والد
کامیت
0e826d4b35
4فایلهای تغییر یافته به همراه31 افزوده شده و 0 حذف شده
  1. 15 0
      src/bench.rs
  2. 10 0
      src/cli.rs
  3. 4 0
      src/main.rs
  4. 2 0
      src/web/api.rs

+ 15 - 0
src/bench.rs

@@ -40,6 +40,14 @@ pub struct Args {
     /// is uploaded to the host; the report + run package land locally. The
     /// host must have `tool-eval-bench` on PATH and reach the serve.
     pub ssh_host: Option<String>,
+    /// Disable thinking/reasoning in the harness (tool `--no-think`, sets
+    /// `enable_thinking=false`). Needed for thinking-by-default models (Ling,
+    /// GLM-4.5-style) whose long default reasoning blows the per-turn timeout
+    /// and excludes most scenarios.
+    pub no_think: bool,
+    /// Per-turn request timeout seconds forwarded to the harness (tool
+    /// `--timeout`). Raise above the 120 s default for slow/thinking models.
+    pub timeout: Option<u64>,
 }
 
 /// Served model name from the recipe's `--served-model-name` arg.
@@ -259,6 +267,13 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
         args.push("--tokenizer".into());
         args.push(tok.clone());
     }
+    if a.no_think {
+        args.push("--no-think".into());
+    }
+    if let Some(t) = a.timeout {
+        args.push("--timeout".into());
+        args.push(t.to_string());
+    }
 
     if dry_run {
         println!("bench: cwd {}", tool_scratch.display());

+ 10 - 0
src/cli.rs

@@ -208,6 +208,16 @@ pub enum Commands {
         /// `tool-eval-bench` on PATH.
         #[arg(long)]
         ssh_host: Option<String>,
+        /// Disable thinking/reasoning in the harness (`--no-think`, sets
+        /// `enable_thinking=false`). Needed for thinking-by-default models
+        /// (Ling, GLM-4.5-style) whose long default reasoning blows the
+        /// per-turn timeout and excludes most scenarios.
+        #[arg(long)]
+        no_think: bool,
+        /// Per-turn request timeout seconds forwarded to the harness
+        /// (`--timeout`). Raise above the 120 s default for slow models.
+        #[arg(long)]
+        timeout: Option<u64>,
     },
 
     /// Check cluster health

+ 4 - 0
src/main.rs

@@ -256,6 +256,8 @@ async fn main() -> Result<()> {
             depth,
             tokenizer,
             ssh_host,
+            no_think,
+            timeout,
         } => {
             let args = crate::bench::Args {
                 recipe,
@@ -268,6 +270,8 @@ async fn main() -> Result<()> {
                 depth,
                 tokenizer,
                 ssh_host,
+                no_think,
+                timeout,
             };
             crate::bench::run(&config, &args, dry_run).await?;
         }

+ 2 - 0
src/web/api.rs

@@ -163,6 +163,8 @@ pub(crate) async fn bench(
         depth: req.depth,
         tokenizer: None,
         ssh_host: req.ssh_host,
+        no_think: false,
+        timeout: None,
     };
     super::jobs::spawn(app2, id.clone(), async move {
         crate::bench::run(&config, &args, false).await