|
|
@@ -40,6 +40,14 @@ pub struct Args {
|
|
|
/// is uploaded to the host; the report + run package land locally. The
|
|
|
/// host must have `tool-eval-bench` on PATH and reach the serve.
|
|
|
pub ssh_host: Option<String>,
|
|
|
+ /// Disable thinking/reasoning in the harness (tool `--no-think`, sets
|
|
|
+ /// `enable_thinking=false`). Needed for thinking-by-default models (Ling,
|
|
|
+ /// GLM-4.5-style) whose long default reasoning blows the per-turn timeout
|
|
|
+ /// and excludes most scenarios.
|
|
|
+ pub no_think: bool,
|
|
|
+ /// Per-turn request timeout seconds forwarded to the harness (tool
|
|
|
+ /// `--timeout`). Raise above the 120 s default for slow/thinking models.
|
|
|
+ pub timeout: Option<u64>,
|
|
|
}
|
|
|
|
|
|
/// Served model name from the recipe's `--served-model-name` arg.
|
|
|
@@ -259,6 +267,13 @@ pub async fn run(config: &Config, a: &Args, dry_run: bool) -> Result<()> {
|
|
|
args.push("--tokenizer".into());
|
|
|
args.push(tok.clone());
|
|
|
}
|
|
|
+ if a.no_think {
|
|
|
+ args.push("--no-think".into());
|
|
|
+ }
|
|
|
+ if let Some(t) = a.timeout {
|
|
|
+ args.push("--timeout".into());
|
|
|
+ args.push(t.to_string());
|
|
|
+ }
|
|
|
|
|
|
if dry_run {
|
|
|
println!("bench: cwd {}", tool_scratch.display());
|