bench_remote.rs 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116
  1. //! Remote-host bench execution (`sm121 bench --ssh-host <host>`): the
  2. //! tool-eval-bench runs on the remote host over SSH (the local machine only
  3. //! orchestrates), keeping the bench client's own network (a WiFi laptop) off
  4. //! the measurement path. The tokenizer is uploaded to the host; the report +
  5. //! run package still land locally (the remote json/md are downloaded back).
  6. use std::path::Path;
  7. use std::process::Command;
  8. use anyhow::Result;
  9. /// Build the SSH command that runs the tool on the remote host. `ssh -tt`
  10. /// allocates a pty (the rich Live display deadlocks on a non-tty pipe, and a
  11. /// single `-t` is ignored when the launcher is not attached to a terminal);
  12. /// `stty -onlcr` keeps the log LF-only; TTY_COMPATIBLE/NO_COLOR match the
  13. /// local run. The `cwd` is a remote path embedded in the shell string.
  14. pub fn remote_command(host: &str, cwd: &Path, bin: &str, args: &[String]) -> Command {
  15. let remote = format!(
  16. "mkdir -p '{}' && cd '{}' || exit 127; stty -onlcr 2>/dev/null; TTY_COMPATIBLE=0 NO_COLOR=1 exec {} {}",
  17. cwd.display(),
  18. cwd.display(),
  19. bin,
  20. args.join(" ")
  21. );
  22. let mut c = Command::new("ssh");
  23. // `-tt` (not `-t`): force the pty even when the launcher has no terminal.
  24. // With plain `-t` ssh prints "Pseudo-terminal will not be allocated …"
  25. // and runs the harness without one — the bench window then shows that
  26. // warning instead of the run.
  27. c.arg("-tt");
  28. c.arg(host);
  29. c.arg(&remote);
  30. c
  31. }
  32. /// Create the remote scratch dir + upload the tokenizer there so the bench
  33. /// client's own network stays off the measurement path.
  34. pub fn prepare(host: &str, tool_scratch: &Path, tokenizer: &Option<String>) -> Result<()> {
  35. let st = Command::new("ssh")
  36. .arg(host)
  37. .arg(format!("mkdir -p '{}'", tool_scratch.display()))
  38. .output()
  39. .map_err(|e| anyhow::anyhow!("ssh mkdir to {host}: {e}"))?;
  40. if !st.status.success() {
  41. anyhow::bail!("mkdir on {host}: {}", String::from_utf8_lossy(&st.stderr));
  42. }
  43. if let Some(tok) = tokenizer {
  44. let st = Command::new("scp")
  45. .arg("-r")
  46. .arg(tok)
  47. .arg(format!("{host}:{}", tool_scratch.display()))
  48. .output()
  49. .map_err(|e| anyhow::anyhow!("scp tokenizer to {host}: {e}"))?;
  50. if !st.status.success() {
  51. anyhow::bail!(
  52. "scp tokenizer to {host}: {}",
  53. String::from_utf8_lossy(&st.stderr)
  54. );
  55. }
  56. eprintln!("bench: tokenizer uploaded to {host}");
  57. }
  58. Ok(())
  59. }
  60. /// Download the json + the md from the remote scratch dir so the local report
  61. /// generation is unchanged. A download failure warns (the run is not lost);
  62. /// the local json is simply missing (the run package is skipped). Perf-only
  63. /// runs write no JSON by design — pass `expect_json = false` to skip the
  64. /// scp (and the warning) entirely.
  65. pub fn collect(
  66. host: &str,
  67. tool_json: &Path,
  68. json_path: &Path,
  69. tool_scratch: &Path,
  70. scratch: &Path,
  71. expect_json: bool,
  72. ) -> Result<()> {
  73. if expect_json {
  74. let st = Command::new("scp")
  75. .arg(format!("{host}:{}", tool_json.display()))
  76. .arg(json_path)
  77. .output()
  78. .map_err(|e| anyhow::anyhow!("scp json from {host}: {e}"))?;
  79. if !st.status.success() {
  80. eprintln!("bench: WARNING could not download json from {host}");
  81. }
  82. }
  83. let st = Command::new("ssh")
  84. .arg(host)
  85. .arg(format!(
  86. "find '{}' -name '*.md' -printf '%T@\\t%p\\n' 2>/dev/null | sort -n | tail -1 | cut -f2",
  87. tool_scratch.join("runs").display()
  88. ))
  89. .output()
  90. .map_err(|e| anyhow::anyhow!("ssh find md on {host}: {e}"))?;
  91. let remote_md = String::from_utf8_lossy(&st.stdout).trim().to_string();
  92. if !remote_md.is_empty() {
  93. let md_name = remote_md
  94. .rsplit('/')
  95. .next()
  96. .unwrap_or("report.md")
  97. .to_string();
  98. let local_runs = scratch.join("runs");
  99. std::fs::create_dir_all(&local_runs)?;
  100. let local_md_path = local_runs.join(&md_name);
  101. let st = Command::new("scp")
  102. .arg(format!("{host}:{remote_md}"))
  103. .arg(&local_md_path)
  104. .output()
  105. .map_err(|e| anyhow::anyhow!("scp md from {host}: {e}"))?;
  106. if !st.status.success() {
  107. eprintln!("bench: WARNING could not download md from {host}");
  108. }
  109. }
  110. Ok(())
  111. }