Jelajahi Sumber

fix: tmux serve quoting, venv setup, cluster probe on dgx3.lan

- Fix : remove  flag from docker command
  (incompatible with , warning text polluted stderr causing
  falsely-occupied nodes)
- Fix : sanitize parentheses and other shell-unsafe
  chars (zsh glob expansion via  killed session kill/create)
- Fix  quoting: use  instead of inline  so
  recipe JSON args with embedded single quotes survive SSH + tmux
  shell layers
- Fix : run sequentially (no spawn), create venv first
  then run setup commands with  so  is
  set for
- Fix laguna recipe: wrap JSON args (--override-generation-config,
  --speculative-config) in single quotes so zsh doesn't do brace
  expansion on
Alexander Korolev 2 bulan lalu
induk
melakukan
a3bd00eef8

+ 1 - 1
recipes/deepseek-v4-flash-dspark/recipe.toml

@@ -12,7 +12,7 @@
 engine = "vllm"
 name = "DeepSeek V4 Flash DSpark (vLLM)"
 docker_image = "sparkrun-vllm-ds4-gb10:fixed"
-model = "deepseek-ai/DeepSeek-V4-Flash-DSpark"
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
 required_nodes = 2
 tensor_parallel_size = 2
 pipeline_parallel_size = 1

+ 2 - 4
recipes/laguna-s2.1-nvfp4/recipe.toml

@@ -70,11 +70,9 @@ args = [
     "--kv-cache-dtype", "fp8",
     "--enable-flashinfer-autotune",
     "--max-num-seqs", "32",
-    "--override-generation-config", """
-'{"temperature":0.7,"top_p":0.95}'""",
+    "--override-generation-config", "'{\"temperature\":0.7,\"top_p\":0.95}'",
     # DFlash speculative decoding with quantization-matched draft
-    "--speculative-config", """
-'{"model":"poolside/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15,"method":"dflash"}'""",
+    "--speculative-config", "'{\"model\":\"poolside/Laguna-S-2.1-DFlash-NVFP4\",\"num_speculative_tokens\":15,\"method\":\"dflash\"}'",
     "--served-model-name", "laguna-s-2.1", "laguna", "default",
 ]
 

+ 2 - 3
src/cluster_state.rs

@@ -43,9 +43,8 @@ impl ClusterState {
                 let config = config.clone();
                 let node = node.clone();
                 let name = node.name.clone();
-                let cmd =
-                    "docker ps -q --filter label=sm121.cluster --format '{{.Names}}' | head -1"
-                        .to_string();
+                let cmd = "docker ps --filter label=sm121.cluster --format '{{.Names}}' | head -1"
+                    .to_string();
                 handles.push(crate::locale::spawn_with_locale(async move {
                     (name, run_ssh(&config, &node, cmd, false).await)
                 }));

+ 4 - 2
src/tmux/commands.rs

@@ -1,5 +1,6 @@
 use crate::config::{Cluster, Node, Recipe};
 use crate::engine_flags::{EngineFlagsCtx, build_engine_args};
+use crate::ssh::shq;
 use crate::tmux::TmuxManager;
 
 impl TmuxManager {
@@ -106,8 +107,9 @@ impl TmuxManager {
 
         let launch_cmd = self.build_launch_command(recipe, node, cluster);
         cmds.push(format!(
-            "tmux send-keys -t {}:0 '{}' Enter",
-            sess, launch_cmd
+            "tmux send-keys -t {}:0 {} Enter",
+            sess,
+            shq(&launch_cmd)
         ));
 
         // --- Window 1: GPU monitor ---

+ 14 - 1
src/tmux/mod.rs

@@ -17,7 +17,20 @@ impl TmuxManager {
     /// Generates a namespaced session name for a recipe and node rank.
     /// Format: `{recipe_name}-{engine}-r{rank}`
     pub fn session_name(&self, recipe: &Recipe, rank: u32) -> String {
-        let name = recipe.name.replace(' ', "_");
+        // Replace shell-unsafe characters (parentheses, quotes, etc.)
+        // with underscores so the session name doesn't trigger glob
+        // expansion or shell injection when used in tmux commands.
+        let name = recipe
+            .name
+            .chars()
+            .map(|c| {
+                if c.is_ascii_alphanumeric() || c == '-' || c == '_' {
+                    c
+                } else {
+                    '_'
+                }
+            })
+            .collect::<String>();
         format!("{}-{}-r{}", name, recipe.engine, rank)
     }
 

+ 59 - 88
src/tmux/serve.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 use tokio::time::sleep;
 use tracing::{info, warn};
 
-use crate::config::{Config, Node, Recipe};
+use crate::config::{Config, Recipe};
 use crate::tmux::TmuxManager;
 
 /// Launches a cluster via tmux sessions.
@@ -274,9 +274,19 @@ async fn prepare_venv(
 ) -> Result<()> {
     let venv = match &recipe.venv {
         Some(v) => v,
-        None => return Ok(()),
+        None => {
+            info!("prepare_venv: no venv field in recipe — skipping");
+            return Ok(());
+        }
     };
 
+    info!(
+        "prepare_venv: venv={}, has_venv_setup_script={}, has_venv_setup={}",
+        venv,
+        recipe.venv_setup_script.is_some(),
+        !recipe.venv_setup.to_vec().is_empty()
+    );
+
     // Resolve setup commands: script file takes precedence over inline.
     let setup: Vec<String> = if let Some(script_path) = &recipe.venv_setup_script {
         let full_path = format!("recipes/{}/{}", recipe_name, script_path);
@@ -297,104 +307,65 @@ async fn prepare_venv(
     };
 
     if setup.is_empty() {
+        info!("venv setup is empty for {} — skipping", venv);
         return Ok(());
     }
 
-    info!("{}", fluent_i18n::t!("venv-preparing", { "venv" => venv }));
+    info!(
+        "{} ({} commands)",
+        fluent_i18n::t!("venv-preparing", { "venv" => venv }),
+        setup.len()
+    );
 
-    let mut handles = Vec::with_capacity(cluster.nodes.len());
+    // Prepare venv on each node sequentially.
     for node in &cluster.nodes {
-        let config = config.clone();
-        let node = node.clone();
-        let venv = venv.clone();
-        let setup = setup.clone();
-        let dry = dry_run;
-
-        handles.push(crate::locale::spawn_with_locale(async move {
-            (
-                node.name.clone(),
-                prepare_venv_for_node(&config, &node, &venv, &setup, dry).await,
-            )
-        }));
-    }
-
-    let mut any_failed = false;
-    for h in handles {
-        let (name, result) = h.await?;
-        if let Err(e) = result {
-            any_failed = true;
-            warn!(
-                "{}",
-                fluent_i18n::t!("venv-prepare-failed", { "name" => &name, "error" => e.to_string() })
-            );
-        }
-    }
-
-    if any_failed {
-        anyhow::bail!("Venv preparation failed on at least one node (see warnings above)");
-    }
+        let check_cmd = format!("test -f {}/bin/activate", venv);
 
-    Ok(())
-}
-
-/// Checks if the venv exists on a single node and creates it if not.
-async fn prepare_venv_for_node(
-    config: &Config,
-    node: &Node,
-    venv: &str,
-    setup: &[String],
-    dry_run: bool,
-) -> Result<()> {
-    let check_cmd = format!("test -f {}/bin/activate", venv);
-
-    if dry_run {
-        info!(
-            "  ssh {}@{}: {} (dry-run)",
-            config.ssh_user, node.ssh_ip, check_cmd
-        );
-        for cmd in setup {
-            info!("    {} (dry-run)", cmd);
-        }
-        return Ok(());
-    }
-
-    // Check if venv already exists.
-    match crate::ssh::run_ssh(config, node, check_cmd.clone(), false).await {
-        Ok(_) => {
-            info!(
-                "{}",
-                fluent_i18n::t!("venv-exists", { "name" => &node.name })
-            );
-            return Ok(());
-        }
-        Err(_) => {
+        if !dry_run {
+            // Skip if venv already exists.
+            if crate::ssh::run_ssh(config, node, check_cmd.clone(), false)
+                .await
+                .is_ok()
+            {
+                info!(
+                    "{}",
+                    fluent_i18n::t!("venv-exists", { "name" => &node.name })
+                );
+                continue;
+            }
             info!(
                 "{}",
                 fluent_i18n::t!("venv-creating", { "name" => &node.name })
             );
-        }
-    }
 
-    // Run setup commands in order.
-    for cmd in setup {
-        info!("  ssh {}@{}: {}", config.ssh_user, node.ssh_ip, cmd);
-        crate::ssh::run_ssh(config, node, cmd.clone(), false).await?;
-    }
-
-    // Re-check that the venv was created.
-    match crate::ssh::run_ssh(config, node, check_cmd, false).await {
-        Ok(_) => {
-            info!(
-                "{}",
-                fluent_i18n::t!("venv-created", { "name" => &node.name })
-            );
-            Ok(())
-        }
-        Err(_) => {
-            anyhow::bail!(
-                "{}",
-                fluent_i18n::t!("venv-not-created", { "name" => &node.name })
+            // Create venv, then run setup with it activated so that
+            // uv pip install finds VIRTUAL_ENV. All commands are joined
+            // with && so the script halts on the first failure.
+            let joined = setup.join(" && ");
+            let full_script = format!(
+                "uv venv {} -p 3.12 && source {}/bin/activate && {}",
+                venv, venv, joined
             );
+            let script = format!("bash -c '{}'", full_script.replace('\'', "'\\''"));
+            crate::ssh::run_ssh(config, node, script, false).await?;
+
+            // Re-check.
+            if crate::ssh::run_ssh(config, node, check_cmd, false)
+                .await
+                .is_ok()
+            {
+                info!(
+                    "{}",
+                    fluent_i18n::t!("venv-created", { "name" => &node.name })
+                );
+            } else {
+                anyhow::bail!(
+                    "{}",
+                    fluent_i18n::t!("venv-not-created", { "name" => &node.name })
+                );
+            }
         }
     }
+
+    Ok(())
 }