Parcourir la source

refactor: split docker.rs and tmux/mod.rs under 500 lines each

- Move docker.rs test module to docker/tests.rs (mod.rs: 554→374 lines)
- Split tmux/mod.rs into commands.rs (build_launch_command,
  build_session_commands) and session.rs (6 session methods)
  (mod.rs: 555→94 lines, commands.rs: 157, session.rs: 321)
- Add Makefile with quality gates: check, check-fmt, check-test,
  check-clippy, check-lines (line count ≤ 500 per source file)
- All 25 tests pass, clippy clean, fmt clean
Alexander Korolev il y a 4 mois
Parent
commit
38290ec6f5
7 fichiers modifiés avec 709 ajouts et 646 suppressions
  1. 31 0
      Makefile
  2. 10 0
      README.md
  3. 1 181
      src/docker/mod.rs
  4. 189 0
      src/docker/tests.rs
  5. 152 0
      src/tmux/commands.rs
  6. 3 465
      src/tmux/mod.rs
  7. 323 0
      src/tmux/session.rs

+ 31 - 0
Makefile

@@ -0,0 +1,31 @@
+.PHONY: check check-fmt check-test check-clippy check-lines
+
+# Run all quality gates
+check: check-fmt check-test check-clippy check-lines
+
+check-fmt:
+	cargo fmt --check
+
+check-test:
+	cargo test
+
+check-clippy:
+	cargo clippy --all-targets -- -D warnings
+
+# Ensure no Rust source file exceeds 500 lines
+check-lines:
+	@exit_code=0; \
+	max=500; \
+	for f in src/**/*.rs src/*.rs; do \
+		[ -f "$$f" ] || continue; \
+		lines=$$(wc -l < "$$f"); \
+		if [ "$$lines" -gt "$$max" ]; then \
+			echo "FAIL: $$f has $$lines lines (max $$max)"; \
+			exit_code=1; \
+		fi; \
+	done; \
+	if [ "$$exit_code" -eq 0 ]; then \
+		echo "✅ All Rust source files are under $$max lines"; \
+	else \
+		exit "$$exit_code"; \
+	fi

+ 10 - 0
README.md

@@ -12,6 +12,16 @@ Rust CLI tool for launching vLLM / SGLang containers across a DGX Spark cluster
 - **Doctor mode** — pre-flight checks and auto-repair of broken container images
 - Dry-run (`-d`) to preview every SSH command without execution
 
+## Quality Gates
+
+```bash
+make check            # Run all quality gates (fmt, test, clippy, line count)
+make check-fmt        # cargo fmt --check
+make check-test       # cargo test
+make check-clippy     # cargo clippy with -D warnings
+make check-lines      # Ensure no Rust file exceeds 500 lines
+```
+
 ## Quick Start
 
 ```bash

+ 1 - 181
src/docker.rs → src/docker/mod.rs

@@ -371,184 +371,4 @@ fn build_docker_command(
 }
 
 #[cfg(test)]
-mod tests {
-    use super::*;
-    use std::collections::HashMap;
-    fn tcls() -> Cluster {
-        Cluster {
-            name: "t".into(),
-            master_ib_ip: "1".into(),
-            nodes: vec![
-                Node {
-                    name: "n0".into(),
-                    ssh_ip: "10.0.0.1".into(),
-                    ib_ip: "1".into(),
-                    rank: 0,
-                    host_key: None,
-                },
-                Node {
-                    name: "n1".into(),
-                    ssh_ip: "10.0.0.2".into(),
-                    ib_ip: "2".into(),
-                    rank: 1,
-                    host_key: None,
-                },
-            ],
-        }
-    }
-    fn tnccl() -> NcclConfig {
-        NcclConfig {
-            ib_hca: "r".into(),
-            socket_ifname: "e".into(),
-            ib_gid_index: 3,
-            ib_disable: 0,
-        }
-    }
-    fn trec() -> Recipe {
-        Recipe {
-            engine: "vllm".into(),
-            name: "T".into(),
-            docker_image: "img".into(),
-            model: "mdl".into(),
-            port: 8000,
-            required_nodes: Some(2),
-            pipeline_parallel_size: Some(2),
-            tensor_parallel_size: 1,
-            args: vec!["--x".into(), "4096".into()],
-            env: HashMap::from([("K".into(), "V".into())]),
-            cluster: None,
-            entrypoint: vec![],
-            distributed_backend: "mp".into(),
-        }
-    }
-    fn run(recipe: &Recipe, node: &Node, head: bool) -> String {
-        build_docker_command(&tcls(), &tnccl(), recipe, "r", node, head, 2, 2).join(" ")
-    }
-
-    #[test]
-    fn mp() {
-        let cmd = run(&trec(), &tcls().nodes[0], true);
-        assert!(cmd.contains("docker run -d --gpus all"));
-        assert!(cmd.contains("-e NCCL_IB_HCA=r"));
-        assert!(cmd.contains("-e K=V"));
-        assert!(cmd.contains("-p 8000:8000"));
-        assert!(cmd.contains("--name sm121_r_0"));
-        assert!(cmd.contains("--label sm121.recipe=r"));
-        assert!(cmd.contains("--label sm121.rank=0"));
-        assert!(cmd.contains("img"));
-        assert!(cmd.contains("mdl"));
-        assert!(cmd.contains("--tensor-parallel-size 1"));
-        assert!(cmd.contains("--pipeline-parallel-size 2"));
-        assert!(cmd.contains("--nnodes 2"));
-        assert!(cmd.contains("--node-rank 0"));
-        assert!(cmd.contains("--master-addr 1"));
-        assert!(cmd.contains("--distributed-executor-backend mp"));
-        assert!(!cmd.contains("RAY_ADDRESS"));
-    }
-    #[test]
-    fn worker_no_port() {
-        let cmd = run(&trec(), &tcls().nodes[1], false);
-        assert!(!cmd.contains("-p 8000:8000"));
-        assert!(cmd.contains("--node-rank 1"));
-        assert!(cmd.contains("--label sm121.rank=1"));
-        assert!(cmd.contains("--name sm121_r_1"));
-    }
-    #[test]
-    fn ray_backend() {
-        let mut r = trec();
-        r.distributed_backend = "ray".into();
-        let cmd = run(&r, &tcls().nodes[0], true);
-        assert!(cmd.contains("-e RAY_ADDRESS=1:6379"));
-        assert!(cmd.contains("-e RAY_PORT=8000"));
-        assert!(cmd.contains("--node-rank 0"));
-        assert!(cmd.contains("--distributed-executor-backend ray"));
-    }
-    #[test]
-    fn auto_pp_size() {
-        let cmd = run(&trec(), &tcls().nodes[0], true);
-        assert!(cmd.contains("--pipeline-parallel-size 2"));
-    }
-    #[test]
-    fn entrypoint() {
-        let mut r = trec();
-        r.entrypoint = vec!["py".into(), "-m".into(), "mod".into()];
-        let cmd = run(&r, &tcls().nodes[0], true);
-        let ip = cmd.find("img").unwrap();
-        let mp = cmd.find("mdl").unwrap();
-        let ep = cmd.find("py -m mod").unwrap();
-        assert!(
-            ip < ep && ep < mp,
-            "entrypoint should be between image and model"
-        );
-    }
-
-    // ── SGLang engine tests ──────────────────────────────────────
-    fn sglang_recipe() -> Recipe {
-        Recipe {
-            engine: "sglang".into(),
-            name: "S".into(),
-            docker_image: "sglang-img".into(),
-            model: "org/sgl-model".into(),
-            port: 8000,
-            required_nodes: Some(4),
-            pipeline_parallel_size: None,
-            tensor_parallel_size: 4,
-            args: vec!["--mem-fraction-static".into(), "0.8".into()],
-            env: HashMap::from([("TORCH_NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())]),
-            cluster: None,
-            entrypoint: vec!["sglang".into(), "serve".into()],
-            distributed_backend: "mp".into(),
-        }
-    }
-    fn srun(recipe: &Recipe, node: &Node, head: bool) -> String {
-        build_docker_command(&tcls(), &tnccl(), recipe, "sg", node, head, 4, 4).join(" ")
-    }
-
-    #[test]
-    fn sglang_host_ip() {
-        let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
-        assert!(cmd.contains("-e SGLANG_WORKER_IP=1"));
-        assert!(!cmd.contains("VLLM_HOST_IP"));
-    }
-
-    #[test]
-    fn sglang_head() {
-        let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
-        assert!(cmd.contains("--model-path"));
-        assert!(cmd.contains("org/sgl-model"));
-        assert!(cmd.contains("--tp 4"));
-        assert!(cmd.contains("--nnodes 4"));
-        assert!(cmd.contains("--node-rank 0"));
-        assert!(cmd.contains("--master-addr 1"));
-        assert!(!cmd.contains("--tensor-parallel-size"));
-        assert!(!cmd.contains("--pipeline-parallel-size"));
-        assert!(!cmd.contains("--distributed-executor-backend"));
-        assert!(cmd.contains("-p 8000:8000"));
-        assert!(cmd.contains("-e TORCH_NCCL_ASYNC_ERROR_HANDLING=1"));
-        assert!(cmd.contains("--port 8000"));
-    }
-
-    #[test]
-    fn sglang_worker() {
-        let cmd = srun(&sglang_recipe(), &tcls().nodes[1], false);
-        assert!(!cmd.contains("-p 8000:8000"));
-        assert!(cmd.contains("--node-rank 1"));
-        assert!(cmd.contains("--port 8000"));
-    }
-
-    #[test]
-    fn sglang_entrypoint_model_path_order() {
-        let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
-        let ep = cmd.find("sglang serve").unwrap();
-        let mp = cmd.find("--model-path").unwrap();
-        let mdl = cmd.find("org/sgl-model").unwrap();
-        let tp = cmd.find("--tp 4").unwrap();
-        assert!(
-            ep < mp && mp < mdl,
-            "entrypoint before model-path before model"
-        );
-        assert!(mdl < tp, "model before --tp");
-        let port = cmd.find("--port 8000").unwrap();
-        assert!(tp < port, "--tp before --port");
-    }
-}
+mod tests;

+ 189 - 0
src/docker/tests.rs

@@ -0,0 +1,189 @@
+use super::*;
+use std::collections::HashMap;
+
+fn tcls() -> Cluster {
+    Cluster {
+        name: "t".into(),
+        master_ib_ip: "1".into(),
+        nodes: vec![
+            Node {
+                name: "n0".into(),
+                ssh_ip: "10.0.0.1".into(),
+                ib_ip: "1".into(),
+                rank: 0,
+                host_key: None,
+            },
+            Node {
+                name: "n1".into(),
+                ssh_ip: "10.0.0.2".into(),
+                ib_ip: "2".into(),
+                rank: 1,
+                host_key: None,
+            },
+        ],
+    }
+}
+
+fn tnccl() -> NcclConfig {
+    NcclConfig {
+        ib_hca: "r".into(),
+        socket_ifname: "e".into(),
+        ib_gid_index: 3,
+        ib_disable: 0,
+    }
+}
+
+fn trec() -> Recipe {
+    Recipe {
+        engine: "vllm".into(),
+        name: "T".into(),
+        docker_image: "img".into(),
+        model: "mdl".into(),
+        port: 8000,
+        required_nodes: Some(2),
+        pipeline_parallel_size: Some(2),
+        tensor_parallel_size: 1,
+        args: vec!["--x".into(), "4096".into()],
+        env: HashMap::from([("K".into(), "V".into())]),
+        cluster: None,
+        entrypoint: vec![],
+        distributed_backend: "mp".into(),
+    }
+}
+
+fn run(recipe: &Recipe, node: &Node, head: bool) -> String {
+    build_docker_command(&tcls(), &tnccl(), recipe, "r", node, head, 2, 2).join(" ")
+}
+
+#[test]
+fn mp() {
+    let cmd = run(&trec(), &tcls().nodes[0], true);
+    assert!(cmd.contains("docker run -d --gpus all"));
+    assert!(cmd.contains("-e NCCL_IB_HCA=r"));
+    assert!(cmd.contains("-e K=V"));
+    assert!(cmd.contains("-p 8000:8000"));
+    assert!(cmd.contains("--name sm121_r_0"));
+    assert!(cmd.contains("--label sm121.recipe=r"));
+    assert!(cmd.contains("--label sm121.rank=0"));
+    assert!(cmd.contains("img"));
+    assert!(cmd.contains("mdl"));
+    assert!(cmd.contains("--tensor-parallel-size 1"));
+    assert!(cmd.contains("--pipeline-parallel-size 2"));
+    assert!(cmd.contains("--nnodes 2"));
+    assert!(cmd.contains("--node-rank 0"));
+    assert!(cmd.contains("--master-addr 1"));
+    assert!(cmd.contains("--distributed-executor-backend mp"));
+    assert!(!cmd.contains("RAY_ADDRESS"));
+}
+
+#[test]
+fn worker_no_port() {
+    let cmd = run(&trec(), &tcls().nodes[1], false);
+    assert!(!cmd.contains("-p 8000:8000"));
+    assert!(cmd.contains("--node-rank 1"));
+    assert!(cmd.contains("--label sm121.rank=1"));
+    assert!(cmd.contains("--name sm121_r_1"));
+}
+
+#[test]
+fn ray_backend() {
+    let mut r = trec();
+    r.distributed_backend = "ray".into();
+    let cmd = run(&r, &tcls().nodes[0], true);
+    assert!(cmd.contains("-e RAY_ADDRESS=1:6379"));
+    assert!(cmd.contains("-e RAY_PORT=8000"));
+    assert!(cmd.contains("--node-rank 0"));
+    assert!(cmd.contains("--distributed-executor-backend ray"));
+}
+
+#[test]
+fn auto_pp_size() {
+    let cmd = run(&trec(), &tcls().nodes[0], true);
+    assert!(cmd.contains("--pipeline-parallel-size 2"));
+}
+
+#[test]
+fn entrypoint() {
+    let mut r = trec();
+    r.entrypoint = vec!["py".into(), "-m".into(), "mod".into()];
+    let cmd = run(&r, &tcls().nodes[0], true);
+    let ip = cmd.find("img").unwrap();
+    let mp = cmd.find("mdl").unwrap();
+    let ep = cmd.find("py -m mod").unwrap();
+    assert!(
+        ip < ep && ep < mp,
+        "entrypoint should be between image and model"
+    );
+}
+
+// ── SGLang engine tests ──────────────────────────────────────
+
+fn sglang_recipe() -> Recipe {
+    Recipe {
+        engine: "sglang".into(),
+        name: "S".into(),
+        docker_image: "sglang-img".into(),
+        model: "org/sgl-model".into(),
+        port: 8000,
+        required_nodes: Some(4),
+        pipeline_parallel_size: None,
+        tensor_parallel_size: 4,
+        args: vec!["--mem-fraction-static".into(), "0.8".into()],
+        env: HashMap::from([("TORCH_NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())]),
+        cluster: None,
+        entrypoint: vec!["sglang".into(), "serve".into()],
+        distributed_backend: "mp".into(),
+    }
+}
+
+fn srun(recipe: &Recipe, node: &Node, head: bool) -> String {
+    build_docker_command(&tcls(), &tnccl(), recipe, "sg", node, head, 4, 4).join(" ")
+}
+
+#[test]
+fn sglang_host_ip() {
+    let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
+    assert!(cmd.contains("-e SGLANG_WORKER_IP=1"));
+    assert!(!cmd.contains("VLLM_HOST_IP"));
+}
+
+#[test]
+fn sglang_head() {
+    let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
+    assert!(cmd.contains("--model-path"));
+    assert!(cmd.contains("org/sgl-model"));
+    assert!(cmd.contains("--tp 4"));
+    assert!(cmd.contains("--nnodes 4"));
+    assert!(cmd.contains("--node-rank 0"));
+    assert!(cmd.contains("--master-addr 1"));
+    assert!(!cmd.contains("--tensor-parallel-size"));
+    assert!(!cmd.contains("--pipeline-parallel-size"));
+    assert!(!cmd.contains("--distributed-executor-backend"));
+    assert!(cmd.contains("-p 8000:8000"));
+    assert!(cmd.contains("-e TORCH_NCCL_ASYNC_ERROR_HANDLING=1"));
+    assert!(cmd.contains("--port 8000"));
+}
+
+#[test]
+fn sglang_worker() {
+    let cmd = srun(&sglang_recipe(), &tcls().nodes[1], false);
+    assert!(!cmd.contains("-p 8000:8000"));
+    assert!(cmd.contains("--node-rank 1"));
+    assert!(cmd.contains("--port 8000"));
+}
+
+#[test]
+fn sglang_entrypoint_model_path_order() {
+    let cmd = srun(&sglang_recipe(), &tcls().nodes[0], true);
+    let ep = cmd.find("sglang serve").unwrap();
+    let mp = cmd.find("--model-path").unwrap();
+    let mdl = cmd.find("org/sgl-model").unwrap();
+    let tp = cmd.find("--tp 4").unwrap();
+    assert!(
+        ep < mp && mp < mdl,
+        "entrypoint before model-path before model"
+    );
+    assert!(mdl < tp, "model before --tp");
+    let port = cmd.find("--port 8000").unwrap();
+    assert!(tp < port, "--tp before --port");
+}

+ 152 - 0
src/tmux/commands.rs

@@ -0,0 +1,152 @@
+use crate::config::{Cluster, NcclConfig, Node, Recipe};
+use crate::tmux::TmuxManager;
+
+impl TmuxManager {
+    /// Builds the full model launch command for a given node.
+    pub fn build_launch_command(&self, recipe: &Recipe, node: &Node, cluster: &Cluster) -> String {
+        let nnodes = cluster.nodes.len() as u32;
+        let pp_size = recipe.pipeline_parallel_size.unwrap_or(nnodes);
+        let is_sglang = recipe.engine == "sglang";
+
+        let mut parts: Vec<String> = Vec::new();
+
+        // Engine entrypoint
+        match recipe.engine.as_str() {
+            "vllm" => {
+                parts.push("python -m vllm.entrypoints.openai.api_server".to_string());
+            }
+            "sglang" => {
+                parts.push("python -m sglang.launch_server".to_string());
+            }
+            other => {
+                parts.push(format!("python -m {}", other));
+            }
+        }
+
+        // Model — engine-specific
+        if is_sglang {
+            parts.push("--model-path".to_string());
+        }
+        parts.push(recipe.model.clone());
+
+        // Recipe-specific args
+        parts.extend(recipe.args.clone());
+
+        // Distributed flags — engine-specific
+        if is_sglang {
+            parts.extend(vec![
+                "--tp".to_string(),
+                recipe.tensor_parallel_size.to_string(),
+                "--nnodes".to_string(),
+                nnodes.to_string(),
+                "--node-rank".to_string(),
+                node.rank.to_string(),
+                "--master-addr".to_string(),
+                cluster.master_ib_ip.clone(),
+            ]);
+        } else {
+            match recipe.distributed_backend.as_str() {
+                "ray" => {
+                    parts.extend(vec![
+                        "--tensor-parallel-size".to_string(),
+                        "1".to_string(),
+                        "--pipeline-parallel-size".to_string(),
+                        pp_size.to_string(),
+                        "--nnodes".to_string(),
+                        nnodes.to_string(),
+                        "--node-rank".to_string(),
+                        node.rank.to_string(),
+                        "--master-addr".to_string(),
+                        cluster.master_ib_ip.clone(),
+                        "--distributed-executor-backend".to_string(),
+                        "ray".to_string(),
+                    ]);
+                }
+                _ => {
+                    parts.extend(vec![
+                        "--tensor-parallel-size".to_string(),
+                        "1".to_string(),
+                        "--pipeline-parallel-size".to_string(),
+                        pp_size.to_string(),
+                        "--nnodes".to_string(),
+                        nnodes.to_string(),
+                        "--node-rank".to_string(),
+                        node.rank.to_string(),
+                        "--master-addr".to_string(),
+                        cluster.master_ib_ip.clone(),
+                        "--distributed-executor-backend".to_string(),
+                        recipe.distributed_backend.clone(),
+                    ]);
+                }
+            }
+        }
+
+        // Port — SGLang needs it on all nodes, vLLM only on head node
+        if is_sglang || node.rank == 0 {
+            parts.extend(vec!["--port".to_string(), recipe.port.to_string()]);
+        }
+
+        parts.join(" ")
+    }
+
+    /// Builds the full sequence of tmux commands for a session, with multi-window layout.
+    pub fn build_session_commands(
+        &self,
+        node: &Node,
+        recipe: &Recipe,
+        nccl: &NcclConfig,
+        cluster: &Cluster,
+    ) -> Vec<String> {
+        let sess = self.session_name(recipe, node.rank);
+        let mut cmds: Vec<String> = Vec::new();
+
+        // Kill existing session if it exists
+        cmds.push(format!("tmux kill-session -t {} 2>/dev/null || true", sess));
+
+        // Create new detached session (window 0)
+        cmds.push(format!("tmux new-session -d -s {} -n model", sess));
+
+        // --- Window 0: model process ---
+        for export in self.build_nccl_setup(nccl) {
+            cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, export));
+        }
+        for export in self.build_host_ip_export(&recipe.engine, node) {
+            cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, export));
+        }
+        if recipe.distributed_backend == "ray" && node.rank > 0 {
+            cmds.push(format!(
+                "tmux send-keys -t {}:0 'export RAY_ADDRESS={}:6379' Enter",
+                sess, cluster.master_ib_ip
+            ));
+        }
+        for (k, v) in &recipe.env {
+            cmds.push(format!(
+                "tmux send-keys -t {}:0 'export {}=\"{}\"' Enter",
+                sess, k, v
+            ));
+        }
+
+        let launch_cmd = self.build_launch_command(recipe, node, cluster);
+        cmds.push(format!(
+            "tmux send-keys -t {}:0 '{}' Enter",
+            sess, launch_cmd
+        ));
+
+        // --- Window 1: GPU monitor ---
+        cmds.push(format!("tmux new-window -t {}:1 -n gpu", sess));
+        cmds.push(format!(
+            "tmux send-keys -t {}:1 'nvidia-smi -l 5' Enter",
+            sess
+        ));
+
+        // --- Window 2: shell (debug) ---
+        cmds.push(format!("tmux new-window -t {}:2 -n shell", sess));
+        cmds.push(format!(
+            "tmux send-keys -t {}:2 'echo \"ready for debugging\"' Enter",
+            sess
+        ));
+        cmds.push(format!("tmux select-window -t {}:0", sess));
+
+        cmds
+    }
+}

+ 3 - 465
src/tmux/mod.rs

@@ -1,14 +1,12 @@
+pub mod commands;
 pub mod ops;
 pub mod serve;
+pub mod session;
 
 use anyhow::Result;
-use std::time::Duration;
-use tokio::time::sleep;
-use tracing::{info, warn};
 
 use crate::cli::TmuxCommands;
-use crate::config::{Cluster, Config, NcclConfig, Node, Recipe};
-use crate::ssh::*;
+use crate::config::{Config, NcclConfig, Node, Recipe};
 
 /// Manages tmux sessions for model inference across a DGX Spark cluster.
 #[derive(Clone)]
@@ -46,466 +44,6 @@ impl TmuxManager {
             ],
         }
     }
-
-    /// Builds the full model launch command for a given node.
-    pub fn build_launch_command(&self, recipe: &Recipe, node: &Node, cluster: &Cluster) -> String {
-        let nnodes = cluster.nodes.len() as u32;
-        let pp_size = recipe.pipeline_parallel_size.unwrap_or(nnodes);
-        let is_sglang = recipe.engine == "sglang";
-
-        let mut parts: Vec<String> = Vec::new();
-
-        // Engine entrypoint
-        match recipe.engine.as_str() {
-            "vllm" => {
-                parts.push("python -m vllm.entrypoints.openai.api_server".to_string());
-            }
-            "sglang" => {
-                parts.push("python -m sglang.launch_server".to_string());
-            }
-            other => {
-                parts.push(format!("python -m {}", other));
-            }
-        }
-
-        // Model — engine-specific
-        if is_sglang {
-            parts.push("--model-path".to_string());
-        }
-        parts.push(recipe.model.clone());
-
-        // Recipe-specific args
-        parts.extend(recipe.args.clone());
-
-        // Distributed flags — engine-specific
-        if is_sglang {
-            // SGLang uses --tp (not --tensor-parallel-size),
-            // and has no --pipeline-parallel-size / --distributed-executor-backend
-            parts.extend(vec![
-                "--tp".to_string(),
-                recipe.tensor_parallel_size.to_string(),
-                "--nnodes".to_string(),
-                nnodes.to_string(),
-                "--node-rank".to_string(),
-                node.rank.to_string(),
-                "--master-addr".to_string(),
-                cluster.master_ib_ip.clone(),
-            ]);
-        } else {
-            match recipe.distributed_backend.as_str() {
-                "ray" => {
-                    parts.extend(vec![
-                        "--tensor-parallel-size".to_string(),
-                        "1".to_string(),
-                        "--pipeline-parallel-size".to_string(),
-                        pp_size.to_string(),
-                        "--nnodes".to_string(),
-                        nnodes.to_string(),
-                        "--node-rank".to_string(),
-                        node.rank.to_string(),
-                        "--master-addr".to_string(),
-                        cluster.master_ib_ip.clone(),
-                        "--distributed-executor-backend".to_string(),
-                        "ray".to_string(),
-                    ]);
-                }
-                _ => {
-                    // MP mode (default)
-                    parts.extend(vec![
-                        "--tensor-parallel-size".to_string(),
-                        "1".to_string(),
-                        "--pipeline-parallel-size".to_string(),
-                        pp_size.to_string(),
-                        "--nnodes".to_string(),
-                        nnodes.to_string(),
-                        "--node-rank".to_string(),
-                        node.rank.to_string(),
-                        "--master-addr".to_string(),
-                        cluster.master_ib_ip.clone(),
-                        "--distributed-executor-backend".to_string(),
-                        recipe.distributed_backend.clone(),
-                    ]);
-                }
-            }
-        }
-
-        // Port — SGLang needs it on all nodes, vLLM only on head node
-        if is_sglang || node.rank == 0 {
-            parts.extend(vec!["--port".to_string(), recipe.port.to_string()]);
-        }
-
-        parts.join(" ")
-    }
-    /// Builds the full sequence of tmux commands for a session, with multi-window layout.
-    pub fn build_session_commands(
-        &self,
-        node: &Node,
-        recipe: &Recipe,
-        nccl: &NcclConfig,
-        cluster: &Cluster,
-    ) -> Vec<String> {
-        let sess = self.session_name(recipe, node.rank);
-        let mut cmds: Vec<String> = Vec::new();
-
-        // Kill existing session if it exists
-        cmds.push(format!("tmux kill-session -t {} 2>/dev/null || true", sess));
-
-        // Create new detached session (window 0)
-        cmds.push(format!("tmux new-session -d -s {} -n model", sess));
-
-        // --- Window 0: model process ---
-        for export in self.build_nccl_setup(nccl) {
-            cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, export));
-        }
-        for export in self.build_host_ip_export(&recipe.engine, node) {
-            cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, export));
-        }
-        if recipe.distributed_backend == "ray" && node.rank > 0 {
-            cmds.push(format!(
-                "tmux send-keys -t {}:0 'export RAY_ADDRESS={}:6379' Enter",
-                sess, cluster.master_ib_ip
-            ));
-        }
-        for (k, v) in &recipe.env {
-            cmds.push(format!(
-                "tmux send-keys -t {}:0 'export {}=\"{}\"' Enter",
-                sess, k, v
-            ));
-        }
-
-        let launch_cmd = self.build_launch_command(recipe, node, cluster);
-        cmds.push(format!(
-            "tmux send-keys -t {}:0 '{}' Enter",
-            sess, launch_cmd
-        ));
-
-        // --- Window 1: GPU monitor ---
-        cmds.push(format!("tmux new-window -t {}:1 -n gpu", sess));
-        cmds.push(format!(
-            "tmux send-keys -t {}:1 'nvidia-smi -l 5' Enter",
-            sess
-        ));
-
-        // --- Window 2: shell (debug) ---
-        cmds.push(format!("tmux new-window -t {}:2 -n shell", sess));
-        cmds.push(format!(
-            "tmux send-keys -t {}:2 'echo \"ready for debugging\"' Enter",
-            sess
-        ));
-        cmds.push(format!("tmux select-window -t {}:0", sess));
-
-        cmds
-    }
-    /// Executes a list of commands via SSH on a node to create a tmux session.
-    pub async fn create_session(
-        &self,
-        config: &Config,
-        node: &Node,
-        commands: &[String],
-        dry_run: bool,
-    ) -> Result<()> {
-        if dry_run {
-            info!("  ssh {}@{}:", config.ssh_user, node.ssh_ip);
-            for cmd in commands {
-                info!("    {}", cmd);
-            }
-            return Ok(());
-        }
-
-        let script = commands.join(" && ");
-        let output = run_ssh(config, node, script, dry_run).await?;
-
-        if output.trim().is_empty() {
-            info!("  ✅ tmux session created on {}", node.name);
-        } else {
-            info!("  tmux session output on {}: {}", node.name, output.trim());
-        }
-
-        Ok(())
-    }
-
-    /// Checks whether a tmux session exists on the given node.
-    pub async fn check_session(
-        &self,
-        config: &Config,
-        node: &Node,
-        session_name: &str,
-        dry_run: bool,
-    ) -> Result<bool> {
-        if dry_run {
-            info!(
-                "  ssh {}@{}: tmux list-sessions 2>/dev/null | grep '{}'",
-                config.ssh_user, node.ssh_ip, session_name
-            );
-            return Ok(true);
-        }
-
-        let cmd = format!(
-            "tmux list-sessions 2>/dev/null | grep -c '{}'",
-            session_name
-        );
-        let output = run_ssh(config, node, cmd, dry_run).await?;
-
-        let found = output.trim().parse::<usize>().unwrap_or(0) > 0;
-        if found {
-            info!("  Session '{}' found on {}", session_name, node.name);
-        } else {
-            warn!("  Session '{}' NOT found on {}", session_name, node.name);
-        }
-
-        Ok(found)
-    }
-    /// Performs a deep health check on a tmux session.
-    pub async fn deep_check_session(
-        &self,
-        config: &Config,
-        node: &Node,
-        _cluster: &Cluster,
-        session_name: &str,
-        recipe: &Recipe,
-        dry_run: bool,
-    ) -> Result<SessionHealth> {
-        if dry_run {
-            return Ok(SessionHealth {
-                session_exists: true,
-                process_alive: true,
-                gpu_ok: true,
-                api_ok: true,
-            });
-        }
-
-        // 1. Check session exists
-        let exists = self
-            .check_session(config, node, session_name, dry_run)
-            .await?;
-
-        // 2. Check process is alive inside tmux
-        let process_alive = if exists {
-            let pid_cmd = format!(
-                "tmux list-panes -t {}:0 -F '#{{pane_pid}}' | head -1",
-                session_name
-            );
-            match run_ssh(config, node, pid_cmd, dry_run).await {
-                Ok(pid_str) => {
-                    let pid = pid_str.trim();
-                    if !pid.is_empty() && pid != "0" {
-                        let ps_cmd = format!("ps -p {} -o pid= 2>/dev/null", pid);
-                        run_ssh(config, node, ps_cmd, dry_run)
-                            .await
-                            .map(|o| !o.trim().is_empty())
-                            .unwrap_or(false)
-                    } else {
-                        false
-                    }
-                }
-                Err(_) => false,
-            }
-        } else {
-            false
-        };
-
-        // 3. Check GPU is available
-        let gpu_ok = match run_ssh(
-            config,
-            node,
-            "nvidia-smi --query-gpu=index,utilization.gpu,memory.used --format=csv,noheader 2>&1 | head -5"
-                .to_string(),
-            dry_run,
-        )
-        .await
-        {
-            Ok(out) => !out.contains("failed") && !out.contains("error") && !out.trim().is_empty(),
-            Err(_) => false,
-        };
-
-        // 4. Check API endpoint (head node only)
-        let api_ok = if node.rank == 0 && process_alive {
-            let api_check = format!(
-                "curl -s -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{}/v1/models 2>/dev/null || echo 'fail'",
-                recipe.port
-            );
-            match run_ssh(config, node, api_check, dry_run).await {
-                Ok(out) => {
-                    let code = out.trim();
-                    code == "200" || !code.contains("fail")
-                }
-                Err(_) => false,
-            }
-        } else {
-            node.rank != 0 || process_alive
-        };
-
-        Ok(SessionHealth {
-            session_exists: exists,
-            process_alive,
-            gpu_ok,
-            api_ok,
-        })
-    }
-
-    /// Gracefully stops a tmux session with timeout escalation.
-    pub async fn graceful_stop(
-        &self,
-        config: &Config,
-        node: &Node,
-        session_name: &str,
-        dry_run: bool,
-    ) -> Result<()> {
-        if dry_run {
-            info!(
-                "  ssh {}@{}: graceful stop of session '{}'",
-                config.ssh_user, node.ssh_ip, session_name
-            );
-            return Ok(());
-        }
-
-        info!(
-            "  Sending Ctrl+C to session '{}' on {}",
-            session_name, node.name
-        );
-        let _ = run_ssh(
-            config,
-            node,
-            format!("tmux send-keys -t {}:0 C-c", session_name),
-            dry_run,
-        )
-        .await;
-
-        let mut exited = false;
-        for i in 0..15 {
-            sleep(Duration::from_secs(2)).await;
-            let check_cmd = format!(
-                "tmux list-panes -t {}:0 -F '#{{pane_pid}}' 2>/dev/null | head -1",
-                session_name
-            );
-            match run_ssh(config, node, check_cmd, dry_run).await {
-                Ok(pid_str) => {
-                    let pid = pid_str.trim();
-                    if pid.is_empty() || pid == "0" || pid == "No" {
-                        exited = true;
-                        info!(
-                            "  Session '{}' exited gracefully (attempt {})",
-                            session_name,
-                            i + 1
-                        );
-                        break;
-                    }
-                    let ps_cmd = format!("ps -p {} -o pid= 2>/dev/null", pid);
-                    match run_ssh(config, node, ps_cmd, dry_run).await {
-                        Ok(ps_out) if ps_out.trim().is_empty() => {
-                            exited = true;
-                            info!(
-                                "  Session '{}' process exited (attempt {})",
-                                session_name,
-                                i + 1
-                            );
-                            break;
-                        }
-                        _ => continue,
-                    }
-                }
-                Err(_) => {
-                    exited = true;
-                    break;
-                }
-            }
-        }
-
-        if !exited {
-            warn!(
-                "  Session '{}' did not exit gracefully, force killing...",
-                session_name
-            );
-            let _ = run_ssh(
-                config,
-                node,
-                format!("tmux kill-session -t {}", session_name),
-                dry_run,
-            )
-            .await;
-        }
-
-        Ok(())
-    }
-    /// Streams the content of a tmux pane, optionally following new output.
-    pub async fn stream_pane(
-        &self,
-        config: &Config,
-        node: &Node,
-        session_name: &str,
-        window: u32,
-        follow: bool,
-        dry_run: bool,
-    ) -> Result<String> {
-        if dry_run {
-            if follow {
-                info!(
-                    "  ssh {}@{}: tail -f (streaming from session '{}' window {})",
-                    config.ssh_user, node.ssh_ip, session_name, window
-                );
-            } else {
-                info!(
-                    "  ssh {}@{}: tmux capture-pane -t {}:{} -p -S -100",
-                    config.ssh_user, node.ssh_ip, session_name, window
-                );
-            }
-            return Ok(String::new());
-        }
-
-        let cmd = if follow {
-            format!(
-                "tmux pipe-pane -o -t {}:{} 'cat > /tmp/sm121_pipe_{}_{}.log' && tail -f /tmp/sm121_pipe_{}_{}.log",
-                session_name,
-                window,
-                session_name.replace('-', "_"),
-                window,
-                session_name.replace('-', "_"),
-                window
-            )
-        } else {
-            format!(
-                "tmux capture-pane -t {}:{} -p -S -200 2>/dev/null || echo 'no output'",
-                session_name, window
-            )
-        };
-
-        run_ssh_with_retry(config, node, cmd, dry_run, 1).await
-    }
-
-    /// Attaches to a tmux session on a node via SSH (interactive).
-    pub async fn attach_session(
-        &self,
-        config: &Config,
-        node: &Node,
-        session_name: &str,
-        dry_run: bool,
-    ) -> Result<()> {
-        if dry_run {
-            info!(
-                "  Would attach: ssh -t {}@{} tmux attach-session -t {}",
-                config.ssh_user, node.ssh_ip, session_name
-            );
-            return Ok(());
-        }
-
-        info!(
-            "  Attaching to '{}' on {}... Press Ctrl+B D to detach.",
-            session_name, node.name
-        );
-        info!("  (russh does not support interactive PTY. Use the following command directly:)");
-        println!(
-            "  ssh -t {}@{} tmux attach-session -t {}",
-            config.ssh_user, node.ssh_ip, session_name
-        );
-        println!();
-        info!("  Streaming pane content instead:");
-        let output = self
-            .stream_pane(config, node, session_name, 0, true, dry_run)
-            .await?;
-        println!("{}", output);
-
-        Ok(())
-    }
 }
 
 /// Detailed health status of a tmux session.

+ 323 - 0
src/tmux/session.rs

@@ -0,0 +1,323 @@
+use anyhow::Result;
+use std::time::Duration;
+use tokio::time::sleep;
+use tracing::{info, warn};
+
+use crate::config::{Cluster, Config, Node, Recipe};
+use crate::ssh::*;
+use crate::tmux::{SessionHealth, TmuxManager};
+
+impl TmuxManager {
+    /// Executes a list of commands via SSH on a node to create a tmux session.
+    pub async fn create_session(
+        &self,
+        config: &Config,
+        node: &Node,
+        commands: &[String],
+        dry_run: bool,
+    ) -> Result<()> {
+        if dry_run {
+            info!("  ssh {}@{}:", config.ssh_user, node.ssh_ip);
+            for cmd in commands {
+                info!("    {}", cmd);
+            }
+            return Ok(());
+        }
+
+        let script = commands.join(" && ");
+        let output = run_ssh(config, node, script, dry_run).await?;
+
+        if output.trim().is_empty() {
+            info!("  ✅ tmux session created on {}", node.name);
+        } else {
+            info!("  tmux session output on {}: {}", node.name, output.trim());
+        }
+
+        Ok(())
+    }
+
+    /// Checks whether a tmux session exists on the given node.
+    pub async fn check_session(
+        &self,
+        config: &Config,
+        node: &Node,
+        session_name: &str,
+        dry_run: bool,
+    ) -> Result<bool> {
+        if dry_run {
+            info!(
+                "  ssh {}@{}: tmux list-sessions 2>/dev/null | grep '{}'",
+                config.ssh_user, node.ssh_ip, session_name
+            );
+            return Ok(true);
+        }
+
+        let cmd = format!(
+            "tmux list-sessions 2>/dev/null | grep -c '{}'",
+            session_name
+        );
+        let output = run_ssh(config, node, cmd, dry_run).await?;
+
+        let found = output.trim().parse::<usize>().unwrap_or(0) > 0;
+        if found {
+            info!("  Session '{}' found on {}", session_name, node.name);
+        } else {
+            warn!("  Session '{}' NOT found on {}", session_name, node.name);
+        }
+
+        Ok(found)
+    }
+
+    /// Performs a deep health check on a tmux session.
+    pub async fn deep_check_session(
+        &self,
+        config: &Config,
+        node: &Node,
+        _cluster: &Cluster,
+        session_name: &str,
+        recipe: &Recipe,
+        dry_run: bool,
+    ) -> Result<SessionHealth> {
+        if dry_run {
+            return Ok(SessionHealth {
+                session_exists: true,
+                process_alive: true,
+                gpu_ok: true,
+                api_ok: true,
+            });
+        }
+
+        // 1. Check session exists
+        let exists = self
+            .check_session(config, node, session_name, dry_run)
+            .await?;
+
+        // 2. Check process is alive inside tmux
+        let process_alive = if exists {
+            let pid_cmd = format!(
+                "tmux list-panes -t {}:0 -F '#{{pane_pid}}' | head -1",
+                session_name
+            );
+            match run_ssh(config, node, pid_cmd, dry_run).await {
+                Ok(pid_str) => {
+                    let pid = pid_str.trim();
+                    if !pid.is_empty() && pid != "0" {
+                        let ps_cmd = format!("ps -p {} -o pid= 2>/dev/null", pid);
+                        run_ssh(config, node, ps_cmd, dry_run)
+                            .await
+                            .map(|o| !o.trim().is_empty())
+                            .unwrap_or(false)
+                    } else {
+                        false
+                    }
+                }
+                Err(_) => false,
+            }
+        } else {
+            false
+        };
+
+        // 3. Check GPU is available
+        let gpu_ok = match run_ssh(
+            config,
+            node,
+            "nvidia-smi --query-gpu=index,utilization.gpu,memory.used --format=csv,noheader 2>&1 | head -5"
+                .to_string(),
+            dry_run,
+        )
+        .await
+        {
+            Ok(out) => !out.contains("failed") && !out.contains("error") && !out.trim().is_empty(),
+            Err(_) => false,
+        };
+
+        // 4. Check API endpoint (head node only)
+        let api_ok = if node.rank == 0 && process_alive {
+            let api_check = format!(
+                "curl -s -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{}/v1/models 2>/dev/null || echo 'fail'",
+                recipe.port
+            );
+            match run_ssh(config, node, api_check, dry_run).await {
+                Ok(out) => {
+                    let code = out.trim();
+                    code == "200" || !code.contains("fail")
+                }
+                Err(_) => false,
+            }
+        } else {
+            node.rank != 0 || process_alive
+        };
+
+        Ok(SessionHealth {
+            session_exists: exists,
+            process_alive,
+            gpu_ok,
+            api_ok,
+        })
+    }
+
+    /// Gracefully stops a tmux session with timeout escalation.
+    pub async fn graceful_stop(
+        &self,
+        config: &Config,
+        node: &Node,
+        session_name: &str,
+        dry_run: bool,
+    ) -> Result<()> {
+        if dry_run {
+            info!(
+                "  ssh {}@{}: graceful stop of session '{}'",
+                config.ssh_user, node.ssh_ip, session_name
+            );
+            return Ok(());
+        }
+
+        info!(
+            "  Sending Ctrl+C to session '{}' on {}",
+            session_name, node.name
+        );
+        let _ = run_ssh(
+            config,
+            node,
+            format!("tmux send-keys -t {}:0 C-c", session_name),
+            dry_run,
+        )
+        .await;
+
+        let mut exited = false;
+        for i in 0..15 {
+            sleep(Duration::from_secs(2)).await;
+            let check_cmd = format!(
+                "tmux list-panes -t {}:0 -F '#{{pane_pid}}' 2>/dev/null | head -1",
+                session_name
+            );
+            match run_ssh(config, node, check_cmd, dry_run).await {
+                Ok(pid_str) => {
+                    let pid = pid_str.trim();
+                    if pid.is_empty() || pid == "0" || pid == "No" {
+                        exited = true;
+                        info!(
+                            "  Session '{}' exited gracefully (attempt {})",
+                            session_name,
+                            i + 1
+                        );
+                        break;
+                    }
+                    let ps_cmd = format!("ps -p {} -o pid= 2>/dev/null", pid);
+                    match run_ssh(config, node, ps_cmd, dry_run).await {
+                        Ok(ps_out) if ps_out.trim().is_empty() => {
+                            exited = true;
+                            info!(
+                                "  Session '{}' process exited (attempt {})",
+                                session_name,
+                                i + 1
+                            );
+                            break;
+                        }
+                        _ => continue,
+                    }
+                }
+                Err(_) => {
+                    exited = true;
+                    break;
+                }
+            }
+        }
+
+        if !exited {
+            warn!(
+                "  Session '{}' did not exit gracefully, force killing...",
+                session_name
+            );
+            let _ = run_ssh(
+                config,
+                node,
+                format!("tmux kill-session -t {}", session_name),
+                dry_run,
+            )
+            .await;
+        }
+
+        Ok(())
+    }
+
+    /// Streams the content of a tmux pane, optionally following new output.
+    pub async fn stream_pane(
+        &self,
+        config: &Config,
+        node: &Node,
+        session_name: &str,
+        window: u32,
+        follow: bool,
+        dry_run: bool,
+    ) -> Result<String> {
+        if dry_run {
+            if follow {
+                info!(
+                    "  ssh {}@{}: tail -f (streaming from session '{}' window {})",
+                    config.ssh_user, node.ssh_ip, session_name, window
+                );
+            } else {
+                info!(
+                    "  ssh {}@{}: tmux capture-pane -t {}:{} -p -S -100",
+                    config.ssh_user, node.ssh_ip, session_name, window
+                );
+            }
+            return Ok(String::new());
+        }
+
+        let cmd = if follow {
+            format!(
+                "tmux pipe-pane -o -t {}:{} 'cat > /tmp/sm121_pipe_{}_{}.log' && tail -f /tmp/sm121_pipe_{}_{}.log",
+                session_name,
+                window,
+                session_name.replace('-', "_"),
+                window,
+                session_name.replace('-', "_"),
+                window
+            )
+        } else {
+            format!(
+                "tmux capture-pane -t {}:{} -p -S -200 2>/dev/null || echo 'no output'",
+                session_name, window
+            )
+        };
+
+        run_ssh_with_retry(config, node, cmd, dry_run, 1).await
+    }
+
+    /// Attaches to a tmux session on a node via SSH (interactive).
+    pub async fn attach_session(
+        &self,
+        config: &Config,
+        node: &Node,
+        session_name: &str,
+        dry_run: bool,
+    ) -> Result<()> {
+        if dry_run {
+            info!(
+                "  Would attach: ssh -t {}@{} tmux attach-session -t {}",
+                config.ssh_user, node.ssh_ip, session_name
+            );
+            return Ok(());
+        }
+
+        info!(
+            "  Attaching to '{}' on {}... Press Ctrl+B D to detach.",
+            session_name, node.name
+        );
+        info!("  (russh does not support interactive PTY. Use the following command directly:)");
+        println!(
+            "  ssh -t {}@{} tmux attach-session -t {}",
+            config.ssh_user, node.ssh_ip, session_name
+        );
+        println!();
+        info!("  Streaming pane content instead:");
+        let output = self
+            .stream_pane(config, node, session_name, 0, true, dry_run)
+            .await?;
+        println!("{}", output);
+
+        Ok(())
+    }
+}