refactor: split codebase into <500-line modules, fix review issues

Project structure after refactor:
  src/
  ├── main.rs      (88)  — entry point only
  ├── cli.rs       (107) — Cli, Commands, TmuxCommands
  ├── config.rs    (386) — Config types, select_cluster, resolve_cluster
  ├── docker.rs    (234) — Docker serve/stop/logs/health/build_docker_command
  ├── doctor.rs    (332) — Doctor mode, repair, Dockerfile generation
  ├── health.rs    (131) — tmux health check (pure data function, integrated)
  ├── ssh.rs       (180) — SSH session management
  └── tmux/
      ├── mod.rs   (500) — TmuxManager, SessionHealth, handle_tmux
      ├── ops.rs   (261) — tmux_stop, tmux_logs, tmux_status, tmux_attach
      └── serve.rs (188) — tmux_serve, wait_for_head_ready

Changes:
- health.rs: removed dead_code silencers, made check_cluster_health a pure
  data function (no println), integrated as data backend for tmux_status()
- Removed duplicate select_cluster from tmux.rs (now uses crate::select_cluster)
- Fixed Ray backend flags: added --pipeline-parallel-size, --nnodes,
  --distributed-executor-backend ray in both docker and tmux paths
- Added #[serde(default)] for nccl field in Config
- 21 unit tests split across config/docker/doctor modules
- Fixed all clippy warnings from previous review

src/cli.rs

@@ -0,0 +1,107 @@
+use clap::{Parser, Subcommand};
+
+#[derive(Parser)]
+#[command(author, version, about = "DGX Spark Launcher для vLLM и SGLang")]
+pub struct Cli {
+    #[arg(long, short, default_value = "config.toml")]
+    pub config: String,
+
+    #[arg(long, short, help = "Показать команды без выполнения")]
+    pub dry_run: bool,
+
+    #[command(subcommand)]
+    pub command: Commands,
+}
+
+#[derive(Subcommand)]
+pub enum Commands {
+    /// Запустить кластер
+    Serve {
+        #[arg(long, required = true)]
+        recipe: String,
+
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+
+    /// Показать логи контейнеров
+    Logs {
+        #[arg(long, required = true)]
+        recipe: String,
+
+        #[arg(long, action = clap::ArgAction::SetTrue)]
+        follow: bool,
+    },
+
+    /// Остановить кластер
+    Stop {
+        #[arg(long, required = true)]
+        recipe: String,
+    },
+
+    /// Проверить здоровье кластера
+    Health {
+        #[arg(
+            long,
+            help = "Имя кластера (например, 4nodes-switch). Если не указано - проверяет все."
+        )]
+        cluster: Option<String>,
+        #[arg(long, help = "Показать хост-ключи узлов (для добавления в config.toml)")]
+        fingerprint: bool,
+    },
+
+    /// Проверить и исправить рецепт (doctor mode)
+    Doctor {
+        #[arg(long, required = true, help = "Имя рецепта для проверки")]
+        recipe: String,
+        #[arg(long, default_value = "auto", help = "Кластер (если не auto — использует recipe.cluster)")]
+        cluster: String,
+        #[arg(long, help = "Автоматически исправить проблемы (создать Dockerfile + собрать образ)")]
+        repair: bool,
+    },
+
+    /// Управление tmux-сессиями (альтернатива Docker)
+    #[command(subcommand)]
+    Tmux(TmuxCommands),
+}
+
+#[derive(Subcommand)]
+pub enum TmuxCommands {
+    /// Запустить кластер через tmux
+    Serve {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+    /// Остановить tmux-сессии
+    Stop {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+    /// Статус tmux-сессий (подробный)
+    Status {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+    /// Показать логи tmux-сессий
+    Logs {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, action = clap::ArgAction::SetTrue)]
+        follow: bool,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+    /// Подключиться к tmux-сессии головного узла
+    Attach {
+        #[arg(long, required = true)]
+        recipe: String,
+        #[arg(long, default_value = "auto")]
+        cluster: String,
+    },
+}

src/config.rs

@@ -0,0 +1,386 @@
+use anyhow::{Context, Result};
+use serde::Deserialize;
+use std::collections::HashMap;
+use tracing::info;
+
+#[derive(Clone, Deserialize)]
+pub struct Node {
+    pub name: String,
+    pub ssh_ip: String,
+    pub ib_ip: String,
+    pub rank: u32,
+    #[serde(default)]
+    pub host_key: Option<String>,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct NcclConfig {
+    pub ib_hca: String,
+    pub socket_ifname: String,
+    pub ib_gid_index: u32,
+    pub ib_disable: u32,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct Cluster {
+    pub name: String,
+    pub master_ib_ip: String,
+    pub nodes: Vec<Node>,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct Config {
+    pub ssh_user: String,
+    pub ssh_key_path: String,
+    pub clusters: Vec<Cluster>,
+    #[serde(default)]
+    pub nccl: HashMap<String, NcclConfig>,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct Recipe {
+    #[serde(default = "default_engine")]
+    pub engine: String,
+    pub name: String,
+    #[serde(default)]
+    pub docker_image: String,
+    #[serde(default)]
+    pub model: String,
+    #[serde(default)]
+    pub port: u16,
+    #[serde(default)]
+    pub required_nodes: Option<u32>,
+    #[serde(default)]
+    pub pipeline_parallel_size: Option<u32>,
+    #[serde(default)]
+    pub args: Vec<String>,
+    #[serde(default)]
+    pub env: HashMap<String, String>,
+    #[serde(default)]
+    pub cluster: Option<String>,
+    #[serde(default)]
+    pub entrypoint: Vec<String>,
+    #[serde(default = "default_distributed_backend")]
+    pub distributed_backend: String,
+}
+
+fn default_engine() -> String {
+    "vllm".to_string()
+}
+
+fn default_distributed_backend() -> String {
+    "mp".to_string()
+}
+
+/// Selects a cluster and its NCCL config by name, or auto-picks the first
+/// cluster that has NCCL config.
+pub fn select_cluster<'c>(config: &'c Config, cluster_name: &str) -> Result<(Cluster, &'c NcclConfig)> {
+    if cluster_name != "auto" {
+        let cluster = config
+            .clusters
+            .iter()
+            .find(|c| c.name == cluster_name)
+            .context(format!("Кластер {} не найден", cluster_name))?;
+        let nccl = config
+            .nccl
+            .get(&cluster.name)
+            .context("NCCL конфиг не найден")?;
+        return Ok((cluster.clone(), nccl));
+    }
+
+    for cluster in &config.clusters {
+        if let Some(nccl) = config.nccl.get(&cluster.name) {
+            info!("✅ Автоподбор кластера: {}", cluster.name);
+            return Ok((cluster.clone(), nccl));
+        }
+    }
+
+    anyhow::bail!("Не удалось подобрать подходящий кластер");
+}
+
+/// Looks up a cluster by name without requiring NCCL config.
+pub fn select_cluster_by_name(config: &Config, cluster_name: &str) -> Result<Cluster> {
+    let cluster = config
+        .clusters
+        .iter()
+        .find(|c| c.name == cluster_name)
+        .context(format!("Кластер {} не найден", cluster_name))?;
+    Ok(cluster.clone())
+}
+
+/// Resolves the cluster for a recipe, applying recipe-level cluster pinning.
+pub fn resolve_cluster<'c>(
+    config: &'c Config,
+    recipe: &Recipe,
+    cluster_name: &str,
+) -> Result<(Cluster, &'c NcclConfig)> {
+    let choice = if cluster_name != "auto" {
+        cluster_name.to_string()
+    } else if let Some(ref rc) = recipe.cluster {
+        rc.clone()
+    } else {
+        "auto".to_string()
+    };
+    select_cluster(config, &choice)
+}
+
+/// Loads a recipe from disk.
+pub async fn load_recipe(recipe_name: &str) -> Result<Recipe> {
+    let path = format!("recipes/{}/recipe.toml", recipe_name);
+    let content = tokio::fs::read_to_string(&path).await?;
+    Ok(toml::from_str(&content)?)
+}
+
+/// Returns the cluster and recipe name for a recipe (for stop/logs commands
+/// that only receive a recipe name).
+pub async fn resolve_cluster_for_recipe(config: &Config, recipe_name: &str) -> Result<(Cluster, Recipe)> {
+    let recipe = load_recipe(recipe_name).await?;
+    let (cluster, _) = resolve_cluster(config, &recipe, "auto")?;
+    Ok((cluster, recipe))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn test_cluster() -> Cluster {
+        Cluster {
+            name: "test-cluster".to_string(),
+            master_ib_ip: "192.168.1.1".to_string(),
+            nodes: vec![
+                Node {
+                    name: "node0".to_string(),
+                    ssh_ip: "10.0.0.1".to_string(),
+                    ib_ip: "192.168.1.1".to_string(),
+                    rank: 0,
+                    host_key: None,
+                },
+                Node {
+                    name: "node1".to_string(),
+                    ssh_ip: "10.0.0.2".to_string(),
+                    ib_ip: "192.168.1.2".to_string(),
+                    rank: 1,
+                    host_key: None,
+                },
+            ],
+        }
+    }
+
+    fn test_nccl() -> NcclConfig {
+        NcclConfig {
+            ib_hca: "rocep1s0f0".to_string(),
+            socket_ifname: "enp1s0f0np0".to_string(),
+            ib_gid_index: 3,
+            ib_disable: 0,
+        }
+    }
+
+    // ── Deserialization tests ──────────────────────────────────
+
+    #[test]
+    fn test_recipe_deserialize_minimal() {
+        let toml_str = r#"
+name = "Minimal"
+model = "test/model"
+"#;
+        let recipe: Recipe = toml::from_str(toml_str).unwrap();
+        assert_eq!(recipe.name, "Minimal");
+        assert_eq!(recipe.model, "test/model");
+        assert_eq!(recipe.engine, "vllm");
+        assert_eq!(recipe.port, 0);
+        assert!(recipe.args.is_empty());
+        assert!(recipe.env.is_empty());
+        assert_eq!(recipe.distributed_backend, "mp");
+    }
+
+    #[test]
+    fn test_recipe_deserialize_full() {
+        let toml_str = r#"
+engine = "sglang"
+name = "Full Recipe"
+docker_image = "org/image:tag"
+model = "org/model"
+port = 30000
+required_nodes = 4
+pipeline_parallel_size = 2
+distributed_backend = "ray"
+cluster = "my-cluster"
+
+args = ["--arg1", "val1", "--arg2"]
+
+[env]
+KEY = "value"
+OTHER = "123"
+"#;
+        let recipe: Recipe = toml::from_str(toml_str).unwrap();
+        assert_eq!(recipe.engine, "sglang");
+        assert_eq!(recipe.name, "Full Recipe");
+        assert_eq!(recipe.docker_image, "org/image:tag");
+        assert_eq!(recipe.model, "org/model");
+        assert_eq!(recipe.port, 30000);
+        assert_eq!(recipe.required_nodes, Some(4));
+        assert_eq!(recipe.pipeline_parallel_size, Some(2));
+        assert_eq!(recipe.distributed_backend, "ray");
+        assert_eq!(recipe.cluster, Some("my-cluster".to_string()));
+        assert_eq!(recipe.args, vec!["--arg1", "val1", "--arg2"]);
+        assert_eq!(recipe.env.get("KEY").unwrap(), "value");
+        assert_eq!(recipe.env.get("OTHER").unwrap(), "123");
+    }
+
+    #[test]
+    fn test_recipe_deserialize_no_required_nodes() {
+        let toml_str = r#"
+name = "Flexible"
+model = "test/model"
+"#;
+        let recipe: Recipe = toml::from_str(toml_str).unwrap();
+        assert_eq!(recipe.required_nodes, None);
+        assert_eq!(recipe.pipeline_parallel_size, None);
+    }
+
+    #[test]
+    fn test_config_deserialize() {
+        let toml_str = r#"
+ssh_user = "admin"
+ssh_key_path = "/home/admin/.ssh/id_rsa"
+
+[[clusters]]
+name = "prod"
+master_ib_ip = "10.0.0.1"
+nodes = [
+    { name = "node1", ssh_ip = "10.0.0.10", ib_ip = "10.0.0.1", rank = 0 },
+    { name = "node2", ssh_ip = "10.0.0.11", ib_ip = "10.0.0.2", rank = 1 },
+]
+
+[nccl."prod"]
+ib_hca = "mlx5_0"
+socket_ifname = "eth0"
+ib_gid_index = 3
+ib_disable = 0
+"#;
+        let config: Config = toml::from_str(toml_str).unwrap();
+        assert_eq!(config.ssh_user, "admin");
+        assert_eq!(config.clusters.len(), 1);
+        assert_eq!(config.clusters[0].nodes.len(), 2);
+        assert_eq!(config.clusters[0].nodes[0].host_key, None);
+        assert!(config.nccl.contains_key("prod"));
+        assert_eq!(config.nccl["prod"].ib_hca, "mlx5_0");
+    }
+
+    #[test]
+    fn test_config_deserialize_with_host_key() {
+        let toml_str = r#"
+ssh_user = "admin"
+ssh_key_path = "/path/to/key"
+
+[[clusters]]
+name = "secure"
+master_ib_ip = "1.1.1.1"
+nodes = [
+    { name = "n1", ssh_ip = "2.2.2.2", ib_ip = "1.1.1.1", rank = 0, host_key = "SHA256:abc123" },
+]
+"#;
+        let config: Config = toml::from_str(toml_str).unwrap();
+        assert_eq!(
+            config.clusters[0].nodes[0].host_key,
+            Some("SHA256:abc123".to_string())
+        );
+    }
+
+    // ── select_cluster tests ───────────────────────────────────
+
+    #[test]
+    fn test_select_cluster_by_name() {
+        let mut config = Config {
+            ssh_user: "user".to_string(),
+            ssh_key_path: "/path/to/key".to_string(),
+            clusters: vec![test_cluster()],
+            nccl: HashMap::new(),
+        };
+        config.nccl.insert("test-cluster".to_string(), test_nccl());
+
+        let (cluster, nccl) = select_cluster(&config, "test-cluster").unwrap();
+        assert_eq!(cluster.name, "test-cluster");
+        assert_eq!(nccl.ib_hca, "rocep1s0f0");
+    }
+
+    #[test]
+    fn test_select_cluster_auto() {
+        let mut config = Config {
+            ssh_user: "user".to_string(),
+            ssh_key_path: "/path/to/key".to_string(),
+            clusters: vec![test_cluster()],
+            nccl: HashMap::new(),
+        };
+        config.nccl.insert("test-cluster".to_string(), test_nccl());
+
+        let (cluster, nccl) = select_cluster(&config, "auto").unwrap();
+        assert_eq!(cluster.name, "test-cluster");
+        assert_eq!(nccl.ib_hca, "rocep1s0f0");
+    }
+
+    #[test]
+    fn test_select_cluster_auto_no_match() {
+        let config = Config {
+            ssh_user: "user".to_string(),
+            ssh_key_path: "/path/to/key".to_string(),
+            clusters: vec![test_cluster()],
+            nccl: HashMap::new(),
+        };
+        let result = select_cluster(&config, "auto");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_select_cluster_not_found() {
+        let config = Config {
+            ssh_user: "user".to_string(),
+            ssh_key_path: "/path/to/key".to_string(),
+            clusters: vec![],
+            nccl: HashMap::new(),
+        };
+        let result = select_cluster(&config, "nonexistent");
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_select_cluster_by_name_found() {
+        let mut nccl_map = HashMap::new();
+        nccl_map.insert("test-cluster".to_string(), test_nccl());
+        let config = Config {
+            ssh_user: "u".to_string(),
+            ssh_key_path: "/k".to_string(),
+            clusters: vec![test_cluster()],
+            nccl: nccl_map,
+        };
+        let cluster = select_cluster_by_name(&config, "test-cluster").unwrap();
+        assert_eq!(cluster.name, "test-cluster");
+        assert_eq!(cluster.nodes.len(), 2);
+    }
+
+    #[test]
+    fn test_select_cluster_by_name_not_found() {
+        let config = Config {
+            ssh_user: "u".to_string(),
+            ssh_key_path: "/k".to_string(),
+            clusters: vec![test_cluster()],
+            nccl: HashMap::new(),
+        };
+        let result = select_cluster_by_name(&config, "no-such-cluster");
+        assert!(result.is_err());
+    }
+
+    // ── Defaults ───────────────────────────────────────────────
+
+    #[test]
+    fn test_default_engine_is_vllm() {
+        let recipe: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"").unwrap();
+        assert_eq!(recipe.engine, "vllm");
+    }
+
+    #[test]
+    fn test_default_distributed_backend_is_mp() {
+        let recipe: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"").unwrap();
+        assert_eq!(recipe.distributed_backend, "mp");
+    }
+}

src/docker.rs

@@ -0,0 +1,234 @@
+use anyhow::{Context, Result};
+use tokio::time::{Duration, sleep};
+use tracing::info;
+
+use crate::config::*;
+use crate::doctor::try_repaired_image;
+use crate::ssh::*;
+
+/// Launches containers across the cluster for a recipe.
+pub async fn serve(config: &Config, recipe_name: &str, cluster_name: &str, dry_run: bool) -> Result<()> {
+    let recipe = load_recipe(recipe_name).await?;
+    info!("🚀 Запуск рецепта: {} ({}) [backend: {}]", recipe.name, recipe.engine, recipe.distributed_backend);
+    let (cluster, nccl) = resolve_cluster(config, &recipe, cluster_name)?;
+
+    if let Some(required) = recipe.required_nodes {
+        let actual = cluster.nodes.len() as u32;
+        if actual != required {
+            anyhow::bail!("Recipe '{}' requires {} nodes, but cluster '{}' has {} nodes", recipe.name, required, cluster.name, actual);
+        }
+    }
+
+    let image = try_repaired_image(config, &cluster, &recipe).await?;
+    let mut recipe = recipe;
+    if image != recipe.docker_image { recipe.docker_image = image.clone(); }
+
+    start_cluster(config, &cluster, nccl, recipe_name, &recipe, dry_run).await?;
+    if !dry_run {
+        info!("\n📜 Последние логи контейнеров:");
+        show_logs(config, &cluster, recipe_name, false, dry_run).await?;
+    }
+    Ok(())
+}
+
+/// Shows Docker logs for a recipe across all nodes (or follows head node).
+pub async fn show_logs(config: &Config, cluster: &Cluster, recipe_name: &str, follow: bool, dry_run: bool) -> Result<()> {
+    if follow {
+        let head_node = cluster.nodes.first().context("No nodes in cluster")?;
+        info!("📜 Following logs for '{}' (head node)...", head_node.name);
+        let cmd = format!("docker ps -q --filter \"label=sm121.recipe={}\" --filter \"label=sm121.rank=0\" | xargs -r docker logs -f", recipe_name);
+        run_ssh_follow(config, head_node, cmd, dry_run).await?;
+    } else {
+        let flag = "--tail=100";
+        info!("📜 Логи кластера '{}' ({} узлов):", cluster.name, cluster.nodes.len());
+        let mut handles = Vec::new();
+        for node in &cluster.nodes {
+            let cmd = format!("docker ps -q --filter \"label=sm121.recipe={}\" --filter \"label=sm121.rank={}\" | xargs -r docker logs {} 2>&1", recipe_name, node.rank, flag);
+            let config = config.clone(); let node = node.clone(); let dry = dry_run;
+            let name = node.name.clone();
+            handles.push(tokio::spawn(async move { (name, run_ssh_with_retry(&config, &node, cmd, dry, 1).await) }));
+        }
+        for result in futures::future::join_all(handles).await {
+            if let Ok((name, Ok(output))) = result && !output.trim().is_empty() {
+                println!("\n--- {} ---", name); println!("{}", output);
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Stops and removes containers for a recipe on all nodes.
+pub async fn stop_cluster(config: &Config, cluster: &Cluster, recipe_name: &str, dry_run: bool) -> Result<()> {
+    info!("🛑 Останавливаем кластер '{}'...", cluster.name);
+    let mut failures = Vec::new();
+
+    for node in &cluster.nodes {
+        let stop_cmd = format!("docker ps -q --filter \"label=sm121.recipe={}\" | xargs -r docker stop", recipe_name);
+        if let Err(e) = run_ssh(config, node, stop_cmd, dry_run).await {
+            tracing::warn!("Stop failed on {}: {}", node.name, e);
+            failures.push((node.name.clone(), "stop".to_string(), e));
+        }
+        let rm_cmd = format!("docker ps -qa --filter \"label=sm121.recipe={}\" | xargs -r docker rm", recipe_name);
+        if let Err(e) = run_ssh(config, node, rm_cmd, dry_run).await {
+            tracing::warn!("Remove failed on {}: {}", node.name, e);
+            failures.push((node.name.clone(), "rm".to_string(), e));
+        }
+    }
+
+    if !failures.is_empty() {
+        for (name, action, e) in &failures { tracing::warn!("Failed to {} on {}: {}", action, name, e); }
+        anyhow::bail!("Failed to stop containers on {} node(s)", failures.len());
+    }
+    info!("✅ Все контейнеры остановлены.");
+    Ok(())
+}
+
+/// Checks Docker container health on a cluster.
+pub async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) -> Result<()> {
+    println!("\n=== Кластер: {} ===", cluster.name);
+    let mut handles = Vec::new();
+    for node in &cluster.nodes {
+        let config = config.clone(); let node = node.clone(); let dry = dry_run;
+        handles.push(tokio::spawn(async move {
+            let result = run_ssh_with_retry(&config, &node, "docker ps --filter \"label=sm121\" --format '{{.Names}} {{.Status}}' 2>&1".to_string(), dry, 1).await;
+            (node.name, node.ib_ip, result)
+        }));
+    }
+    let mut all_healthy = true;
+    for result in futures::future::join_all(handles).await {
+        match result {
+            Ok((name, ib_ip, Ok(check))) => {
+                print!("{} ({}): ", name, ib_ip);
+                if check.is_empty() && dry_run { continue; }
+                if check.starts_with("docker:") || check.contains("permission denied") {
+                    println!("✗ Docker недоступен");
+                    if let Some(line) = check.lines().next() { println!("  └─ {}", line); }
+                    all_healthy = false;
+                } else {
+                    let containers: Vec<&str> = check.lines().filter(|l| !l.is_empty()).collect();
+                    if containers.is_empty() { println!("✗ Нет контейнеров"); all_healthy = false; }
+                    else { println!("✓ {} контейнер(ов)", containers.len()); for c in containers { println!("  └─ {}", c); } }
+                }
+            }
+            Ok((name, ib_ip, Err(e))) => { println!("{} ({}): ✗ SSH ошибка: {}", name, ib_ip, e); all_healthy = false; }
+            Err(join_err) => { println!("✗ Task join error: {}", join_err); all_healthy = false; }
+        }
+    }
+    if !dry_run && all_healthy { info!("✅ Кластер здоров!"); } else if !dry_run { info!("⚠️ Кластер имеет проблемы"); }
+    Ok(())
+}
+
+/// Starts containers on all nodes in parallel.
+async fn start_cluster(config: &Config, cluster: &Cluster, nccl: &NcclConfig, recipe_name: &str, recipe: &Recipe, dry_run: bool) -> Result<()> {
+    let nnodes = cluster.nodes.len() as u32;
+    let pp_size = if recipe.pipeline_parallel_size.unwrap_or(0) == 0 { nnodes } else { recipe.pipeline_parallel_size.unwrap() };
+    let mut handles = Vec::new();
+
+    for node in &cluster.nodes {
+        let node = node.clone(); let cluster = cluster.clone(); let nccl = nccl.clone();
+        let recipe = recipe.clone(); let config = config.clone();
+        let rn = recipe_name.to_string(); let dry = dry_run; let engine = recipe.engine.clone();
+        let is_head = node.rank == 0;
+        let cleanup = format!("docker rm -f sm121_{}_{} 2>/dev/null; true", recipe_name, node.rank);
+
+        handles.push(tokio::spawn(async move {
+            if !dry { let _ = run_ssh(&config, &node, cleanup, dry).await; }
+            let cmd = build_docker_command(&cluster, &nccl, &recipe, &rn, &node, is_head, pp_size, nnodes).join(" ");
+            let result = run_ssh_with_retry(&config, &node, cmd, dry, 2).await;
+            (node.name, node.rank, engine, result)
+        }));
+    }
+
+    for result in futures::future::join_all(handles).await {
+        match result {
+            Ok((name, rank, engine, Ok(_))) => info!("✅ {} запущен на {} (rank {})", engine.to_uppercase(), name, rank),
+            Ok((name, rank, engine, Err(e))) => anyhow::bail!("Failed to start {} on {} (rank {}): {}", engine.to_uppercase(), name, rank, e),
+            Err(e) => anyhow::bail!("Task join error: {}", e),
+        }
+    }
+
+    if !dry_run { sleep(Duration::from_secs(10)).await; info!("🎉 Кластер успешно запущен! API → http://{}:{}", cluster.master_ib_ip, recipe.port); }
+    Ok(())
+}
+
+#[allow(clippy::too_many_arguments)]
+fn build_docker_command(cluster: &Cluster, nccl: &NcclConfig, recipe: &Recipe, recipe_name: &str, node: &Node, is_head: bool, pp_size: u32, nnodes: u32) -> Vec<String> {
+    let mut cmd = vec![
+        "docker run -d --gpus all --ipc=host --privileged --network=host".to_string(),
+        format!("-e NCCL_IB_DISABLE={}", nccl.ib_disable), format!("-e NCCL_IB_HCA={}", nccl.ib_hca),
+        format!("-e NCCL_SOCKET_IFNAME={}", nccl.socket_ifname), format!("-e NCCL_IB_GID_INDEX={}", nccl.ib_gid_index),
+        format!("-e VLLM_HOST_IP={}", node.ib_ip),
+    ];
+    for (k, v) in &recipe.env { cmd.push(format!("-e {}={}", k, v)); }
+    if is_head { cmd.push(format!("-p {}:{}", recipe.port, recipe.port)); }
+    cmd.push(format!("--name sm121_{}_{}", recipe_name, node.rank));
+    cmd.push(format!("--label sm121.recipe={}", recipe_name));
+    cmd.push(format!("--label sm121.rank={}", node.rank));
+    if recipe.distributed_backend == "ray" {
+        cmd.push(format!("-e RAY_ADDRESS={}:6379", cluster.master_ib_ip));
+        cmd.push(format!("-e RAY_PORT={}", recipe.port));
+    }
+    cmd.push(recipe.docker_image.clone());
+    if !recipe.entrypoint.is_empty() { cmd.extend(recipe.entrypoint.clone()); }
+    cmd.push(recipe.model.clone());
+    cmd.extend(recipe.args.clone());
+
+    let flags: Vec<String> = match recipe.distributed_backend.as_str() {
+        "ray" | "mp" => vec![
+            "--tensor-parallel-size".into(), "1".into(),
+            "--pipeline-parallel-size".into(), pp_size.to_string(),
+            "--nnodes".into(), nnodes.to_string(),
+            "--node-rank".into(), node.rank.to_string(),
+            "--master-addr".into(), cluster.master_ib_ip.clone(),
+            "--distributed-executor-backend".into(), recipe.distributed_backend.clone(),
+        ],
+        _ => vec![],
+    };
+    cmd.extend(flags);
+    cmd
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashMap;
+    fn tcls() -> Cluster { Cluster { name: "t".into(), master_ib_ip: "1".into(), nodes: vec![
+        Node { name: "n0".into(), ssh_ip: "10.0.0.1".into(), ib_ip: "1".into(), rank: 0, host_key: None },
+        Node { name: "n1".into(), ssh_ip: "10.0.0.2".into(), ib_ip: "2".into(), rank: 1, host_key: None },
+    ] } }
+    fn tnccl() -> NcclConfig { NcclConfig { ib_hca: "r".into(), socket_ifname: "e".into(), ib_gid_index: 3, ib_disable: 0 } }
+    fn trec() -> Recipe { Recipe {
+        engine: "vllm".into(), name: "T".into(), docker_image: "img".into(), model: "mdl".into(), port: 8000,
+        required_nodes: Some(2), pipeline_parallel_size: Some(2),
+        args: vec!["--x".into(), "4096".into()], env: HashMap::from([("K".into(), "V".into())]),
+        cluster: None, entrypoint: vec![], distributed_backend: "mp".into(),
+    } }
+    fn run(recipe: &Recipe, node: &Node, head: bool) -> String {
+        build_docker_command(&tcls(), &tnccl(), recipe, "r", node, head, 2, 2).join(" ")
+    }
+
+    #[test] fn mp() { let cmd = run(&trec(), &tcls().nodes[0], true);
+        assert!(cmd.contains("docker run -d --gpus all")); assert!(cmd.contains("-e NCCL_IB_HCA=r"));
+        assert!(cmd.contains("-e K=V")); assert!(cmd.contains("-p 8000:8000")); assert!(cmd.contains("--name sm121_r_0"));
+        assert!(cmd.contains("--label sm121.recipe=r")); assert!(cmd.contains("--label sm121.rank=0"));
+        assert!(cmd.contains("img")); assert!(cmd.contains("mdl")); assert!(cmd.contains("--tensor-parallel-size 1"));
+        assert!(cmd.contains("--pipeline-parallel-size 2")); assert!(cmd.contains("--nnodes 2"));
+        assert!(cmd.contains("--node-rank 0")); assert!(cmd.contains("--master-addr 1"));
+        assert!(cmd.contains("--distributed-executor-backend mp")); assert!(!cmd.contains("RAY_ADDRESS"));
+    }
+    #[test] fn worker_no_port() { let cmd = run(&trec(), &tcls().nodes[1], false);
+        assert!(!cmd.contains("-p 8000:8000")); assert!(cmd.contains("--node-rank 1"));
+        assert!(cmd.contains("--label sm121.rank=1")); assert!(cmd.contains("--name sm121_r_1"));
+    }
+    #[test] fn ray_backend() { let mut r = trec(); r.distributed_backend = "ray".into();
+        let cmd = run(&r, &tcls().nodes[0], true);
+        assert!(cmd.contains("-e RAY_ADDRESS=1:6379")); assert!(cmd.contains("-e RAY_PORT=8000"));
+        assert!(cmd.contains("--node-rank 0")); assert!(cmd.contains("--distributed-executor-backend ray"));
+    }
+    #[test] fn auto_pp_size() { let cmd = run(&trec(), &tcls().nodes[0], true); assert!(cmd.contains("--pipeline-parallel-size 2")); }
+    #[test] fn entrypoint() { let mut r = trec(); r.entrypoint = vec!["py".into(), "-m".into(), "mod".into()];
+        let cmd = run(&r, &tcls().nodes[0], true);
+        let ip = cmd.find("img").unwrap(); let mp = cmd.find("mdl").unwrap(); let ep = cmd.find("py -m mod").unwrap();
+        assert!(ip < ep && ep < mp, "entrypoint should be between image and model");
+    }
+}

src/doctor.rs

@@ -0,0 +1,332 @@
+use anyhow::{Context, Result};
+use tracing::info;
+
+use crate::config::*;
+use crate::ssh::*;
+
+/// Doctor check result for a single node/container.
+pub struct DoctorResult {
+    pub node_name: String,
+    pub checks: Vec<DoctorCheck>,
+}
+
+pub(crate) struct DoctorCheck {
+    name: String,
+    status: DoctorStatus,
+    detail: String,
+}
+
+#[derive(PartialEq)]
+enum DoctorStatus {
+    Ok,
+    Warn,
+    Fail,
+}
+
+impl std::fmt::Display for DoctorStatus {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            DoctorStatus::Ok => write!(f, "✓"),
+            DoctorStatus::Warn => write!(f, "⚠"),
+            DoctorStatus::Fail => write!(f, "✗"),
+        }
+    }
+}
+
+/// Run doctor checks on the cluster for a recipe.
+pub async fn doctor(
+    config: &Config,
+    recipe_name: &str,
+    cluster_name: &str,
+    repair: bool,
+    dry_run: bool,
+) -> Result<()> {
+    let recipe = load_recipe(recipe_name).await?;
+    println!("\n🩺 Doctor: {} ({}) [backend: {}]", recipe.name, recipe.engine, recipe.distributed_backend);
+
+    let (cluster, _nccl) = resolve_cluster(config, &recipe, cluster_name)?;
+
+    // Check each node
+    let mut all_results: Vec<DoctorResult> = Vec::new();
+    let mut handles = Vec::new();
+
+    for node in &cluster.nodes {
+        let config = config.clone();
+        let node = node.clone();
+        let engine = recipe.engine.clone();
+        let backend = recipe.distributed_backend.clone();
+        let image = recipe.docker_image.clone();
+        handles.push(tokio::spawn(async move {
+            check_node(&config, &node, &engine, &backend, &image).await
+        }));
+    }
+
+    let results = futures::future::join_all(handles).await;
+    for r in results.into_iter().flatten() {
+        all_results.push(r);
+    }
+
+    // Print results
+    let mut has_failures = false;
+    for result in &all_results {
+        println!("\n--- {} ---", result.node_name);
+        for check in &result.checks {
+            println!("  {} {} {}", check.status, check.name, check.detail);
+            if check.status == DoctorStatus::Fail {
+                has_failures = true;
+            }
+        }
+    }
+
+    if has_failures {
+        if repair {
+            println!("\n🔧 Repairing image...");
+            let repaired = run_cluster_repair(config, &cluster, &recipe, dry_run).await?;
+            println!("✅ Repaired image: {}", repaired);
+        } else {
+            println!("\n⚠ Some checks failed. Run with --repair to auto-fix.");
+            std::process::exit(1);
+        }
+    } else {
+        println!("\n✅ All checks passed!");
+    }
+
+    Ok(())
+}
+
+/// Check a single node for required software.
+async fn check_node(
+    config: &Config,
+    node: &Node,
+    engine: &str,
+    backend: &str,
+    image: &str,
+) -> DoctorResult {
+    let mut checks = Vec::new();
+
+    checks.push(node_check(config, node, "Docker", "docker --version").await);
+    checks.push(node_check(config, node, "NVIDIA GPU", "nvidia-smi").await);
+    checks.push(node_check(config, node, "NCCL libs", "ldconfig -p | grep -q nccl && echo OK").await);
+
+    checks.push(container_check(config, node, image, "Engine", format!("{} version", engine)).await);
+
+    if backend == "ray" {
+        checks.push(container_check(config, node, image, "Ray", "ray --version".to_string()).await);
+    }
+
+    checks.push(container_entrypoint_check(config, node, image).await);
+
+    DoctorResult {
+        node_name: node.name.clone(),
+        checks,
+    }
+}
+
+/// Run a command on the node via SSH and return a check result.
+async fn node_check(config: &Config, node: &Node, name: &str, command: &str) -> DoctorCheck {
+    match run_ssh(config, node, command.to_string(), false).await {
+        Ok(output) if !output.trim().is_empty() => DoctorCheck {
+            name: name.to_string(),
+            status: DoctorStatus::Ok,
+            detail: format!("OK: {}", output.trim().lines().next().unwrap_or(output.trim())),
+        },
+        Ok(_) => DoctorCheck {
+            name: name.to_string(),
+            status: DoctorStatus::Fail,
+            detail: "not found".to_string(),
+        },
+        Err(e) => DoctorCheck {
+            name: name.to_string(),
+            status: DoctorStatus::Fail,
+            detail: format!("error: {}", e),
+        },
+    }
+}
+
+/// Run a command inside a container on the node and return a check result.
+async fn container_check(config: &Config, node: &Node, image: &str, name: &str, command: String) -> DoctorCheck {
+    let docker_cmd = format!(
+        "docker run --rm {} {} 2>&1 | head -1",
+        image, command
+    );
+    match run_ssh(config, node, docker_cmd, false).await {
+        Ok(output) if !output.contains("not found") && !output.contains("No such file") => DoctorCheck {
+            name: name.to_string(),
+            status: DoctorStatus::Ok,
+            detail: format!("OK: {}", output.trim()),
+        },
+        Ok(output) => DoctorCheck {
+            name: name.to_string(),
+            status: DoctorStatus::Fail,
+            detail: format!("not found: {}", output.trim()),
+        },
+        Err(e) => DoctorCheck {
+            name: name.to_string(),
+            status: DoctorStatus::Fail,
+            detail: format!("error: {}", e),
+        },
+    }
+}
+
+/// Check the container entrypoint.
+async fn container_entrypoint_check(config: &Config, node: &Node, image: &str) -> DoctorCheck {
+    let docker_cmd = format!(
+        "docker inspect --format '{{{{json .Config.Entrypoint}}}}' {} 2>/dev/null || echo 'null'",
+        image
+    );
+    match run_ssh(config, node, docker_cmd, false).await {
+        Ok(output) => {
+            let entrypoint = output.trim();
+            if entrypoint == "null" || entrypoint == "[]" {
+                DoctorCheck {
+                    name: "Entrypoint".to_string(),
+                    status: DoctorStatus::Ok,
+                    detail: "default (shell) entrypoint".to_string(),
+                }
+            } else {
+                DoctorCheck {
+                    name: "Entrypoint".to_string(),
+                    status: DoctorStatus::Warn,
+                    detail: format!("custom: {}", entrypoint),
+                }
+            }
+        }
+        Err(e) => DoctorCheck {
+            name: "Entrypoint".to_string(),
+            status: DoctorStatus::Fail,
+            detail: format!("error: {}", e),
+        },
+    }
+}
+
+/// Generate Dockerfile and build repaired image on head node.
+async fn run_cluster_repair(
+    config: &Config,
+    cluster: &Cluster,
+    recipe: &Recipe,
+    dry_run: bool,
+) -> Result<String> {
+    let head_node = cluster.nodes.first().context("No nodes in cluster")?;
+    let repaired_tag = format!("{}.repaired", recipe.docker_image);
+    let dockerfile = generate_repair_dockerfile(&repaired_tag, recipe);
+
+    let dockerfile_path = format!("recipes/{}/Dockerfile.repaired", "temp");
+    tokio::fs::write(&dockerfile_path, &dockerfile).await
+        .context("Failed to write Dockerfile")?;
+
+    let create_dockerfile = format!(
+        "mkdir -p /tmp/sm121_docker && cat > /tmp/sm121_docker/Dockerfile << 'DOCKERFILE_END'\n{}DOCKERFILE_END",
+        dockerfile
+    );
+
+    if !dry_run {
+        run_ssh(config, head_node, create_dockerfile, false).await?;
+
+        let build_cmd = format!(
+            "cd /tmp/sm121_docker && docker build -t {} .",
+            repaired_tag
+        );
+        let output = run_ssh_with_retry(config, head_node, build_cmd, false, 1).await
+            .context("Failed to build repaired image")?;
+        println!("Build output: {}", output.lines().last().unwrap_or(&output));
+    } else {
+        println!("  (dry-run) Would build {} on {}", repaired_tag, head_node.name);
+    }
+
+    tokio::fs::remove_file(&dockerfile_path).await.ok();
+    Ok(repaired_tag)
+}
+
+/// Generate a Dockerfile to repair a container.
+fn generate_repair_dockerfile(base_image: &str, recipe: &Recipe) -> String {
+    let mut lines = vec![format!("FROM {}", base_image)];
+
+    if recipe.distributed_backend == "ray" {
+        lines.push("RUN pip install ray[default] || pip install ray".to_string());
+    }
+
+    let engine_install = match recipe.engine.as_str() {
+        "vllm" => "pip install vllm",
+        "sglang" => "pip install sglang",
+        _ => "",
+    };
+    if !engine_install.is_empty() {
+        lines.push(format!("RUN {} || true", engine_install));
+    }
+
+    lines.push("ENTRYPOINT [\"/bin/bash\", \"-c\"]".to_string());
+    lines.join("\n")
+}
+
+/// Check if a repaired image exists on the head node.
+pub async fn try_repaired_image(config: &Config, cluster: &Cluster, recipe: &Recipe) -> Result<String> {
+    let head_node = cluster.nodes.first().context("No nodes in cluster")?;
+    let repaired_tag = format!("{}.repaired", recipe.docker_image);
+
+    let check_cmd = format!("docker images -q {} | head -1", repaired_tag);
+
+    match run_ssh(config, head_node, check_cmd, false).await {
+        Ok(output) if !output.trim().is_empty() => {
+            info!("🩺 Found repaired image: {}", repaired_tag);
+            Ok(repaired_tag)
+        }
+        _ => Ok(recipe.docker_image.clone()),
+    }
+}
+
+// ====================== Tests ======================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashMap;
+
+    fn test_recipe() -> Recipe {
+        Recipe {
+            engine: "vllm".to_string(),
+            name: "Test Model".to_string(),
+            docker_image: "test/image:latest".to_string(),
+            model: "test/model".to_string(),
+            port: 8000,
+            required_nodes: Some(2),
+            pipeline_parallel_size: Some(2),
+            args: vec!["--max-model-len".to_string(), "4096".to_string()],
+            env: HashMap::new(),
+            cluster: None,
+            entrypoint: vec![],
+            distributed_backend: "mp".to_string(),
+        }
+    }
+
+    #[test]
+    fn test_generate_repair_dockerfile_vllm() {
+        let recipe = test_recipe();
+        let dockerfile = generate_repair_dockerfile("base/image:tag", &recipe);
+
+        assert!(dockerfile.contains("FROM base/image:tag"));
+        assert!(dockerfile.contains("pip install vllm"));
+        assert!(!dockerfile.contains("ray"));
+        assert!(dockerfile.contains("ENTRYPOINT"));
+    }
+
+    #[test]
+    fn test_generate_repair_dockerfile_ray() {
+        let mut recipe = test_recipe();
+        recipe.distributed_backend = "ray".to_string();
+        let dockerfile = generate_repair_dockerfile("base/image:tag", &recipe);
+
+        assert!(dockerfile.contains("FROM base/image:tag"));
+        assert!(dockerfile.contains("pip install vllm"));
+        assert!(dockerfile.contains("pip install ray"));
+    }
+
+    #[test]
+    fn test_generate_repair_dockerfile_sglang() {
+        let mut recipe = test_recipe();
+        recipe.engine = "sglang".to_string();
+        let dockerfile = generate_repair_dockerfile("base/image:tag", &recipe);
+
+        assert!(dockerfile.contains("pip install sglang"));
+        assert!(!dockerfile.contains("vllm"));
+    }
+}

src/health.rs

@@ -1,7 +1,7 @@
 use anyhow::Result;
 
+use crate::config::{Cluster, Config, Recipe};
 use crate::tmux::{SessionHealth, TmuxManager};
-use crate::{Cluster, Config, Recipe};
 
 /// Health status of a node's tmux session
 #[derive(Debug, Clone, PartialEq)]

src/main.rs

@@ -1,212 +1,38 @@
-use anyhow::{Context, Result};
-use clap::{Parser, Subcommand};
-use serde::Deserialize;
-use std::collections::HashMap;
-use tokio::fs;
-use tokio::time::{Duration, sleep};
-use tracing::info;
+use anyhow::Result;
+use clap::Parser;
 
+mod cli;
+mod config;
+mod docker;
+mod doctor;
 mod health;
 mod ssh;
 mod tmux;
 
-use crate::ssh::*;
-
-#[derive(Parser)]
-#[command(author, version, about = "DGX Spark Launcher для vLLM и SGLang")]
-struct Cli {
-    #[arg(long, short, default_value = "config.toml")]
-    config: String,
-
-    #[arg(long, short, help = "Показать команды без выполнения")]
-    dry_run: bool,
-
-    #[command(subcommand)]
-    command: Commands,
-}
-
-#[derive(Subcommand)]
-enum Commands {
-    /// Запустить кластер
-    Serve {
-        #[arg(long, required = true)]
-        recipe: String,
-
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-
-    /// Показать логи контейнеров
-    Logs {
-        #[arg(long, required = true)]
-        recipe: String,
-
-        #[arg(long, action = clap::ArgAction::SetTrue)]
-        follow: bool,
-    },
-
-    /// Остановить кластер
-    Stop {
-        #[arg(long, required = true)]
-        recipe: String,
-    },
-
-    /// Проверить здоровье кластера
-    Health {
-        #[arg(
-            long,
-            help = "Имя кластера (например, 4nodes-switch). Если не указано - проверяет все."
-        )]
-        cluster: Option<String>,
-        #[arg(long, help = "Показать хост-ключи узлов (для добавления в config.toml)")]
-        fingerprint: bool,
-    },
-
-    /// Проверить и исправить рецепт (doctor mode)
-    Doctor {
-        #[arg(long, required = true, help = "Имя рецепта для проверки")]
-        recipe: String,
-        #[arg(long, default_value = "auto", help = "Кластер (если не auto — использует recipe.cluster)")]
-        cluster: String,
-        #[arg(long, help = "Автоматически исправить проблемы (создать Dockerfile + собрать образ)")]
-        repair: bool,
-    },
-
-    /// Управление tmux-сессиями (альтернатива Docker)
-    #[command(subcommand)]
-    Tmux(TmuxCommands),
-}
-
-#[derive(Subcommand)]
-pub enum TmuxCommands {
-    /// Запустить кластер через tmux
-    Serve {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-    /// Остановить tmux-сессии
-    Stop {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-    /// Статус tmux-сессий (подробный)
-    Status {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-    /// Показать логи tmux-сессий
-    Logs {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, action = clap::ArgAction::SetTrue)]
-        follow: bool,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-    /// Подключиться к tmux-сессии головного узла
-    Attach {
-        #[arg(long, required = true)]
-        recipe: String,
-        #[arg(long, default_value = "auto")]
-        cluster: String,
-    },
-}
-
-#[derive(Clone, Deserialize)]
-struct Node {
-    name: String,
-    ssh_ip: String,
-    ib_ip: String,
-    rank: u32,
-    #[serde(default)]
-    host_key: Option<String>,
-}
-
-#[derive(Clone, Deserialize)]
-struct NcclConfig {
-    ib_hca: String,
-    socket_ifname: String,
-    ib_gid_index: u32,
-    ib_disable: u32,
-}
-
-#[derive(Clone, Deserialize)]
-struct Cluster {
-    name: String,
-    master_ib_ip: String,
-    nodes: Vec<Node>,
-}
-
-#[derive(Clone, Deserialize)]
-struct Config {
-    ssh_user: String,
-    ssh_key_path: String,
-    clusters: Vec<Cluster>,
-    #[serde(default)]
-    nccl: HashMap<String, NcclConfig>,
-}
-
-#[derive(Clone, Deserialize)]
-struct Recipe {
-    #[serde(default = "default_engine")]
-    engine: String,
-    name: String,
-    #[serde(default)]
-    docker_image: String,
-    #[serde(default)]
-    model: String,
-    #[serde(default)]
-    port: u16,
-    #[serde(default)]
-    required_nodes: Option<u32>,
-    #[serde(default)]
-    pipeline_parallel_size: Option<u32>,
-    #[serde(default)]
-    args: Vec<String>,
-    #[serde(default)]
-    env: HashMap<String, String>,
-    #[serde(default)]
-    cluster: Option<String>,
-    #[serde(default)]
-    entrypoint: Vec<String>,
-    #[serde(default = "default_distributed_backend")]
-    distributed_backend: String,
-}
-
-fn default_engine() -> String {
-    "vllm".to_string()
-}
-
-fn default_distributed_backend() -> String {
-    "mp".to_string()
-}
+use crate::cli::{Cli, Commands};
+use crate::config::{Config, Cluster, select_cluster_by_name, resolve_cluster_for_recipe};
+use crate::ssh::fetch_host_key;
 
 #[tokio::main]
 async fn main() -> Result<()> {
     tracing_subscriber::fmt::init();
     let cli = Cli::parse();
 
-    let config: Config = toml::from_str(&fs::read_to_string(&cli.config).await?)?;
-
+    let raw_config = tokio::fs::read_to_string(&cli.config).await?;
+    let config: Config = toml::from_str(&raw_config)?;
     let dry_run = cli.dry_run;
 
     match cli.command {
         Commands::Serve { recipe, cluster } => {
-            serve(&config, &recipe, &cluster, dry_run).await?;
+            docker::serve(&config, &recipe, &cluster, dry_run).await?;
         }
         Commands::Logs { recipe, follow } => {
-            let (cluster, recipe_name) = get_cluster_for_recipe(&config, &recipe).await?;
-            show_logs(&config, &cluster, &recipe_name, follow, dry_run).await?;
+            let (cluster, recipe) = resolve_cluster_for_recipe(&config, &recipe).await?;
+            docker::show_logs(&config, &cluster, &recipe.name, follow, dry_run).await?;
         }
         Commands::Stop { recipe } => {
-            let (cluster, recipe_name) = get_cluster_for_recipe(&config, &recipe).await?;
-            stop_cluster(&config, &cluster, &recipe_name, dry_run).await?;
+            let (cluster, recipe) = resolve_cluster_for_recipe(&config, &recipe).await?;
+            docker::stop_cluster(&config, &cluster, &recipe.name, dry_run).await?;
         }
         Commands::Health {
             cluster: cluster_name,
@@ -221,8 +47,14 @@ async fn main() -> Result<()> {
                     println!("\n=== Кластер: {} ===", cluster.name);
                     for node in &cluster.nodes {
                         match fetch_host_key(&config, node).await {
-                            Ok(fp) => println!("{} ({}) -> host_key = \"{}\"", node.name, node.ssh_ip, fp),
-                            Err(e) => println!("{} ({}) -> error: {}", node.name, node.ssh_ip, e),
+                            Ok(fp) => println!(
+                                "{} ({}) -> host_key = \"{}\"",
+                                node.name, node.ssh_ip, fp
+                            ),
+                            Err(e) => println!(
+                                "{} ({}) -> error: {}",
+                                node.name, node.ssh_ip, e
+                            ),
                         }
                     }
                 }
@@ -230,1145 +62,27 @@ async fn main() -> Result<()> {
                 match cluster_name {
                     Some(ref name) => {
                         let cluster = select_cluster_by_name(&config, name)?;
-                        health_check(&config, &cluster, dry_run).await?;
+                        docker::health_check(&config, &cluster, dry_run).await?;
                     }
                     None => {
                         for cluster in &config.clusters {
-                            health_check(&config, cluster, dry_run).await?;
-                        }
-                    }
-                }
-            }
-        },
-        Commands::Doctor { recipe, cluster, repair } => {
-            doctor(&config, &recipe, &cluster, repair, dry_run).await?;
-        },
-        Commands::Tmux(tmux_cmd) => {
-            tmux::handle_tmux(&config, tmux_cmd, dry_run).await?;
-        },
-    }
-
-    Ok(())
-}
-
-// ====================== Serve ======================
-async fn serve(
-    config: &Config,
-    recipe_name: &str,
-    cluster_name: &str,
-    dry_run: bool,
-) -> Result<()> {
-    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
-    let recipe: Recipe = toml::from_str(&fs::read_to_string(&recipe_path).await?)?;
-
-    info!("🚀 Запуск рецепта: {} ({}) [backend: {}]", recipe.name, recipe.engine, recipe.distributed_backend);
-
-    let cluster_choice = if cluster_name != "auto" {
-        cluster_name.to_string()
-    } else if let Some(ref rc) = recipe.cluster {
-        rc.clone()
-    } else {
-        "auto".to_string()
-    };
-    let (cluster, nccl) = select_cluster(config, &cluster_choice)?;
-
-    if let Some(required) = recipe.required_nodes {
-        let actual = cluster.nodes.len() as u32;
-        if actual != required {
-            anyhow::bail!(
-                "Recipe '{}' requires {} nodes, but cluster '{}' has {} nodes",
-                recipe.name, required, cluster.name, actual
-            );
-        }
-    }
-
-    // Check for repaired image
-    let image = try_repaired_image(config, &cluster, &recipe).await?;
-    let mut recipe = recipe;
-    if image != recipe.docker_image {
-        recipe.docker_image = image.clone();
-    }
-
-    start_cluster(config, &cluster, nccl, recipe_name, &recipe, dry_run).await?;
-
-    if !dry_run {
-        info!("\n📜 Последние логи контейнеров:");
-        show_logs(config, &cluster, recipe_name, false, dry_run).await?;
-    }
-
-    Ok(())
-}
-
-// ====================== Логи ======================
-async fn show_logs(config: &Config, cluster: &Cluster, recipe_name: &str, follow: bool, dry_run: bool) -> Result<()> {
-    if follow {
-        let head_node = cluster.nodes.first().context("No nodes in cluster")?;
-        info!("📜 Following logs for '{}' (head node)...", head_node.name);
-        let cmd = format!(
-            "docker ps -q --filter \"label=sm121.recipe={}\" --filter \"label=sm121.rank=0\" | xargs -r docker logs -f",
-            recipe_name
-        );
-        run_ssh_follow(config, head_node, cmd, dry_run).await?;
-    } else {
-        let flag = "--tail=100";
-        info!(
-            "📜 Логи кластера '{}' ({} узлов):",
-            cluster.name,
-            cluster.nodes.len()
-        );
-
-        let mut handles = Vec::new();
-        for node in &cluster.nodes {
-            let cmd = format!(
-                "docker ps -q --filter \"label=sm121.recipe={}\" --filter \"label=sm121.rank={}\" | xargs -r docker logs {} 2>&1",
-                recipe_name, node.rank, flag
-            );
-            let config = config.clone();
-            let node = node.clone();
-            let dry = dry_run;
-            handles.push(tokio::spawn(async move {
-                let output = run_ssh_with_retry(&config, &node, cmd, dry, 1).await;
-                (node.name, output)
-            }));
-        }
-
-        let results = futures::future::join_all(handles).await;
-        for result in results {
-            if let Ok((name, Ok(output))) = result
-                && !output.trim().is_empty()
-            {
-                println!("\n--- {} ---", name);
-                println!("{}", output);
-            }
-        }
-    }
-    Ok(())
-}
-
-// ====================== Stop ======================
-async fn stop_cluster(config: &Config, cluster: &Cluster, recipe_name: &str, dry_run: bool) -> Result<()> {
-    info!("🛑 Останавливаем кластер '{}'...", cluster.name);
-    let mut failures = Vec::new();
-
-    for node in &cluster.nodes {
-        let stop_cmd = format!(
-            "docker ps -q --filter \"label=sm121.recipe={}\" | xargs -r docker stop",
-            recipe_name
-        );
-        if let Err(e) = run_ssh(config, node, stop_cmd, dry_run).await {
-            tracing::warn!("Stop failed on {}: {}", node.name, e);
-            failures.push((node.name.clone(), "stop".to_string(), e));
-        }
-
-        let rm_cmd = format!(
-            "docker ps -qa --filter \"label=sm121.recipe={}\" | xargs -r docker rm",
-            recipe_name
-        );
-        if let Err(e) = run_ssh(config, node, rm_cmd, dry_run).await {
-            tracing::warn!("Remove failed on {}: {}", node.name, e);
-            failures.push((node.name.clone(), "rm".to_string(), e));
-        }
-    }
-
-    if !failures.is_empty() {
-        for (name, action, e) in &failures {
-            tracing::warn!("Failed to {} on {}: {}", action, name, e);
-        }
-        anyhow::bail!("Failed to stop containers on {} node(s)", failures.len());
-    }
-
-    info!("✅ Все контейнеры остановлены.");
-    Ok(())
-}
-
-// ====================== Health ======================
-async fn health_check(config: &Config, cluster: &Cluster, dry_run: bool) -> Result<()> {
-    println!("\n=== Кластер: {} ===", cluster.name);
-
-    let mut handles = Vec::new();
-    for node in &cluster.nodes {
-        let config = config.clone();
-        let node = node.clone();
-        let dry = dry_run;
-        handles.push(tokio::spawn(async move {
-            let result = run_ssh_with_retry(
-                &config,
-                &node,
-                "docker ps --filter \"label=sm121\" --format '{{.Names}} {{.Status}}' 2>&1".to_string(),
-                dry,
-                1,
-            ).await;
-            (node.name, node.ib_ip, result)
-        }));
-    }
-
-    let mut all_healthy = true;
-    let results = futures::future::join_all(handles).await;
-    for result in results {
-        match result {
-            Ok((name, ib_ip, Ok(docker_check))) => {
-                print!("{} ({}): ", name, ib_ip);
-                if docker_check.is_empty() && dry_run {
-                    continue;
-                }
-                if docker_check.starts_with("docker:") || docker_check.contains("permission denied") {
-                    println!("✗ Docker недоступен");
-                    if let Some(line) = docker_check.lines().next() {
-                        println!("  └─ {}", line);
-                    }
-                    all_healthy = false;
-                } else {
-                    let containers: Vec<&str> = docker_check.lines().filter(|l| !l.is_empty()).collect();
-                    if containers.is_empty() {
-                        println!("✗ Нет контейнеров");
-                        all_healthy = false;
-                    } else {
-                        println!("✓ {} контейнер(ов)", containers.len());
-                        for c in containers {
-                            println!("  └─ {}", c);
+                            docker::health_check(&config, cluster, dry_run).await?;
                         }
                     }
                 }
             }
-            Ok((name, ib_ip, Err(e))) => {
-                println!("{} ({}): ✗ SSH ошибка: {}", name, ib_ip, e);
-                all_healthy = false;
-            }
-            Err(join_err) => {
-                println!("✗ Task join error: {}", join_err);
-                all_healthy = false;
-            }
-        }
-    }
-
-    if !dry_run {
-        if all_healthy {
-            info!("✅ Кластер здоров!");
-        } else {
-            info!("⚠️ Кластер имеет проблемы");
-        }
-    }
-    Ok(())
-}
-
-fn select_cluster_by_name(config: &Config, cluster_name: &str) -> Result<Cluster> {
-    let cluster = config
-        .clusters
-        .iter()
-        .find(|c| c.name == cluster_name)
-        .context(format!("Кластер {} не найден", cluster_name))?;
-    Ok(cluster.clone())
-}
-
-// ====================== Вспомогательные функции ======================
-async fn get_cluster_for_recipe(config: &Config, recipe_name: &str) -> Result<(Cluster, String)> {
-    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
-    let recipe: Recipe = toml::from_str(&fs::read_to_string(&recipe_path).await?)?;
-    let cluster_choice = recipe.cluster.as_deref().unwrap_or("auto");
-    let (cluster, _) = select_cluster(config, cluster_choice)?;
-    Ok((cluster, recipe_name.to_string()))
-}
-
-fn select_cluster<'c>(config: &'c Config, cluster_name: &str) -> Result<(Cluster, &'c NcclConfig)> {
-    if cluster_name != "auto" {
-        let cluster = config
-            .clusters
-            .iter()
-            .find(|c| c.name == cluster_name)
-            .context(format!("Кластер {} не найден", cluster_name))?;
-        let nccl = config
-            .nccl
-            .get(&cluster.name)
-            .context("NCCL конфиг не найден")?;
-
-        return Ok((cluster.clone(), nccl));
-    }
-
-    for cluster in &config.clusters {
-        if let Some(nccl) = config.nccl.get(&cluster.name) {
-            info!("✅ Автоподбор кластера: {}", cluster.name);
-            return Ok((cluster.clone(), nccl));
-        }
-    }
-
-    anyhow::bail!("Не удалось подобрать подходящий кластер");
-}
-
-async fn start_cluster(
-    config: &Config,
-    cluster: &Cluster,
-    nccl: &NcclConfig,
-    recipe_name: &str,
-    recipe: &Recipe,
-    dry_run: bool,
-) -> Result<()> {
-    let nnodes = cluster.nodes.len() as u32;
-    let pp_size = recipe.pipeline_parallel_size.unwrap_or(0);
-    let pp_size = if pp_size == 0 { nnodes } else { pp_size };
-
-    let recipe_clone = (*recipe).clone();
-
-    let mut handles = Vec::new();
-    for node in &cluster.nodes {
-        let is_head = node.rank == 0;
-        let cleanup_cmd = format!("docker rm -f sm121_{}_{} 2>/dev/null; true", recipe_name, node.rank);
-
-        let config = config.clone();
-        let cluster = cluster.clone();
-        let nccl = nccl.clone();
-        let node = node.clone();
-        let recipe = recipe_clone.clone();
-        let recipe_name = recipe_name.to_string();
-        let dry = dry_run;
-        let engine = recipe.engine.clone();
-
-        handles.push(tokio::spawn(async move {
-            // Cleanup leftover container
-            if !dry {
-                let _ = run_ssh(&config, &node, cleanup_cmd, dry).await;
-            }
-
-            let docker_cmd = build_docker_command(
-                &cluster, &nccl, &recipe, &recipe_name, &node, is_head, pp_size, nnodes,
-            );
-            let full_cmd = docker_cmd.join(" ");
-
-            let result = run_ssh_with_retry(&config, &node, full_cmd, dry, 2).await;
-            (node.name, node.rank, engine, result)
-        }));
-    }
-
-    let results = futures::future::join_all(handles).await;
-    for result in results {
-        match result {
-            Ok((name, rank, engine, Ok(_output))) => {
-                info!("✅ {} запущен на {} (rank {})", engine.to_uppercase(), name, rank);
-            }
-            Ok((name, rank, engine, Err(e))) => {
-                anyhow::bail!("Failed to start {} on {} (rank {}): {}", engine.to_uppercase(), name, rank, e);
-            }
-            Err(join_err) => {
-                anyhow::bail!("Task join error: {}", join_err);
-            }
-        }
-    }
-
-    if !dry_run {
-        sleep(Duration::from_secs(10)).await;
-        info!(
-            "🎉 Кластер успешно запущен! API → http://{}:{}",
-            cluster.master_ib_ip, recipe.port
-        );
-    }
-    Ok(())
-}
-
-#[allow(clippy::too_many_arguments)]
-fn build_docker_command(
-    cluster: &Cluster,
-    nccl: &NcclConfig,
-    recipe: &Recipe,
-    recipe_name: &str,
-    node: &Node,
-    is_head: bool,
-    pp_size: u32,
-    nnodes: u32,
-) -> Vec<String> {
-    let mut cmd = vec![
-        "docker run -d --gpus all --ipc=host --privileged --network=host".to_string(),
-        format!("-e NCCL_IB_DISABLE={}", nccl.ib_disable),
-        format!("-e NCCL_IB_HCA={}", nccl.ib_hca),
-        format!("-e NCCL_SOCKET_IFNAME={}", nccl.socket_ifname),
-        format!("-e NCCL_IB_GID_INDEX={}", nccl.ib_gid_index),
-        format!("-e VLLM_HOST_IP={}", node.ib_ip),
-    ];
-
-    for (k, v) in &recipe.env {
-        cmd.push(format!("-e {}={}", k, v));
-    }
-
-    if is_head {
-        cmd.push(format!("-p {}:{}" , recipe.port, recipe.port));
-    }
-
-    // Container name and labels
-    cmd.push(format!("--name sm121_{}_{}", recipe_name, node.rank));
-    cmd.push(format!("--label sm121.recipe={}", recipe_name));
-    cmd.push(format!("--label sm121.rank={}", node.rank));
-
-    // Ray backend: need RAY_ADDRESS + custom entrypoint that starts Ray before vLLM
-    if recipe.distributed_backend == "ray" {
-        cmd.push(format!("-e RAY_ADDRESS={}:6379", cluster.master_ib_ip));
-        cmd.push(format!("-e RAY_PORT={}", recipe.port));
-    }
-
-    cmd.push(recipe.docker_image.clone());
-
-    // Engine entrypoint (optional — empty for vLLM)
-    if !recipe.entrypoint.is_empty() {
-        cmd.extend(recipe.entrypoint.clone());
-    }
-
-    cmd.push(recipe.model.clone());
-    cmd.extend(recipe.args.clone());
-
-    // Backend-specific flags
-    match recipe.distributed_backend.as_str() {
-        "ray" => {
-            cmd.extend(vec![
-                "--tensor-parallel-size".to_string(),
-                "1".to_string(),
-                "--pipeline-parallel-size".to_string(),
-                pp_size.to_string(),
-                "--nnodes".to_string(),
-                nnodes.to_string(),
-                "--node-rank".to_string(),
-                node.rank.to_string(),
-                "--master-addr".to_string(),
-                cluster.master_ib_ip.clone(),
-                "--distributed-executor-backend".to_string(),
-                "ray".to_string(),
-            ]);
-        }
-        "mp" => {
-            cmd.extend(vec![
-                "--tensor-parallel-size".to_string(),
-                "1".to_string(),
-                "--pipeline-parallel-size".to_string(),
-                pp_size.to_string(),
-                "--nnodes".to_string(),
-                nnodes.to_string(),
-                "--node-rank".to_string(),
-                node.rank.to_string(),
-                "--master-addr".to_string(),
-                cluster.master_ib_ip.clone(),
-                "--distributed-executor-backend".to_string(),
-                "mp".to_string(),
-            ]);
-        }
-        _ => {}
-    }
-
-    cmd
-}
-
-// ====================== Doctor ======================
-
-/// Doctor check result for a single node/container
-struct DoctorResult {
-    node_name: String,
-    checks: Vec<DoctorCheck>,
-}
-
-struct DoctorCheck {
-    name: String,
-    status: DoctorStatus,
-    detail: String,
-}
-
-#[derive(PartialEq)]
-enum DoctorStatus {
-    Ok,
-    Warn,
-    Fail,
-}
-
-impl std::fmt::Display for DoctorStatus {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            DoctorStatus::Ok => write!(f, "✓"),
-            DoctorStatus::Warn => write!(f, "⚠"),
-            DoctorStatus::Fail => write!(f, "✗"),
         }
-    }
-}
-
-/// Run doctor checks on the cluster for a recipe
-async fn doctor(
-    config: &Config,
-    recipe_name: &str,
-    cluster_name: &str,
-    repair: bool,
-    dry_run: bool,
-) -> Result<()> {
-    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
-    let recipe: Recipe = toml::from_str(&fs::read_to_string(&recipe_path).await?)?;
-
-    println!("\n🩺 Doctor: {} ({}) [backend: {}]", recipe.name, recipe.engine, recipe.distributed_backend);
-
-    // Resolve cluster
-    let cluster_choice = if cluster_name != "auto" {
-        cluster_name.to_string()
-    } else if let Some(ref rc) = recipe.cluster {
-        rc.clone()
-    } else {
-        "auto".to_string()
-    };
-    let (cluster, _nccl) = select_cluster(config, &cluster_choice)?;
-
-    // Check each node
-    let mut all_results: Vec<DoctorResult> = Vec::new();
-    let mut handles = Vec::new();
-
-    for node in &cluster.nodes {
-        let config = config.clone();
-        let node = node.clone();
-        let engine = recipe.engine.clone();
-        let backend = recipe.distributed_backend.clone();
-        let image = recipe.docker_image.clone();
-        handles.push(tokio::spawn(async move {
-            check_node(&config, &node, &engine, &backend, &image).await
-        }));
-    }
-
-    let results = futures::future::join_all(handles).await;
-    for r in results.into_iter().flatten() {
-        all_results.push(r);
-    }
-
-    // Print results
-    let mut has_failures = false;
-    for result in &all_results {
-        println!("\n--- {} ---", result.node_name);
-        for check in &result.checks {
-            println!("  {} {} {}", check.status, check.name, check.detail);
-            if check.status == DoctorStatus::Fail {
-                has_failures = true;
-            }
+        Commands::Doctor {
+            recipe,
+            cluster,
+            repair,
+        } => {
+            doctor::doctor(&config, &recipe, &cluster, repair, dry_run).await?;
         }
-    }
-
-    if has_failures {
-        if repair {
-            println!("\n🔧 Repairing image...");
-            let repaired = run_cluster_repair(config, &cluster, &recipe, dry_run).await?;
-            println!("✅ Repaired image: {}", repaired);
-        } else {
-            println!("\n⚠ Some checks failed. Run with --repair to auto-fix.");
-            std::process::exit(1);
+        Commands::Tmux(tmux_cmd) => {
+            crate::tmux::handle_tmux(&config, tmux_cmd, dry_run).await?;
         }
-    } else {
-        println!("\n✅ All checks passed!");
     }
 
     Ok(())
 }
-
-/// Check a single node for required software
-async fn check_node(
-    config: &Config,
-    node: &Node,
-    engine: &str,
-    backend: &str,
-    image: &str,
-) -> DoctorResult {
-    let mut checks = Vec::new();
-
-    // Node-level checks
-    checks.push(node_check(config, node, "Docker", "docker --version").await);
-    checks.push(node_check(config, node, "NVIDIA GPU", "nvidia-smi").await);
-    checks.push(node_check(config, node, "NCCL libs", "ldconfig -p | grep -q nccl && echo OK").await);
-
-    // Container-level checks
-    checks.push(container_check(config, node, image, "Engine", format!("{} version", engine)).await);
-
-    if backend == "ray" {
-        checks.push(container_check(config, node, image, "Ray", "ray --version".to_string()).await);
-    }
-
-    // Check container entrypoint
-    checks.push(container_entrypoint_check(config, node, image).await);
-
-    DoctorResult {
-        node_name: node.name.clone(),
-        checks,
-    }
-}
-
-/// Run a command on the node via SSH and return a check result
-async fn node_check(config: &Config, node: &Node, name: &str, command: &str) -> DoctorCheck {
-    match run_ssh(config, node, command.to_string(), false).await {
-        Ok(output) if !output.trim().is_empty() => DoctorCheck {
-            name: name.to_string(),
-            status: DoctorStatus::Ok,
-            detail: format!("OK: {}", output.trim().lines().next().unwrap_or(output.trim())),
-        },
-        Ok(_) => DoctorCheck {
-            name: name.to_string(),
-            status: DoctorStatus::Fail,
-            detail: "not found".to_string(),
-        },
-        Err(e) => DoctorCheck {
-            name: name.to_string(),
-            status: DoctorStatus::Fail,
-            detail: format!("error: {}", e),
-        },
-    }
-}
-
-/// Run a command inside a container on the node and return a check result
-async fn container_check(config: &Config, node: &Node, image: &str, name: &str, command: String) -> DoctorCheck {
-    let docker_cmd = format!(
-        "docker run --rm {} {} 2>&1 | head -1",
-        image, command
-    );
-    match run_ssh(config, node, docker_cmd, false).await {
-        Ok(output) if !output.contains("not found") && !output.contains("No such file") => DoctorCheck {
-            name: name.to_string(),
-            status: DoctorStatus::Ok,
-            detail: format!("OK: {}", output.trim()),
-        },
-        Ok(output) => DoctorCheck {
-            name: name.to_string(),
-            status: DoctorStatus::Fail,
-            detail: format!("not found: {}", output.trim()),
-        },
-        Err(e) => DoctorCheck {
-            name: name.to_string(),
-            status: DoctorStatus::Fail,
-            detail: format!("error: {}", e),
-        },
-    }
-}
-
-/// Check the container entrypoint
-async fn container_entrypoint_check(config: &Config, node: &Node, image: &str) -> DoctorCheck {
-    let docker_cmd = format!(
-        "docker inspect --format '{{{{json .Config.Entrypoint}}}}' {} 2>/dev/null || echo 'null'",
-        image
-    );
-    match run_ssh(config, node, docker_cmd, false).await {
-        Ok(output) => {
-            let entrypoint = output.trim();
-            if entrypoint == "null" || entrypoint == "[]" {
-                DoctorCheck {
-                    name: "Entrypoint".to_string(),
-                    status: DoctorStatus::Ok,
-                    detail: "default (shell) entrypoint".to_string(),
-                }
-            } else {
-                DoctorCheck {
-                    name: "Entrypoint".to_string(),
-                    status: DoctorStatus::Warn,
-                    detail: format!("custom: {}", entrypoint),
-                }
-            }
-        }
-        Err(e) => DoctorCheck {
-            name: "Entrypoint".to_string(),
-            status: DoctorStatus::Fail,
-            detail: format!("error: {}", e),
-        },
-    }
-}
-
-/// Generate Dockerfile and build repaired image on head node
-async fn run_cluster_repair(
-    config: &Config,
-    cluster: &Cluster,
-    recipe: &Recipe,
-    dry_run: bool,
-) -> Result<String> {
-    let head_node = cluster.nodes.first().context("No nodes in cluster")?;
-
-    // Generate repaired image name
-    let repaired_tag = format!("{}.repaired", recipe.docker_image);
-
-    // Generate Dockerfile
-    let dockerfile = generate_repair_dockerfile(&repaired_tag, recipe);
-
-    // Write Dockerfile locally
-    let dockerfile_path = format!("recipes/{}/Dockerfile.repaired", "temp");
-    fs::write(&dockerfile_path, &dockerfile).await
-        .context("Failed to write Dockerfile")?;
-
-    // SSH to head node, create Dockerfile, build image
-    let create_dockerfile = format!(
-        "mkdir -p /tmp/sm121_docker && cat > /tmp/sm121_docker/Dockerfile << 'DOCKERFILE_END'\n{}DOCKERFILE_END",
-        dockerfile
-    );
-
-    if !dry_run {
-        run_ssh(config, head_node, create_dockerfile, false).await?;
-
-        let build_cmd = format!(
-            "cd /tmp/sm121_docker && docker build -t {} .",
-            repaired_tag
-        );
-        let output = run_ssh_with_retry(config, head_node, build_cmd, false, 1).await
-            .context("Failed to build repaired image")?;
-        println!("Build output: {}", output.lines().last().unwrap_or(&output));
-    } else {
-        println!("  (dry-run) Would build {} on {}", repaired_tag, head_node.name);
-    }
-
-    // Cleanup
-    fs::remove_file(&dockerfile_path).await.ok();
-
-    Ok(repaired_tag)
-}
-
-/// Generate a Dockerfile to repair a container
-fn generate_repair_dockerfile(base_image: &str, recipe: &Recipe) -> String {
-    let mut lines = vec![format!("FROM {}", base_image)];
-
-    // Install ray if needed
-    if recipe.distributed_backend == "ray" {
-        lines.push("RUN pip install ray[default] || pip install ray".to_string());
-    }
-
-    // Install engine if needed
-    let engine_install = match recipe.engine.as_str() {
-        "vllm" => "pip install vllm",
-        "sglang" => "pip install sglang",
-        _ => "",
-    };
-    if !engine_install.is_empty() {
-        lines.push(format!("RUN {} || true", engine_install));
-    }
-
-    // Ensure entrypoint is shell-compatible
-    lines.push("ENTRYPOINT [\"/bin/bash\", \"-c\"]".to_string());
-
-    lines.join("\n")
-}
-
-// ====================== serve() repair image check ======================
-
-async fn try_repaired_image(config: &Config, cluster: &Cluster, recipe: &Recipe) -> Result<String> {
-    let head_node = cluster.nodes.first().context("No nodes in cluster")?;
-    let repaired_tag = format!("{}.repaired", recipe.docker_image);
-
-    // Check if repaired image exists on head node
-    let check_cmd = format!(
-        "docker images -q {} | head -1",
-        repaired_tag
-    );
-
-    match run_ssh(config, head_node, check_cmd, false).await {
-        Ok(output) if !output.trim().is_empty() => {
-            info!("🩺 Found repaired image: {}", repaired_tag);
-            Ok(repaired_tag)
-        }
-        _ => Ok(recipe.docker_image.clone()),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::collections::HashMap;
-
-    fn test_cluster() -> Cluster {
-        Cluster {
-            name: "test-cluster".to_string(),
-            master_ib_ip: "192.168.1.1".to_string(),
-            nodes: vec![
-                Node {
-                    name: "node0".to_string(),
-                    ssh_ip: "10.0.0.1".to_string(),
-                    ib_ip: "192.168.1.1".to_string(),
-                    rank: 0,
-                    host_key: None,
-                },
-                Node {
-                    name: "node1".to_string(),
-                    ssh_ip: "10.0.0.2".to_string(),
-                    ib_ip: "192.168.1.2".to_string(),
-                    rank: 1,
-                    host_key: None,
-                },
-            ],
-        }
-    }
-
-    fn test_nccl() -> NcclConfig {
-        NcclConfig {
-            ib_hca: "rocep1s0f0".to_string(),
-            socket_ifname: "enp1s0f0np0".to_string(),
-            ib_gid_index: 3,
-            ib_disable: 0,
-        }
-    }
-
-    fn test_recipe() -> Recipe {
-        Recipe {
-            engine: "vllm".to_string(),
-            name: "Test Model".to_string(),
-            docker_image: "test/image:latest".to_string(),
-            model: "test/model".to_string(),
-            port: 8000,
-            required_nodes: Some(2),
-            pipeline_parallel_size: Some(2),
-            args: vec!["--max-model-len".to_string(), "4096".to_string()],
-            env: HashMap::from([("NCCL_DEBUG".to_string(), "INFO".to_string())]),
-            cluster: None,
-            entrypoint: vec![],
-            distributed_backend: "mp".to_string(),
-        }
-    }
-
-    // ── build_docker_command tests ──────────────────────────────
-
-    #[test]
-    fn test_build_docker_command_mp_backend() {
-        let cluster = test_cluster();
-        let nccl = test_nccl();
-        let recipe = test_recipe();
-        let node = &cluster.nodes[0];
-
-        let cmd = build_docker_command(&cluster, &nccl, &recipe, "test-recipe", node, true, 2, 2);
-
-        let joined = cmd.join(" ");
-
-        // Basic docker flags
-        assert!(joined.contains("docker run -d --gpus all --ipc=host --privileged --network=host"));
-        // NCCL env
-        assert!(joined.contains("-e NCCL_IB_HCA=rocep1s0f0"));
-        assert!(joined.contains("-e NCCL_SOCKET_IFNAME=enp1s0f0np0"));
-        // Recipe env
-        assert!(joined.contains("-e NCCL_DEBUG=INFO"));
-        // Port only on head
-        assert!(joined.contains("-p 8000:8000"));
-        // Container name & labels
-        assert!(joined.contains("--name sm121_test-recipe_0"));
-        assert!(joined.contains("--label sm121.recipe=test-recipe"));
-        assert!(joined.contains("--label sm121.rank=0"));
-        // Image & model
-        assert!(joined.contains("test/image:latest"));
-        assert!(joined.contains("test/model"));
-        // Args
-        assert!(joined.contains("--max-model-len 4096"));
-        // MP-specific flags
-        assert!(joined.contains("--tensor-parallel-size 1"));
-        assert!(joined.contains("--pipeline-parallel-size 2"));
-        assert!(joined.contains("--nnodes 2"));
-        assert!(joined.contains("--node-rank 0"));
-        assert!(joined.contains("--master-addr 192.168.1.1"));
-        assert!(joined.contains("--distributed-executor-backend mp"));
-        // No Ray env
-        assert!(!joined.contains("RAY_ADDRESS"));
-    }
-
-    #[test]
-    fn test_build_docker_command_worker_no_port() {
-        let cluster = test_cluster();
-        let nccl = test_nccl();
-        let recipe = test_recipe();
-        let node = &cluster.nodes[1]; // rank 1
-
-        let cmd = build_docker_command(&cluster, &nccl, &recipe, "test-recipe", node, false, 2, 2);
-        let joined = cmd.join(" ");
-
-        // No port mapping on worker
-        assert!(!joined.contains("-p 8000:8000"));
-        // Correct rank
-        assert!(joined.contains("--node-rank 1"));
-        assert!(joined.contains("--label sm121.rank=1"));
-        assert!(joined.contains("--name sm121_test-recipe_1"));
-    }
-
-    #[test]
-    fn test_build_docker_command_ray_backend() {
-        let cluster = test_cluster();
-        let nccl = test_nccl();
-        let mut recipe = test_recipe();
-        recipe.distributed_backend = "ray".to_string();
-        let node = &cluster.nodes[0];
-
-        let cmd = build_docker_command(&cluster, &nccl, &recipe, "test-recipe", node, true, 2, 2);
-        let joined = cmd.join(" ");
-
-        // Ray-specific env
-        assert!(joined.contains("-e RAY_ADDRESS=192.168.1.1:6379"));
-        assert!(joined.contains("-e RAY_PORT=8000"));
-        // Ray-specific flags
-        assert!(joined.contains("--tensor-parallel-size 1"));
-        assert!(joined.contains("--pipeline-parallel-size 2"));
-        assert!(joined.contains("--nnodes 2"));
-        assert!(joined.contains("--node-rank 0"));
-        assert!(joined.contains("--master-addr 192.168.1.1"));
-        assert!(joined.contains("--distributed-executor-backend ray"));
-    }
-
-    #[test]
-    fn test_build_docker_command_auto_pp_size() {
-        let cluster = test_cluster();
-        let nccl = test_nccl();
-        let mut recipe = test_recipe();
-        recipe.pipeline_parallel_size = Some(0); // auto
-        let node = &cluster.nodes[0];
-
-        // pp_size = 0 should become nnodes (2) in start_cluster, but build_docker_command
-        // takes pp_size directly. test with pp_size = nnodes = 2.
-        let cmd = build_docker_command(&cluster, &nccl, &recipe, "test-recipe", node, true, 2, 2);
-        let joined = cmd.join(" ");
-
-        assert!(joined.contains("--pipeline-parallel-size 2"));
-    }
-
-    #[test]
-    fn test_build_docker_command_entrypoint() {
-        let cluster = test_cluster();
-        let nccl = test_nccl();
-        let mut recipe = test_recipe();
-        recipe.entrypoint = vec!["python".to_string(), "-m".to_string(), "custom.module".to_string()];
-        let node = &cluster.nodes[0];
-
-        let cmd = build_docker_command(&cluster, &nccl, &recipe, "test-recipe", node, true, 2, 2);
-        let joined = cmd.join(" ");
-
-        // Entrypoint should appear after image name, before model
-        let image_pos = joined.find("test/image:latest").unwrap();
-        let model_pos = joined.find("test/model").unwrap();
-        let entrypoint_pos = joined.find("python -m custom.module").unwrap();
-        assert!(image_pos < entrypoint_pos && entrypoint_pos < model_pos,
-            "entrypoint should be between image and model");
-    }
-
-    // ── select_cluster tests ────────────────────────────────────
-
-    #[test]
-    fn test_select_cluster_by_name() {
-        let mut config = Config {
-            ssh_user: "user".to_string(),
-            ssh_key_path: "/path/to/key".to_string(),
-            clusters: vec![test_cluster()],
-            nccl: HashMap::new(),
-        };
-        config.nccl.insert("test-cluster".to_string(), test_nccl());
-
-        let (cluster, nccl) = select_cluster(&config, "test-cluster").unwrap();
-        assert_eq!(cluster.name, "test-cluster");
-        assert_eq!(nccl.ib_hca, "rocep1s0f0");
-    }
-
-    #[test]
-    fn test_select_cluster_auto() {
-        let mut config = Config {
-            ssh_user: "user".to_string(),
-            ssh_key_path: "/path/to/key".to_string(),
-            clusters: vec![test_cluster()],
-            nccl: HashMap::new(),
-        };
-        config.nccl.insert("test-cluster".to_string(), test_nccl());
-
-        let (cluster, nccl) = select_cluster(&config, "auto").unwrap();
-        assert_eq!(cluster.name, "test-cluster");
-        assert_eq!(nccl.ib_hca, "rocep1s0f0");
-    }
-
-    #[test]
-    fn test_select_cluster_auto_no_match() {
-        let config = Config {
-            ssh_user: "user".to_string(),
-            ssh_key_path: "/path/to/key".to_string(),
-            clusters: vec![test_cluster()],
-            nccl: HashMap::new(),
-        };
-        // No NCCL config for test-cluster → auto should fail
-        let result = select_cluster(&config, "auto");
-        assert!(result.is_err());
-    }
-
-    #[test]
-    fn test_select_cluster_not_found() {
-        let config = Config {
-            ssh_user: "user".to_string(),
-            ssh_key_path: "/path/to/key".to_string(),
-            clusters: vec![],
-            nccl: HashMap::new(),
-        };
-        let result = select_cluster(&config, "nonexistent");
-        assert!(result.is_err());
-    }
-
-    // ── Recipe deserialization tests ─────────────────────────────
-
-    #[test]
-    fn test_recipe_deserialize_minimal() {
-        let toml_str = r#"
-name = "Minimal"
-model = "test/model"
-"#;
-        let recipe: Recipe = toml::from_str(toml_str).unwrap();
-        assert_eq!(recipe.name, "Minimal");
-        assert_eq!(recipe.model, "test/model");
-        assert_eq!(recipe.engine, "vllm"); // default
-        assert_eq!(recipe.port, 0); // default
-        assert!(recipe.args.is_empty());
-        assert!(recipe.env.is_empty());
-        assert_eq!(recipe.distributed_backend, "mp"); // default
-    }
-
-    #[test]
-    fn test_recipe_deserialize_full() {
-        let toml_str = r#"
-engine = "sglang"
-name = "Full Recipe"
-docker_image = "org/image:tag"
-model = "org/model"
-port = 30000
-required_nodes = 4
-pipeline_parallel_size = 2
-distributed_backend = "ray"
-cluster = "my-cluster"
-
-args = ["--arg1", "val1", "--arg2"]
-
-[env]
-KEY = "value"
-OTHER = "123"
-"#;
-        let recipe: Recipe = toml::from_str(toml_str).unwrap();
-        assert_eq!(recipe.engine, "sglang");
-        assert_eq!(recipe.name, "Full Recipe");
-        assert_eq!(recipe.docker_image, "org/image:tag");
-        assert_eq!(recipe.model, "org/model");
-        assert_eq!(recipe.port, 30000);
-        assert_eq!(recipe.required_nodes, Some(4));
-        assert_eq!(recipe.pipeline_parallel_size, Some(2));
-        assert_eq!(recipe.distributed_backend, "ray");
-        assert_eq!(recipe.cluster, Some("my-cluster".to_string()));
-        assert_eq!(recipe.args, vec!["--arg1", "val1", "--arg2"]);
-        assert_eq!(recipe.env.get("KEY").unwrap(), "value");
-        assert_eq!(recipe.env.get("OTHER").unwrap(), "123");
-    }
-
-    #[test]
-    fn test_recipe_deserialize_no_required_nodes() {
-        let toml_str = r#"
-name = "Flexible"
-model = "test/model"
-"#;
-        let recipe: Recipe = toml::from_str(toml_str).unwrap();
-        assert_eq!(recipe.required_nodes, None); // None, not Some(0)
-        assert_eq!(recipe.pipeline_parallel_size, None);
-    }
-
-    // ── Config deserialization tests ───────────────────────────────
-
-    #[test]
-    fn test_config_deserialize() {
-        let toml_str = r#"
-ssh_user = "admin"
-ssh_key_path = "/home/admin/.ssh/id_rsa"
-
-[[clusters]]
-name = "prod"
-master_ib_ip = "10.0.0.1"
-nodes = [
-    { name = "node1", ssh_ip = "10.0.0.10", ib_ip = "10.0.0.1", rank = 0 },
-    { name = "node2", ssh_ip = "10.0.0.11", ib_ip = "10.0.0.2", rank = 1 },
-]
-
-[nccl."prod"]
-ib_hca = "mlx5_0"
-socket_ifname = "eth0"
-ib_gid_index = 3
-ib_disable = 0
-"#;
-        let config: Config = toml::from_str(toml_str).unwrap();
-        assert_eq!(config.ssh_user, "admin");
-        assert_eq!(config.clusters.len(), 1);
-        assert_eq!(config.clusters[0].nodes.len(), 2);
-        assert_eq!(config.clusters[0].nodes[0].host_key, None);
-        assert!(config.nccl.contains_key("prod"));
-        assert_eq!(config.nccl["prod"].ib_hca, "mlx5_0");
-    }
-
-    #[test]
-    fn test_config_deserialize_with_host_key() {
-        let toml_str = r#"
-ssh_user = "admin"
-ssh_key_path = "/path/to/key"
-
-[[clusters]]
-name = "secure"
-master_ib_ip = "1.1.1.1"
-nodes = [
-    { name = "n1", ssh_ip = "2.2.2.2", ib_ip = "1.1.1.1", rank = 0, host_key = "SHA256:abc123" },
-]
-"#;
-        let config: Config = toml::from_str(toml_str).unwrap();
-        assert_eq!(
-            config.clusters[0].nodes[0].host_key,
-            Some("SHA256:abc123".to_string())
-        );
-    }
-
-    // ── generate_repair_dockerfile tests ───────────────────────────
-
-    #[test]
-    fn test_generate_repair_dockerfile_vllm() {
-        let recipe = test_recipe();
-        let dockerfile = generate_repair_dockerfile("base/image:tag", &recipe);
-
-        assert!(dockerfile.contains("FROM base/image:tag"));
-        assert!(dockerfile.contains("pip install vllm"));
-        assert!(!dockerfile.contains("ray"));
-        assert!(dockerfile.contains("ENTRYPOINT"));
-    }
-
-    #[test]
-    fn test_generate_repair_dockerfile_ray() {
-        let mut recipe = test_recipe();
-        recipe.distributed_backend = "ray".to_string();
-        let dockerfile = generate_repair_dockerfile("base/image:tag", &recipe);
-
-        assert!(dockerfile.contains("FROM base/image:tag"));
-        assert!(dockerfile.contains("pip install vllm"));
-        assert!(dockerfile.contains("pip install ray"));
-    }
-
-    #[test]
-    fn test_generate_repair_dockerfile_sglang() {
-        let mut recipe = test_recipe();
-        recipe.engine = "sglang".to_string();
-        let dockerfile = generate_repair_dockerfile("base/image:tag", &recipe);
-
-        assert!(dockerfile.contains("pip install sglang"));
-        assert!(!dockerfile.contains("vllm"));
-    }
-
-    // ── select_cluster_by_name tests ───────────────────────────────
-
-    #[test]
-    fn test_select_cluster_by_name_found() {
-        let mut nccl_map = HashMap::new();
-        nccl_map.insert("test-cluster".to_string(), test_nccl());
-        let config = Config {
-            ssh_user: "u".to_string(),
-            ssh_key_path: "/k".to_string(),
-            clusters: vec![test_cluster()],
-            nccl: nccl_map,
-        };
-        let cluster = select_cluster_by_name(&config, "test-cluster").unwrap();
-        assert_eq!(cluster.name, "test-cluster");
-        assert_eq!(cluster.nodes.len(), 2);
-    }
-
-    #[test]
-    fn test_select_cluster_by_name_not_found() {
-        let config = Config {
-            ssh_user: "u".to_string(),
-            ssh_key_path: "/k".to_string(),
-            clusters: vec![test_cluster()],
-            nccl: HashMap::new(),
-        };
-        let result = select_cluster_by_name(&config, "no-such-cluster");
-        assert!(result.is_err());
-    }
-
-    // ── Defaults ───────────────────────────────────────────────────
-
-    #[test]
-    fn test_default_engine() {
-        assert_eq!(default_engine(), "vllm");
-    }
-
-    #[test]
-    fn test_default_distributed_backend() {
-        assert_eq!(default_distributed_backend(), "mp");
-    }
-}

src/ssh.rs

@@ -8,7 +8,7 @@ use std::sync::Arc;
 use tokio::time::{Duration, sleep};
 use tracing::warn;
 
-use crate::{Config, Node};
+use crate::config::{Config, Node};
 
 /// Establishes an SSH session to a node using the config credentials.
 pub async fn ssh_session(config: &Config, node: &Node) -> Result<client::Handle<VerifyingHandler>> {

src/tmux.rs

@@ -1,981 +0,0 @@
-use anyhow::{Context, Result};
-use std::time::Duration;
-use tokio::time::sleep;
-use tracing::{info, warn};
-
-use crate::ssh::*;
-use crate::{health, Cluster, Config, NcclConfig, Node, Recipe, TmuxCommands};
-
-/// Manages tmux sessions for model inference across a DGX Spark cluster.
-#[derive(Clone)]
-pub struct TmuxManager {
-    #[allow(dead_code)]
-    pub session_prefix: String,
-}
-
-impl TmuxManager {
-    /// Generates a namespaced session name for a recipe and node rank.
-    /// Format: `{recipe_name}-{engine}-r{rank}`
-    pub fn session_name(&self, recipe: &Recipe, rank: u32) -> String {
-        let name = recipe.name.replace(' ', "_");
-        format!("{}-{}-r{}", name, recipe.engine, rank)
-    }
-
-    /// Builds the list of `export` commands for NCCL environment variables.
-    pub fn build_nccl_setup(&self, nccl: &NcclConfig) -> Vec<String> {
-        vec![
-            format!("export NCCL_IB_HCA=\"{}\"", nccl.ib_hca),
-            format!("export NCCL_SOCKET_IFNAME=\"{}\"", nccl.socket_ifname),
-            format!("export NCCL_IB_GID_INDEX={}", nccl.ib_gid_index),
-            format!("export NCCL_IB_DISABLE={}", nccl.ib_disable),
-        ]
-    }
-
-    /// Builds engine-specific host IP environment variable.
-    pub fn build_host_ip_export(&self, engine: &str, node: &Node) -> Vec<String> {
-        match engine {
-            "vllm" => vec![format!("export VLLM_HOST_IP={}", node.ib_ip)],
-            "sglang" => vec![format!("export SGLANG_WORKER_IP={}", node.ib_ip)],
-            _ => vec![
-                format!("export VLLM_HOST_IP={}", node.ib_ip),
-                format!("export SGLANG_WORKER_IP={}", node.ib_ip),
-            ],
-        }
-    }
-
-    /// Builds the full model launch command for a given node.
-    pub fn build_launch_command(
-        &self,
-        recipe: &Recipe,
-        node: &Node,
-        cluster: &Cluster,
-    ) -> String {
-        let nnodes = cluster.nodes.len() as u32;
-        let pp_size = recipe.pipeline_parallel_size.unwrap_or(nnodes);
-
-        let mut parts: Vec<String> = Vec::new();
-
-        // Engine entrypoint
-        match recipe.engine.as_str() {
-            "vllm" => {
-                parts.push("python -m vllm.entrypoints.openai.api_server".to_string());
-            }
-            "sglang" => {
-                parts.push("python -m sglang.launch_server".to_string());
-            }
-            other => {
-                parts.push(format!("python -m {}", other));
-            }
-        }
-
-        // Model
-        parts.push(recipe.model.clone());
-
-        // Recipe-specific args
-        parts.extend(recipe.args.clone());
-
-        // Distributed flags
-        match recipe.distributed_backend.as_str() {
-            "ray" => {
-                // Ray mode: workers connect via RAY_ADDRESS
-                parts.extend(vec![
-                    "--tensor-parallel-size".to_string(),
-                    "1".to_string(),
-                    "--pipeline-parallel-size".to_string(),
-                    pp_size.to_string(),
-                    "--nnodes".to_string(),
-                    nnodes.to_string(),
-                    "--node-rank".to_string(),
-                    node.rank.to_string(),
-                    "--master-addr".to_string(),
-                    cluster.master_ib_ip.clone(),
-                    "--distributed-executor-backend".to_string(),
-                    "ray".to_string(),
-                ]);
-            }
-            _ => {
-                // MP mode (default)
-                parts.extend(vec![
-                    "--tensor-parallel-size".to_string(),
-                    "1".to_string(),
-                    "--pipeline-parallel-size".to_string(),
-                    pp_size.to_string(),
-                    "--nnodes".to_string(),
-                    nnodes.to_string(),
-                    "--node-rank".to_string(),
-                    node.rank.to_string(),
-                    "--master-addr".to_string(),
-                    cluster.master_ib_ip.clone(),
-                    "--distributed-executor-backend".to_string(),
-                    recipe.distributed_backend.clone(),
-                ]);
-            }
-        }
-
-        // Port (only on head node / rank 0)
-        if node.rank == 0 {
-            parts.extend(vec!["--port".to_string(), recipe.port.to_string()]);
-        }
-
-        parts.join(" ")
-    }
-
-    /// Builds the full sequence of tmux commands for a session, with multi-window layout.
-    ///
-    /// Layout:
-    ///   Window 0: model process (main vLLM/SGLang server)
-    ///   Window 1: GPU monitor (nvidia-smi -l 5)
-    ///   Window 2: shell (debug access)
-    pub fn build_session_commands(
-        &self,
-        node: &Node,
-        recipe: &Recipe,
-        nccl: &NcclConfig,
-        cluster: &Cluster,
-    ) -> Vec<String> {
-        let sess = self.session_name(recipe, node.rank);
-        let mut cmds: Vec<String> = Vec::new();
-
-        // Kill existing session if it exists
-        cmds.push(format!("tmux kill-session -t {} 2>/dev/null || true", sess));
-
-        // Create new detached session (window 0)
-        cmds.push(format!("tmux new-session -d -s {} -n model", sess));
-
-        // --- Window 0: model process ---
-
-        // Send NCCL environment variables
-        for export in self.build_nccl_setup(nccl) {
-            cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, export));
-        }
-
-        // Send engine-specific host IP
-        for export in self.build_host_ip_export(&recipe.engine, node) {
-            cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, export));
-        }
-
-        // Send RAY_ADDRESS for ray backend
-        if recipe.distributed_backend == "ray" && node.rank > 0 {
-            // Workers get RAY_ADDRESS pointing to head node
-            cmds.push(format!(
-                "tmux send-keys -t {}:0 'export RAY_ADDRESS={}:6379' Enter",
-                sess, cluster.master_ib_ip
-            ));
-        }
-
-        // Send recipe env vars
-        for (k, v) in &recipe.env {
-            cmds.push(format!(
-                "tmux send-keys -t {}:0 'export {}=\"{}\"' Enter",
-                sess, k, v
-            ));
-        }
-
-        // Send the launch command
-        let launch_cmd = self.build_launch_command(recipe, node, cluster);
-        cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, launch_cmd));
-
-        // --- Window 1: GPU monitor ---
-        cmds.push(format!("tmux new-window -t {}:1 -n gpu", sess));
-        cmds.push(format!(
-            "tmux send-keys -t {}:1 'nvidia-smi -l 5' Enter",
-            sess
-        ));
-
-        // --- Window 2: shell (debug) ---
-        cmds.push(format!("tmux new-window -t {}:2 -n shell", sess));
-        cmds.push(format!(
-            "tmux send-keys -t {}:2 'echo \"ready for debugging\"' Enter",
-            sess
-        ));
-
-        // Focus back to window 0 (model process)
-        cmds.push(format!("tmux select-window -t {}:0", sess));
-
-        cmds
-    }
-
-    /// Executes a list of commands via SSH on a node to create a tmux session.
-    pub async fn create_session(
-        &self,
-        config: &Config,
-        node: &Node,
-        commands: &[String],
-        dry_run: bool,
-    ) -> Result<()> {
-        if dry_run {
-            info!("  ssh {}@{}:", config.ssh_user, node.ssh_ip);
-            for cmd in commands {
-                info!("    {}", cmd);
-            }
-            return Ok(());
-        }
-
-        let script = commands.join(" && ");
-        let output = run_ssh(config, node, script, dry_run).await?;
-
-        if output.trim().is_empty() {
-            info!("  ✅ tmux session created on {}", node.name);
-        } else {
-            info!("  tmux session output on {}: {}", node.name, output.trim());
-        }
-
-        Ok(())
-    }
-
-    /// Checks whether a tmux session exists and is running on the given node.
-    pub async fn check_session(
-        &self,
-        config: &Config,
-        node: &Node,
-        session_name: &str,
-        dry_run: bool,
-    ) -> Result<bool> {
-        if dry_run {
-            info!(
-                "  ssh {}@{}: tmux list-sessions 2>/dev/null | grep '{}'",
-                config.ssh_user, node.ssh_ip, session_name
-            );
-            return Ok(true);
-        }
-
-        let cmd = format!(
-            "tmux list-sessions 2>/dev/null | grep -c '{}'",
-            session_name
-        );
-        let output = run_ssh(config, node, cmd, dry_run).await?;
-
-        let found = output.trim().parse::<usize>().unwrap_or(0) > 0;
-        if found {
-            info!("  Session '{}' found on {}", session_name, node.name);
-        } else {
-            warn!("  Session '{}' NOT found on {}", session_name, node.name);
-        }
-
-        Ok(found)
-    }
-
-    /// Performs a deep health check on a tmux session.
-    /// Checks: session exists, process alive, GPU status, API endpoint.
-    pub async fn deep_check_session(
-        &self,
-        config: &Config,
-        node: &Node,
-        _cluster: &Cluster,
-        session_name: &str,
-        recipe: &Recipe,
-        dry_run: bool,
-    ) -> Result<SessionHealth> {
-        if dry_run {
-            return Ok(SessionHealth {
-                session_exists: true,
-                process_alive: true,
-                gpu_ok: true,
-                api_ok: true,
-            });
-        }
-
-        // 1. Check session exists
-        let exists = self
-            .check_session(config, node, session_name, dry_run)
-            .await?;
-
-        // 2. Check process is alive inside tmux
-        let process_alive = if exists {
-            let pid_cmd = format!(
-                "tmux list-panes -t {}:0 -F '#{{pane_pid}}' | head -1",
-                session_name
-            );
-            match run_ssh(config, node, pid_cmd, dry_run).await {
-                Ok(pid_str) => {
-                    let pid = pid_str.trim();
-                    if !pid.is_empty() && pid != "0" {
-                        // Check if the process (or its child) is running
-                        let ps_cmd = format!("ps -p {} -o pid= 2>/dev/null", pid);
-                        run_ssh(config, node, ps_cmd, dry_run)
-                            .await
-                            .map(|o| !o.trim().is_empty())
-                            .unwrap_or(false)
-                    } else {
-                        false
-                    }
-                }
-                Err(_) => false,
-            }
-        } else {
-            false
-        };
-
-        // 3. Check GPU is available and not in error state
-        let gpu_ok = match run_ssh(config, node, "nvidia-smi --query-gpu=index,utilization.gpu,memory.used --format=csv,noheader 2>&1 | head -5".to_string(), dry_run).await {
-            Ok(out) => !out.contains("failed") && !out.contains("error") && !out.trim().is_empty(),
-            Err(_) => false,
-        };
-
-        // 4. Check API endpoint (head node only)
-        let api_ok = if node.rank == 0 && process_alive {
-            let api_check = format!(
-                "curl -s -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{}/v1/models 2>/dev/null || echo 'fail'",
-                recipe.port
-            );
-            match run_ssh(config, node, api_check, dry_run).await {
-                Ok(out) => {
-                    let code = out.trim();
-                    code == "200" || !code.contains("fail")
-                }
-                Err(_) => false,
-            }
-        } else {
-            // Non-head nodes or no process: skip API check
-            node.rank != 0 || process_alive
-        };
-
-        Ok(SessionHealth {
-            session_exists: exists,
-            process_alive,
-            gpu_ok,
-            api_ok,
-        })
-    }
-
-    /// Gracefully stops a tmux session with timeout escalation.
-    /// Sends Ctrl+C, polls for exit up to 30s, then SIGTERM, then SIGKILL.
-    pub async fn graceful_stop(
-        &self,
-        config: &Config,
-        node: &Node,
-        session_name: &str,
-        dry_run: bool,
-    ) -> Result<()> {
-        if dry_run {
-            info!("  ssh {}@{}: graceful stop of session '{}'", config.ssh_user, node.ssh_ip, session_name);
-            return Ok(());
-        }
-
-        // Phase 1: Ctrl+C (graceful shutdown)
-        info!("  Sending Ctrl+C to session '{}' on {}", session_name, node.name);
-        let stop_cmd = format!("tmux send-keys -t {}:0 C-c", session_name);
-        let _ = run_ssh(config, node, stop_cmd, dry_run).await;
-
-        // Phase 2: Poll for exit up to 30 seconds
-        let mut exited = false;
-        for i in 0..15 {
-            sleep(Duration::from_secs(2)).await;
-            let check_cmd = format!(
-                "tmux list-panes -t {}:0 -F '#{{pane_pid}}' 2>/dev/null | head -1",
-                session_name
-            );
-            match run_ssh(config, node, check_cmd, dry_run).await {
-                Ok(pid_str) => {
-                    let pid = pid_str.trim();
-                    if pid.is_empty() || pid == "0" || pid == "No" {
-                        // "No" from "no target session" — session already gone
-                        exited = true;
-                        info!("  Session '{}' exited gracefully (attempt {})", session_name, i + 1);
-                        break;
-                    }
-                    // Check process still alive
-                    let ps_cmd = format!("ps -p {} -o pid= 2>/dev/null", pid);
-                    match run_ssh(config, node, ps_cmd, dry_run).await {
-                        Ok(ps_out) if ps_out.trim().is_empty() => {
-                            exited = true;
-                            info!("  Session '{}' process exited (attempt {})", session_name, i + 1);
-                            break;
-                        }
-                        _ => continue,
-                    }
-                }
-                Err(_) => {
-                    // SSH error — session may already be gone
-                    exited = true;
-                    break;
-                }
-            }
-        }
-
-        // Phase 3: Force kill tmux session
-        if !exited {
-            warn!("  Session '{}' did not exit gracefully, force killing...", session_name);
-            let kill_cmd = format!("tmux kill-session -t {}", session_name);
-            let _ = run_ssh(config, node, kill_cmd, dry_run).await;
-        }
-
-        Ok(())
-    }
-
-    /// Streams the content of a tmux pane, optionally following new output.
-    pub async fn stream_pane(
-        &self,
-        config: &Config,
-        node: &Node,
-        session_name: &str,
-        window: u32,
-        follow: bool,
-        dry_run: bool,
-    ) -> Result<String> {
-        if dry_run {
-            if follow {
-                info!(
-                    "  ssh {}@{}: tail -f (streaming from session '{}' window {})",
-                    config.ssh_user, node.ssh_ip, session_name, window
-                );
-            } else {
-                info!(
-                    "  ssh {}@{}: tmux capture-pane -t {}:{} -p -S -100",
-                    config.ssh_user, node.ssh_ip, session_name, window
-                );
-            }
-            return Ok(String::new());
-        }
-
-        let cmd = if follow {
-            format!("tmux pipe-pane -o -t {}:{} 'cat > /tmp/sm121_pipe_{}_{}.log' && tail -f /tmp/sm121_pipe_{}_{}.log",
-                session_name, window, session_name.replace('-', "_"), window,
-                session_name.replace('-', "_"), window)
-        } else {
-            format!(
-                "tmux capture-pane -t {}:{} -p -S -200 2>/dev/null || echo 'no output'",
-                session_name, window
-            )
-        };
-
-        let output = run_ssh_with_retry(config, node, cmd, dry_run, 1).await?;
-        Ok(output)
-    }
-
-    /// Attaches to a tmux session on a node via SSH (interactive).
-    pub async fn attach_session(
-        &self,
-        config: &Config,
-        node: &Node,
-        session_name: &str,
-        dry_run: bool,
-    ) -> Result<()> {
-        if dry_run {
-            info!(
-                "  Would attach: ssh -t {}@{} tmux attach-session -t {}",
-                config.ssh_user, node.ssh_ip, session_name
-            );
-            return Ok(());
-        }
-
-        info!(
-            "  Attaching to '{}' on {}... Press Ctrl+B D to detach.",
-            session_name, node.name
-        );
-
-        // For non-interactive SSH (russh), we can't really do a proper attach.
-        // Instead, print instructions and stream the pane.
-        info!("  (russh does not support interactive PTY. Use the following command directly:)");
-        println!(
-            "  ssh -t {}@{} tmux attach-session -t {}",
-            config.ssh_user, node.ssh_ip, session_name
-        );
-        println!();
-        info!("  Streaming pane content instead:");
-        let output = self
-            .stream_pane(config, node, session_name, 0, true, dry_run)
-            .await?;
-        println!("{}", output);
-
-        Ok(())
-    }
-}
-
-// ====================== Health Report ======================
-
-/// Detailed health status of a tmux session.
-#[derive(Debug, Clone)]
-pub struct SessionHealth {
-    pub session_exists: bool,
-    pub process_alive: bool,
-    pub gpu_ok: bool,
-    pub api_ok: bool,
-}
-
-impl SessionHealth {
-    pub fn is_healthy(&self) -> bool {
-        self.session_exists && self.process_alive && self.gpu_ok && self.api_ok
-    }
-
-    #[allow(dead_code)]
-    pub fn is_degraded(&self) -> bool {
-        self.session_exists && !self.process_alive
-    }
-}
-
-// ====================== CLI Dispatch ======================
-
-/// Top-level handler for all `cargo run tmux <subcommand>` commands.
-pub async fn handle_tmux(
-    config: &Config,
-    cmd: TmuxCommands,
-    dry_run: bool,
-) -> Result<()> {
-    match cmd {
-        TmuxCommands::Serve { recipe, cluster } => {
-            tmux_serve(config, &recipe, &cluster, dry_run).await?;
-        }
-        TmuxCommands::Stop { recipe, cluster } => {
-            tmux_stop(config, &recipe, &cluster, dry_run).await?;
-        }
-        TmuxCommands::Status { recipe, cluster } => {
-            tmux_status(config, &recipe, &cluster, dry_run).await?;
-        }
-        TmuxCommands::Logs { recipe, follow, cluster } => {
-            tmux_logs(config, &recipe, &cluster, follow, dry_run).await?;
-        }
-        TmuxCommands::Attach { recipe, cluster } => {
-            tmux_attach(config, &recipe, &cluster, dry_run).await?;
-        }
-    }
-    Ok(())
-}
-
-// ====================== Serve (tmux) ======================
-
-/// Launches a cluster via tmux sessions.
-/// For Ray backend: starts head node first, waits for readiness, then workers.
-async fn tmux_serve(
-    config: &Config,
-    recipe_name: &str,
-    cluster_name: &str,
-    dry_run: bool,
-) -> Result<()> {
-    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
-    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
-
-    info!(
-        "🚀 TMUX launch: {} ({}) [backend: {}] [engine: {}]",
-        recipe.name, recipe_name, recipe.distributed_backend, recipe.engine
-    );
-
-    let cluster_choice = if cluster_name != "auto" {
-        cluster_name.to_string()
-    } else if let Some(ref rc) = recipe.cluster {
-        rc.clone()
-    } else {
-        "auto".to_string()
-    };
-    let (cluster, nccl) = crate::select_cluster(config, &cluster_choice)?;
-
-    if let Some(required) = recipe.required_nodes {
-        let actual = cluster.nodes.len() as u32;
-        if actual != required {
-            anyhow::bail!(
-                "Recipe '{}' requires {} nodes, but cluster '{}' has {} nodes",
-                recipe.name,
-                required,
-                cluster.name,
-                actual
-            );
-        }
-    }
-
-    let manager = TmuxManager {
-        session_prefix: recipe_name.to_string(),
-    };
-
-    let is_ray = recipe.distributed_backend == "ray";
-
-    if is_ray {
-        // Sequential startup for Ray: head first, then workers
-        info!("🐍 Ray backend detected — starting head node first...");
-
-        let head_node = cluster
-            .nodes
-            .iter()
-            .find(|n| n.rank == 0)
-            .context("No head node (rank 0) in cluster")?;
-
-        // Start head node
-        let head_cmds = manager.build_session_commands(head_node, &recipe, nccl, &cluster);
-        manager
-            .create_session(config, head_node, &head_cmds, dry_run)
-            .await?;
-
-        if !dry_run {
-            info!("⏳ Waiting for head node to be ready...");
-            let sess = manager.session_name(&recipe, 0);
-            let head_ready = wait_for_head_ready(config, head_node, &sess, &recipe).await?;
-            if head_ready {
-                info!("✅ Head node ready. Starting worker nodes...");
-            } else {
-                warn!("⚠️ Head node may not be fully ready — starting workers anyway.");
-            }
-        }
-
-        // Start worker nodes
-        for node in &cluster.nodes {
-            if node.rank == 0 {
-                continue;
-            }
-            let worker_cmds = manager.build_session_commands(node, &recipe, nccl, &cluster);
-            manager
-                .create_session(config, node, &worker_cmds, dry_run)
-                .await?;
-        }
-    } else {
-        // MP mode: start all nodes in parallel
-        info!("🔗 MP backend detected — starting all nodes in parallel...");
-
-        let mut handles = Vec::new();
-        for node in &cluster.nodes {
-            let config = config.clone();
-            let node = node.clone();
-            let recipe = recipe.clone();
-            let nccl = nccl.clone();
-            let cluster = cluster.clone();
-            let manager = manager.clone();
-            let dry = dry_run;
-
-            handles.push(tokio::spawn(async move {
-                let cmds = manager.build_session_commands(&node, &recipe, &nccl, &cluster);
-                manager
-                    .create_session(&config, &node, &cmds, dry)
-                    .await
-                    .map(|_| node.name.clone())
-            }));
-        }
-
-        let results = futures::future::join_all(handles).await;
-        let mut failures = Vec::new();
-        for result in results {
-            match result {
-                Ok(Ok(name)) => info!("  ✅ Session started on {}", name),
-                Ok(Err(e)) => failures.push(e),
-                Err(e) => failures.push(anyhow::anyhow!("Task join: {}", e)),
-            }
-        }
-
-        if !failures.is_empty() {
-            for e in &failures {
-                warn!("  ✗ {}", e);
-            }
-            anyhow::bail!("Failed to start sessions on {} node(s)", failures.len());
-        }
-    }
-
-    if !dry_run {
-        sleep(Duration::from_secs(5)).await;
-        info!(
-            "🎉 Cluster launched via tmux! API → http://{}:{}",
-            cluster.master_ib_ip, recipe.port
-        );
-        info!("📌 To view sessions: `cargo run tmux status --recipe {}`", recipe_name);
-        info!("📌 To attach: `cargo run tmux attach --recipe {}`", recipe_name);
-    }
-
-    Ok(())
-}
-
-/// Polls the head node until the API endpoint responds or timeout.
-async fn wait_for_head_ready(
-    config: &Config,
-    head_node: &Node,
-    session_name: &str,
-    recipe: &Recipe,
-) -> Result<bool> {
-    let max_attempts = 30; // 30 * 5s = 150s max wait
-    for i in 0..max_attempts {
-        // Quick check: is the session process still alive?
-        let pid_cmd = format!(
-            "tmux list-panes -t {}:0 -F '#{{pane_pid}}' | head -1",
-            session_name
-        );
-        match run_ssh(config, head_node, pid_cmd, false).await {
-            Ok(pid_str) => {
-                let pid = pid_str.trim();
-                if pid.is_empty() || pid == "0" {
-                    sleep(Duration::from_secs(5)).await;
-                    continue;
-                }
-            }
-            Err(_) => {
-                sleep(Duration::from_secs(5)).await;
-                continue;
-            }
-        }
-
-        // API readiness check via curl from head node
-        let api_check = format!(
-            "curl -s -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{}/v1/models 2>/dev/null || echo 'not_ready'",
-            recipe.port
-        );
-        if let Ok(out) = run_ssh(config, head_node, api_check, false).await {
-            let code = out.trim();
-            if code == "200" {
-                info!("  Head node API ready (attempt {})", i + 1);
-                return Ok(true);
-            }
-        }
-
-        // Also check stderr/output for model load progress
-        let log_check = format!(
-            "tmux capture-pane -t {}:0 -p -S -5 2>/dev/null | grep -c 'Application startup complete' || true",
-            session_name
-        );
-        let _ = run_ssh(config, head_node, log_check, false).await;
-
-        if (i + 1) % 5 == 0 {
-            info!(
-                "  Still waiting for head node... ({}/{} attempts)",
-                i + 1,
-                max_attempts
-            );
-        }
-        sleep(Duration::from_secs(5)).await;
-    }
-
-    warn!("  Head node did not become ready within timeout");
-    Ok(false)
-}
-
-// ====================== Stop (tmux) ======================
-
-/// Stops all tmux sessions for a recipe across all nodes in a cluster.
-async fn tmux_stop(
-    config: &Config,
-    recipe_name: &str,
-    cluster_name: &str,
-    dry_run: bool,
-) -> Result<()> {
-    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
-    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
-
-    let cluster_choice = if cluster_name != "auto" {
-        cluster_name.to_string()
-    } else if let Some(ref rc) = recipe.cluster {
-        rc.clone()
-    } else {
-        "auto".to_string()
-    };
-    let (cluster, _) = crate::select_cluster(config, &cluster_choice)?;
-
-    let manager = TmuxManager {
-        session_prefix: recipe_name.to_string(),
-    };
-
-    info!("🛑 Stopping tmux sessions for '{}' on cluster '{}'...", recipe_name, cluster.name);
-
-    let mut failures = Vec::new();
-    for node in &cluster.nodes {
-        let sess = manager.session_name(&recipe, node.rank);
-        match manager.graceful_stop(config, node, &sess, dry_run).await {
-            Ok(()) => info!("  ✅ Stopped '{}' on {}", sess, node.name),
-            Err(e) => {
-                warn!("  ✗ Failed to stop '{}' on {}: {}", sess, node.name, e);
-                failures.push((node.name.clone(), e));
-            }
-        }
-    }
-
-    if !failures.is_empty() {
-        for (name, e) in &failures {
-            warn!("  Failed on {}: {}", name, e);
-        }
-        anyhow::bail!("Failed to stop sessions on {} node(s)", failures.len());
-    }
-
-    info!("✅ All tmux sessions stopped.");
-    Ok(())
-}
-
-// ====================== Status (tmux) ======================
-
-/// Checks health status of tmux sessions across the cluster.
-async fn tmux_status(
-    config: &Config,
-    recipe_name: &str,
-    cluster_name: &str,
-    dry_run: bool,
-) -> Result<()> {
-    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
-    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
-
-    let cluster_choice = if cluster_name != "auto" {
-        cluster_name.to_string()
-    } else if let Some(ref rc) = recipe.cluster {
-        rc.clone()
-    } else {
-        "auto".to_string()
-    };
-    let (cluster, _) = crate::select_cluster(config, &cluster_choice)?;
-
-    let manager = TmuxManager {
-        session_prefix: recipe_name.to_string(),
-    };
-
-    println!("\n📊 TMUX Status: {} on cluster {}", recipe.name, cluster.name);
-    println!("{}", "─".repeat(60));
-
-    // Use health::check_cluster_health for data collection
-    let report = health::check_cluster_health(&manager, config, &cluster, &recipe, dry_run).await?;
-
-    for node_health in &report.nodes {
-        let status_icon = if node_health.status == health::SessionStatus::Running {
-            "✓".to_string()
-        } else if node_health.session_exists {
-            "⚠".to_string()
-        } else {
-            "✗".to_string()
-        };
-
-        print!("  {} {} (rank {}, {}): ", status_icon, node_health.name, node_health.rank, node_health.ib_ip);
-
-        if node_health.status == health::SessionStatus::Running {
-            println!("✅ ALL OK");
-        } else {
-            let mut issues = Vec::new();
-            if !node_health.session_exists {
-                issues.push("no session");
-            }
-            if !node_health.process_running {
-                issues.push("process dead");
-            }
-            if !node_health.port_accessible && node_health.rank == 0 {
-                issues.push("API down");
-            }
-            println!("⚠️  {}", issues.join(", "));
-        }
-
-        if node_health.session_exists {
-            println!("     Session: alive | Process: {} | API: {}",
-                if node_health.process_running { "✓" } else { "✗" },
-                if node_health.port_accessible { "✓" } else { "✗" },
-            );
-        }
-    }
-
-    println!("{}", "─".repeat(60));
-    match report.overall {
-        health::HealthStatus::Healthy => info!("✅ Cluster is healthy!"),
-        health::HealthStatus::Degraded => warn!("⚠️ Cluster is degraded"),
-        health::HealthStatus::Unhealthy => warn!("⚠️ Cluster has issues — check details above."),
-    }
-
-    Ok(())
-}
-
-// ====================== Logs (tmux) ======================
-
-/// Shows or follows logs from tmux pane content.
-async fn tmux_logs(
-    config: &Config,
-    recipe_name: &str,
-    cluster_name: &str,
-    follow: bool,
-    dry_run: bool,
-) -> Result<()> {
-    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
-    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
-
-    let cluster_choice = if cluster_name != "auto" {
-        cluster_name.to_string()
-    } else if let Some(ref rc) = recipe.cluster {
-        rc.clone()
-    } else {
-        "auto".to_string()
-    };
-    let (cluster, _) = crate::select_cluster(config, &cluster_choice)?;
-
-    let manager = TmuxManager {
-        session_prefix: recipe_name.to_string(),
-    };
-
-    if follow {
-        // Follow head node logs only (most useful)
-        let head_node = cluster
-            .nodes
-            .first()
-            .context("No nodes in cluster")?;
-        let sess = manager.session_name(&recipe, 0);
-
-        info!("📜 Following logs from '{}' on {}...", sess, head_node.name);
-        let output = manager
-            .stream_pane(config, head_node, &sess, 0, true, dry_run)
-            .await?;
-        println!("{}", output);
-    } else {
-        info!(
-            "📜 Logs for '{}' on cluster '{}' ({} nodes):",
-            recipe.name,
-            cluster.name,
-            cluster.nodes.len()
-        );
-
-        let mut handles = Vec::new();
-        for node in &cluster.nodes {
-            let config = config.clone();
-            let node = node.clone();
-            let manager = manager.clone();
-            let recipe = recipe.clone();
-            let dry = dry_run;
-
-            handles.push(tokio::spawn(async move {
-                let sess = manager.session_name(&recipe, node.rank);
-                let output = manager
-                    .stream_pane(&config, &node, &sess, 0, false, dry)
-                    .await;
-                (node.name, sess, output)
-            }));
-        }
-
-        let results = futures::future::join_all(handles).await;
-        for result in results {
-            if let Ok((name, sess, Ok(output))) = result {
-                let trimmed = output.trim();
-                if !trimmed.is_empty() {
-                    println!("\n--- {} ({}) ---", name, sess);
-                    println!("{}", trimmed);
-                } else {
-                    println!("\n--- {} — no output ---", name);
-                }
-            }
-        }
-    }
-
-    Ok(())
-}
-
-// ====================== Attach (tmux) ======================
-
-/// Provides instructions and streamed output for attaching to a tmux session.
-async fn tmux_attach(
-    config: &Config,
-    recipe_name: &str,
-    cluster_name: &str,
-    dry_run: bool,
-) -> Result<()> {
-    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
-    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
-
-    let cluster_choice = if cluster_name != "auto" {
-        cluster_name.to_string()
-    } else if let Some(ref rc) = recipe.cluster {
-        rc.clone()
-    } else {
-        "auto".to_string()
-    };
-    let (cluster, _) = crate::select_cluster(config, &cluster_choice)?;
-
-    let head_node = cluster
-        .nodes
-        .iter()
-        .find(|n| n.rank == 0)
-        .context("No head node (rank 0) in cluster")?;
-
-    let manager = TmuxManager {
-        session_prefix: recipe_name.to_string(),
-    };
-    let sess = manager.session_name(&recipe, 0);
-
-    info!("🔗 Attaching to head node '{}' session '{}'", head_node.name, sess);
-    manager
-        .attach_session(config, head_node, &sess, dry_run)
-        .await?;
-
-    Ok(())
-}
-
-

src/tmux/mod.rs

@@ -0,0 +1,500 @@
+pub mod ops;
+pub mod serve;
+
+use anyhow::Result;
+use std::time::Duration;
+use tokio::time::sleep;
+use tracing::{info, warn};
+
+use crate::cli::TmuxCommands;
+use crate::config::{Cluster, Config, NcclConfig, Node, Recipe};
+use crate::ssh::*;
+
+/// Manages tmux sessions for model inference across a DGX Spark cluster.
+#[derive(Clone)]
+pub struct TmuxManager {
+    #[allow(dead_code)]
+    pub session_prefix: String,
+}
+
+impl TmuxManager {
+    /// Generates a namespaced session name for a recipe and node rank.
+    /// Format: `{recipe_name}-{engine}-r{rank}`
+    pub fn session_name(&self, recipe: &Recipe, rank: u32) -> String {
+        let name = recipe.name.replace(' ', "_");
+        format!("{}-{}-r{}", name, recipe.engine, rank)
+    }
+
+    /// Builds the list of `export` commands for NCCL environment variables.
+    pub fn build_nccl_setup(&self, nccl: &NcclConfig) -> Vec<String> {
+        vec![
+            format!("export NCCL_IB_HCA=\"{}\"", nccl.ib_hca),
+            format!("export NCCL_SOCKET_IFNAME=\"{}\"", nccl.socket_ifname),
+            format!("export NCCL_IB_GID_INDEX={}", nccl.ib_gid_index),
+            format!("export NCCL_IB_DISABLE={}", nccl.ib_disable),
+        ]
+    }
+
+    /// Builds engine-specific host IP environment variable.
+    pub fn build_host_ip_export(&self, engine: &str, node: &Node) -> Vec<String> {
+        match engine {
+            "vllm" => vec![format!("export VLLM_HOST_IP={}", node.ib_ip)],
+            "sglang" => vec![format!("export SGLANG_WORKER_IP={}", node.ib_ip)],
+            _ => vec![
+                format!("export VLLM_HOST_IP={}", node.ib_ip),
+                format!("export SGLANG_WORKER_IP={}", node.ib_ip),
+            ],
+        }
+    }
+
+    /// Builds the full model launch command for a given node.
+    pub fn build_launch_command(
+        &self,
+        recipe: &Recipe,
+        node: &Node,
+        cluster: &Cluster,
+    ) -> String {
+        let nnodes = cluster.nodes.len() as u32;
+        let pp_size = recipe.pipeline_parallel_size.unwrap_or(nnodes);
+
+        let mut parts: Vec<String> = Vec::new();
+
+        // Engine entrypoint
+        match recipe.engine.as_str() {
+            "vllm" => {
+                parts.push("python -m vllm.entrypoints.openai.api_server".to_string());
+            }
+            "sglang" => {
+                parts.push("python -m sglang.launch_server".to_string());
+            }
+            other => {
+                parts.push(format!("python -m {}", other));
+            }
+        }
+
+        // Model
+        parts.push(recipe.model.clone());
+
+        // Recipe-specific args
+        parts.extend(recipe.args.clone());
+
+        // Distributed flags
+        match recipe.distributed_backend.as_str() {
+            "ray" => {
+                parts.extend(vec![
+                    "--tensor-parallel-size".to_string(),
+                    "1".to_string(),
+                    "--pipeline-parallel-size".to_string(),
+                    pp_size.to_string(),
+                    "--nnodes".to_string(),
+                    nnodes.to_string(),
+                    "--node-rank".to_string(),
+                    node.rank.to_string(),
+                    "--master-addr".to_string(),
+                    cluster.master_ib_ip.clone(),
+                    "--distributed-executor-backend".to_string(),
+                    "ray".to_string(),
+                ]);
+            }
+            _ => {
+                // MP mode (default)
+                parts.extend(vec![
+                    "--tensor-parallel-size".to_string(),
+                    "1".to_string(),
+                    "--pipeline-parallel-size".to_string(),
+                    pp_size.to_string(),
+                    "--nnodes".to_string(),
+                    nnodes.to_string(),
+                    "--node-rank".to_string(),
+                    node.rank.to_string(),
+                    "--master-addr".to_string(),
+                    cluster.master_ib_ip.clone(),
+                    "--distributed-executor-backend".to_string(),
+                    recipe.distributed_backend.clone(),
+                ]);
+            }
+        }
+
+        // Port (only on head node / rank 0)
+        if node.rank == 0 {
+            parts.extend(vec!["--port".to_string(), recipe.port.to_string()]);
+        }
+
+        parts.join(" ")
+    }
+    /// Builds the full sequence of tmux commands for a session, with multi-window layout.
+    pub fn build_session_commands(
+        &self,
+        node: &Node,
+        recipe: &Recipe,
+        nccl: &NcclConfig,
+        cluster: &Cluster,
+    ) -> Vec<String> {
+        let sess = self.session_name(recipe, node.rank);
+        let mut cmds: Vec<String> = Vec::new();
+
+        // Kill existing session if it exists
+        cmds.push(format!("tmux kill-session -t {} 2>/dev/null || true", sess));
+
+        // Create new detached session (window 0)
+        cmds.push(format!("tmux new-session -d -s {} -n model", sess));
+
+        // --- Window 0: model process ---
+        for export in self.build_nccl_setup(nccl) {
+            cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, export));
+        }
+        for export in self.build_host_ip_export(&recipe.engine, node) {
+            cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, export));
+        }
+        if recipe.distributed_backend == "ray" && node.rank > 0 {
+            cmds.push(format!(
+                "tmux send-keys -t {}:0 'export RAY_ADDRESS={}:6379' Enter",
+                sess, cluster.master_ib_ip
+            ));
+        }
+        for (k, v) in &recipe.env {
+            cmds.push(format!(
+                "tmux send-keys -t {}:0 'export {}=\"{}\"' Enter",
+                sess, k, v
+            ));
+        }
+
+        let launch_cmd = self.build_launch_command(recipe, node, cluster);
+        cmds.push(format!("tmux send-keys -t {}:0 '{}' Enter", sess, launch_cmd));
+
+        // --- Window 1: GPU monitor ---
+        cmds.push(format!("tmux new-window -t {}:1 -n gpu", sess));
+        cmds.push(format!(
+            "tmux send-keys -t {}:1 'nvidia-smi -l 5' Enter",
+            sess
+        ));
+
+        // --- Window 2: shell (debug) ---
+        cmds.push(format!("tmux new-window -t {}:2 -n shell", sess));
+        cmds.push(format!(
+            "tmux send-keys -t {}:2 'echo \"ready for debugging\"' Enter",
+            sess
+        ));
+        cmds.push(format!("tmux select-window -t {}:0", sess));
+
+        cmds
+    }
+    /// Executes a list of commands via SSH on a node to create a tmux session.
+    pub async fn create_session(
+        &self,
+        config: &Config,
+        node: &Node,
+        commands: &[String],
+        dry_run: bool,
+    ) -> Result<()> {
+        if dry_run {
+            info!("  ssh {}@{}:", config.ssh_user, node.ssh_ip);
+            for cmd in commands {
+                info!("    {}", cmd);
+            }
+            return Ok(());
+        }
+
+        let script = commands.join(" && ");
+        let output = run_ssh(config, node, script, dry_run).await?;
+
+        if output.trim().is_empty() {
+            info!("  ✅ tmux session created on {}", node.name);
+        } else {
+            info!("  tmux session output on {}: {}", node.name, output.trim());
+        }
+
+        Ok(())
+    }
+
+    /// Checks whether a tmux session exists on the given node.
+    pub async fn check_session(
+        &self,
+        config: &Config,
+        node: &Node,
+        session_name: &str,
+        dry_run: bool,
+    ) -> Result<bool> {
+        if dry_run {
+            info!(
+                "  ssh {}@{}: tmux list-sessions 2>/dev/null | grep '{}'",
+                config.ssh_user, node.ssh_ip, session_name
+            );
+            return Ok(true);
+        }
+
+        let cmd = format!(
+            "tmux list-sessions 2>/dev/null | grep -c '{}'",
+            session_name
+        );
+        let output = run_ssh(config, node, cmd, dry_run).await?;
+
+        let found = output.trim().parse::<usize>().unwrap_or(0) > 0;
+        if found {
+            info!("  Session '{}' found on {}", session_name, node.name);
+        } else {
+            warn!("  Session '{}' NOT found on {}", session_name, node.name);
+        }
+
+        Ok(found)
+    }
+    /// Performs a deep health check on a tmux session.
+    pub async fn deep_check_session(
+        &self,
+        config: &Config,
+        node: &Node,
+        _cluster: &Cluster,
+        session_name: &str,
+        recipe: &Recipe,
+        dry_run: bool,
+    ) -> Result<SessionHealth> {
+        if dry_run {
+            return Ok(SessionHealth {
+                session_exists: true,
+                process_alive: true,
+                gpu_ok: true,
+                api_ok: true,
+            });
+        }
+
+        // 1. Check session exists
+        let exists = self
+            .check_session(config, node, session_name, dry_run)
+            .await?;
+
+        // 2. Check process is alive inside tmux
+        let process_alive = if exists {
+            let pid_cmd = format!(
+                "tmux list-panes -t {}:0 -F '#{{pane_pid}}' | head -1",
+                session_name
+            );
+            match run_ssh(config, node, pid_cmd, dry_run).await {
+                Ok(pid_str) => {
+                    let pid = pid_str.trim();
+                    if !pid.is_empty() && pid != "0" {
+                        let ps_cmd = format!("ps -p {} -o pid= 2>/dev/null", pid);
+                        run_ssh(config, node, ps_cmd, dry_run)
+                            .await
+                            .map(|o| !o.trim().is_empty())
+                            .unwrap_or(false)
+                    } else {
+                        false
+                    }
+                }
+                Err(_) => false,
+            }
+        } else {
+            false
+        };
+
+        // 3. Check GPU is available
+        let gpu_ok = match run_ssh(
+            config,
+            node,
+            "nvidia-smi --query-gpu=index,utilization.gpu,memory.used --format=csv,noheader 2>&1 | head -5"
+                .to_string(),
+            dry_run,
+        )
+        .await
+        {
+            Ok(out) => !out.contains("failed") && !out.contains("error") && !out.trim().is_empty(),
+            Err(_) => false,
+        };
+
+        // 4. Check API endpoint (head node only)
+        let api_ok = if node.rank == 0 && process_alive {
+            let api_check = format!(
+                "curl -s -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{}/v1/models 2>/dev/null || echo 'fail'",
+                recipe.port
+            );
+            match run_ssh(config, node, api_check, dry_run).await {
+                Ok(out) => {
+                    let code = out.trim();
+                    code == "200" || !code.contains("fail")
+                }
+                Err(_) => false,
+            }
+        } else {
+            node.rank != 0 || process_alive
+        };
+
+        Ok(SessionHealth {
+            session_exists: exists,
+            process_alive,
+            gpu_ok,
+            api_ok,
+        })
+    }
+
+    /// Gracefully stops a tmux session with timeout escalation.
+    pub async fn graceful_stop(
+        &self,
+        config: &Config,
+        node: &Node,
+        session_name: &str,
+        dry_run: bool,
+    ) -> Result<()> {
+        if dry_run {
+            info!(
+                "  ssh {}@{}: graceful stop of session '{}'",
+                config.ssh_user, node.ssh_ip, session_name
+            );
+            return Ok(());
+        }
+
+        info!("  Sending Ctrl+C to session '{}' on {}", session_name, node.name);
+        let _ = run_ssh(config, node, format!("tmux send-keys -t {}:0 C-c", session_name), dry_run).await;
+
+        let mut exited = false;
+        for i in 0..15 {
+            sleep(Duration::from_secs(2)).await;
+            let check_cmd = format!(
+                "tmux list-panes -t {}:0 -F '#{{pane_pid}}' 2>/dev/null | head -1",
+                session_name
+            );
+            match run_ssh(config, node, check_cmd, dry_run).await {
+                Ok(pid_str) => {
+                    let pid = pid_str.trim();
+                    if pid.is_empty() || pid == "0" || pid == "No" {
+                        exited = true;
+                        info!("  Session '{}' exited gracefully (attempt {})", session_name, i + 1);
+                        break;
+                    }
+                    let ps_cmd = format!("ps -p {} -o pid= 2>/dev/null", pid);
+                    match run_ssh(config, node, ps_cmd, dry_run).await {
+                        Ok(ps_out) if ps_out.trim().is_empty() => {
+                            exited = true;
+                            info!("  Session '{}' process exited (attempt {})", session_name, i + 1);
+                            break;
+                        }
+                        _ => continue,
+                    }
+                }
+                Err(_) => {
+                    exited = true;
+                    break;
+                }
+            }
+        }
+
+        if !exited {
+            warn!("  Session '{}' did not exit gracefully, force killing...", session_name);
+            let _ = run_ssh(config, node, format!("tmux kill-session -t {}", session_name), dry_run).await;
+        }
+
+        Ok(())
+    }
+    /// Streams the content of a tmux pane, optionally following new output.
+    pub async fn stream_pane(
+        &self,
+        config: &Config,
+        node: &Node,
+        session_name: &str,
+        window: u32,
+        follow: bool,
+        dry_run: bool,
+    ) -> Result<String> {
+        if dry_run {
+            if follow {
+                info!(
+                    "  ssh {}@{}: tail -f (streaming from session '{}' window {})",
+                    config.ssh_user, node.ssh_ip, session_name, window
+                );
+            } else {
+                info!(
+                    "  ssh {}@{}: tmux capture-pane -t {}:{} -p -S -100",
+                    config.ssh_user, node.ssh_ip, session_name, window
+                );
+            }
+            return Ok(String::new());
+        }
+
+        let cmd = if follow {
+            format!(
+                "tmux pipe-pane -o -t {}:{} 'cat > /tmp/sm121_pipe_{}_{}.log' && tail -f /tmp/sm121_pipe_{}_{}.log",
+                session_name, window,
+                session_name.replace('-', "_"), window,
+                session_name.replace('-', "_"), window
+            )
+        } else {
+            format!(
+                "tmux capture-pane -t {}:{} -p -S -200 2>/dev/null || echo 'no output'",
+                session_name, window
+            )
+        };
+
+        run_ssh_with_retry(config, node, cmd, dry_run, 1).await
+    }
+
+    /// Attaches to a tmux session on a node via SSH (interactive).
+    pub async fn attach_session(
+        &self,
+        config: &Config,
+        node: &Node,
+        session_name: &str,
+        dry_run: bool,
+    ) -> Result<()> {
+        if dry_run {
+            info!(
+                "  Would attach: ssh -t {}@{} tmux attach-session -t {}",
+                config.ssh_user, node.ssh_ip, session_name
+            );
+            return Ok(());
+        }
+
+        info!("  Attaching to '{}' on {}... Press Ctrl+B D to detach.", session_name, node.name);
+        info!("  (russh does not support interactive PTY. Use the following command directly:)");
+        println!(
+            "  ssh -t {}@{} tmux attach-session -t {}",
+            config.ssh_user, node.ssh_ip, session_name
+        );
+        println!();
+        info!("  Streaming pane content instead:");
+        let output = self.stream_pane(config, node, session_name, 0, true, dry_run).await?;
+        println!("{}", output);
+
+        Ok(())
+    }
+}
+
+/// Detailed health status of a tmux session.
+#[derive(Debug, Clone)]
+pub struct SessionHealth {
+    pub session_exists: bool,
+    pub process_alive: bool,
+    pub gpu_ok: bool,
+    pub api_ok: bool,
+}
+
+impl SessionHealth {
+    pub fn is_healthy(&self) -> bool {
+        self.session_exists && self.process_alive && self.gpu_ok && self.api_ok
+    }
+
+    #[allow(dead_code)]
+    pub fn is_degraded(&self) -> bool {
+        self.session_exists && !self.process_alive
+    }
+}
+
+/// Top-level handler for all `cargo run tmux <subcommand>` commands.
+pub async fn handle_tmux(config: &Config, cmd: TmuxCommands, dry_run: bool) -> Result<()> {
+    match cmd {
+        TmuxCommands::Serve { recipe, cluster } => {
+            serve::tmux_serve(config, &recipe, &cluster, dry_run).await?;
+        }
+        TmuxCommands::Stop { recipe, cluster } => {
+            ops::tmux_stop(config, &recipe, &cluster, dry_run).await?;
+        }
+        TmuxCommands::Status { recipe, cluster } => {
+            ops::tmux_status(config, &recipe, &cluster, dry_run).await?;
+        }
+        TmuxCommands::Logs { recipe, follow, cluster } => {
+            ops::tmux_logs(config, &recipe, &cluster, follow, dry_run).await?;
+        }
+        TmuxCommands::Attach { recipe, cluster } => {
+            ops::tmux_attach(config, &recipe, &cluster, dry_run).await?;
+        }
+    }
+    Ok(())
+}

src/tmux/ops.rs

@@ -0,0 +1,261 @@
+use anyhow::{Context, Result};
+use tracing::{info, warn};
+
+use crate::config::{Config, Recipe};
+use crate::health;
+use crate::tmux::TmuxManager;
+
+/// Stops all tmux sessions for a recipe across all nodes in a cluster.
+pub async fn tmux_stop(
+    config: &Config,
+    recipe_name: &str,
+    cluster_name: &str,
+    dry_run: bool,
+) -> Result<()> {
+    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
+    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
+
+    let choice = if cluster_name != "auto" {
+        cluster_name.to_string()
+    } else if let Some(ref rc) = recipe.cluster {
+        rc.clone()
+    } else {
+        "auto".to_string()
+    };
+    let (cluster, _) = crate::config::select_cluster(config, &choice)?;
+
+    let manager = TmuxManager {
+        session_prefix: recipe_name.to_string(),
+    };
+
+    info!(
+        "🛑 Stopping tmux sessions for '{}' on cluster '{}'...",
+        recipe_name, cluster.name
+    );
+
+    let mut failures = Vec::new();
+    for node in &cluster.nodes {
+        let sess = manager.session_name(&recipe, node.rank);
+        match manager
+            .graceful_stop(config, node, &sess, dry_run)
+            .await
+        {
+            Ok(()) => info!("  ✅ Stopped '{}' on {}", sess, node.name),
+            Err(e) => {
+                warn!("  ✗ Failed to stop '{}' on {}: {}", sess, node.name, e);
+                failures.push((node.name.clone(), e));
+            }
+        }
+    }
+
+    if !failures.is_empty() {
+        for (name, e) in &failures {
+            warn!("  Failed on {}: {}", name, e);
+        }
+        anyhow::bail!("Failed to stop sessions on {} node(s)", failures.len());
+    }
+
+    info!("✅ All tmux sessions stopped.");
+    Ok(())
+}
+
+/// Checks health status of tmux sessions across the cluster.
+pub async fn tmux_status(
+    config: &Config,
+    recipe_name: &str,
+    cluster_name: &str,
+    dry_run: bool,
+) -> Result<()> {
+    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
+    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
+
+    let choice = if cluster_name != "auto" {
+        cluster_name.to_string()
+    } else if let Some(ref rc) = recipe.cluster {
+        rc.clone()
+    } else {
+        "auto".to_string()
+    };
+    let (cluster, _) = crate::config::select_cluster(config, &choice)?;
+
+    let manager = TmuxManager {
+        session_prefix: recipe_name.to_string(),
+    };
+
+    println!("\n📊 TMUX Status: {} on cluster {}", recipe.name, cluster.name);
+    println!("{}", "─".repeat(60));
+
+    let report = health::check_cluster_health(&manager, config, &cluster, &recipe, dry_run).await?;
+
+    for node_health in &report.nodes {
+        let status_icon = if node_health.status == health::SessionStatus::Running {
+            "✓".to_string()
+        } else if node_health.session_exists {
+            "⚠".to_string()
+        } else {
+            "✗".to_string()
+        };
+
+        print!(
+            "  {} {} (rank {}, {}): ",
+            status_icon, node_health.name, node_health.rank, node_health.ib_ip
+        );
+
+        if node_health.status == health::SessionStatus::Running {
+            println!("✅ ALL OK");
+        } else {
+            let mut issues = Vec::new();
+            if !node_health.session_exists {
+                issues.push("no session");
+            }
+            if !node_health.process_running {
+                issues.push("process dead");
+            }
+            if !node_health.port_accessible && node_health.rank == 0 {
+                issues.push("API down");
+            }
+            println!("⚠️  {}", issues.join(", "));
+        }
+
+        if node_health.session_exists {
+            println!(
+                "     Session: alive | Process: {} | API: {}",
+                if node_health.process_running {
+                    "✓"
+                } else {
+                    "✗"
+                },
+                if node_health.port_accessible {
+                    "✓"
+                } else {
+                    "✗"
+                },
+            );
+        }
+    }
+
+    println!("{}", "─".repeat(60));
+    match report.overall {
+        health::HealthStatus::Healthy => info!("✅ Cluster is healthy!"),
+        health::HealthStatus::Degraded => warn!("⚠️ Cluster is degraded"),
+        health::HealthStatus::Unhealthy => warn!("⚠️ Cluster has issues — check details above."),
+    }
+
+    Ok(())
+}
+
+/// Shows or follows logs from tmux pane content.
+pub async fn tmux_logs(
+    config: &Config,
+    recipe_name: &str,
+    cluster_name: &str,
+    follow: bool,
+    dry_run: bool,
+) -> Result<()> {
+    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
+    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
+
+    let choice = if cluster_name != "auto" {
+        cluster_name.to_string()
+    } else if let Some(ref rc) = recipe.cluster {
+        rc.clone()
+    } else {
+        "auto".to_string()
+    };
+    let (cluster, _) = crate::config::select_cluster(config, &choice)?;
+
+    let manager = TmuxManager {
+        session_prefix: recipe_name.to_string(),
+    };
+
+    if follow {
+        let head_node = cluster.nodes.first().context("No nodes in cluster")?;
+        let sess = manager.session_name(&recipe, 0);
+
+        info!("📜 Following logs from '{}' on {}...", sess, head_node.name);
+        let output = manager
+            .stream_pane(config, head_node, &sess, 0, true, dry_run)
+            .await?;
+        println!("{}", output);
+    } else {
+        info!(
+            "📜 Logs for '{}' on cluster '{}' ({} nodes):",
+            recipe.name,
+            cluster.name,
+            cluster.nodes.len()
+        );
+
+        let mut handles = Vec::new();
+        for node in &cluster.nodes {
+            let config = config.clone();
+            let node = node.clone();
+            let manager = manager.clone();
+            let recipe = recipe.clone();
+            let dry = dry_run;
+
+            handles.push(tokio::spawn(async move {
+                let sess = manager.session_name(&recipe, node.rank);
+                let output = manager
+                    .stream_pane(&config, &node, &sess, 0, false, dry)
+                    .await;
+                (node.name, sess, output)
+            }));
+        }
+
+        let results = futures::future::join_all(handles).await;
+        for result in results {
+            if let Ok((name, sess, Ok(output))) = result {
+                let trimmed = output.trim();
+                if !trimmed.is_empty() {
+                    println!("\n--- {} ({}) ---", name, sess);
+                    println!("{}", trimmed);
+                } else {
+                    println!("\n--- {} — no output ---", name);
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Provides instructions and streamed output for attaching to a tmux session.
+pub async fn tmux_attach(
+    config: &Config,
+    recipe_name: &str,
+    cluster_name: &str,
+    dry_run: bool,
+) -> Result<()> {
+    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
+    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
+
+    let choice = if cluster_name != "auto" {
+        cluster_name.to_string()
+    } else if let Some(ref rc) = recipe.cluster {
+        rc.clone()
+    } else {
+        "auto".to_string()
+    };
+    let (cluster, _) = crate::config::select_cluster(config, &choice)?;
+
+    let head_node = cluster
+        .nodes
+        .iter()
+        .find(|n| n.rank == 0)
+        .context("No head node (rank 0) in cluster")?;
+
+    let manager = TmuxManager {
+        session_prefix: recipe_name.to_string(),
+    };
+    let sess = manager.session_name(&recipe, 0);
+
+    info!(
+        "🔗 Attaching to head node '{}' session '{}'",
+        head_node.name, sess
+    );
+    manager
+        .attach_session(config, head_node, &sess, dry_run)
+        .await?;
+
+    Ok(())
+}

src/tmux/serve.rs

@@ -0,0 +1,188 @@
+use anyhow::{Context, Result};
+use std::time::Duration;
+use tokio::time::sleep;
+use tracing::{info, warn};
+
+use crate::config::{Config, Recipe};
+use crate::tmux::TmuxManager;
+
+/// Launches a cluster via tmux sessions.
+/// For Ray backend: starts head node first, waits for readiness, then workers.
+pub async fn tmux_serve(
+    config: &Config,
+    recipe_name: &str,
+    cluster_name: &str,
+    dry_run: bool,
+) -> Result<()> {
+    let recipe_path = format!("recipes/{}/recipe.toml", recipe_name);
+    let recipe: Recipe = toml::from_str(&tokio::fs::read_to_string(&recipe_path).await?)?;
+
+    info!(
+        "🚀 TMUX launch: {} ({}) [backend: {}] [engine: {}]",
+        recipe.name, recipe_name, recipe.distributed_backend, recipe.engine
+    );
+
+    let choice = if cluster_name != "auto" {
+        cluster_name.to_string()
+    } else if let Some(ref rc) = recipe.cluster {
+        rc.clone()
+    } else {
+        "auto".to_string()
+    };
+    let (cluster, nccl) = crate::config::select_cluster(config, &choice)?;
+
+    if let Some(required) = recipe.required_nodes {
+        let actual = cluster.nodes.len() as u32;
+        if actual != required {
+            anyhow::bail!(
+                "Recipe '{}' requires {} nodes, but cluster '{}' has {} nodes",
+                recipe.name, required, cluster.name, actual
+            );
+        }
+    }
+
+    let manager = TmuxManager {
+        session_prefix: recipe_name.to_string(),
+    };
+
+    let is_ray = recipe.distributed_backend == "ray";
+
+    if is_ray {
+        info!("🐍 Ray backend detected — starting head node first...");
+
+        let head_node = cluster
+            .nodes
+            .iter()
+            .find(|n| n.rank == 0)
+            .context("No head node (rank 0) in cluster")?;
+
+        let head_cmds = manager.build_session_commands(head_node, &recipe, nccl, &cluster);
+        manager
+            .create_session(config, head_node, &head_cmds, dry_run)
+            .await?;
+
+        if !dry_run {
+            info!("⏳ Waiting for head node to be ready...");
+            let sess = manager.session_name(&recipe, 0);
+            let head_ready = wait_for_head_ready(config, head_node, &sess, &recipe).await?;
+            if head_ready {
+                info!("✅ Head node ready. Starting worker nodes...");
+            } else {
+                warn!("⚠️ Head node may not be fully ready — starting workers anyway.");
+            }
+        }
+
+        for node in &cluster.nodes {
+            if node.rank == 0 {
+                continue;
+            }
+            let worker_cmds = manager.build_session_commands(node, &recipe, nccl, &cluster);
+            manager
+                .create_session(config, node, &worker_cmds, dry_run)
+                .await?;
+        }
+    } else {
+        info!("🔗 MP backend detected — starting all nodes in parallel...");
+
+        let mut handles = Vec::new();
+        for node in &cluster.nodes {
+            let config = config.clone();
+            let node = node.clone();
+            let recipe = recipe.clone();
+            let nccl = nccl.clone();
+            let cluster = cluster.clone();
+            let manager = manager.clone();
+            let dry = dry_run;
+
+            handles.push(tokio::spawn(async move {
+                let cmds = manager.build_session_commands(&node, &recipe, &nccl, &cluster);
+                manager
+                    .create_session(&config, &node, &cmds, dry)
+                    .await
+                    .map(|_| node.name.clone())
+            }));
+        }
+
+        let results = futures::future::join_all(handles).await;
+        let mut failures = Vec::new();
+        for result in results {
+            match result {
+                Ok(Ok(name)) => info!("  ✅ Session started on {}", name),
+                Ok(Err(e)) => failures.push(e),
+                Err(e) => failures.push(anyhow::anyhow!("Task join: {}", e)),
+            }
+        }
+
+        if !failures.is_empty() {
+            for e in &failures {
+                warn!("  ✗ {}", e);
+            }
+            anyhow::bail!("Failed to start sessions on {} node(s)", failures.len());
+        }
+    }
+
+    if !dry_run {
+        sleep(Duration::from_secs(5)).await;
+        info!(
+            "🎉 Cluster launched via tmux! API → http://{}:{}",
+            cluster.master_ib_ip, recipe.port
+        );
+        info!("📌 To view sessions: `cargo run tmux status --recipe {}`", recipe_name);
+        info!("📌 To attach: `cargo run tmux attach --recipe {}`", recipe_name);
+    }
+
+    Ok(())
+}
+
+/// Polls the head node until the API endpoint responds or timeout.
+async fn wait_for_head_ready(
+    config: &Config,
+    head_node: &crate::config::Node,
+    session_name: &str,
+    recipe: &Recipe,
+) -> Result<bool> {
+    let max_attempts = 30;
+    for i in 0..max_attempts {
+        let pid_cmd = format!(
+            "tmux list-panes -t {}:0 -F '#{{pane_pid}}' | head -1",
+            session_name
+        );
+        match crate::ssh::run_ssh(config, head_node, pid_cmd, false).await {
+            Ok(pid_str) => {
+                let pid = pid_str.trim();
+                if pid.is_empty() || pid == "0" {
+                    sleep(Duration::from_secs(5)).await;
+                    continue;
+                }
+            }
+            Err(_) => {
+                sleep(Duration::from_secs(5)).await;
+                continue;
+            }
+        }
+
+        let api_check = format!(
+            "curl -s -o /dev/null -w '%{{http_code}}' http://127.0.0.1:{}/v1/models 2>/dev/null || echo 'not_ready'",
+            recipe.port
+        );
+        if let Ok(out) = crate::ssh::run_ssh(config, head_node, api_check, false).await {
+            let code = out.trim();
+            if code == "200" {
+                info!("  Head node API ready (attempt {})", i + 1);
+                return Ok(true);
+            }
+        }
+
+        if (i + 1) % 5 == 0 {
+            info!(
+                "  Still waiting for head node... ({}/{})",
+                i + 1,
+                max_attempts
+            );
+        }
+        sleep(Duration::from_secs(5)).await;
+    }
+
+    warn!("  Head node did not become ready within timeout");
+    Ok(false)
+}