Global config: recipe_sources + cluster_sources

sm121 previously required the repo dir as cwd (./config.toml + ./recipes).
Now the global config (~/.config/sm121 or ~/Library/Application Support/
sm121 on macOS — existing fallback) can drive everything:

- `recipe_sources`: extra recipe dirs (each `<name>/recipe.toml`), searched
  after ./recipes, before the global-config recipes fallback. `~` expanded
  locally. Lets `sm121` run from any cwd.
- `cluster_sources`: extra cluster definition files; each may carry
  `[[clusters]]` + per-cluster `[nccl.<name>]` / `[before_serve.<name>]`
  tables, merged by name (main config wins on conflict; among sources,
  first wins). Non-cluster keys in source files are ignored, so the repo's
  config.toml itself can serve as a cluster source.
- `Config.clusters` is now `serde(default)` so a sources-only global config
  parses without its own `[[clusters]]`.

load_recipe(name, sources) threads the sources through all call sites
(serve/stop/build/logs/doctor/exec/tmux-discover). AGENTS.md documents the
resolution order + both fields.

Verified: dry-run serve from /tmp with a sources-only config, and from
$HOME via the real global config pointing at the repo's recipes/ and
config.toml.

AGENTS.md

@@ -415,10 +415,21 @@ why each decision was made, and how to re-benchmark.
 
 ## Config fields
 
+Config resolution: `--config <path>` > `./config.toml` (cwd) > the global
+config (`~/.config/sm121/config.toml` on Linux, `~/Library/Application
+Support/sm121/config.toml` on macOS). The global config exists so `sm121`
+runs from **any cwd**: it holds `ssh_user`/`ssh_key_path` plus the two
+source lists below (it may carry no `[[clusters]]` of its own — clusters
+may come entirely from `cluster_sources`).
+
 ```toml
 ssh_user = "u"
 ssh_key_path = "/path"              # PathBuf
 
+recipe_sources = ["/path/to/recipes"]       # extra recipe dirs (each `<name>/recipe.toml`); searched after ./recipes
+cluster_sources = ["/path/clusters.toml"]   # extra cluster def files: `[[clusters]]` (+ `[nccl.<name>]` /
+                                            # `[before_serve.<name>]` tables); merged by name, this file wins
+
 [[clusters]]
 name = "c"
 master_ib_ip = "10.0.0.1"

src/after_stop.rs

@@ -176,6 +176,8 @@ mod tests {
             home_assistant: None,
             drop_caches: true,
             mount_aliases: std::collections::HashMap::new(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
             global_mods_dir: None,
         }
     }

src/before_serve.rs

@@ -220,6 +220,8 @@ mod tests {
             global_mods_dir: None,
             drop_caches: true,
             mount_aliases: HashMap::new(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
         }
     }
 

src/cluster_state.rs

@@ -342,6 +342,8 @@ mod tests {
             home_assistant: None,
             drop_caches: true,
             mount_aliases: HashMap::new(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
             global_mods_dir: None,
         };
 
@@ -369,6 +371,8 @@ mod tests {
             home_assistant: None,
             drop_caches: true,
             mount_aliases: HashMap::new(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
             global_mods_dir: None,
         };
         assert!(select_cluster(&cfg, "nonexistent", None, &state).is_err());
@@ -400,6 +404,8 @@ mod tests {
             home_assistant: None,
             drop_caches: true,
             mount_aliases: HashMap::new(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
             global_mods_dir: None,
         };
         assert!(select_cluster(&cfg, "auto", Some(1), &state).is_err());
@@ -427,6 +433,8 @@ mod tests {
             home_assistant: None,
             drop_caches: true,
             mount_aliases: HashMap::new(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
             global_mods_dir: None,
         };
         // Single-node recipe on a cluster without NCCL config → OK
@@ -464,6 +472,8 @@ mod tests {
             home_assistant: None,
             drop_caches: true,
             mount_aliases: HashMap::new(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
             global_mods_dir: None,
         };
         // Multi-node recipe on a cluster without NCCL config → error

src/config.rs

@@ -1,5 +1,6 @@
-use anyhow::Result;
-use std::collections::HashSet;
+use anyhow::{Context, Result};
+use serde::Deserialize;
+use std::collections::{HashMap, HashSet};
 use tracing::info;
 
 mod types;
@@ -53,6 +54,55 @@ pub fn resolve_cluster_name<'a>(clusters: &'a [Cluster], name: &str) -> Result<&
     }
 }
 
+/// A `cluster_sources` file: extra `[[clusters]]` entries, optionally with
+/// per-cluster `[nccl.<name>]` / `[before_serve.<name>]` tables. Merged into
+/// the main [`Config`] by [`merge_cluster_sources`].
+#[derive(Deserialize, Default)]
+pub struct ClusterSource {
+    #[serde(default)]
+    pub clusters: Vec<Cluster>,
+    #[serde(default)]
+    pub nccl: HashMap<String, NcclConfig>,
+    #[serde(default)]
+    pub before_serve: HashMap<String, BeforeServeTable>,
+}
+
+/// Merge each `Config.cluster_sources` file into `config`: extra clusters
+/// (main-config entries win on name conflict; among sources, first wins),
+/// plus per-cluster `nccl` / `before_serve` tables.
+pub async fn merge_cluster_sources(config: &mut Config) -> Result<()> {
+    for raw in &config.cluster_sources {
+        let path = local_path(raw);
+        let content = tokio::fs::read_to_string(&path)
+            .await
+            .with_context(|| format!("cluster source '{}'", path.display()))?;
+        let src: ClusterSource = toml::from_str(&content)
+            .with_context(|| format!("invalid cluster source '{}'", path.display()))?;
+        for c in src.clusters {
+            if !config.clusters.iter().any(|x| x.name == c.name) {
+                config.clusters.push(c);
+            }
+        }
+        for (name, v) in src.nccl {
+            config.nccl.entry(name).or_insert(v);
+        }
+        for (name, v) in src.before_serve {
+            config.before_serve.entry(name).or_insert(v);
+        }
+    }
+    Ok(())
+}
+
+/// Expand `~/...` against the local user's home (unlike `expand_tilde`,
+/// which targets the ssh user's home *on the node*). Absolute paths pass
+/// through.
+fn local_path(s: &str) -> std::path::PathBuf {
+    if let (Some(rest), Some(home)) = (s.strip_prefix("~/"), dirs::home_dir()) {
+        return home.join(rest);
+    }
+    std::path::PathBuf::from(s)
+}
+
 /// Whether NCCL configuration is required for cluster selection.
 ///
 /// NCCL config is only needed for multi-node distributed runs — for
@@ -172,11 +222,20 @@ pub async fn resolve_mods_dir(name: &str) -> Option<std::path::PathBuf> {
     cfg.is_dir().then_some(cfg)
 }
 
-pub async fn load_recipe(recipe_name: &str) -> Result<Recipe> {
+/// Load a recipe by name. Search order: `./recipes/<name>/recipe.toml` (cwd),
+/// then each `sources` entry (a dir containing `<name>/recipe.toml`, `~`
+/// expanded), then `<default_config_dir>/recipes/<name>/recipe.toml`.
+pub async fn load_recipe(recipe_name: &str, sources: &[String]) -> Result<Recipe> {
     let cwd_path = format!("recipes/{}/recipe.toml", recipe_name);
     if let Ok(content) = tokio::fs::read_to_string(&cwd_path).await {
         return Ok(toml::from_str(&content)?);
     }
+    for src in sources {
+        let p = local_path(src).join(recipe_name).join("recipe.toml");
+        if let Ok(content) = tokio::fs::read_to_string(&p).await {
+            return Ok(toml::from_str(&content)?);
+        }
+    }
     if let Some(dir) = default_config_dir() {
         let in_default = dir.join("recipes").join(recipe_name).join("recipe.toml");
         if let Ok(content) = tokio::fs::read_to_string(&in_default).await {

src/config/tests.rs

@@ -42,6 +42,8 @@ fn tcfg() -> Config {
         global_mods_dir: None,
         drop_caches: true,
         mount_aliases: HashMap::new(),
+        recipe_sources: Vec::new(),
+        cluster_sources: Vec::new(),
     }
 }
 
@@ -321,3 +323,67 @@ fn default_config_dir_appends_sm121() {
     };
     assert!(d.ends_with("sm121") && d.parent().is_some());
 }
+
+#[tokio::test]
+async fn test_merge_cluster_sources_adds_new_keeps_main() {
+    let dir = std::env::temp_dir().join(format!("sm121-csrc-{}", std::process::id()));
+    std::fs::create_dir_all(&dir).unwrap();
+    let src_path = dir.join("clusters-extra.toml");
+    // "extra" is new; "test-cluster" collides with tcfg's cluster (main wins).
+    std::fs::write(
+        &src_path,
+        r#"
+[[clusters]]
+name = "extra"
+nodes = [{ name = "e0", ssh_ip = "10.9.0.1", rank = 0 }]
+
+[[clusters]]
+name = "test-cluster"
+nodes = [{ name = "intruder", ssh_ip = "10.66.0.1", rank = 0 }]
+
+[nccl."extra"]
+ib_hca = "mlx5_0"
+socket_ifname = "eth0"
+ib_disable = 0
+
+[before_serve."extra"]
+commands = ["echo hi"]
+"#,
+    )
+    .unwrap();
+    let mut cfg = tcfg();
+    cfg.cluster_sources = vec![src_path.to_string_lossy().into_owned()];
+    merge_cluster_sources(&mut cfg).await.unwrap();
+    assert_eq!(
+        cfg.clusters.len(),
+        2,
+        "extra added, conflicting name not duplicated"
+    );
+    let main = cfg
+        .clusters
+        .iter()
+        .find(|c| c.name == "test-cluster")
+        .unwrap();
+    assert_eq!(
+        main.nodes[0].name, "node0",
+        "main-config cluster wins on conflict"
+    );
+    assert!(cfg.clusters.iter().any(|c| c.name == "extra"));
+    assert!(cfg.nccl.contains_key("extra"));
+    assert!(cfg.before_serve.contains_key("extra"));
+}
+
+#[tokio::test]
+async fn test_load_recipe_from_recipe_sources() {
+    let dir = std::env::temp_dir().join(format!("sm121-rsrc-{}", std::process::id()));
+    let rdir = dir.join("remote-recipes").join("remote-recipe");
+    std::fs::create_dir_all(&rdir).unwrap();
+    std::fs::write(rdir.join("recipe.toml"), "name = \"remote\"\n").unwrap();
+    let recipe = load_recipe(
+        "remote-recipe",
+        &[dir.join("remote-recipes").to_string_lossy().into_owned()],
+    )
+    .await
+    .unwrap();
+    assert_eq!(recipe.name, "remote");
+}

src/config/types.rs

@@ -57,6 +57,7 @@ pub struct Cluster {
 pub struct Config {
     pub ssh_user: String,
     pub ssh_key_path: std::path::PathBuf,
+    #[serde(default)]
     pub clusters: Vec<Cluster>,
     #[serde(default)]
     pub nccl: HashMap<String, NcclConfig>,
@@ -82,6 +83,12 @@ pub struct Config {
     /// overrides it.
     #[serde(default)]
     pub mount_aliases: HashMap<String, String>,
+    /// Extra recipe dirs (each `<name>/recipe.toml`) searched after `./recipes`; `~` expanded.
+    #[serde(default)]
+    pub recipe_sources: Vec<String>,
+    /// Extra cluster def files (`[[clusters]]`, `[nccl.<name>]` tables); this file wins.
+    #[serde(default)]
+    pub cluster_sources: Vec<String>,
 }
 
 #[derive(Clone, Deserialize)]

src/detect.rs

@@ -155,6 +155,8 @@ mod tests {
             global_mods_dir: None,
             drop_caches: false,
             mount_aliases: Default::default(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
         };
         for dry in [false, true] {
             let r = resolve_nccl_for_node(&config, &node, Some(&nccl), dry)

src/docker/command_tests.rs

@@ -49,6 +49,8 @@ fn minimal_config() -> Config {
         global_mods_dir: None,
         drop_caches: true,
         mount_aliases: HashMap::from([("models".to_string(), "/mnt/nas/models".to_string())]),
+        recipe_sources: Vec::new(),
+        cluster_sources: Vec::new(),
     }
 }
 

src/docker/mirror.rs

@@ -161,6 +161,8 @@ mod tests {
             global_mods_dir: None,
             drop_caches: true,
             mount_aliases: HashMap::new(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
         }
     }
 

src/docker/mod.rs

@@ -58,7 +58,7 @@ pub async fn serve(
     node_name: Option<&str>,
     opts: ServeOpts,
 ) -> Result<Cluster> {
-    let mut recipe = load_recipe(recipe_name).await?;
+    let mut recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
     let rn = recipe.name.clone();
     let re = recipe.engine.clone();
     info!(

src/doctor.rs

@@ -42,7 +42,7 @@ pub async fn doctor(
     repair: bool,
     dry_run: bool,
 ) -> Result<()> {
-    let recipe = load_recipe(recipe_name).await?;
+    let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
     let dn = recipe.name.clone();
     let de = recipe.engine.clone();
     let db = recipe

src/exec.rs

@@ -23,7 +23,7 @@ pub async fn resolve_target(
             Ok((cluster, None))
         }
         (None, Some(rname)) => {
-            let recipe = crate::config::load_recipe(rname).await?;
+            let recipe = crate::config::load_recipe(rname, &config.recipe_sources).await?;
             let cname = recipe.cluster.as_deref().unwrap_or("auto");
             let (cluster, _) =
                 select_cluster(config, cname, recipe.required_nodes, &HashSet::new())
@@ -170,6 +170,8 @@ mod tests {
             home_assistant: None,
             drop_caches: true,
             mount_aliases: HashMap::new(),
+            recipe_sources: Vec::new(),
+            cluster_sources: Vec::new(),
             global_mods_dir: None,
         }
     }

src/main.rs

@@ -42,8 +42,7 @@ use std::collections::HashSet;
 /// 2. `./config.toml` in cwd
 /// 3. `<default_config_dir>/config.toml`
 ///
-/// Returns an error if nothing is found, with a hint pointing the user
-/// to copy the example config to one of the two paths.
+/// Returns an error if nothing is found (hint: copy the example config).
 fn resolve_config_path(cli_config: &str) -> std::path::PathBuf {
     use std::path::PathBuf;
     let explicit = PathBuf::from(cli_config);
@@ -86,7 +85,8 @@ async fn main() -> Result<()> {
 
     let config_path = resolve_config_path(&cli.config);
     let raw_config = tokio::fs::read_to_string(&config_path).await?;
-    let config: Config = toml::from_str(&raw_config)?;
+    let mut config: Config = toml::from_str(&raw_config)?;
+    crate::config::merge_cluster_sources(&mut config).await?;
     let dry_run = cli.dry_run;
 
     match cli.command {
@@ -221,7 +221,7 @@ async fn handle_serve(
     follow: bool,
     dry_run: bool,
 ) -> Result<()> {
-    let recipe_obj = load_recipe(recipe).await?;
+    let recipe_obj = load_recipe(recipe, &config.recipe_sources).await?;
     match recipe_obj.deploy {
         DeployTarget::Docker => {
             let cluster = docker::serve(config, recipe, cluster, node.as_deref(), opts).await?;
@@ -282,7 +282,7 @@ async fn handle_build_cmd(
     force_build: bool,
     dry_run: bool,
 ) -> Result<()> {
-    let recipe_obj = crate::config::load_recipe(recipe).await?;
+    let recipe_obj = crate::config::load_recipe(recipe, &config.recipe_sources).await?;
     let cluster = crate::config::resolve_build_cluster(config, cluster, build_node)?;
     let opts = docker::ServeOpts {
         keep: false,
@@ -376,7 +376,7 @@ async fn handle_stop(
     // Run recipe `after_stop` hooks (e.g. restore_dgx_services) on the
     // nodes that were actually stopped. Best-effort against the loaded
     // recipe; if the recipe file is unavailable, skip silently.
-    if let Ok(recipe) = crate::config::load_recipe(recipe_name).await {
+    if let Ok(recipe) = crate::config::load_recipe(recipe_name, &config.recipe_sources).await {
         crate::after_stop::run(config, &stopped_cluster, &recipe, dry_run).await?;
     }
 
@@ -485,7 +485,7 @@ async fn resolve_cluster_for_op(
     cluster_name: &str,
 ) -> Result<Cluster> {
     if cluster_name != "auto" {
-        let recipe = load_recipe(recipe_name).await?;
+        let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
         let (cluster, _) = resolve_cluster(config, &recipe, cluster_name, &HashSet::new())?;
         return Ok(cluster);
     }
@@ -494,7 +494,7 @@ async fn resolve_cluster_for_op(
         return Ok(c.clone());
     }
     // Fall back to config-based auto (dry-run / containers not running)
-    let recipe = load_recipe(recipe_name).await?;
+    let recipe = load_recipe(recipe_name, &config.recipe_sources).await?;
     let (cluster, _) = resolve_cluster(config, &recipe, "auto", &HashSet::new())?;
     Ok(cluster)
 }

src/orchestrate/tests.rs

@@ -89,6 +89,8 @@ fn test_config(clusters: Vec<Cluster>) -> Config {
         home_assistant: None,
         drop_caches: true,
         mount_aliases: HashMap::new(),
+        recipe_sources: Vec::new(),
+        cluster_sources: Vec::new(),
         global_mods_dir: None,
     }
 }

src/tmux/discover.rs

@@ -31,12 +31,13 @@ pub fn list_recipe_dirs() -> Vec<String> {
     recipes
 }
 
-/// Finds all recipes with `deploy = "tmux"`.
-pub async fn find_tmux_recipes() -> Vec<(String, Recipe)> {
+/// Finds all recipes with `deploy = "tmux"` (in `./recipes`, resolved via
+/// `load_recipe` with the given extra sources).
+pub async fn find_tmux_recipes(sources: &[String]) -> Vec<(String, Recipe)> {
     let dirs = list_recipe_dirs();
     let mut recipes = Vec::new();
     for dir in dirs {
-        let Ok(recipe) = crate::config::load_recipe(&dir).await else {
+        let Ok(recipe) = crate::config::load_recipe(&dir, sources).await else {
             continue;
         };
         if recipe.deploy == DeployTarget::Tmux {
@@ -61,7 +62,7 @@ pub struct TmuxHealthEntry {
 /// (session existence, process liveness, GPU, API) on every node.
 /// Returns one entry per node per recipe.
 pub async fn tmux_health_for_cluster(config: &Config, cluster: &Cluster) -> Vec<TmuxHealthEntry> {
-    let tmux_recipes = find_tmux_recipes().await;
+    let tmux_recipes = find_tmux_recipes(&config.recipe_sources).await;
     let manager = TmuxManager;
     let mut entries = Vec::new();
 
@@ -156,7 +157,7 @@ mod tests {
     async fn find_tmux_recipes_finds_tmux_deployed() {
         // This test runs in the project directory, so it should find
         // the laguna recipe which has deploy = "tmux"
-        let recipes = find_tmux_recipes().await;
+        let recipes = find_tmux_recipes(&[]).await;
         assert!(recipes.iter().any(|(dir, _)| dir == "laguna-s2.1-nvfp4"));
         for (_dir, recipe) in &recipes {
             assert_eq!(recipe.deploy, DeployTarget::Tmux);