|
@@ -16,7 +16,6 @@ pub struct Node {
|
|
|
#[serde(default)]
|
|
#[serde(default)]
|
|
|
pub entity: Option<String>,
|
|
pub entity: Option<String>,
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
#[derive(Clone, Deserialize)]
|
|
#[derive(Clone, Deserialize)]
|
|
|
pub struct NcclConfig {
|
|
pub struct NcclConfig {
|
|
|
pub ib_hca: String,
|
|
pub ib_hca: String,
|
|
@@ -24,19 +23,15 @@ pub struct NcclConfig {
|
|
|
pub ib_gid_index: u32,
|
|
pub ib_gid_index: u32,
|
|
|
pub ib_disable: u32,
|
|
pub ib_disable: u32,
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
#[derive(Clone, Deserialize)]
|
|
#[derive(Clone, Deserialize)]
|
|
|
pub struct Cluster {
|
|
pub struct Cluster {
|
|
|
pub name: String,
|
|
pub name: String,
|
|
|
pub master_ib_ip: String,
|
|
pub master_ib_ip: String,
|
|
|
pub nodes: Vec<Node>,
|
|
pub nodes: Vec<Node>,
|
|
|
- /// Extra Home Assistant switches to toggle alongside the nodes
|
|
|
|
|
- /// during `power on`/`power off`. E.g. a network switch that powers
|
|
|
|
|
- /// the cluster. Toggled together with per-node entities.
|
|
|
|
|
|
|
+ /// Extra Home Assistant switches to toggle alongside the nodes during power on/off.
|
|
|
#[serde(default)]
|
|
#[serde(default)]
|
|
|
pub extra_entities: Vec<String>,
|
|
pub extra_entities: Vec<String>,
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
/// Top-level config (clusters + per-cluster pre-serve commands).
|
|
/// Top-level config (clusters + per-cluster pre-serve commands).
|
|
|
#[derive(Clone, Deserialize)]
|
|
#[derive(Clone, Deserialize)]
|
|
|
pub struct Config {
|
|
pub struct Config {
|
|
@@ -144,10 +139,12 @@ fn default_tensor_parallel_size() -> u32 {
|
|
|
1
|
|
1
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-/// Selects a cluster + NCCL config by name or auto-picks the first one with NCCL.
|
|
|
|
|
|
|
+/// Selects a cluster + NCCL config by name or auto-picks the smallest
|
|
|
|
|
+/// cluster with NCCL config satisfying `required_nodes`.
|
|
|
pub fn select_cluster<'c>(
|
|
pub fn select_cluster<'c>(
|
|
|
config: &'c Config,
|
|
config: &'c Config,
|
|
|
cluster_name: &str,
|
|
cluster_name: &str,
|
|
|
|
|
+ required_nodes: Option<u32>,
|
|
|
) -> Result<(Cluster, &'c NcclConfig)> {
|
|
) -> Result<(Cluster, &'c NcclConfig)> {
|
|
|
if cluster_name != "auto" {
|
|
if cluster_name != "auto" {
|
|
|
let cluster = config
|
|
let cluster = config
|
|
@@ -162,11 +159,22 @@ pub fn select_cluster<'c>(
|
|
|
return Ok((cluster.clone(), nccl));
|
|
return Ok((cluster.clone(), nccl));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- for cluster in &config.clusters {
|
|
|
|
|
- if let Some(nccl) = config.nccl.get(&cluster.name) {
|
|
|
|
|
- info!("✅ Автоподбор кластера: {}", cluster.name);
|
|
|
|
|
- return Ok((cluster.clone(), nccl));
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ let mut candidates: Vec<(&Cluster, &NcclConfig)> = config
|
|
|
|
|
+ .clusters
|
|
|
|
|
+ .iter()
|
|
|
|
|
+ .filter_map(|c| Some((c, config.nccl.get(&c.name)?)))
|
|
|
|
|
+ .filter(|(c, _)| required_nodes.is_none_or(|r| (c.nodes.len() as u32) == r))
|
|
|
|
|
+ .collect();
|
|
|
|
|
+
|
|
|
|
|
+ candidates.sort_by_key(|(c, _)| c.nodes.len());
|
|
|
|
|
+
|
|
|
|
|
+ if let Some((cluster, nccl)) = candidates.into_iter().next() {
|
|
|
|
|
+ info!(
|
|
|
|
|
+ "✅ Автоподбор кластера: {} ({} узлов)",
|
|
|
|
|
+ cluster.name,
|
|
|
|
|
+ cluster.nodes.len()
|
|
|
|
|
+ );
|
|
|
|
|
+ return Ok((cluster.clone(), nccl));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
anyhow::bail!("Не удалось подобрать подходящий кластер");
|
|
anyhow::bail!("Не удалось подобрать подходящий кластер");
|
|
@@ -195,7 +203,7 @@ pub fn resolve_cluster<'c>(
|
|
|
} else {
|
|
} else {
|
|
|
"auto".to_string()
|
|
"auto".to_string()
|
|
|
};
|
|
};
|
|
|
- select_cluster(config, &choice)
|
|
|
|
|
|
|
+ select_cluster(config, &choice, recipe.required_nodes)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/// Loads a recipe (cwd first, then `<default_config_dir>/recipes/<name>/recipe.toml`,
|
|
/// Loads a recipe (cwd first, then `<default_config_dir>/recipes/<name>/recipe.toml`,
|
|
@@ -286,7 +294,6 @@ mod tests {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// ── Deserialization tests ──────────────────────────────────
|
|
// ── Deserialization tests ──────────────────────────────────
|
|
|
-
|
|
|
|
|
#[test]
|
|
#[test]
|
|
|
fn test_recipe_deserialize_minimal() {
|
|
fn test_recipe_deserialize_minimal() {
|
|
|
let toml_str = r#"
|
|
let toml_str = r#"
|
|
@@ -398,7 +405,6 @@ nodes = [
|
|
|
Some("SHA256:abc123".to_string())
|
|
Some("SHA256:abc123".to_string())
|
|
|
);
|
|
);
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
#[test]
|
|
#[test]
|
|
|
fn test_select_cluster_by_name() {
|
|
fn test_select_cluster_by_name() {
|
|
|
let mut config = Config {
|
|
let mut config = Config {
|
|
@@ -414,11 +420,10 @@ nodes = [
|
|
|
};
|
|
};
|
|
|
config.nccl.insert("test-cluster".to_string(), test_nccl());
|
|
config.nccl.insert("test-cluster".to_string(), test_nccl());
|
|
|
|
|
|
|
|
- let (cluster, nccl) = select_cluster(&config, "test-cluster").unwrap();
|
|
|
|
|
|
|
+ let (cluster, nccl) = select_cluster(&config, "test-cluster", None).unwrap();
|
|
|
assert_eq!(cluster.name, "test-cluster");
|
|
assert_eq!(cluster.name, "test-cluster");
|
|
|
assert_eq!(nccl.ib_hca, "rocep1s0f0");
|
|
assert_eq!(nccl.ib_hca, "rocep1s0f0");
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
#[test]
|
|
#[test]
|
|
|
fn test_select_cluster_auto() {
|
|
fn test_select_cluster_auto() {
|
|
|
let mut config = Config {
|
|
let mut config = Config {
|
|
@@ -434,15 +439,14 @@ nodes = [
|
|
|
};
|
|
};
|
|
|
config.nccl.insert("test-cluster".to_string(), test_nccl());
|
|
config.nccl.insert("test-cluster".to_string(), test_nccl());
|
|
|
|
|
|
|
|
- let (cluster, nccl) = select_cluster(&config, "auto").unwrap();
|
|
|
|
|
|
|
+ let (cluster, nccl) = select_cluster(&config, "auto", None).unwrap();
|
|
|
assert_eq!(cluster.name, "test-cluster");
|
|
assert_eq!(cluster.name, "test-cluster");
|
|
|
assert_eq!(nccl.ib_hca, "rocep1s0f0");
|
|
assert_eq!(nccl.ib_hca, "rocep1s0f0");
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
#[test]
|
|
#[test]
|
|
|
fn test_select_cluster_auto_no_match() {
|
|
fn test_select_cluster_auto_no_match() {
|
|
|
let config = test_config();
|
|
let config = test_config();
|
|
|
- let result = select_cluster(&config, "auto");
|
|
|
|
|
|
|
+ let result = select_cluster(&config, "auto", None);
|
|
|
assert!(result.is_err());
|
|
assert!(result.is_err());
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -450,10 +454,9 @@ nodes = [
|
|
|
fn test_select_cluster_not_found() {
|
|
fn test_select_cluster_not_found() {
|
|
|
let mut config = test_config();
|
|
let mut config = test_config();
|
|
|
config.clusters = vec![];
|
|
config.clusters = vec![];
|
|
|
- let result = select_cluster(&config, "nonexistent");
|
|
|
|
|
|
|
+ let result = select_cluster(&config, "nonexistent", None);
|
|
|
assert!(result.is_err());
|
|
assert!(result.is_err());
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
#[test]
|
|
#[test]
|
|
|
fn test_select_cluster_by_name_found() {
|
|
fn test_select_cluster_by_name_found() {
|
|
|
let mut nccl_map = HashMap::new();
|
|
let mut nccl_map = HashMap::new();
|
|
@@ -464,14 +467,12 @@ nodes = [
|
|
|
assert_eq!(cluster.name, "test-cluster");
|
|
assert_eq!(cluster.name, "test-cluster");
|
|
|
assert_eq!(cluster.nodes.len(), 2);
|
|
assert_eq!(cluster.nodes.len(), 2);
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
#[test]
|
|
#[test]
|
|
|
fn test_select_cluster_by_name_not_found() {
|
|
fn test_select_cluster_by_name_not_found() {
|
|
|
let config = test_config();
|
|
let config = test_config();
|
|
|
let result = select_cluster_by_name(&config, "no-such-cluster");
|
|
let result = select_cluster_by_name(&config, "no-such-cluster");
|
|
|
assert!(result.is_err());
|
|
assert!(result.is_err());
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
// ── Defaults ───────────────────────────────────────────────
|
|
// ── Defaults ───────────────────────────────────────────────
|
|
|
|
|
|
|
|
#[test]
|
|
#[test]
|
|
@@ -485,7 +486,6 @@ nodes = [
|
|
|
let recipe: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"").unwrap();
|
|
let recipe: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"").unwrap();
|
|
|
assert_eq!(recipe.distributed_backend, "mp");
|
|
assert_eq!(recipe.distributed_backend, "mp");
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
// ── Default config dir ─────────────────────────────────────
|
|
// ── Default config dir ─────────────────────────────────────
|
|
|
|
|
|
|
|
#[test]
|
|
#[test]
|