flare_nodes.rs 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106
  1. //! Which of our nodes runs the `flare orbit` CLI.
  2. //!
  3. //! FLARE names its peers itself (`gx10-dgx5`…), so the launcher *finds* the
  4. //! controller: a cluster's rank-0 node is the candidate and the first one
  5. //! whose `orbit status` answers is it. `[flare.clusters.<name>].controller_node`
  6. //! pins it per cluster; the flat `[flare].controller_node` is the
  7. //! single-cluster fallback.
  8. //!
  9. //! Only clusters the config actually **pairs** with an orbit are candidates:
  10. //! several of our clusters share nodes (`dgx1` is rank 0 of `4nodes-switch`,
  11. //! `6nodes-switch`, `2nodes-dac`, `qwen38`, `1node-1`), so probing an unpaired
  12. //! cluster with the flat `sparks` default made another cluster's lane look
  13. //! like its own.
  14. use crate::config::{Cluster, Config, Node};
  15. /// The pinned controller of ONE cluster: a per-cluster override wins, then the
  16. /// flat `[flare]` field — and either must actually be a node of that cluster.
  17. pub fn pinned_for<'a>(config: &'a Config, cluster: &'a Cluster) -> Option<&'a Node> {
  18. let f = config.flare.as_ref()?;
  19. let want = f
  20. .clusters
  21. .get(&cluster.name)
  22. .and_then(|c| c.controller_node.clone())
  23. .or_else(|| f.controller_node.clone())?;
  24. cluster.nodes.iter().find(|n| n.name == want)
  25. }
  26. /// Rank-0 candidates of ONE cluster (pinned controller first) — the fleet
  27. /// probe must ask each cluster with its own orbit name, so `prod4` groups are
  28. /// not looked for in the `sparks` orbit (that silently showed nothing).
  29. pub fn candidates_for<'a>(
  30. config: &'a Config,
  31. cluster: &'a crate::config::Cluster,
  32. ) -> Vec<&'a Node> {
  33. let mut out: Vec<&Node> = Vec::new();
  34. if let Some(n) = pinned_for(config, cluster) {
  35. out.push(n);
  36. }
  37. for n in &cluster.nodes {
  38. if n.rank == 0 && !out.iter().any(|x| x.ssh_ip == n.ssh_ip) {
  39. out.push(n);
  40. }
  41. }
  42. out
  43. }
  44. #[cfg(test)]
  45. mod tests {
  46. use super::*;
  47. fn config() -> Config {
  48. toml::from_str(
  49. r#"
  50. ssh_user = "u"
  51. ssh_key_path = "/k"
  52. [flare]
  53. orbit_cluster = "sparks"
  54. controller_node = "n5"
  55. [flare.clusters.prod4]
  56. orbit_cluster = "prod4"
  57. controller_node = "n1"
  58. [[clusters]]
  59. name = "prod4"
  60. nodes = [
  61. { name = "n1", ssh_ip = "10.0.0.1", rank = 0 },
  62. { name = "n2", ssh_ip = "10.0.0.2", rank = 1 },
  63. ]
  64. [[clusters]]
  65. name = "six"
  66. nodes = [
  67. { name = "n1", ssh_ip = "10.0.0.1", rank = 0 },
  68. { name = "n2", ssh_ip = "10.0.0.2", rank = 1 },
  69. { name = "n5", ssh_ip = "10.0.0.5", rank = 2 },
  70. ]
  71. "#,
  72. )
  73. .expect("config toml")
  74. }
  75. #[test]
  76. fn pinned_is_per_cluster_and_in_that_cluster() {
  77. let cfg = config();
  78. let prod4 = cfg.clusters.iter().find(|c| c.name == "prod4").unwrap();
  79. let six = cfg.clusters.iter().find(|c| c.name == "six").unwrap();
  80. assert_eq!(pinned_for(&cfg, prod4).map(|n| n.name.as_str()), Some("n1"));
  81. // `six` has no per-cluster pin; the flat one (n5) is one of its nodes
  82. assert_eq!(pinned_for(&cfg, six).map(|n| n.name.as_str()), Some("n5"));
  83. }
  84. #[test]
  85. fn pairing_only_names_clusters_the_config_mentions() {
  86. use crate::native::flare::orbit_cluster_paired;
  87. let cfg = config();
  88. assert_eq!(
  89. orbit_cluster_paired(&cfg, "prod4").as_deref(),
  90. Some("prod4")
  91. );
  92. // dgx5 is six's rank 2, not its controller — no pairing
  93. assert_eq!(orbit_cluster_paired(&cfg, "six"), None);
  94. }
  95. }