tests.rs 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460
  1. use crate::config::*;
  2. use crate::pre_serve::*;
  3. use std::collections::{HashMap, HashSet};
  4. fn tcls() -> Cluster {
  5. Cluster {
  6. name: "test-cluster".to_string(),
  7. master_ib_ip: Some("192.168.1.1".to_string()),
  8. nodes: vec![
  9. Node {
  10. name: "node0".to_string(),
  11. ssh_ip: "10.0.0.1".to_string(),
  12. ib_ip: Some("192.168.1.1".to_string()),
  13. rank: 0,
  14. host_key: None,
  15. entity: None,
  16. },
  17. Node {
  18. name: "node1".to_string(),
  19. ssh_ip: "10.0.0.2".to_string(),
  20. ib_ip: Some("192.168.1.2".to_string()),
  21. rank: 1,
  22. host_key: None,
  23. entity: None,
  24. },
  25. ],
  26. extra_entities: vec![],
  27. }
  28. }
  29. fn tcfg() -> Config {
  30. Config {
  31. ssh_user: "u".to_string(),
  32. ssh_key_path: "/k".into(),
  33. clusters: vec![tcls()],
  34. nccl: HashMap::new(),
  35. hf_cache_dir: None,
  36. init_port: 25000,
  37. before_serve: HashMap::new(),
  38. exec_aliases: HashMap::new(),
  39. home_assistant: None,
  40. global_mods_dir: None,
  41. drop_caches: true,
  42. mount_aliases: HashMap::new(),
  43. recipe_sources: Vec::new(),
  44. docker_image_sources: Vec::new(),
  45. cluster_sources: Vec::new(),
  46. }
  47. }
  48. fn tnccl() -> NcclConfig {
  49. NcclConfig {
  50. ib_hca: "rocep1s0f0".to_string(),
  51. socket_ifname: "enp1s0f0np0".to_string(),
  52. ib_gid_index: Some(3),
  53. ib_disable: 0,
  54. gloo_socket_ifname: None,
  55. ucx_net_devices: None,
  56. nccl_cross_nic: None,
  57. nccl_proto: None,
  58. nccl_p2p_level: None,
  59. }
  60. }
  61. #[test]
  62. fn test_recipe_deserialize_minimal() {
  63. let r: Recipe = toml::from_str("\nname = \"M\"\nmodel = \"t/m\"\n").unwrap();
  64. assert_eq!(r.name, "M");
  65. assert_eq!(r.engine, "vllm");
  66. assert_eq!(r.port, 30000);
  67. }
  68. #[test]
  69. fn test_recipe_deserialize_full() {
  70. let r: Recipe = toml::from_str("engine = \"sglang\"\nname = \"Full\"\ndocker_image = \"o/i:t\"\nmodel = \"o/m\"\nport = 30000\nrequired_nodes = 4\npipeline_parallel_size = 2\ndistributed_backend = \"ray\"\ncluster = \"c\"\nargs = [\"--a\", \"v\"]\n[env]\nK = \"V\"\n").unwrap();
  71. assert_eq!(r.engine, "sglang");
  72. assert_eq!(r.required_nodes, Some(4));
  73. assert_eq!(r.distributed_backend, Some(DistributedBackend::Ray));
  74. }
  75. #[test]
  76. fn test_recipe_deserialize_no_required_nodes() {
  77. let r: Recipe = toml::from_str("name = \"F\"\nmodel = \"t/m\"\n").unwrap();
  78. assert_eq!(r.required_nodes, None);
  79. assert_eq!(r.pipeline_parallel_size, None);
  80. }
  81. #[test]
  82. fn test_composed_source_exclude_deserialize() {
  83. let toml = r#"
  84. name = "C"
  85. model = "t/m"
  86. [composed_model]
  87. mount = "/root/model"
  88. [[composed_model.sources]]
  89. repo = "base"
  90. [[composed_model.sources]]
  91. repo = "overlay"
  92. glob = "L*.safetensors"
  93. exclude = ["L5.safetensors", "L8.safetensors"]
  94. [[composed_model.files]]
  95. name = "cfg.json"
  96. content = "{}"
  97. "#;
  98. let r: Recipe = toml::from_str(toml).unwrap();
  99. let composed = r.composed_model.unwrap();
  100. assert_eq!(composed.sources.len(), 2);
  101. let overlay = &composed.sources[1];
  102. assert_eq!(overlay.repo, "overlay");
  103. assert_eq!(overlay.glob.as_deref(), Some("L*.safetensors"));
  104. let exclude = overlay
  105. .exclude
  106. .as_ref()
  107. .expect("exclude should deserialize");
  108. assert_eq!(exclude.len(), 2);
  109. assert!(exclude.contains(&"L5.safetensors".to_string()));
  110. assert!(exclude.contains(&"L8.safetensors".to_string()));
  111. }
  112. #[test]
  113. fn test_config_deserialize() {
  114. let c: Config = toml::from_str("ssh_user = \"a\"\nssh_key_path = \"/p\"\n[[clusters]]\nname = \"p\"\nmaster_ib_ip = \"1\"\nnodes = [{name=\"n1\",ssh_ip=\"10.0.0.10\",ib_ip=\"1\",rank=0},{name=\"n2\",ssh_ip=\"10.0.0.11\",ib_ip=\"2\",rank=1}]\n[nccl.\"p\"]\nib_hca = \"m\"\nsocket_ifname = \"e\"\nib_gid_index = 3\nib_disable = 0\n").unwrap();
  115. assert_eq!(c.clusters.len(), 1);
  116. assert_eq!(c.clusters[0].nodes.len(), 2);
  117. }
  118. #[test]
  119. fn test_config_deserialize_with_host_key() {
  120. let c: Config = toml::from_str("ssh_user = \"a\"\nssh_key_path = \"/k\"\n[[clusters]]\nname = \"s\"\nmaster_ib_ip = \"1\"\nnodes = [\n { name = \"n\", ssh_ip = \"2\", ib_ip = \"1\", rank = 0, host_key = \"SHA256:abc\" },\n]\n").unwrap();
  121. assert_eq!(c.clusters[0].nodes[0].host_key, Some("SHA256:abc".into()));
  122. }
  123. #[test]
  124. fn test_select_cluster_by_name() {
  125. let mut cfg = tcfg();
  126. cfg.nccl.insert("test-cluster".into(), tnccl());
  127. let (c, n) = select_cluster(&cfg, "test-cluster", None, &HashSet::new()).unwrap();
  128. assert_eq!(c.name, "test-cluster");
  129. assert_eq!(n.unwrap().ib_hca, "rocep1s0f0");
  130. }
  131. #[test]
  132. fn test_select_cluster_auto() {
  133. let mut cfg = tcfg();
  134. cfg.nccl.insert("test-cluster".into(), tnccl());
  135. let (c, _n) = select_cluster(&cfg, "auto", None, &HashSet::new()).unwrap();
  136. assert_eq!(c.name, "test-cluster");
  137. }
  138. #[test]
  139. fn test_select_cluster_auto_no_nccl_allows_single_node() {
  140. // With required_nodes = None, NCCL config is not required — the cluster
  141. // is selected even without NCCL config (the node count is unknown at
  142. // selection time, so we can't require it).
  143. let cfg = tcfg();
  144. let (c, n) = select_cluster(&cfg, "auto", None, &HashSet::new()).unwrap();
  145. assert_eq!(c.name, "test-cluster");
  146. assert!(n.is_none());
  147. }
  148. #[test]
  149. fn test_select_cluster_multi_node_requires_nccl() {
  150. // Multi-node (required_nodes > 1) without NCCL config → error.
  151. assert!(select_cluster(&tcfg(), "auto", Some(2), &HashSet::new()).is_err());
  152. }
  153. #[test]
  154. fn test_select_cluster_not_found() {
  155. let mut cfg = tcfg();
  156. cfg.clusters = vec![];
  157. assert!(select_cluster(&cfg, "x", None, &HashSet::new()).is_err());
  158. }
  159. #[test]
  160. fn test_select_cluster_by_name_found() {
  161. let mut nccl = HashMap::new();
  162. nccl.insert("test-cluster".into(), tnccl());
  163. let mut cfg = tcfg();
  164. cfg.nccl = nccl;
  165. let c = select_cluster_by_name(&cfg, "test-cluster").unwrap();
  166. assert_eq!(c.name, "test-cluster");
  167. }
  168. #[test]
  169. fn test_select_cluster_by_name_not_found() {
  170. assert!(select_cluster_by_name(&tcfg(), "x").is_err());
  171. }
  172. #[test]
  173. fn test_select_cluster_by_prefix() {
  174. let mut cfg = tcfg();
  175. cfg.nccl.insert("test-cluster".into(), tnccl());
  176. let c = select_cluster_by_name(&cfg, "test-c").unwrap();
  177. assert_eq!(c.name, "test-cluster");
  178. assert!(select_cluster_by_name(&cfg, "no-such").is_err());
  179. }
  180. #[test]
  181. fn test_default_engine_is_vllm() {
  182. let r: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"\n").unwrap();
  183. assert_eq!(r.engine, "vllm");
  184. }
  185. #[test]
  186. fn test_default_distributed_backend_is_none() {
  187. let r: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"\n").unwrap();
  188. assert_eq!(r.distributed_backend, None);
  189. }
  190. #[test]
  191. fn test_deploy_defaults_to_docker() {
  192. let r: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"\n").unwrap();
  193. assert_eq!(r.deploy, DeployTarget::Docker);
  194. }
  195. #[test]
  196. fn test_deploy_explicit_tmux() {
  197. let r: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"\ndeploy = \"tmux\"\n").unwrap();
  198. assert_eq!(r.deploy, DeployTarget::Tmux);
  199. }
  200. #[test]
  201. fn test_deploy_explicit_docker() {
  202. let r: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"\ndeploy = \"docker\"\n").unwrap();
  203. assert_eq!(r.deploy, DeployTarget::Docker);
  204. }
  205. #[test]
  206. fn test_venv_field() {
  207. let r: Recipe = toml::from_str(
  208. "name = \"x\"\nmodel = \"m\"\ndeploy = \"tmux\"\nvenv = \"~/venvs/vllm025\"\n",
  209. )
  210. .unwrap();
  211. assert_eq!(r.deploy, DeployTarget::Tmux);
  212. assert_eq!(r.venv.as_deref(), Some("~/venvs/vllm025"));
  213. }
  214. #[test]
  215. fn test_venv_defaults_to_none() {
  216. let r: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"\n").unwrap();
  217. assert_eq!(r.venv, None);
  218. }
  219. #[test]
  220. fn test_venv_setup_list_deserializes() {
  221. let r: Recipe = toml::from_str(
  222. r#"name = "x"
  223. model = "m"
  224. deploy = "tmux"
  225. venv = "~/venvs/vllm025"
  226. venv_setup = [
  227. "uv venv ~/venvs/vllm025 -p 3.12",
  228. "uv pip install vllm==0.25.1 --torch-backend=cu130",
  229. ]
  230. "#,
  231. )
  232. .unwrap();
  233. assert_eq!(r.venv.as_deref(), Some("~/venvs/vllm025"));
  234. let setup = r.venv_setup.to_vec();
  235. assert_eq!(setup.len(), 2);
  236. assert_eq!(setup[0], "uv venv ~/venvs/vllm025 -p 3.12");
  237. assert_eq!(
  238. setup[1],
  239. "uv pip install vllm==0.25.1 --torch-backend=cu130"
  240. );
  241. }
  242. #[test]
  243. fn test_venv_setup_inline_deserializes() {
  244. let r: Recipe = toml::from_str(
  245. r#"name = "x"
  246. model = "m"
  247. deploy = "tmux"
  248. venv = "~/venvs/vllm025"
  249. venv_setup = """
  250. uv venv ~/venvs/vllm025 -p 3.12
  251. uv pip install vllm==0.25.1 --torch-backend=cu130
  252. """
  253. "#,
  254. )
  255. .unwrap();
  256. let setup = r.venv_setup.to_vec();
  257. assert_eq!(setup.len(), 2);
  258. assert_eq!(setup[0], "uv venv ~/venvs/vllm025 -p 3.12");
  259. assert_eq!(
  260. setup[1],
  261. "uv pip install vllm==0.25.1 --torch-backend=cu130"
  262. );
  263. }
  264. #[test]
  265. fn test_venv_setup_inline_skips_comments_and_blanks() {
  266. let r: Recipe = toml::from_str(
  267. r#"name = "x"
  268. model = "m"
  269. deploy = "tmux"
  270. venv = "~/venvs/vllm025"
  271. venv_setup = """
  272. # create the venv
  273. uv venv ~/venvs/vllm025 -p 3.12
  274. # install deps
  275. uv pip install vllm==0.25.1
  276. """
  277. "#,
  278. )
  279. .unwrap();
  280. let setup = r.venv_setup.to_vec();
  281. assert_eq!(setup.len(), 2);
  282. assert_eq!(setup[0], "uv venv ~/venvs/vllm025 -p 3.12");
  283. assert_eq!(setup[1], "uv pip install vllm==0.25.1");
  284. }
  285. #[test]
  286. fn test_venv_setup_defaults_to_empty() {
  287. let r: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"\n").unwrap();
  288. assert!(r.venv_setup.to_vec().is_empty());
  289. }
  290. #[test]
  291. fn default_config_dir_appends_sm121() {
  292. let Some(d) = default_config_dir() else {
  293. return;
  294. };
  295. assert!(d.ends_with("sm121") && d.parent().is_some());
  296. }
  297. #[tokio::test]
  298. async fn test_merge_cluster_sources_adds_new_keeps_main() {
  299. let dir = std::env::temp_dir().join(format!("sm121-csrc-{}", std::process::id()));
  300. std::fs::create_dir_all(&dir).unwrap();
  301. let src_path = dir.join("clusters-extra.toml");
  302. // "extra" is new; "test-cluster" collides with tcfg's cluster (main wins).
  303. std::fs::write(
  304. &src_path,
  305. r#"
  306. ssh_user = "srcuser"
  307. ssh_key_path = "/src/key"
  308. [[clusters]]
  309. name = "extra"
  310. nodes = [{ name = "e0", ssh_ip = "10.9.0.1", rank = 0 }]
  311. [[clusters]]
  312. name = "test-cluster"
  313. nodes = [{ name = "intruder", ssh_ip = "10.66.0.1", rank = 0 }]
  314. [nccl."extra"]
  315. ib_hca = "mlx5_0"
  316. socket_ifname = "eth0"
  317. ib_disable = 0
  318. [before_serve."extra"]
  319. commands = ["echo hi"]
  320. [home_assistant]
  321. url = "http://ha.lan:8123"
  322. token = "sufficiently-long-test-token"
  323. shutdown_delay_seconds = 30
  324. "#,
  325. )
  326. .unwrap();
  327. let mut cfg = tcfg();
  328. cfg.cluster_sources = vec![src_path.to_string_lossy().into_owned()];
  329. cfg.ssh_user = String::new();
  330. cfg.ssh_key_path = std::path::PathBuf::new();
  331. merge_cluster_sources(&mut cfg).await.unwrap();
  332. assert_eq!(cfg.ssh_user, "srcuser", "empty ssh_user filled from source");
  333. assert_eq!(cfg.ssh_key_path, std::path::PathBuf::from("/src/key"));
  334. assert_eq!(
  335. cfg.clusters.len(),
  336. 2,
  337. "extra added, conflicting name not duplicated"
  338. );
  339. let main = cfg
  340. .clusters
  341. .iter()
  342. .find(|c| c.name == "test-cluster")
  343. .unwrap();
  344. assert_eq!(
  345. main.nodes[0].name, "node0",
  346. "main-config cluster wins on conflict"
  347. );
  348. assert!(cfg.clusters.iter().any(|c| c.name == "extra"));
  349. assert!(cfg.nccl.contains_key("extra"));
  350. assert!(cfg.before_serve.contains_key("extra"));
  351. // HA config merges from the source when main has none (power on from
  352. // other cwds depends on this).
  353. assert!(
  354. cfg.home_assistant.is_some(),
  355. "home_assistant filled from source"
  356. );
  357. assert_eq!(
  358. cfg.home_assistant.as_ref().unwrap().url,
  359. "http://ha.lan:8123"
  360. );
  361. // A main-config ssh_user is never overridden by a source; same rule
  362. // for home_assistant.
  363. let mut cfg2 = tcfg();
  364. cfg2.cluster_sources = vec![src_path.to_string_lossy().into_owned()];
  365. cfg2.home_assistant = Some(crate::config::HomeAssistantConfig {
  366. url: "http://main-ha:8123".into(),
  367. token: "main".into(),
  368. shutdown_delay_seconds: 5,
  369. });
  370. merge_cluster_sources(&mut cfg2).await.unwrap();
  371. assert_eq!(cfg2.ssh_user, "u", "main ssh_user wins over source");
  372. assert_eq!(
  373. cfg2.home_assistant.as_ref().unwrap().url,
  374. "http://main-ha:8123",
  375. "main home_assistant wins over source"
  376. );
  377. }
  378. #[tokio::test]
  379. async fn test_load_recipe_from_recipe_sources() {
  380. let dir = std::env::temp_dir().join(format!("sm121-rsrc-{}", std::process::id()));
  381. let rdir = dir.join("remote-recipes").join("remote-recipe");
  382. std::fs::create_dir_all(&rdir).unwrap();
  383. std::fs::write(rdir.join("recipe.toml"), "name = \"remote\"\n").unwrap();
  384. let recipe = load_recipe(
  385. "remote-recipe",
  386. &[dir.join("remote-recipes").to_string_lossy().into_owned()],
  387. )
  388. .await
  389. .unwrap();
  390. assert_eq!(recipe.name, "remote");
  391. }
  392. #[test]
  393. fn effective_master_ip_falls_back_to_rank0_ib_ip() {
  394. let mut c = tcls();
  395. c.master_ib_ip = None;
  396. assert_eq!(c.effective_master_ip(), "192.168.1.1"); // rank-0 ib_ip
  397. }
  398. #[test]
  399. fn effective_master_ip_falls_back_to_ssh_ip_when_ib_missing() {
  400. let mut c = tcls();
  401. c.master_ib_ip = None;
  402. for n in &mut c.nodes {
  403. n.ib_ip = None;
  404. }
  405. assert_eq!(c.effective_master_ip(), "10.0.0.1"); // rank-0 ssh_ip
  406. }
  407. #[test]
  408. fn effective_master_ip_uses_first_node_without_rank0() {
  409. let mut c = tcls();
  410. c.master_ib_ip = None;
  411. for n in &mut c.nodes {
  412. n.rank = 3; // no rank 0 → first node wins
  413. }
  414. assert_eq!(c.effective_master_ip(), "192.168.1.1");
  415. }
  416. #[test]
  417. fn effective_master_ip_prefers_explicit_config() {
  418. assert_eq!(tcls().effective_master_ip(), "192.168.1.1");
  419. }