command_tests.rs 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392
  1. use super::*;
  2. use crate::config::{
  3. Cluster, DeployTarget, DistributedBackend, EnvValue, ExtraMount, NcclConfig, Node, Recipe,
  4. };
  5. use crate::pre_serve::{RecipeArgs, VenvSetup};
  6. use std::collections::HashMap;
  7. fn tcls() -> Cluster {
  8. Cluster {
  9. name: "t".into(),
  10. master_ib_ip: Some("1".into()),
  11. nodes: vec![Node {
  12. name: "n0".into(),
  13. ssh_ip: "10.0.0.1".into(),
  14. ib_ip: Some("1".into()),
  15. rank: 0,
  16. host_key: None,
  17. entity: None,
  18. }],
  19. extra_entities: vec![],
  20. }
  21. }
  22. fn tnccl() -> NcclConfig {
  23. NcclConfig {
  24. ib_hca: "r".into(),
  25. socket_ifname: "e".into(),
  26. ib_gid_index: Some(3),
  27. ib_disable: 0,
  28. gloo_socket_ifname: None,
  29. ucx_net_devices: None,
  30. nccl_cross_nic: None,
  31. nccl_proto: None,
  32. nccl_p2p_level: None,
  33. }
  34. }
  35. fn minimal_config() -> Config {
  36. Config {
  37. cache: None,
  38. flare: None,
  39. ssh_user: "u".into(),
  40. ssh_key_path: "/tmp/key".into(),
  41. clusters: vec![],
  42. nccl: HashMap::new(),
  43. hf_cache_dir: None,
  44. init_port: 25000,
  45. before_serve: HashMap::new(),
  46. exec_aliases: HashMap::new(),
  47. home_assistant: None,
  48. mikrotik: None,
  49. global_mods_dir: None,
  50. drop_caches: true,
  51. mount_aliases: HashMap::from([("models".to_string(), "/mnt/nas/models".to_string())]),
  52. recipe_sources: Vec::new(),
  53. recipe_repos: Vec::new(),
  54. docker_image_sources: Vec::new(),
  55. cluster_sources: Vec::new(),
  56. registry: None,
  57. }
  58. }
  59. fn run_with(recipe: &Recipe) -> String {
  60. let resolved = resolve_extra_mounts(&minimal_config(), recipe).unwrap();
  61. let ctx = DockerCommandCtx {
  62. cluster: tcls(),
  63. nccl: Some(tnccl()),
  64. recipe: recipe.clone(),
  65. recipe_name: "r".to_string(),
  66. pp_size: 2,
  67. tp_size: 1,
  68. nnodes: 2,
  69. cache_dir: "/home/user/.cache/huggingface".to_string(),
  70. keep: false,
  71. init_port: 25000,
  72. mods_remote_path: String::new(),
  73. extra_mounts: resolved.static_mounts,
  74. };
  75. build_docker_command(&ctx, &tcls().nodes[0], true)
  76. .unwrap()
  77. .join(" ")
  78. }
  79. fn base_recipe() -> Recipe {
  80. Recipe {
  81. flare: None,
  82. build_profile: None,
  83. parent_model: None,
  84. archived: false,
  85. engine: "vllm".into(),
  86. name: "T".into(),
  87. docker_image: "img".into(),
  88. model: "mdl".into(),
  89. port: 8000,
  90. required_nodes: Some(2),
  91. pipeline_parallel_size: Some(2),
  92. tensor_parallel_size: 1,
  93. args: RecipeArgs::List(vec!["--x".into(), "4096".into()]),
  94. cold_args: RecipeArgs::default(),
  95. env: HashMap::new(),
  96. cluster: None,
  97. entrypoint: vec![],
  98. distributed_backend: Some(DistributedBackend::Mp),
  99. before_serve: vec![],
  100. after_stop: vec![],
  101. solo: None,
  102. deploy: DeployTarget::Docker,
  103. venv: None,
  104. venv_setup: VenvSetup::default(),
  105. venv_setup_script: None,
  106. mods: Default::default(),
  107. shm_size: String::new(),
  108. devices: vec![],
  109. memory_limit: None,
  110. cpus: None,
  111. sync_image: true,
  112. docker_image_dir: None,
  113. extra_mounts: vec![],
  114. extra_mounts_manifest: None,
  115. build: Default::default(),
  116. composed_model: None,
  117. staging: vec![],
  118. dgpp: None,
  119. }
  120. }
  121. #[test]
  122. fn env_ref_resolves_to_recipe_port() {
  123. let mut r = base_recipe();
  124. r.port = 8123;
  125. r.env = HashMap::from([
  126. (
  127. "PORT".into(),
  128. crate::config::EnvValue::Ref {
  129. from: "port".into(),
  130. },
  131. ),
  132. ("PLAIN".into(), crate::config::EnvValue::Plain("x".into())),
  133. ]);
  134. let env = r.resolved_env().unwrap();
  135. assert_eq!(env["PORT"], "8123");
  136. assert_eq!(env["PLAIN"], "x");
  137. }
  138. #[test]
  139. fn env_ref_unknown_target_errors() {
  140. let mut r = base_recipe();
  141. r.env = HashMap::from([(
  142. "K".into(),
  143. crate::config::EnvValue::Ref {
  144. from: "bogus".into(),
  145. },
  146. )]);
  147. let err = r.resolved_env().unwrap_err().to_string();
  148. assert!(err.contains("bogus"), "err: {err}");
  149. assert!(err.contains("K"), "err: {err}");
  150. }
  151. #[test]
  152. fn env_toml_parses_mixed_plain_and_ref() {
  153. let r: Recipe = toml::from_str(
  154. r#"
  155. engine = "vllm"
  156. name = "T"
  157. docker_image = "img"
  158. model = "mdl"
  159. port = 31337
  160. [env]
  161. A = "1"
  162. PORT = { from = "port" }
  163. "#,
  164. )
  165. .unwrap();
  166. assert_eq!(r.env.len(), 2);
  167. let env = r.resolved_env().unwrap();
  168. assert_eq!(env["PORT"], "31337");
  169. assert_eq!(env["A"], "1");
  170. }
  171. #[test]
  172. fn env_ref_renders_in_docker_env() {
  173. let mut r = base_recipe();
  174. r.port = 39999;
  175. r.env = HashMap::from([(
  176. "PORT".into(),
  177. crate::config::EnvValue::Ref {
  178. from: "port".into(),
  179. },
  180. )]);
  181. let cmd = run_with(&r);
  182. assert!(cmd.contains("-e 'PORT=39999'"), "cmd: {cmd}");
  183. }
  184. #[test]
  185. fn extra_mounts_are_mounted() {
  186. let mut r = base_recipe();
  187. r.extra_mounts.push(ExtraMount {
  188. alias: Some("models".into()),
  189. host: Some("~/.cache/models".into()),
  190. repo: None,
  191. subfolder: None,
  192. container: "/models".into(),
  193. read_only: true,
  194. env: Some("MODEL_DIR".into()),
  195. });
  196. let cmd = run_with(&r);
  197. // alias present in minimal_config() → alias host wins. Host/container
  198. // and env are now shell-quoted so a crafted value can't inject.
  199. assert!(cmd.contains("-v '/mnt/nas/models':'/models':ro"));
  200. assert!(cmd.contains("-v '/home/user/.cache/huggingface':/root/.cache/huggingface"));
  201. assert!(!cmd.contains("-v /mnt/nas/models:/models:ro")); // unquoted form gone
  202. // env injected pointing at the container path, before recipe env
  203. assert!(cmd.contains("-e 'MODEL_DIR=/models'"));
  204. }
  205. #[test]
  206. fn extra_mounts_fallback_to_host_when_alias_absent() {
  207. let mut r = base_recipe();
  208. r.extra_mounts.push(ExtraMount {
  209. alias: Some("no_such_alias".into()),
  210. host: Some("~/.cache/jit".into()),
  211. repo: None,
  212. subfolder: None,
  213. container: "/root/.cache/jit".into(),
  214. read_only: false,
  215. env: None,
  216. });
  217. let cmd = run_with(&r);
  218. // alias not in config → host default used with ~ expansion (ssh_user=u)
  219. assert!(cmd.contains("-v '/home/u/.cache/jit':'/root/.cache/jit'"));
  220. assert!(!cmd.contains(":ro"));
  221. }
  222. #[test]
  223. fn malicious_env_value_cannot_inject() {
  224. // A crafted `[env]` value with shell metachars must stay a single
  225. // quoted argument — never break out of the remote `docker run`.
  226. let mut r = base_recipe();
  227. r.env
  228. .insert("EVIL".to_string(), "x; touch /tmp/pwned".into());
  229. let cmd = run_with(&r);
  230. assert!(cmd.contains("-e 'EVIL=x; touch /tmp/pwned'"));
  231. // No raw `; touch` escape: the whole K=V is inside quotes.
  232. assert!(!cmd.contains("-e EVIL=x;"));
  233. }
  234. #[test]
  235. fn malicious_mount_container_cannot_inject() {
  236. let mut r = base_recipe();
  237. r.extra_mounts.push(ExtraMount {
  238. alias: None,
  239. host: Some("/safe".into()),
  240. repo: None,
  241. subfolder: None,
  242. container: "/x; touch /tmp/pwned".into(),
  243. read_only: false,
  244. env: None,
  245. });
  246. let cmd = run_with(&r);
  247. // Container path is quoted; the metacharacter is inert inside quotes.
  248. assert!(cmd.contains("'/x; touch /tmp/pwned'"));
  249. // Never appear as the unquoted concatenation `-v /safe:/x; touch`.
  250. assert!(!cmd.contains("-v /safe:/x; touch"));
  251. }
  252. #[test]
  253. fn model_path_ref_is_partitioned_not_mounted() {
  254. // repo-based mounts are path references, NOT bind-mounts. They are
  255. // partitioned into path_refs and resolved per-node via substitution.
  256. let mut r = base_recipe();
  257. r.extra_mounts.push(ExtraMount {
  258. alias: None,
  259. host: None,
  260. repo: Some("aidendle94/GLM-5.2-Hybrid-FP8-MXPF4".into()),
  261. subfolder: Some("mtp-draft".into()),
  262. container: "mtp_draft".into(),
  263. read_only: true,
  264. env: None,
  265. });
  266. let resolved = resolve_extra_mounts(&minimal_config(), &r).unwrap();
  267. assert_eq!(resolved.path_refs.len(), 1);
  268. assert_eq!(resolved.static_mounts.len(), 0);
  269. let pr = &resolved.path_refs[0];
  270. assert_eq!(pr.repo, "aidendle94/GLM-5.2-Hybrid-FP8-MXPF4");
  271. assert_eq!(pr.subfolder.as_deref(), Some("mtp-draft"));
  272. assert_eq!(pr.name, "mtp_draft");
  273. // No -v flag for path refs in build_docker_command (substitution only)
  274. let cmd = run_with(&r);
  275. assert!(!cmd.contains("-v /models/mtp-draft"));
  276. }
  277. #[test]
  278. fn substitute_path_refs_replaces_placeholders() {
  279. let mut r = base_recipe();
  280. r.args = RecipeArgs::List(vec![
  281. "--speculative-config".into(),
  282. "{\"model\":\"{{mtp_draft}}\"}".into(),
  283. ]);
  284. let path_refs = vec![(
  285. "mtp_draft".to_string(),
  286. "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft".to_string(),
  287. )];
  288. let substituted = substitute_path_refs(&r, &path_refs);
  289. if let RecipeArgs::List(args) = &substituted.args {
  290. assert_eq!(
  291. args[1],
  292. "{\"model\":\"/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft\"}"
  293. );
  294. } else {
  295. panic!("expected list args");
  296. }
  297. }
  298. #[test]
  299. fn substitute_path_refs_in_env() {
  300. let mut r = base_recipe();
  301. r.env.insert("MTP_PATH".to_string(), "{{mtp_draft}}".into());
  302. let path_refs = vec![(
  303. "mtp_draft".to_string(),
  304. "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft".to_string(),
  305. )];
  306. let substituted = substitute_path_refs(&r, &path_refs);
  307. assert_eq!(
  308. substituted.env["MTP_PATH"],
  309. EnvValue::Plain(
  310. "/root/.cache/huggingface/hub/models--aidendle94--GLM-5.2-Hybrid-FP8-MXPF4/snapshots/bb0d0858/mtp-draft".to_string()
  311. )
  312. );
  313. }
  314. #[test]
  315. fn clear_paths_dedups_and_sorts() {
  316. let m = |host: &str| ResolvedMount {
  317. host: host.into(),
  318. container: "/c".into(),
  319. read_only: false,
  320. env: None,
  321. };
  322. let mounts = vec![m("/b/.cache/jit"), m("/a/.cache/jit"), m("/b/.cache/jit")];
  323. assert_eq!(clear_paths(&mounts), vec!["/a/.cache/jit", "/b/.cache/jit"]);
  324. assert!(clear_paths(&[]).is_empty());
  325. }
  326. #[test]
  327. fn devices_pass_through_as_docker_flags() {
  328. let mut r = base_recipe();
  329. r.devices = vec!["/dev/dri/card0".into()];
  330. let cmd = run_with(&r);
  331. assert!(
  332. cmd.contains("--device /dev/dri/card0:/dev/dri/card0"),
  333. "missing --device passthrough in: {cmd}"
  334. );
  335. }
  336. #[test]
  337. fn no_devices_no_flag() {
  338. let cmd = run_with(&base_recipe());
  339. assert!(!cmd.contains("--device"));
  340. }
  341. #[test]
  342. fn memory_and_cpus_cgroup_flags() {
  343. let mut r = base_recipe();
  344. r.memory_limit = Some("9g".into());
  345. r.cpus = Some(6.0);
  346. let cmd = run_with(&r);
  347. assert!(cmd.contains("--memory 9g"), "in: {cmd}");
  348. assert!(cmd.contains("--memory-swap 9g"), "in: {cmd}");
  349. assert!(cmd.contains("--cpus 6"), "in: {cmd}");
  350. }
  351. #[test]
  352. fn no_memory_limit_no_flags() {
  353. let cmd = run_with(&base_recipe());
  354. assert!(!cmd.contains("--memory"));
  355. assert!(!cmd.contains("--cpus"));
  356. }
  357. #[test]
  358. fn entrypoint_flag_overrides_image_entrypoint() {
  359. let mut r = base_recipe();
  360. r.entrypoint = vec![
  361. "/opt/ds41-venv/bin/python".into(),
  362. "/opt/ds41-serving/serve.py".into(),
  363. ];
  364. let cmd = run_with(&r);
  365. assert!(
  366. cmd.contains("--entrypoint '/opt/ds41-venv/bin/python'"),
  367. "in: {cmd}"
  368. );
  369. }