|
|
@@ -1,4 +1,6 @@
|
|
|
-pub use crate::pre_serve::{BeforeServeTable, ModsTable, OnFailure, PreServeCommand};
|
|
|
+pub use crate::pre_serve::{
|
|
|
+ BeforeServeTable, DistributedBackend, ModsTable, OnFailure, PreServeCommand,
|
|
|
+};
|
|
|
use anyhow::{Context, Result};
|
|
|
use serde::Deserialize;
|
|
|
use std::collections::HashMap;
|
|
|
@@ -112,8 +114,8 @@ pub struct Recipe {
|
|
|
pub cluster: Option<String>,
|
|
|
#[serde(default)]
|
|
|
pub entrypoint: Vec<String>,
|
|
|
- #[serde(default = "default_distributed_backend")]
|
|
|
- pub distributed_backend: String,
|
|
|
+ #[serde(default)]
|
|
|
+ pub distributed_backend: DistributedBackend,
|
|
|
/// Shared memory size for the container (e.g. "64gb"). Omitted when empty.
|
|
|
#[serde(default = "default_shm_size")]
|
|
|
pub shm_size: String,
|
|
|
@@ -152,9 +154,6 @@ fn resolve_cluster_name<'a>(clusters: &'a [Cluster], name: &str) -> Result<&'a C
|
|
|
fn default_engine() -> String {
|
|
|
"vllm".to_string()
|
|
|
}
|
|
|
-fn default_distributed_backend() -> String {
|
|
|
- "mp".to_string()
|
|
|
-}
|
|
|
fn default_shm_size() -> String {
|
|
|
String::new()
|
|
|
}
|
|
|
@@ -309,7 +308,7 @@ mod tests {
|
|
|
assert_eq!(recipe.port, 30000);
|
|
|
assert!(recipe.args.is_empty());
|
|
|
assert!(recipe.env.is_empty());
|
|
|
- assert_eq!(recipe.distributed_backend, "mp");
|
|
|
+ assert_eq!(recipe.distributed_backend, DistributedBackend::Mp);
|
|
|
assert_eq!(recipe.tensor_parallel_size, 1);
|
|
|
}
|
|
|
#[test]
|
|
|
@@ -337,7 +336,7 @@ OTHER = "123"
|
|
|
assert_eq!(recipe.port, 30000);
|
|
|
assert_eq!(recipe.required_nodes, Some(4));
|
|
|
assert_eq!(recipe.pipeline_parallel_size, Some(2));
|
|
|
- assert_eq!(recipe.distributed_backend, "ray");
|
|
|
+ assert_eq!(recipe.distributed_backend, DistributedBackend::Ray);
|
|
|
assert_eq!(recipe.cluster, Some("my-cluster".to_string()));
|
|
|
assert_eq!(recipe.args, vec!["--arg1", "val1", "--arg2"]);
|
|
|
assert_eq!(recipe.env.get("KEY").unwrap(), "value");
|
|
|
@@ -483,7 +482,7 @@ ib_disable = 0
|
|
|
#[test]
|
|
|
fn test_default_distributed_backend_is_mp() {
|
|
|
let recipe: Recipe = toml::from_str("name = \"x\"\nmodel = \"m\"").unwrap();
|
|
|
- assert_eq!(recipe.distributed_backend, "mp");
|
|
|
+ assert_eq!(recipe.distributed_backend, DistributedBackend::Mp);
|
|
|
}
|
|
|
#[test]
|
|
|
fn default_config_dir_appends_sm121_launcher() {
|