|
@@ -24,6 +24,9 @@ pub struct EngineFlagsCtx<'a> {
|
|
|
pub nnodes: u32,
|
|
pub nnodes: u32,
|
|
|
pub init_port: u16,
|
|
pub init_port: u16,
|
|
|
pub master_ib_ip: &'a str,
|
|
pub master_ib_ip: &'a str,
|
|
|
|
|
+ /// Ordered launch-node IPs (rank order). gr8's `--peers` list; vLLM/SGLang
|
|
|
|
|
+ /// ignore it.
|
|
|
|
|
+ pub peers: &'a [String],
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/// Per-engine launch semantics. See the module docs.
|
|
/// Per-engine launch semantics. See the module docs.
|
|
@@ -60,6 +63,13 @@ pub trait EngineDriver: Send + Sync {
|
|
|
false
|
|
false
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ /// Whether the deployment path appends `--host 0.0.0.0 --port <port>` to
|
|
|
|
|
+ /// the engine command. vLLM/SGLang take them; engines with a fixed/built-in
|
|
|
|
|
+ /// API port (gr8 :8100, DGPP :18080) do not.
|
|
|
|
|
+ fn takes_host_port(&self) -> bool {
|
|
|
|
|
+ true
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
/// tmux entrypoint fallback when the recipe sets none.
|
|
/// tmux entrypoint fallback when the recipe sets none.
|
|
|
fn entrypoint_default(&self) -> Vec<String>;
|
|
fn entrypoint_default(&self) -> Vec<String>;
|
|
|
|
|
|
|
@@ -156,12 +166,47 @@ impl EngineDriver for SglangDriver {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+/// gr8 (`kindlingai/gr8`): a from-scratch CUDA engine for full GLM-5.3,
|
|
|
|
|
+/// tensor-parallel across 3/4/6 Sparks. Per-node launch (every box runs the
|
|
|
|
|
+/// same `gr8 glm53 up` command; each rank finds itself in `--peers`). Model
|
|
|
|
|
+/// via `--model`, rendezvous via `--tp`/`--peers` (mgmt-LAN IPs), no headless,
|
|
|
|
|
+/// no bind-IP env, fixed API port :8100 (no `--host`/`--port`).
|
|
|
|
|
+pub struct Gr8Driver;
|
|
|
|
|
+
|
|
|
|
|
+impl EngineDriver for Gr8Driver {
|
|
|
|
|
+ fn model_args(&self, recipe: &Recipe) -> Vec<String> {
|
|
|
|
|
+ if recipe.model.is_empty() {
|
|
|
|
|
+ Vec::new()
|
|
|
|
|
+ } else {
|
|
|
|
|
+ vec!["--model".into(), recipe.model.clone()]
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fn distributed_args(&self, ctx: &EngineFlagsCtx<'_>, _node: &Node) -> Vec<String> {
|
|
|
|
|
+ vec![
|
|
|
|
|
+ "--tp".into(),
|
|
|
|
|
+ ctx.tp_size.to_string(),
|
|
|
|
|
+ "--peers".into(),
|
|
|
|
|
+ ctx.peers.join(","),
|
|
|
|
|
+ ]
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fn takes_host_port(&self) -> bool {
|
|
|
|
|
+ false
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fn entrypoint_default(&self) -> Vec<String> {
|
|
|
|
|
+ vec!["/opt/gr8/gr8".into(), "glm53".into(), "up".into()]
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
/// Resolve the driver for a recipe's `engine` string. Unknown engines fall
|
|
/// Resolve the driver for a recipe's `engine` string. Unknown engines fall
|
|
|
/// back to the vLLM driver (matching the historical default: an engine with no
|
|
/// back to the vLLM driver (matching the historical default: an engine with no
|
|
|
/// dedicated driver still gets the vLLM-style tail).
|
|
/// dedicated driver still gets the vLLM-style tail).
|
|
|
pub fn driver_for(engine: &str) -> &'static dyn EngineDriver {
|
|
pub fn driver_for(engine: &str) -> &'static dyn EngineDriver {
|
|
|
match engine {
|
|
match engine {
|
|
|
"sglang" => &SglangDriver,
|
|
"sglang" => &SglangDriver,
|
|
|
|
|
+ "gr8" => &Gr8Driver,
|
|
|
"vllm" => &VllmDriver,
|
|
"vllm" => &VllmDriver,
|
|
|
_ => &VllmDriver,
|
|
_ => &VllmDriver,
|
|
|
}
|
|
}
|
|
@@ -270,6 +315,24 @@ mod tests {
|
|
|
nnodes,
|
|
nnodes,
|
|
|
init_port: 25000,
|
|
init_port: 25000,
|
|
|
master_ib_ip: "10.0.0.1",
|
|
master_ib_ip: "10.0.0.1",
|
|
|
|
|
+ peers: &[],
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fn ctx_peers<'a>(
|
|
|
|
|
+ r: &'a Recipe,
|
|
|
|
|
+ tp: u32,
|
|
|
|
|
+ nnodes: u32,
|
|
|
|
|
+ peers: &'a [String],
|
|
|
|
|
+ ) -> EngineFlagsCtx<'a> {
|
|
|
|
|
+ EngineFlagsCtx {
|
|
|
|
|
+ recipe: r,
|
|
|
|
|
+ tp_size: tp,
|
|
|
|
|
+ pp_size: 1,
|
|
|
|
|
+ nnodes,
|
|
|
|
|
+ init_port: 25000,
|
|
|
|
|
+ master_ib_ip: "10.0.0.1",
|
|
|
|
|
+ peers,
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -335,4 +398,22 @@ mod tests {
|
|
|
// Unknown engines fall back to the vLLM driver (historical default).
|
|
// Unknown engines fall back to the vLLM driver (historical default).
|
|
|
assert!(driver_for("something-new").headless_workers());
|
|
assert!(driver_for("something-new").headless_workers());
|
|
|
}
|
|
}
|
|
|
|
|
+
|
|
|
|
|
+ #[test]
|
|
|
|
|
+ fn gr8_uses_tp_peers_and_model_flag() {
|
|
|
|
|
+ let mut r = base_recipe();
|
|
|
|
|
+ r.engine = "gr8".into();
|
|
|
|
|
+ r.entrypoint = vec!["/opt/gr8/gr8".into(), "glm53".into(), "up".into()];
|
|
|
|
|
+ let peers = vec!["10.0.0.1".to_string(), "10.0.0.2".to_string()];
|
|
|
|
|
+ let c = ctx_peers(&r, 2, 2, &peers);
|
|
|
|
|
+ let args = build_engine_args(&c, &tnode(0));
|
|
|
|
|
+ // model via --model, rendezvous via --tp/--peers, no --headless.
|
|
|
|
|
+ assert!(args.contains(&"--model".into()));
|
|
|
|
|
+ assert!(args.contains(&"--tp".into()));
|
|
|
|
|
+ assert!(args.contains(&"--peers".into()));
|
|
|
|
|
+ assert!(args.contains(&"10.0.0.1,10.0.0.2".into()));
|
|
|
|
|
+ assert!(!args.contains(&"--headless".into()));
|
|
|
|
|
+ // gr8 does not take --host/--port (fixed API port).
|
|
|
|
|
+ assert!(!driver_for("gr8").takes_host_port());
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|