|
|
@@ -0,0 +1,357 @@
|
|
|
+//! FLARE deploy (`engine = "flare"`, `deploy = "native"`).
|
|
|
+//!
|
|
|
+//! FLARE (`local-inference-lab/flare`) serves a *signed package* instead of a
|
|
|
+//! raw HF checkpoint: `flare download <repo> --revision <rev>` fetches it, and
|
|
|
+//! `flare orbit run "<controller>:<path>" --cluster <orbit cluster> …` brings
|
|
|
+//! up one rank per paired agent and serves the API on the controller. The
|
|
|
+//! launcher therefore does what a human would: preflight the agents, download
|
|
|
+//! the package once on rank 0 (stamped, so a repeat serve costs nothing), run
|
|
|
+//! orbit detached, then probe `/health` + `/v1/models`.
|
|
|
+//!
|
|
|
+//! Recipe block:
|
|
|
+//! ```toml
|
|
|
+//! engine = "flare"
|
|
|
+//! deploy = "native"
|
|
|
+//! [flare]
|
|
|
+//! package = "local-inference-lab/DeepSeek-V4.1-Flash-1017-FLARE"
|
|
|
+//! revision = "gb10"
|
|
|
+//! name = "deepseek1017"
|
|
|
+//! http_port = 8000
|
|
|
+//! serve_flags = ["--max-concurrency", "16", "--mtp-draft-tokens", "5"]
|
|
|
+//! ```
|
|
|
+//! Credentials (`--dangerously-approve-key`, publisher) live in the private
|
|
|
+//! config's `[flare]` block, never in the recipe repo.
|
|
|
+
|
|
|
+use anyhow::{Result, bail};
|
|
|
+use tracing::info;
|
|
|
+
|
|
|
+use crate::config::{Config, Node, Recipe};
|
|
|
+use crate::docker::{ServeOpts, shq};
|
|
|
+
|
|
|
+/// FLARE API port on the controller (recipe `[flare].http_port`, else 8000).
|
|
|
+pub fn http_port(recipe: &Recipe) -> u16 {
|
|
|
+ recipe
|
|
|
+ .flare
|
|
|
+ .as_ref()
|
|
|
+ .and_then(|f| f.http_port)
|
|
|
+ .unwrap_or(8000)
|
|
|
+}
|
|
|
+
|
|
|
+fn orbit_cluster(config: &Config) -> String {
|
|
|
+ config
|
|
|
+ .flare
|
|
|
+ .as_ref()
|
|
|
+ .and_then(|f| f.orbit_cluster.clone())
|
|
|
+ .unwrap_or_else(|| "sparks".to_string())
|
|
|
+}
|
|
|
+
|
|
|
+/// The orbit-side name for one of our nodes (rank order → `orbit_nodes`,
|
|
|
+/// falling back to the cluster's own name).
|
|
|
+fn orbit_node_name(config: &Config, node: &Node) -> String {
|
|
|
+ config
|
|
|
+ .flare
|
|
|
+ .as_ref()
|
|
|
+ .and_then(|f| f.orbit_nodes.get(node.rank as usize).cloned())
|
|
|
+ .unwrap_or_else(|| node.name.clone())
|
|
|
+}
|
|
|
+
|
|
|
+fn binary(config: &Config) -> String {
|
|
|
+ config
|
|
|
+ .flare
|
|
|
+ .as_ref()
|
|
|
+ .and_then(|f| f.binary.clone())
|
|
|
+ .unwrap_or_else(|| "flare".to_string())
|
|
|
+}
|
|
|
+
|
|
|
+/// `flare download <package> --revision <rev>` — prints the package path.
|
|
|
+pub fn download_cmd(config: &Config, recipe: &Recipe) -> Result<String> {
|
|
|
+ let f = recipe
|
|
|
+ .flare
|
|
|
+ .as_ref()
|
|
|
+ .ok_or_else(|| anyhow::anyhow!("engine=flare requires a [flare] block"))?;
|
|
|
+ let mut cmd = format!("{} download {}", binary(config), shq(&f.package));
|
|
|
+ if let Some(rev) = &f.revision {
|
|
|
+ cmd.push_str(&format!(" --revision {}", shq(rev)));
|
|
|
+ }
|
|
|
+ if let Some(key) = config.flare.as_ref().and_then(|c| c.accept_key.clone()) {
|
|
|
+ cmd.push_str(&format!(" --dangerously-approve-key {}", shq(&key)));
|
|
|
+ }
|
|
|
+ Ok(cmd)
|
|
|
+}
|
|
|
+
|
|
|
+/// `flare orbit run "<node>:<package>" --cluster <c> --name <n> … --detach`.
|
|
|
+pub fn run_cmd(
|
|
|
+ config: &Config,
|
|
|
+ recipe: &Recipe,
|
|
|
+ controller: &str,
|
|
|
+ package: &str,
|
|
|
+) -> Result<String> {
|
|
|
+ let f = recipe
|
|
|
+ .flare
|
|
|
+ .as_ref()
|
|
|
+ .ok_or_else(|| anyhow::anyhow!("engine=flare requires a [flare] block"))?;
|
|
|
+ let model_name = f.name.clone().unwrap_or_else(|| recipe.model.clone());
|
|
|
+ let mut cmd = format!(
|
|
|
+ "{} orbit run {} --cluster {} --name {} --host 0.0.0.0 --port {} --detach",
|
|
|
+ binary(config),
|
|
|
+ shq(&format!("{controller}:{package}")),
|
|
|
+ shq(&orbit_cluster(config)),
|
|
|
+ shq(&model_name),
|
|
|
+ http_port(recipe),
|
|
|
+ );
|
|
|
+ if let Some(k) = config.flare.as_ref().and_then(|c| c.accept_key.clone()) {
|
|
|
+ cmd.push_str(&format!(" --dangerously-approve-key {}", shq(&k)));
|
|
|
+ }
|
|
|
+ if let Some(p) = config
|
|
|
+ .flare
|
|
|
+ .as_ref()
|
|
|
+ .and_then(|c| c.accept_publisher.clone())
|
|
|
+ {
|
|
|
+ cmd.push_str(&format!(" --dangerously-accept-publisher {}", shq(&p)));
|
|
|
+ }
|
|
|
+ for flag in &f.serve_flags {
|
|
|
+ cmd.push_str(&format!(" {}", shq(flag)));
|
|
|
+ }
|
|
|
+ Ok(cmd)
|
|
|
+}
|
|
|
+
|
|
|
+/// `flare orbit stop <name> --cluster <c>`.
|
|
|
+pub fn stop_cmd(config: &Config, recipe: &Recipe) -> Result<String> {
|
|
|
+ let f = recipe
|
|
|
+ .flare
|
|
|
+ .as_ref()
|
|
|
+ .ok_or_else(|| anyhow::anyhow!("engine=flare requires a [flare] block"))?;
|
|
|
+ let model_name = f.name.clone().unwrap_or_else(|| recipe.model.clone());
|
|
|
+ Ok(format!(
|
|
|
+ "{} orbit stop {} --cluster {}",
|
|
|
+ binary(config),
|
|
|
+ shq(&model_name),
|
|
|
+ shq(&orbit_cluster(config)),
|
|
|
+ ))
|
|
|
+}
|
|
|
+
|
|
|
+/// `flare orbit attach <name> --cluster <c>` (log tail; the CLI's `logs`).
|
|
|
+pub fn attach_cmd(config: &Config, recipe: &Recipe) -> Result<String> {
|
|
|
+ let f = recipe
|
|
|
+ .flare
|
|
|
+ .as_ref()
|
|
|
+ .ok_or_else(|| anyhow::anyhow!("engine=flare requires a [flare] block"))?;
|
|
|
+ let model_name = f.name.clone().unwrap_or_else(|| recipe.model.clone());
|
|
|
+ Ok(format!(
|
|
|
+ "{} orbit attach {} --cluster {}",
|
|
|
+ binary(config),
|
|
|
+ shq(&model_name),
|
|
|
+ shq(&orbit_cluster(config)),
|
|
|
+ ))
|
|
|
+}
|
|
|
+
|
|
|
+/// Preflight on every launch node: the agent installed and the orbit cluster
|
|
|
+/// visible. Runs before any download or launch — a peer that never paired
|
|
|
+/// otherwise fails minutes into a 500 GB fetch.
|
|
|
+pub fn preflight_cmd(config: &Config) -> String {
|
|
|
+ format!(
|
|
|
+ "{} --version >/dev/null && systemctl --user is-active flare-agent >/dev/null && {} orbit status --cluster {} >/dev/null",
|
|
|
+ binary(config),
|
|
|
+ binary(config),
|
|
|
+ shq(&orbit_cluster(config)),
|
|
|
+ )
|
|
|
+}
|
|
|
+
|
|
|
+fn head(cluster: &crate::config::Cluster) -> Result<&Node> {
|
|
|
+ cluster
|
|
|
+ .nodes
|
|
|
+ .iter()
|
|
|
+ .find(|n| n.rank == 0)
|
|
|
+ .ok_or_else(|| anyhow::anyhow!("no head node (rank 0) in cluster"))
|
|
|
+}
|
|
|
+
|
|
|
+/// Launch a FLARE recipe: preflight, package download on the controller, then
|
|
|
+/// `orbit run` detached. Readiness (`/health`, `/v1/models`) is polled by the
|
|
|
+/// caller, like every other engine.
|
|
|
+pub async fn serve(
|
|
|
+ config: &Config,
|
|
|
+ recipe: &Recipe,
|
|
|
+ cluster: &crate::config::Cluster,
|
|
|
+ dry_run: bool,
|
|
|
+) -> Result<()> {
|
|
|
+ let controller = head(cluster)?;
|
|
|
+ let pre = preflight_cmd(config);
|
|
|
+ if dry_run {
|
|
|
+ println!(
|
|
|
+ " [dry-run] preflight on {} nodes: {pre}",
|
|
|
+ cluster.nodes.len()
|
|
|
+ );
|
|
|
+ } else {
|
|
|
+ for node in &cluster.nodes {
|
|
|
+ if let Err(e) = crate::ssh::run_ssh(config, node, pre.clone(), false).await {
|
|
|
+ bail!(
|
|
|
+ "FLARE preflight failed on {}: {e} — install `flare`, then `flare agent install` and pair the node with the controller",
|
|
|
+ node.name
|
|
|
+ );
|
|
|
+ }
|
|
|
+ }
|
|
|
+ info!("FLARE agents ok on {} node(s)", cluster.nodes.len());
|
|
|
+ }
|
|
|
+ let dl = download_cmd(config, recipe)?;
|
|
|
+ let package = crate::ssh::run_ssh(config, controller, dl, dry_run).await?;
|
|
|
+ let package = package
|
|
|
+ .lines()
|
|
|
+ .last()
|
|
|
+ .unwrap_or_default()
|
|
|
+ .trim()
|
|
|
+ .to_string();
|
|
|
+ if !dry_run && package.is_empty() {
|
|
|
+ bail!(
|
|
|
+ "flare download printed no package path on {}",
|
|
|
+ controller.name
|
|
|
+ );
|
|
|
+ }
|
|
|
+ let orbit_name = orbit_node_name(config, controller);
|
|
|
+ let run = run_cmd(config, recipe, &orbit_name, &package)?;
|
|
|
+ crate::ssh::run_ssh(config, controller, run.clone(), dry_run).await?;
|
|
|
+ if dry_run {
|
|
|
+ println!(" [dry-run] {}: {run}", controller.name);
|
|
|
+ } else {
|
|
|
+ info!("FLARE group launched on {}", controller.name);
|
|
|
+ }
|
|
|
+ Ok(())
|
|
|
+}
|
|
|
+
|
|
|
+/// Post-launch: wait for the API (the package load takes many minutes) and
|
|
|
+/// point the operator at the log command.
|
|
|
+pub async fn after_serve(
|
|
|
+ config: &Config,
|
|
|
+ recipe: &Recipe,
|
|
|
+ cluster: &crate::config::Cluster,
|
|
|
+ recipe_name: &str,
|
|
|
+ opts: &ServeOpts,
|
|
|
+ follow: bool,
|
|
|
+) -> Result<()> {
|
|
|
+ let port = http_port(recipe);
|
|
|
+ if !opts.no_wait_ready {
|
|
|
+ crate::readiness::wait_engine(config, cluster, port, 30 * 60, recipe_name).await?;
|
|
|
+ }
|
|
|
+ if follow {
|
|
|
+ info!(
|
|
|
+ "FLARE serving on {}:{} — logs: sm121 logs --recipe {recipe_name}",
|
|
|
+ cluster
|
|
|
+ .nodes
|
|
|
+ .first()
|
|
|
+ .map(|n| n.ssh_ip.as_str())
|
|
|
+ .unwrap_or("?"),
|
|
|
+ port
|
|
|
+ );
|
|
|
+ }
|
|
|
+ Ok(())
|
|
|
+}
|
|
|
+
|
|
|
+/// Stop the FLARE group on the controller.
|
|
|
+pub async fn stop(
|
|
|
+ config: &Config,
|
|
|
+ recipe: &Recipe,
|
|
|
+ cluster: &crate::config::Cluster,
|
|
|
+ dry_run: bool,
|
|
|
+) -> Result<()> {
|
|
|
+ let controller = head(cluster)?;
|
|
|
+ let cmd = stop_cmd(config, recipe)?;
|
|
|
+ crate::ssh::run_ssh(config, controller, cmd, dry_run).await?;
|
|
|
+ Ok(())
|
|
|
+}
|
|
|
+
|
|
|
+#[cfg(test)]
|
|
|
+mod tests {
|
|
|
+ use super::*;
|
|
|
+ use crate::config::FlareRecipe;
|
|
|
+
|
|
|
+ fn cfg() -> Config {
|
|
|
+ let c: Config = toml::from_str(
|
|
|
+ "ssh_user = \"u\"\nssh_key_path = \"/k\"\n\n[flare]\naccept_key = \"key-123\"\naccept_publisher = \"local-development\"\norbit_cluster = \"sparks\"\n",
|
|
|
+ )
|
|
|
+ .expect("test config");
|
|
|
+ c
|
|
|
+ }
|
|
|
+
|
|
|
+ fn recipe() -> Recipe {
|
|
|
+ let mut r = crate::engine::tests::base_recipe();
|
|
|
+ r.engine = "flare".into();
|
|
|
+ r.model = "deepseek-ai/DeepSeek-V4.1-Flash-1017".into();
|
|
|
+ r.flare = Some(FlareRecipe {
|
|
|
+ package: "local-inference-lab/DeepSeek-V4.1-Flash-1017-FLARE".into(),
|
|
|
+ revision: Some("gb10".into()),
|
|
|
+ name: Some("deepseek1017".into()),
|
|
|
+ http_port: Some(8000),
|
|
|
+ serve_flags: vec![
|
|
|
+ "--max-concurrency".into(),
|
|
|
+ "16".into(),
|
|
|
+ "--mtp-adaptive-verification".into(),
|
|
|
+ ],
|
|
|
+ });
|
|
|
+ r
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn download_carries_revision_and_key() {
|
|
|
+ let cmd = download_cmd(&cfg(), &recipe()).unwrap();
|
|
|
+ assert!(cmd.starts_with("flare download "), "{cmd}");
|
|
|
+ // shq() shell-quotes every value, so compare the quoted forms
|
|
|
+ assert!(cmd.contains("--revision 'gb10'"), "{cmd}");
|
|
|
+ assert!(cmd.contains("--dangerously-approve-key 'key-123'"), "{cmd}");
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn run_passes_flags_verbatim_and_the_controller_package() {
|
|
|
+ let cmd = run_cmd(&cfg(), &recipe(), "dgx5", "/pkgs/dsv41.safetensors").unwrap();
|
|
|
+ assert!(
|
|
|
+ cmd.contains("orbit run 'dgx5:/pkgs/dsv41.safetensors'"),
|
|
|
+ "{cmd}"
|
|
|
+ );
|
|
|
+ assert!(cmd.contains("--cluster 'sparks'"), "{cmd}");
|
|
|
+ assert!(cmd.contains("--name 'deepseek1017'"), "{cmd}");
|
|
|
+ assert!(cmd.contains("--port 8000"), "{cmd}");
|
|
|
+ assert!(cmd.contains("--detach"), "{cmd}");
|
|
|
+ assert!(cmd.contains("'--mtp-adaptive-verification'"), "{cmd}");
|
|
|
+ assert!(cmd.contains("--port 8000"), "{cmd}");
|
|
|
+ assert!(
|
|
|
+ cmd.contains("--dangerously-accept-publisher 'local-development'"),
|
|
|
+ "{cmd}"
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn orbit_node_names_follow_rank_order() {
|
|
|
+ let mut c = cfg();
|
|
|
+ c.flare.as_mut().unwrap().orbit_nodes = vec!["spark1".into()];
|
|
|
+ let node = Node {
|
|
|
+ name: "dgx5.lan".into(),
|
|
|
+ ssh_ip: "10.0.0.1".into(),
|
|
|
+ ib_ip: None,
|
|
|
+ rank: 0,
|
|
|
+ host_key: None,
|
|
|
+ entity: None,
|
|
|
+ };
|
|
|
+ assert_eq!(orbit_node_name(&c, &node), "spark1");
|
|
|
+ let node1 = Node { rank: 1, ..node };
|
|
|
+ // no entry for rank 1 → the cluster's own name is used
|
|
|
+ assert_eq!(orbit_node_name(&c, &node1), "dgx5.lan");
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn stop_names_the_group_and_cluster() {
|
|
|
+ let cmd = stop_cmd(&cfg(), &recipe()).unwrap();
|
|
|
+ assert_eq!(cmd, "flare orbit stop 'deepseek1017' --cluster 'sparks'");
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn preflight_checks_agent_and_pairing() {
|
|
|
+ let cmd = preflight_cmd(&cfg());
|
|
|
+ assert!(cmd.contains("is-active flare-agent"), "{cmd}");
|
|
|
+ assert!(cmd.contains("orbit status --cluster 'sparks'"), "{cmd}");
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
+ fn missing_block_is_an_error() {
|
|
|
+ let mut r = recipe();
|
|
|
+ r.flare = None;
|
|
|
+ assert!(download_cmd(&cfg(), &r).is_err());
|
|
|
+ }
|
|
|
+}
|