|
|
@@ -5,6 +5,7 @@ pub mod deploy;
|
|
|
pub mod health;
|
|
|
pub mod logs;
|
|
|
pub mod mods;
|
|
|
+pub(crate) mod path_ref;
|
|
|
pub use logs::show_logs;
|
|
|
pub mod stale;
|
|
|
pub mod sync;
|
|
|
@@ -389,7 +390,8 @@ async fn start_cluster(
|
|
|
let resolved_path = if dry {
|
|
|
command::path_ref_placeholder(spec)
|
|
|
} else {
|
|
|
- match resolve_path_ref_on_node(&config, &node, spec, &hf_cache).await {
|
|
|
+ match path_ref::resolve_path_ref_on_node(&config, &node, spec, &hf_cache).await
|
|
|
+ {
|
|
|
Ok(p) => p,
|
|
|
Err(e) => {
|
|
|
return (node.name, node.rank, engine, Err(e));
|
|
|
@@ -459,50 +461,5 @@ async fn start_cluster(
|
|
|
Ok(())
|
|
|
}
|
|
|
|
|
|
-/// Resolve a model path reference's full HF cache path on a specific node by
|
|
|
-/// reading its `refs/main`: `<hf_cache>/hub/<model_cache_dir_name(repo)>/snapshots/<refs/main>[/<subfolder>]`.
|
|
|
-/// `refs/main` always points at the latest commit, so the path survives
|
|
|
-/// `huggingface-cli download` updates (no fragile symlink). If the repo or
|
|
|
-/// subfolder is absent, returns an error pointing at `huggingface-cli download`
|
|
|
-/// (the launcher never downloads).
|
|
|
-///
|
|
|
-/// The resolved path is substituted into the recipe args/env via `{{<name>}}`
|
|
|
-/// — no bind-mount is created (the HF cache is already mounted at
|
|
|
-/// `/root/.cache/huggingface`, so the path's symlinks resolve correctly).
|
|
|
-async fn resolve_path_ref_on_node(
|
|
|
- config: &Config,
|
|
|
- node: &Node,
|
|
|
- spec: &command::PathRef,
|
|
|
- hf_cache: &std::path::Path,
|
|
|
-) -> Result<String> {
|
|
|
- let model_dir = crate::config::model_cache_dir_name(&spec.repo);
|
|
|
- let sub = spec.subfolder.as_deref().unwrap_or("");
|
|
|
- let hub_dir = format!("{}/hub/{}", hf_cache.display(), model_dir);
|
|
|
- let refs_path = format!("{}/refs/main", hub_dir);
|
|
|
- // `cat refs/main` prints the current commit hash; trim trailing newline.
|
|
|
- // All literal path components are shell-quoted (a repo/subfolder with a
|
|
|
- // quote/space would otherwise break the command); `$ref` is our own
|
|
|
- // captured var, so it expands safely inside the printf double-quotes.
|
|
|
- let cmd = format!(
|
|
|
- "ref=\"$(cat {refs_q} 2>/dev/null)\" && test -n \"$ref\" && \
|
|
|
- dir=\"$(printf '%s/snapshots/%s/%s' {hub_q} \"$ref\" {sub_q})\" && \
|
|
|
- test -d \"$dir\" && printf '%s' \"$dir\"",
|
|
|
- refs_q = shq(&refs_path),
|
|
|
- hub_q = shq(&hub_dir),
|
|
|
- sub_q = shq(sub),
|
|
|
- );
|
|
|
- let resolved_path = run_ssh(config, node, cmd, false).await?;
|
|
|
- if resolved_path.trim().is_empty() {
|
|
|
- let hint = format!(
|
|
|
- "Model subfolder '{}' not found in HF cache on '{}': \
|
|
|
- repo '{}' is not downloaded there or 'refs/main' is missing. \
|
|
|
- Run `huggingface-cli download {}` on that node.",
|
|
|
- sub, node.name, spec.repo, spec.repo
|
|
|
- );
|
|
|
- anyhow::bail!(hint);
|
|
|
- }
|
|
|
- Ok(resolved_path.replace(&hf_cache.display().to_string(), "/root/.cache/huggingface"))
|
|
|
-}
|
|
|
-
|
|
|
#[cfg(test)]
|
|
|
mod tests;
|