|
|
@@ -1,7 +1,9 @@
|
|
|
//! Composed model directories: build a /tmp dir per-node with symlinks into
|
|
|
-//! the HF cache (container-side absolute paths) + generated files, then
|
|
|
-//! bind-mount it into the container. Lets a recipe combine multiple HF repos
|
|
|
-//! (e.g. a base checkpoint + an expert overlay) without mutating the HF cache.
|
|
|
+//! the HF cache (container-side absolute paths) + generated files + hardlinks
|
|
|
+//! from node-local files, then bind-mount it into the container. Lets a
|
|
|
+//! recipe combine multiple HF repos (e.g. a base checkpoint + an expert
|
|
|
+//! overlay), and serve selected files from node-local storage (NVMe) instead
|
|
|
+//! of the shared NAS (local_files), without mutating the HF cache.
|
|
|
//!
|
|
|
//! Symlinks point to `/root/.cache/huggingface/...` (the container-side HF
|
|
|
//! cache mount path), NOT the host-side path. They are dangling on the host
|
|
|
@@ -11,7 +13,7 @@
|
|
|
//! relative-path gymnastics across different mount roots.
|
|
|
|
|
|
use super::shq;
|
|
|
-use crate::config::{ComposedModel, ComposedSource};
|
|
|
+use crate::config::{ComposedLocalFile, ComposedModel, ComposedSource, expand_tilde};
|
|
|
use crate::ssh::run_ssh;
|
|
|
use anyhow::{Context, Result};
|
|
|
use base64::Engine;
|
|
|
@@ -44,6 +46,7 @@ pub(crate) async fn build_composed_dir_on_node(
|
|
|
"mount" => container_path,
|
|
|
"sources" => composed.sources.len() as u32,
|
|
|
"files" => composed.files.len() as u32,
|
|
|
+ "local" => composed.local_files.len() as u32,
|
|
|
})
|
|
|
);
|
|
|
return Ok((host_path, container_path));
|
|
|
@@ -59,6 +62,13 @@ pub(crate) async fn build_composed_dir_on_node(
|
|
|
symlink_source(config, node, source, &host_path, &cache_str).await?;
|
|
|
}
|
|
|
|
|
|
+ // Hardlink local files (later wins over the symlinks above). These come
|
|
|
+ // from node-local storage (e.g. an NVMe copy of a table shard) and must
|
|
|
+ // live on the same filesystem as the composed dir (/tmp) to hardlink.
|
|
|
+ for lf in &composed.local_files {
|
|
|
+ link_local_file(config, node, lf, &host_path).await?;
|
|
|
+ }
|
|
|
+
|
|
|
// Write generated files (base64-encoded to avoid quoting issues).
|
|
|
for file in &composed.files {
|
|
|
let b64 = STANDARD.encode(file.content.as_bytes());
|
|
|
@@ -83,6 +93,7 @@ pub(crate) async fn build_composed_dir_on_node(
|
|
|
"node" => node.name.clone(),
|
|
|
"sources" => composed.sources.len() as u32,
|
|
|
"files" => composed.files.len() as u32,
|
|
|
+ "local" => composed.local_files.len() as u32,
|
|
|
})
|
|
|
);
|
|
|
|
|
|
@@ -205,3 +216,54 @@ async fn symlink_source(
|
|
|
})?;
|
|
|
Ok(())
|
|
|
}
|
|
|
+
|
|
|
+/// Hardlink a node-local file into the composed dir. Runs after the repo
|
|
|
+/// symlink pass, so a local file overwrites the repo's symlink at the same
|
|
|
+/// name (local wins — that is the point: e.g. an NVMe table shard instead of
|
|
|
+/// the shared-NAS version). Hardlinked, NOT copied — a sparse local copy
|
|
|
+/// stays sparse and costs zero extra disk — so the host path must be on the
|
|
|
+/// same filesystem as the composed dir (/tmp).
|
|
|
+async fn link_local_file(
|
|
|
+ config: &crate::config::Config,
|
|
|
+ node: &crate::config::Node,
|
|
|
+ lf: &ComposedLocalFile,
|
|
|
+ host_path: &str,
|
|
|
+) -> Result<()> {
|
|
|
+ let host = expand_tilde(&lf.host, &config.ssh_user);
|
|
|
+ let name = lf
|
|
|
+ .name
|
|
|
+ .as_deref()
|
|
|
+ .filter(|n| !n.is_empty())
|
|
|
+ .or_else(|| host.rsplit('/').next())
|
|
|
+ .filter(|s| !s.is_empty())
|
|
|
+ .ok_or_else(|| anyhow::anyhow!("composed_model local_file with empty host path"))?;
|
|
|
+ if name.starts_with('/')
|
|
|
+ || name.contains("..")
|
|
|
+ || !name
|
|
|
+ .chars()
|
|
|
+ .all(|c| c.is_ascii_alphanumeric() || "._-".contains(c))
|
|
|
+ {
|
|
|
+ anyhow::bail!(
|
|
|
+ "composed_model local_file '{}': name '{}' is invalid \
|
|
|
+ (allowed: alphanumeric . _ - ; flat name, no directories)",
|
|
|
+ host,
|
|
|
+ name
|
|
|
+ );
|
|
|
+ }
|
|
|
+ // Existence check + hardlink in one round-trip; `ln -f` replaces an
|
|
|
+ // earlier symlink at the same path.
|
|
|
+ let cmd = format!(
|
|
|
+ "test -e {h} && ln -f {h} {d}/{n}",
|
|
|
+ h = shq(&host),
|
|
|
+ d = shq(host_path),
|
|
|
+ n = shq(name),
|
|
|
+ );
|
|
|
+ run_ssh(config, node, cmd, false).await.with_context(|| {
|
|
|
+ format!(
|
|
|
+ "composed_model local_file: '{}' not found on '{}' — a per-node \
|
|
|
+ local path is required (and it must share a filesystem with /tmp to hardlink)",
|
|
|
+ host, node.name
|
|
|
+ )
|
|
|
+ })?;
|
|
|
+ Ok(())
|
|
|
+}
|
