|
|
@@ -170,6 +170,18 @@ smallest cluster with ≥N free nodes; bare `--fit` + `--cluster auto` picks
|
|
|
the smallest cluster, so pass `--cluster <name> --fit` to scale onto a
|
|
|
specific larger cluster. Incompatible with `--solo`.
|
|
|
|
|
|
+`serve --cold` overlays the recipe's `cold_args` over its `args` (by flag
|
|
|
+name) — the **first-launch / tight-node safety valve** for memory knobs. A
|
|
|
+recipe declares `cold_args` (same flat shape as `args`, engine-agnostic) with
|
|
|
+conservative values (lower `--gpu-memory-utilization`, `--max-model-len`,
|
|
|
+`--max-num-seqs`, `--max-num-batched-tokens`); `--cold` replaces each matching
|
|
|
+flag's value in `args` (or appends it if the recipe omitted the flag), leaving
|
|
|
+all other flags untouched. Boot a memory-tight lane conservatively, then drop
|
|
|
+the flag once it is proven to come up — no per-launch recipe edit. No-op (with
|
|
|
+a warning) if the recipe has no `cold_args`. This is the fix for the class of
|
|
|
+failures where a full-context/GMU config OOMs the KV cache or (on GB10,
|
|
|
+unified memory) *stalls the host* — see `src/cold.rs`.
|
|
|
+
|
|
|
`serve --mirror` (optional value) mounts a **per-node local mirror** of the HF
|
|
|
|
|
|
cache instead of the shared one at `/root/.cache/huggingface`. Bare `--mirror`
|
|
|
@@ -290,6 +302,8 @@ before_serve = [{ command = "echo", on_failure = "abort" }]
|
|
|
after_stop = [{ script = "restore_dgx_services" }] # runs after `sm121 stop`
|
|
|
args = ["--kv-cache-dtype", "fp8"]
|
|
|
|
|
|
+cold_args = ["--gpu-memory-utilization", "0.60", "--max-model-len", "16384"] # optional; overlaid over `args` by flag name under `serve --cold` (first-launch safety valve; engine-agnostic)
|
|
|
+
|
|
|
[[extra_mounts]] # extra bind-mounts for `docker run -v`
|
|
|
alias = "jit_cache" # resolves via [mount_aliases] in config.toml; wins over `host`
|
|
|
host = "~/.cache/jit" # fallback default if alias absent; `~` expanded to ssh_user home
|
