| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399 |
- use crate::runlock::RunlockCommands;
- use clap::{Parser, Subcommand, ValueEnum};
- pub use crate::cli_sub::{
- ConfigCommands, PowerAction, PublishArgs, RegistryCommands, ReposArgs, ReposCommands,
- SyncImageArgs, TmuxCommands,
- };
- /// Output format for human-readable reports (`--format`).
- #[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)]
- pub enum OutputFormat {
- Plain,
- Md,
- Html,
- }
- #[derive(Parser)]
- #[command(author, version, about = "DGX Spark Launcher for vLLM and SGLang")]
- pub struct Cli {
- #[arg(long, short, default_value = "config.toml")]
- pub config: String,
- #[arg(long, short, help = "Show commands without executing")]
- pub dry_run: bool,
- #[command(subcommand)]
- pub command: Commands,
- }
- #[derive(Subcommand)]
- pub enum Commands {
- /// Distribute a recipe's image across the cluster nodes (node-to-node,
- /// one save + parallel fan-out; no serve started).
- SyncImage(SyncImageArgs),
- /// Launch a cluster
- Serve {
- #[arg(long, required = true)]
- recipe: String,
- #[arg(long, default_value = "auto")]
- cluster: String,
- /// Target a specific node within the cluster (default: all).
- #[arg(long)]
- node: Option<String>,
- /// Keep containers after stop (default: auto-remove on stop)
- #[arg(long)]
- keep: bool,
- /// Skip pre-serve hooks (cluster-level + recipe-level).
- #[arg(long)]
- skip_before_serve: bool,
- /// Don't follow logs after serving (default: follow).
- #[arg(long, action = clap::ArgAction::SetTrue)]
- no_follow: bool,
- /// Skip the engine-readiness wait after launch (no /v1/models poll).
- #[arg(long)]
- no_wait: bool,
- /// Power on cluster if switch is off (requires HA config).
- #[arg(short = 'O', long = "on")]
- auto_power_on: bool,
- /// Sync Docker image across nodes before starting.
- #[arg(long, default_value_t = true)]
- sync_image: bool,
- /// Explicit source node for image sync (default: auto-detect).
- #[arg(long)]
- sync_image_source: Option<String>,
- /// Build image on this node if not found (requires docker-images/<recipe>/).
- #[arg(long)]
- build_node: Option<String>,
- /// Force rebuild even if checksum matches.
- #[arg(long)]
- force_build: bool,
- /// Run even on occupied nodes (dangerous for GPU recipes: OOM,
- /// port conflicts). Skips the occupancy check in cluster selection.
- #[arg(long)]
- force: bool,
- /// Serve single-node on the first free node (overrides recipe `solo`);
- /// `--no-solo` forces multi-node.
- #[arg(long)]
- solo: Option<bool>,
- /// Mount a per-node local HF mirror (bare = ~/.cache/hf-models;
- /// `--mirror <path>` = that path; first serve copies per node).
- /// Mutually exclusive with `--hf-cache` (both pick the HF mount source).
- #[arg(long, num_args = 0..=1, default_missing_value = "")]
- mirror: Option<String>,
- /// Override the host-side HF cache dir for this serve (config
- /// `hf_cache_dir`, else `~/.cache/huggingface`). `~` expands to the
- /// node's SSH-user home. Mutually exclusive with `--mirror`.
- #[arg(long)]
- hf_cache: Option<String>,
- /// How hard to run the recipe's `[[staging]]` blocks: `auto`
- /// (ledger-verified, copy what is new), `skip` (assume staged),
- /// `force` (re-verify everything).
- #[arg(long)]
- stage: Option<String>,
- /// Scale the recipe to N nodes (`--fit=N`; bare = whole cluster, TP
- /// tracks it; incompatible with --solo).
- #[arg(long, num_args = 0..=1, default_missing_value = "")]
- fit: Option<String>,
- /// `sudo rm -rf` the host-side extra_mounts folders on the launch
- /// nodes before containers start (repo-based mounts untouched).
- #[arg(long)]
- clear: bool,
- /// Skip the post-build registry push (only meaningful when `[registry]`
- /// is configured and the image needs (re)building).
- #[arg(long)]
- no_push: bool,
- /// Where the image comes from: build from `docker-images/` (default)
- /// or pull the prebuilt `{base}/{name}:{checksum}` from `[registry]`
- /// (skips the build; the image must already be pushed).
- #[arg(long, value_enum, default_value_t = crate::docker::ImageSource::Repo)]
- image_source: crate::docker::ImageSource,
- },
- /// Show container logs
- Logs {
- #[arg(long, required = true)]
- recipe: String,
- #[arg(long, action = clap::ArgAction::SetTrue)]
- follow: bool,
- #[arg(long, default_value = "auto")]
- cluster: String,
- /// Target a specific node within the cluster (default: head node if following, all nodes if not).
- #[arg(long)]
- node: Option<String>,
- },
- /// Stop a cluster
- Stop {
- #[arg(long, required = true)]
- recipe: String,
- /// Target specific nodes (repeatable, e.g. --node dgx1 --node dgx2).
- #[arg(long)]
- node: Vec<String>,
- #[arg(long, default_value = "auto")]
- cluster: String,
- /// Power off cluster after stopping containers (requires HA config).
- #[arg(long = "off")]
- power_off: bool,
- /// Don't wait until the containers are actually stopped and removed
- /// (default: wait and verify, up to 60 s per node, so an immediate
- /// re-serve on the same nodes doesn't race the dying containers).
- #[arg(long)]
- no_wait: bool,
- },
- /// Bench a live serve with tool-eval-bench: reports auto-land in
- /// recipes/<lane>/reports/<UTC-date>_<label>.{md,json,log} (seed 42, hardmode, perf).
- Bench {
- #[arg(long, required = true)]
- recipe: String,
- #[arg(long, required = true)]
- cluster: String,
- #[arg(long, required = true)]
- label: String,
- #[arg(long, default_value = "1,2")]
- concurrency: String,
- #[arg(long, default_value_t = 42)]
- seed: u64,
- /// Run scenario evaluations over N concurrent streams (tool
- /// `--parallel`). Speeds up the scenario phase ~Nx at the cost of
- /// mild engine contention; perf numbers are measured separately.
- #[arg(long)]
- parallel: Option<u32>,
- /// Local tokenizer for llama-benchy (tokenizer.json or a dir with
- /// one) — needed when the served id isn't resolvable offline.
- #[arg(long)]
- tokenizer: Option<String>,
- /// Skip the scenario phase — run only the perf micro-benchmarks
- /// (use to re-baseline the perf table after a cold first row).
- #[arg(long)]
- perf_only: bool,
- /// Run tool-eval-bench on a remote host over SSH instead of locally,
- /// so the bench client's own network (a WiFi laptop) stays off the
- /// measurement path. The tokenizer is uploaded to the host; the
- /// report + run package still land locally. The host must have
- /// `tool-eval-bench` on PATH.
- #[arg(long)]
- ssh_host: Option<String>,
- },
- /// Check cluster health
- Health {
- #[arg(
- long,
- help = "Cluster name (e.g. 4nodes-switch). If omitted — checks all."
- )]
- cluster: Option<String>,
- #[arg(long, help = "Show node host keys (for adding to config.toml)")]
- fingerprint: bool,
- /// Output format (cluster-health mode only; ignored with --fingerprint).
- #[arg(long, value_enum, default_value_t = OutputFormat::Plain)]
- format: OutputFormat,
- },
- /// Check and repair a recipe (doctor mode)
- Doctor {
- #[arg(long, required = true, help = "Recipe name to check")]
- recipe: String,
- #[arg(
- long,
- default_value = "auto",
- help = "Cluster (if not auto — uses recipe.cluster)"
- )]
- cluster: String,
- #[arg(long, help = "Auto-fix issues (create Dockerfile + build image)")]
- repair: bool,
- },
- /// Publish a recipe (+ image overlay) as a standalone reproducible bundle
- Publish(PublishArgs),
- /// Browse and install recipes from the public registry
- #[command(subcommand)]
- Registry(RegistryCommands),
- /// Manage tmux sessions (Docker alternative)
- #[command(subcommand)]
- Tmux(TmuxCommands),
- /// Auto-discover host keys and write them into config.toml
- InitHostKeys {
- #[arg(long, help = "Cluster name (e.g. 4nodes-switch). If omitted — all.")]
- cluster: Option<String>,
- },
- /// Run an arbitrary shell command on a cluster or single node
- Exec {
- /// Cluster name (mutually exclusive with --recipe).
- #[arg(long, conflicts_with = "recipe")]
- cluster: Option<String>,
- /// Recipe name; resolves to the recipe's pinned cluster.
- #[arg(long, conflicts_with = "cluster")]
- recipe: Option<String>,
- /// Target a specific node within the cluster (default: all).
- #[arg(long)]
- node: Option<String>,
- /// Name of an alias from `[exec_aliases]` in config.toml.
- /// Mutually exclusive with a trailing command.
- #[arg(long)]
- alias: Option<String>,
- /// Command to run (everything after `--`).
- #[arg(trailing_var_arg = true)]
- command: Vec<String>,
- },
- /// Manage configuration (global config path, init)
- #[command(subcommand)]
- Config(ConfigCommands),
- /// Run-ledger management (per-recipe .lock; offline)
- #[command(subcommand)]
- Runlock(RunlockCommands),
- /// Manage the configured git recipe repos (clone/pull cache, archive
- /// registry). `sm121 repos` lists state; `sm121 repos sync [--force]`
- /// re-pulls.
- Repos(ReposArgs),
- /// Archive a recipe repo (it is skipped by sync; state in repos.json)
- Archive {
- /// The repo URL (a `[[recipe_repos]]` entry)
- url: String,
- },
- /// Un-archive a recipe repo (it takes part in sync again)
- Unarchive {
- /// The repo URL (a `[[recipe_repos]]` entry)
- url: String,
- },
- /// Move a recipe across recipe repos (git mv + commit + push; the
- /// target is updated first, so a failure duplicates, never deletes).
- /// Default (or `-d`) prints the exact commands; `--apply` executes.
- Promote {
- /// Recipe name (its directory under the repo's `recipes/`)
- recipe: String,
- /// Source repo URL (a `[[recipe_repos]]` entry)
- #[arg(long, required = true)]
- from: String,
- /// Target repo URL (a `[[recipe_repos]]` entry)
- #[arg(long, required = true)]
- to: String,
- /// Execute the move (default: print the plan only)
- #[arg(long)]
- apply: bool,
- },
- /// Launch the web dashboard (axum + SSE) on a local port
- Web {
- #[arg(long, default_value_t = 3100)]
- port: u16,
- #[arg(long, default_value = "127.0.0.1")]
- bind: String,
- /// Bearer token for data endpoints (required when --bind is not loopback)
- #[arg(long)]
- token: Option<String>,
- /// Open the browser once the server is up
- #[arg(long)]
- open: bool,
- },
- /// Table of all tested recipes and their benchmark history
- Report {
- /// Interactive HTML table via --serve (default: print Markdown)
- #[arg(long)]
- serve: bool,
- /// Port for --serve
- #[arg(long, default_value_t = 3101)]
- port: u16,
- /// Open the browser (implies --serve)
- #[arg(long)]
- open: bool,
- #[arg(long, help = "Archive a recipe (hidden from report output)")]
- archive: Option<String>,
- #[arg(long, help = "Un-archive a recipe")]
- unarchive: Option<String>,
- #[arg(long, help = "Show the archived recipes instead of the active ones")]
- archived: bool,
- },
- /// Build Docker image for a recipe on a remote node
- Build {
- #[arg(long, required = true, help = "Recipe name")]
- recipe: String,
- #[arg(long, required = true, help = "Node to build on (e.g. dgx1.lan)")]
- build_node: String,
- #[arg(long, default_value = "auto", help = "Cluster name")]
- cluster: String,
- /// Force rebuild even if checksum matches.
- #[arg(long)]
- force_build: bool,
- /// Skip the post-build registry push (only meaningful when `[registry]`
- /// is configured).
- #[arg(long)]
- no_push: bool,
- /// Where the image comes from: build from `docker-images/` (default)
- /// or pull the prebuilt tag from `[registry]` (no build, no push).
- #[arg(long, value_enum, default_value_t = crate::docker::ImageSource::Repo)]
- image_source: crate::docker::ImageSource,
- },
- /// Power on/off nodes via Home Assistant (smart plugs / Wake-on-LAN).
- Power {
- #[command(subcommand)]
- action: PowerAction,
- },
- /// Validate recipes against the launcher contract (build dirs exist,
- /// port convention, no floating :latest, floating model revisions,
- /// node counts). Errors fail the run (exit 1); archived lanes are
- /// grandfathered (their errors downgrade to warnings). Also runs as a
- /// `cargo test` gate over the repo's own `recipes/`.
- Validate,
- }
- #[cfg(test)]
- #[path = "cli_tests.rs"]
- mod tests;
|