api.rs 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443
  1. //! REST endpoints. P1: `GET /api/state` (latest snapshot; probes once if
  2. //! the supervisor hasn't ticked yet).
  3. use axum::{extract::State, http::StatusCode, response::Json};
  4. use serde::Deserialize;
  5. use super::AppState;
  6. use super::events::Snapshot;
  7. pub(crate) async fn state(app: State<AppState>) -> Json<Snapshot> {
  8. if app.last.lock().unwrap().is_none() {
  9. // First request before the first supervisor tick: probe now so the
  10. // UI gets data immediately (probe is read-only `docker ps` per node
  11. // and degrades to `reachable: false` on failure). Two racing first
  12. // requests may both probe — harmless (idempotent, last write wins).
  13. super::push_state(&app).await;
  14. }
  15. app.last
  16. .lock()
  17. .unwrap()
  18. .as_ref()
  19. .map(|s| Json(s.clone()))
  20. .unwrap()
  21. }
  22. // ---------------------------------------------------------------------------
  23. // P2: actions (serve/stop/power run as tracked jobs via the same handlers
  24. // the CLI uses) + recipe list.
  25. type ApiError = (StatusCode, String);
  26. #[derive(Deserialize)]
  27. pub(crate) struct ServeReq {
  28. pub cluster: String,
  29. pub recipe: String,
  30. #[serde(default)]
  31. pub node: Option<String>,
  32. #[serde(default)]
  33. pub solo: Option<bool>,
  34. #[serde(default)]
  35. pub force: bool,
  36. #[serde(default)]
  37. pub fit: Option<String>,
  38. #[serde(default)]
  39. pub clear: bool,
  40. #[serde(default)]
  41. pub mirror: Option<String>,
  42. #[serde(default)]
  43. pub keep: bool,
  44. #[serde(default)]
  45. pub dry_run: bool,
  46. }
  47. /// `POST /api/serve` — same dispatch as the CLI (docker vs tmux by
  48. /// `recipe.deploy`), run as a tracked job.
  49. pub(crate) async fn serve(
  50. app: State<AppState>,
  51. Json(req): Json<ServeReq>,
  52. ) -> Result<Json<serde_json::Value>, ApiError> {
  53. let id = super::jobs::new_job(&app, "serve", &req.cluster, Some(&req.recipe))?;
  54. let config = app.config.clone();
  55. let app2 = (*app).clone();
  56. let opts = crate::docker::ServeOpts {
  57. keep: req.keep,
  58. skip_before_serve: false,
  59. no_wait_ready: false,
  60. dry_run: req.dry_run,
  61. auto_power_on: false,
  62. sync_image: true,
  63. sync_image_source: None,
  64. no_push: false,
  65. image_source: crate::docker::ImageSource::Repo,
  66. build_node: None,
  67. force_build: false,
  68. force: req.force,
  69. solo: req.solo,
  70. mirror: req.mirror,
  71. hf_cache: None,
  72. stage: None,
  73. fit: req.fit,
  74. clear: req.clear,
  75. };
  76. let recipe = req.recipe.clone();
  77. let cluster = req.cluster.clone();
  78. let node = req.node.clone();
  79. super::jobs::spawn(app2, id.clone(), async move {
  80. crate::handle_serve(&config, &recipe, &cluster, node, opts, false, false).await
  81. });
  82. Ok(Json(serde_json::json!({ "job": id })))
  83. }
  84. #[derive(Deserialize)]
  85. pub(crate) struct StopReq {
  86. pub cluster: String,
  87. pub recipe: String,
  88. #[serde(default)]
  89. pub node: Option<String>,
  90. }
  91. /// `POST /api/stop` — stops the recipe's containers (auto-discovers the
  92. /// cluster when the recipe is running exactly once), as a tracked job.
  93. pub(crate) async fn stop(
  94. app: State<AppState>,
  95. Json(req): Json<StopReq>,
  96. ) -> Result<Json<serde_json::Value>, ApiError> {
  97. let id = super::jobs::new_job(&app, "stop", &req.cluster, Some(&req.recipe))?;
  98. let config = app.config.clone();
  99. let app2 = (*app).clone();
  100. let recipe = req.recipe.clone();
  101. let cluster = req.cluster.clone();
  102. let node = req.node.clone();
  103. super::jobs::spawn(app2, id.clone(), async move {
  104. let nodes = vec![node.unwrap_or_default()];
  105. let nodes = if nodes[0].is_empty() { vec![] } else { nodes };
  106. crate::docker::stop::handle(&config, &recipe, &cluster, &nodes, true, false, false).await
  107. });
  108. Ok(Json(serde_json::json!({ "job": id })))
  109. }
  110. #[derive(Deserialize)]
  111. pub(crate) struct PowerReq {
  112. /// "off" | "on" | "switch"
  113. pub action: String,
  114. pub cluster: String,
  115. /// switch target (required only for `switch`)
  116. #[serde(default)]
  117. pub to: Option<String>,
  118. }
  119. /// `POST /api/power` — HA power actions as a tracked job. The UI enforces
  120. /// the type-the-name confirm; this endpoint trusts the caller (it is
  121. /// token-gated / loopback-bound).
  122. pub(crate) async fn power(
  123. app: State<AppState>,
  124. Json(req): Json<PowerReq>,
  125. ) -> Result<Json<serde_json::Value>, ApiError> {
  126. let id = super::jobs::new_job(&app, "power", &req.cluster, None)?;
  127. let config = app.config.clone();
  128. let app2 = (*app).clone();
  129. let action = req.action.clone();
  130. let cluster = req.cluster.clone();
  131. let to = req.to.clone();
  132. let stale = stale_from_snapshot(&app, &req.cluster);
  133. super::jobs::spawn_report(app2, id.clone(), async move {
  134. match action.as_str() {
  135. "off" => {
  136. let c = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
  137. crate::power::power_off(&config, c, &[], false).await?;
  138. Ok(Vec::new())
  139. }
  140. "on" => {
  141. let c = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
  142. // nodes the last probe saw down: their plugs may be stale-on
  143. // (a graceful shutdown leaves the plug on), so power on
  144. // cycles those instead of issuing a no-op turn_on
  145. crate::power::power_on(&config, c, &[], false, &stale).await
  146. }
  147. "switch" => {
  148. let to = to
  149. .as_deref()
  150. .ok_or_else(|| anyhow::anyhow!("`to` cluster is required for power switch"))?;
  151. let from = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
  152. let to = crate::config::resolve_cluster_name(&config.clusters, to)?;
  153. crate::power::power_switch(&config, from, to, false).await?;
  154. Ok(Vec::new())
  155. }
  156. other => Err(anyhow::anyhow!(
  157. "unknown power action '{other}' (off|on|switch)"
  158. )),
  159. }
  160. });
  161. Ok(Json(serde_json::json!({ "job": id })))
  162. }
  163. /// Nodes of `cluster` that the last fleet probe saw unreachable — the
  164. /// `power on` cycle list (a plug left on by a graceful shutdown needs a
  165. /// cycle, not a no-op `turn_on`).
  166. pub(crate) fn stale_from_snapshot(app: &AppState, cluster: &str) -> Vec<String> {
  167. let snap = app
  168. .last
  169. .lock()
  170. .unwrap_or_else(std::sync::PoisonError::into_inner)
  171. .clone();
  172. let Some(snap) = snap else { return Vec::new() };
  173. let mut out = Vec::new();
  174. for c in &snap.clusters {
  175. if c.name != cluster {
  176. continue;
  177. }
  178. for n in &c.nodes {
  179. if !n.reachable {
  180. out.push(n.name.clone());
  181. }
  182. }
  183. }
  184. out
  185. }
  186. /// `GET /api/recipes` — all discoverable recipes (cwd + `recipe_sources` +
  187. /// the global config dir), with deploy target + node requirements.
  188. pub(crate) async fn recipes(app: State<AppState>) -> Json<serde_json::Value> {
  189. let dirs = crate::config::recipe_dirs(&app.config);
  190. let mut out = Vec::new();
  191. for dir in dirs {
  192. let mut entries = match tokio::fs::read_dir(&dir).await {
  193. Ok(e) => e,
  194. Err(_) => continue,
  195. };
  196. loop {
  197. let Ok(Some(entry)) = entries.next_entry().await else {
  198. break;
  199. };
  200. let path = entry.path();
  201. if !path.is_dir() {
  202. continue;
  203. }
  204. let Some(name) = entry.file_name().to_str().map(String::from) else {
  205. continue;
  206. };
  207. let file = path.join("recipe.toml");
  208. let Ok(toml_str) = tokio::fs::read_to_string(&file).await else {
  209. continue;
  210. };
  211. match toml::from_str::<crate::config::Recipe>(&toml_str) {
  212. Ok(r) => out.push(serde_json::json!({
  213. "name": name,
  214. "engine": r.engine,
  215. "deploy": format!("{:?}", r.deploy).to_lowercase(),
  216. "required_nodes": r.required_nodes,
  217. })),
  218. Err(e) => tracing::debug!("skipping recipe {}: {e}", name),
  219. }
  220. }
  221. }
  222. Json(serde_json::json!({ "recipes": out }))
  223. }
  224. /// `POST /api/repos/sync` — clone/pull every configured recipe repo right
  225. /// now (the same best-effort batch the command start runs; unreachable
  226. /// repos are skipped, never fatal). A single in-flight guard: a second
  227. /// sync while one is running gets a 409, not a second git storm. The
  228. /// recipes panel's ⟳ button calls it; the report refresher's file watcher
  229. /// then picks up the new/changed recipes on its own.
  230. pub(crate) async fn repos_sync(app: State<AppState>) -> Result<Json<serde_json::Value>, ApiError> {
  231. static SYNCING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
  232. if SYNCING.swap(true, std::sync::atomic::Ordering::SeqCst) {
  233. return Err((
  234. StatusCode::CONFLICT,
  235. "a repo sync is already running".into(),
  236. ));
  237. }
  238. let config = app.config.clone();
  239. let roots = crate::recipe_repos::sync_recipe_repos(&config).await;
  240. SYNCING.store(false, std::sync::atomic::Ordering::SeqCst);
  241. Ok(Json(
  242. serde_json::json!({ "ok": true, "repos": roots.len() }),
  243. ))
  244. }
  245. // ---------------------------------------------------------------------------
  246. // P3: log windows — attach a streaming `docker logs -f` per node+recipe+rank
  247. // (idempotent), detach, and fetch ring history for a fresh window.
  248. #[derive(Deserialize)]
  249. pub(crate) struct LogsReq {
  250. pub cluster: String,
  251. pub node: String,
  252. pub recipe: String,
  253. #[serde(default)]
  254. pub rank: u32,
  255. #[serde(default)]
  256. pub lines: usize,
  257. }
  258. /// `POST /api/logs` — attach (reuses a live stream for the same key).
  259. pub(crate) async fn logs_attach(
  260. app: State<AppState>,
  261. Json(req): Json<LogsReq>,
  262. ) -> Result<Json<serde_json::Value>, ApiError> {
  263. let (id, history) = super::logs::attach(
  264. &app,
  265. &app.log_attaches,
  266. &req.cluster,
  267. &req.node,
  268. &req.recipe,
  269. req.rank,
  270. )
  271. .map_err(|e| {
  272. (
  273. if e.starts_with("max") {
  274. StatusCode::CONFLICT
  275. } else {
  276. StatusCode::BAD_REQUEST
  277. },
  278. e,
  279. )
  280. })?;
  281. Ok(Json(
  282. serde_json::json!({ "attach_id": id, "history": history }),
  283. ))
  284. }
  285. /// `DELETE /api/logs` — stop the stream and drop the attach.
  286. #[derive(Deserialize)]
  287. pub(crate) struct DetachReq {
  288. pub attach_id: String,
  289. }
  290. pub(crate) async fn logs_detach(
  291. app: State<AppState>,
  292. Json(req): Json<DetachReq>,
  293. ) -> Result<StatusCode, ApiError> {
  294. let id = req.attach_id;
  295. if super::logs::detach(&app.log_attaches, &id) {
  296. Ok(StatusCode::NO_CONTENT)
  297. } else {
  298. Err((StatusCode::NOT_FOUND, format!("no log attach '{id}'")))
  299. }
  300. }
  301. /// `GET /api/logs?cluster=&node=&recipe=&rank=&lines=` — ring history for a
  302. /// fresh window (0 lines = `logs::TAIL_LINES`).
  303. pub(crate) async fn logs_history(
  304. app: State<AppState>,
  305. axum::extract::Query(q): axum::extract::Query<LogsReq>,
  306. ) -> Result<Json<serde_json::Value>, ApiError> {
  307. let lines = if q.lines == 0 {
  308. super::logs::TAIL_LINES
  309. } else {
  310. q.lines
  311. };
  312. let (id, history) = super::logs::by_key(
  313. &app.log_attaches,
  314. &q.cluster,
  315. &q.node,
  316. &q.recipe,
  317. q.rank,
  318. lines,
  319. )
  320. .ok_or_else(|| {
  321. (
  322. StatusCode::NOT_FOUND,
  323. "no log attach for that node/recipe".to_string(),
  324. )
  325. })?;
  326. Ok(Json(
  327. serde_json::json!({ "attach_id": id, "history": history }),
  328. ))
  329. }
  330. // ---------------------------------------------------------------------------
  331. // P4: raw content — the active config file, a recipe verbatim, and the
  332. // merged effective config (post cluster-source merge) as JSON.
  333. /// `GET /api/raw/config` — the active config file, verbatim.
  334. pub(crate) async fn raw_config() -> Result<axum::response::Response, ApiError> {
  335. let path = crate::gconfig::resolve_config_path("config.toml");
  336. let content = tokio::fs::read_to_string(&path).await.map_err(|e| {
  337. (
  338. StatusCode::NOT_FOUND,
  339. format!("no config file at {}: {e}", path.display()),
  340. )
  341. })?;
  342. Ok(axum::response::Response::builder()
  343. .status(200)
  344. .header("content-type", "text/plain; charset=utf-8")
  345. .body(axum::body::Body::from(content))
  346. .unwrap())
  347. }
  348. /// `GET /api/raw/recipe/<name>` — recipe.toml verbatim (search order as
  349. /// `load_recipe`: cwd, recipe_sources, global dir).
  350. pub(crate) async fn raw_recipe(
  351. app: State<AppState>,
  352. axum::extract::Path(name): axum::extract::Path<String>,
  353. ) -> Result<axum::response::Response, ApiError> {
  354. if name.is_empty() || name.contains('/') || name.contains("..") {
  355. return Err((StatusCode::BAD_REQUEST, "invalid recipe name".to_string()));
  356. }
  357. let dirs = crate::config::recipe_dirs(&app.config);
  358. for dir in &dirs {
  359. let p = dir.join(&name).join("recipe.toml");
  360. if let Ok(content) = tokio::fs::read_to_string(&p).await {
  361. return Ok(axum::response::Response::builder()
  362. .status(200)
  363. .header("content-type", "text/plain; charset=utf-8")
  364. .body(axum::body::Body::from(content))
  365. .unwrap());
  366. }
  367. }
  368. Err((StatusCode::NOT_FOUND, format!("recipe '{name}' not found")))
  369. }
  370. /// `GET /api/raw/effective` — the merged `Config` as JSON (post
  371. /// cluster-source merge), HA token redacted.
  372. pub(crate) async fn raw_effective(app: State<AppState>) -> Json<serde_json::Value> {
  373. use crate::config::NcclConfig;
  374. let ha = app.config.home_assistant.as_ref().map(|h| {
  375. let masked: String = h.token.chars().take(12).collect();
  376. let token = if h.token.len() > 12 {
  377. format!("{masked}…<redacted>")
  378. } else {
  379. "<redacted>".to_string()
  380. };
  381. serde_json::json!({
  382. "url": h.url,
  383. "token": token,
  384. "shutdown_delay_seconds": h.shutdown_delay_seconds,
  385. })
  386. });
  387. Json(serde_json::json!({
  388. "config_path": crate::gconfig::resolve_config_path("config.toml").display().to_string(),
  389. "ssh_user": app.config.ssh_user,
  390. "ssh_key_path": app.config.ssh_key_path.display().to_string(),
  391. "clusters": app.config.clusters.iter().map(|c| serde_json::json!({
  392. "name": c.name,
  393. "nodes": c.nodes.iter().map(|n| serde_json::json!({
  394. "name": n.name,
  395. "ssh_ip": n.ssh_ip,
  396. "rank": n.rank,
  397. "entity": n.entity,
  398. "host_key_set": n.host_key.is_some(),
  399. })).collect::<Vec<_>>(),
  400. "extra_entities": c.extra_entities.iter().map(|e| serde_json::json!({
  401. "entity": e.entity(),
  402. "blocks": e.blocks(),
  403. })).collect::<Vec<_>>(),
  404. "nccl": app.config.nccl.get(&c.name).map(|n: &NcclConfig| serde_json::json!({
  405. "ib_hca": n.ib_hca,
  406. "socket_ifname": n.socket_ifname,
  407. "ib_gid_index": n.ib_gid_index,
  408. "ib_disable": n.ib_disable,
  409. })),
  410. "before_serve": app.config.before_serve.get(&c.name).map(|b| b.commands.len()),
  411. })).collect::<Vec<_>>(),
  412. "home_assistant": ha,
  413. "init_port": app.config.init_port,
  414. "mount_aliases": app.config.mount_aliases,
  415. "recipe_sources": app.config.recipe_sources,
  416. "cluster_sources": app.config.cluster_sources,
  417. "drop_caches": app.config.drop_caches,
  418. }))
  419. }