api.rs 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424
  1. //! REST endpoints. P1: `GET /api/state` (latest snapshot; probes once if
  2. //! the supervisor hasn't ticked yet).
  3. use axum::extract::State;
  4. use axum::http::StatusCode;
  5. use axum::response::Json;
  6. use serde::Deserialize;
  7. use super::AppState;
  8. use super::events::Snapshot;
  9. pub(crate) async fn state(app: State<AppState>) -> Json<Snapshot> {
  10. if app.last.lock().unwrap().is_none() {
  11. // First request before the first supervisor tick: probe now so the
  12. // UI gets data immediately (probe is read-only `docker ps` per node
  13. // and degrades to `reachable: false` on failure). Two racing first
  14. // requests may both probe — harmless (idempotent, last write wins).
  15. super::push_state(&app).await;
  16. }
  17. app.last
  18. .lock()
  19. .unwrap()
  20. .as_ref()
  21. .map(|s| Json(s.clone()))
  22. .unwrap()
  23. }
  24. // ---------------------------------------------------------------------------
  25. // P2: actions (serve/stop/power run as tracked jobs via the same handlers
  26. // the CLI uses) + recipe list.
  27. type ApiError = (StatusCode, String);
  28. #[derive(Deserialize)]
  29. pub(crate) struct ServeReq {
  30. pub cluster: String,
  31. pub recipe: String,
  32. #[serde(default)]
  33. pub node: Option<String>,
  34. #[serde(default)]
  35. pub solo: Option<bool>,
  36. #[serde(default)]
  37. pub force: bool,
  38. #[serde(default)]
  39. pub fit: Option<String>,
  40. #[serde(default)]
  41. pub clear: bool,
  42. #[serde(default)]
  43. pub mirror: Option<String>,
  44. #[serde(default)]
  45. pub keep: bool,
  46. #[serde(default)]
  47. pub dry_run: bool,
  48. }
  49. /// `POST /api/serve` — same dispatch as the CLI (docker vs tmux by
  50. /// `recipe.deploy`), run as a tracked job.
  51. pub(crate) async fn serve(
  52. app: State<AppState>,
  53. Json(req): Json<ServeReq>,
  54. ) -> Result<Json<serde_json::Value>, ApiError> {
  55. let id = super::jobs::new_job(&app, "serve", &req.cluster, Some(&req.recipe))?;
  56. let config = app.config.clone();
  57. let app2 = (*app).clone();
  58. let opts = crate::docker::ServeOpts {
  59. keep: req.keep,
  60. skip_before_serve: false,
  61. no_wait_ready: false,
  62. dry_run: req.dry_run,
  63. auto_power_on: false,
  64. sync_image: true,
  65. sync_image_source: None,
  66. no_push: false,
  67. image_source: crate::docker::ImageSource::Repo,
  68. build_node: None,
  69. force_build: false,
  70. force: req.force,
  71. solo: req.solo,
  72. mirror: req.mirror,
  73. hf_cache: None,
  74. stage: None,
  75. fit: req.fit,
  76. cold: false,
  77. clear: req.clear,
  78. };
  79. let recipe = req.recipe.clone();
  80. let cluster = req.cluster.clone();
  81. let node = req.node.clone();
  82. super::jobs::spawn(app2, id.clone(), async move {
  83. crate::handle_serve(&config, &recipe, &cluster, node, opts, false, false).await
  84. });
  85. Ok(Json(serde_json::json!({ "job": id })))
  86. }
  87. #[derive(Deserialize)]
  88. pub(crate) struct StopReq {
  89. pub cluster: String,
  90. pub recipe: String,
  91. #[serde(default)]
  92. pub node: Option<String>,
  93. }
  94. /// `POST /api/stop` — stops the recipe's containers (auto-discovers the
  95. /// cluster when the recipe is running exactly once), as a tracked job.
  96. pub(crate) async fn stop(
  97. app: State<AppState>,
  98. Json(req): Json<StopReq>,
  99. ) -> Result<Json<serde_json::Value>, ApiError> {
  100. let id = super::jobs::new_job(&app, "stop", &req.cluster, Some(&req.recipe))?;
  101. let config = app.config.clone();
  102. let app2 = (*app).clone();
  103. let recipe = req.recipe.clone();
  104. let cluster = req.cluster.clone();
  105. let node = req.node.clone();
  106. super::jobs::spawn(app2, id.clone(), async move {
  107. let nodes = vec![node.unwrap_or_default()];
  108. let nodes = if nodes[0].is_empty() { vec![] } else { nodes };
  109. crate::docker::stop::handle(&config, &recipe, &cluster, &nodes, true, false, false).await
  110. });
  111. Ok(Json(serde_json::json!({ "job": id })))
  112. }
  113. #[derive(Deserialize)]
  114. pub(crate) struct PowerReq {
  115. /// "off" | "on" | "switch"
  116. pub action: String,
  117. pub cluster: String,
  118. /// switch target (required only for `switch`)
  119. #[serde(default)]
  120. pub to: Option<String>,
  121. }
  122. /// `POST /api/power` — HA power actions as a tracked job. The UI enforces
  123. /// the type-the-name confirm; this endpoint trusts the caller (it is
  124. /// token-gated / loopback-bound).
  125. pub(crate) async fn power(
  126. app: State<AppState>,
  127. Json(req): Json<PowerReq>,
  128. ) -> Result<Json<serde_json::Value>, ApiError> {
  129. let id = super::jobs::new_job(&app, "power", &req.cluster, None)?;
  130. let config = app.config.clone();
  131. let app2 = (*app).clone();
  132. let action = req.action.clone();
  133. let cluster = req.cluster.clone();
  134. let to = req.to.clone();
  135. let stale = stale_from_snapshot(&app, &req.cluster);
  136. super::jobs::spawn_report(app2, id.clone(), async move {
  137. match action.as_str() {
  138. "off" => {
  139. let c = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
  140. crate::power::power_off(&config, c, &[], false).await?;
  141. Ok(Vec::new())
  142. }
  143. "on" => {
  144. let c = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
  145. // nodes the last probe saw down: their plugs may be stale-on
  146. // (a graceful shutdown leaves the plug on), so power on
  147. // cycles those instead of issuing a no-op turn_on
  148. crate::power::power_on(&config, c, &[], false, &stale).await
  149. }
  150. "switch" => {
  151. let to = to
  152. .as_deref()
  153. .ok_or_else(|| anyhow::anyhow!("`to` cluster is required for power switch"))?;
  154. let from = crate::config::resolve_cluster_name(&config.clusters, &cluster)?;
  155. let to = crate::config::resolve_cluster_name(&config.clusters, to)?;
  156. crate::power::power_switch(&config, from, to, false).await?;
  157. Ok(Vec::new())
  158. }
  159. other => Err(anyhow::anyhow!(
  160. "unknown power action '{other}' (off|on|switch)"
  161. )),
  162. }
  163. });
  164. Ok(Json(serde_json::json!({ "job": id })))
  165. }
  166. /// Nodes of `cluster` that the last fleet probe saw unreachable — the
  167. /// `power on` cycle list (a plug left on by a graceful shutdown needs a
  168. /// cycle, not a no-op `turn_on`).
  169. pub(crate) fn stale_from_snapshot(app: &AppState, cluster: &str) -> Vec<String> {
  170. let snap = app
  171. .last
  172. .lock()
  173. .unwrap_or_else(std::sync::PoisonError::into_inner)
  174. .clone();
  175. let Some(snap) = snap else { return Vec::new() };
  176. let mut out = Vec::new();
  177. for c in &snap.clusters {
  178. if c.name != cluster {
  179. continue;
  180. }
  181. for n in &c.nodes {
  182. if !n.reachable {
  183. out.push(n.name.clone());
  184. }
  185. }
  186. }
  187. out
  188. }
  189. /// `GET /api/recipes` — all discoverable recipes (cwd + `recipe_sources` +
  190. /// the global config dir), with deploy target + node requirements.
  191. pub(crate) async fn recipes(app: State<AppState>) -> Json<serde_json::Value> {
  192. let dirs = crate::config::recipe_dirs(&app.config);
  193. let mut out = Vec::new();
  194. for dir in dirs {
  195. let mut entries = match tokio::fs::read_dir(&dir).await {
  196. Ok(e) => e,
  197. Err(_) => continue,
  198. };
  199. loop {
  200. let Ok(Some(entry)) = entries.next_entry().await else {
  201. break;
  202. };
  203. let path = entry.path();
  204. if !path.is_dir() {
  205. continue;
  206. }
  207. let Some(name) = entry.file_name().to_str().map(String::from) else {
  208. continue;
  209. };
  210. let file = path.join("recipe.toml");
  211. let Ok(toml_str) = tokio::fs::read_to_string(&file).await else {
  212. continue;
  213. };
  214. match toml::from_str::<crate::config::Recipe>(&toml_str) {
  215. Ok(r) => out.push(serde_json::json!({
  216. "name": name,
  217. "engine": r.engine,
  218. "deploy": format!("{:?}", r.deploy).to_lowercase(),
  219. "required_nodes": r.required_nodes,
  220. })),
  221. Err(e) => tracing::debug!("skipping recipe {}: {e}", name),
  222. }
  223. }
  224. }
  225. Json(serde_json::json!({ "recipes": out }))
  226. }
  227. // ---------------------------------------------------------------------------
  228. // P3: log windows — attach a streaming `docker logs -f` per node+recipe+rank
  229. // (idempotent), detach, and fetch ring history for a fresh window.
  230. #[derive(Deserialize)]
  231. pub(crate) struct LogsReq {
  232. pub cluster: String,
  233. pub node: String,
  234. pub recipe: String,
  235. #[serde(default)]
  236. pub rank: u32,
  237. #[serde(default)]
  238. pub lines: usize,
  239. }
  240. /// `POST /api/logs` — attach (reuses a live stream for the same key).
  241. pub(crate) async fn logs_attach(
  242. app: State<AppState>,
  243. Json(req): Json<LogsReq>,
  244. ) -> Result<Json<serde_json::Value>, ApiError> {
  245. let (id, history) = super::logs::attach(
  246. &app,
  247. &app.log_attaches,
  248. &req.cluster,
  249. &req.node,
  250. &req.recipe,
  251. req.rank,
  252. )
  253. .map_err(|e| {
  254. (
  255. if e.starts_with("max") {
  256. StatusCode::CONFLICT
  257. } else {
  258. StatusCode::BAD_REQUEST
  259. },
  260. e,
  261. )
  262. })?;
  263. Ok(Json(
  264. serde_json::json!({ "attach_id": id, "history": history }),
  265. ))
  266. }
  267. /// `DELETE /api/logs` — stop the stream and drop the attach.
  268. #[derive(Deserialize)]
  269. pub(crate) struct DetachReq {
  270. pub attach_id: String,
  271. }
  272. pub(crate) async fn logs_detach(
  273. app: State<AppState>,
  274. Json(req): Json<DetachReq>,
  275. ) -> Result<StatusCode, ApiError> {
  276. let id = req.attach_id;
  277. if super::logs::detach(&app.log_attaches, &id) {
  278. Ok(StatusCode::NO_CONTENT)
  279. } else {
  280. Err((StatusCode::NOT_FOUND, format!("no log attach '{id}'")))
  281. }
  282. }
  283. /// `GET /api/logs?cluster=&node=&recipe=&rank=&lines=` — ring history for a
  284. /// fresh window (0 lines = `logs::TAIL_LINES`).
  285. pub(crate) async fn logs_history(
  286. app: State<AppState>,
  287. axum::extract::Query(q): axum::extract::Query<LogsReq>,
  288. ) -> Result<Json<serde_json::Value>, ApiError> {
  289. let lines = if q.lines == 0 {
  290. super::logs::TAIL_LINES
  291. } else {
  292. q.lines
  293. };
  294. let (id, history) = super::logs::by_key(
  295. &app.log_attaches,
  296. &q.cluster,
  297. &q.node,
  298. &q.recipe,
  299. q.rank,
  300. lines,
  301. )
  302. .ok_or_else(|| {
  303. (
  304. StatusCode::NOT_FOUND,
  305. "no log attach for that node/recipe".to_string(),
  306. )
  307. })?;
  308. Ok(Json(
  309. serde_json::json!({ "attach_id": id, "history": history }),
  310. ))
  311. }
  312. // ---------------------------------------------------------------------------
  313. // P4: raw content — the active config file, a recipe verbatim, and the
  314. // merged effective config (post cluster-source merge) as JSON.
  315. /// `GET /api/raw/config` — the active config file, verbatim.
  316. pub(crate) async fn raw_config() -> Result<axum::response::Response, ApiError> {
  317. let path = crate::gconfig::resolve_config_path("config.toml");
  318. let content = tokio::fs::read_to_string(&path).await.map_err(|e| {
  319. (
  320. StatusCode::NOT_FOUND,
  321. format!("no config file at {}: {e}", path.display()),
  322. )
  323. })?;
  324. Ok(axum::response::Response::builder()
  325. .status(200)
  326. .header("content-type", "text/plain; charset=utf-8")
  327. .body(axum::body::Body::from(content))
  328. .unwrap())
  329. }
  330. /// `GET /api/raw/recipe/<name>` — recipe.toml verbatim (search order as
  331. /// `load_recipe`: cwd, recipe_sources, global dir).
  332. pub(crate) async fn raw_recipe(
  333. app: State<AppState>,
  334. axum::extract::Path(name): axum::extract::Path<String>,
  335. ) -> Result<axum::response::Response, ApiError> {
  336. if name.is_empty() || name.contains('/') || name.contains("..") {
  337. return Err((StatusCode::BAD_REQUEST, "invalid recipe name".to_string()));
  338. }
  339. let dirs = crate::config::recipe_dirs(&app.config);
  340. for dir in &dirs {
  341. let p = dir.join(&name).join("recipe.toml");
  342. if let Ok(content) = tokio::fs::read_to_string(&p).await {
  343. return Ok(axum::response::Response::builder()
  344. .status(200)
  345. .header("content-type", "text/plain; charset=utf-8")
  346. .body(axum::body::Body::from(content))
  347. .unwrap());
  348. }
  349. }
  350. Err((StatusCode::NOT_FOUND, format!("recipe '{name}' not found")))
  351. }
  352. /// `GET /api/raw/effective` — the merged `Config` as JSON (post
  353. /// cluster-source merge), HA token redacted.
  354. pub(crate) async fn raw_effective(app: State<AppState>) -> Json<serde_json::Value> {
  355. use crate::config::NcclConfig;
  356. let ha = app.config.home_assistant.as_ref().map(|h| {
  357. let masked: String = h.token.chars().take(12).collect();
  358. let token = if h.token.len() > 12 {
  359. format!("{masked}…<redacted>")
  360. } else {
  361. "<redacted>".to_string()
  362. };
  363. serde_json::json!({
  364. "url": h.url,
  365. "token": token,
  366. "shutdown_delay_seconds": h.shutdown_delay_seconds,
  367. })
  368. });
  369. Json(serde_json::json!({
  370. "config_path": crate::gconfig::resolve_config_path("config.toml").display().to_string(),
  371. "ssh_user": app.config.ssh_user,
  372. "ssh_key_path": app.config.ssh_key_path.display().to_string(),
  373. "clusters": app.config.clusters.iter().map(|c| serde_json::json!({
  374. "name": c.name,
  375. "nodes": c.nodes.iter().map(|n| serde_json::json!({
  376. "name": n.name,
  377. "ssh_ip": n.ssh_ip,
  378. "rank": n.rank,
  379. "entity": n.entity,
  380. "host_key_set": n.host_key.is_some(),
  381. })).collect::<Vec<_>>(),
  382. "extra_entities": c.extra_entities.iter().map(|e| serde_json::json!({
  383. "entity": e.entity(),
  384. "blocks": e.blocks(),
  385. })).collect::<Vec<_>>(),
  386. "nccl": app.config.nccl.get(&c.name).map(|n: &NcclConfig| serde_json::json!({
  387. "ib_hca": n.ib_hca,
  388. "socket_ifname": n.socket_ifname,
  389. "ib_gid_index": n.ib_gid_index,
  390. "ib_disable": n.ib_disable,
  391. })),
  392. "before_serve": app.config.before_serve.get(&c.name).map(|b| b.commands.len()),
  393. })).collect::<Vec<_>>(),
  394. "home_assistant": ha,
  395. "init_port": app.config.init_port,
  396. "mount_aliases": app.config.mount_aliases,
  397. "recipe_sources": app.config.recipe_sources,
  398. "cluster_sources": app.config.cluster_sources,
  399. "drop_caches": app.config.drop_caches,
  400. }))
  401. }