hf.rs 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195
  1. //! Hugging Face catalogue helpers: `sm121 hf search|card`.
  2. //!
  3. //! Read-only, straight against the public HF API — the point is to pick a
  4. //! model **and a revision** for a recipe without opening a browser: search by
  5. //! name, see the card (tags, size, gated/licence, last update) and list the
  6. //! revisions with their resolved commits, then take that `org/name@revision`
  7. //! into the recipe / `[flare].revision`.
  8. use anyhow::{Result, bail};
  9. fn client() -> Result<reqwest::Client> {
  10. Ok(reqwest::Client::builder()
  11. .timeout(std::time::Duration::from_secs(25))
  12. .user_agent("sm121")
  13. .build()?)
  14. }
  15. /// Token from the environment, so gated repos resolve.
  16. fn token() -> Option<String> {
  17. std::env::var("HF_TOKEN").ok().filter(|t| !t.is_empty())
  18. }
  19. async fn get_json(url: &str) -> Result<serde_json::Value> {
  20. let mut req = client()?.get(url);
  21. if let Some(t) = token() {
  22. req = req.bearer_auth(t);
  23. }
  24. let res = req.send().await?;
  25. if !res.status().is_success() {
  26. bail!("{url} → HTTP {}", res.status());
  27. }
  28. Ok(res.json().await?)
  29. }
  30. /// `sm121 hf search <query>`
  31. pub async fn search(query: &str, limit: usize, json: bool) -> Result<()> {
  32. let url = format!(
  33. "https://huggingface.co/api/models?search={}&limit={limit}&sort=downloads&direction=-1&full=true",
  34. urlencode(query)
  35. );
  36. let body = get_json(&url).await?;
  37. let items = body.as_array().cloned().unwrap_or_default();
  38. if json {
  39. println!("{}", serde_json::to_string_pretty(&body)?);
  40. return Ok(());
  41. }
  42. if items.is_empty() {
  43. println!("no models match '{query}'");
  44. return Ok(());
  45. }
  46. println!("{:<52} {:>9} {:>6} updated", "model", "downloads", "likes");
  47. for m in &items {
  48. let id = m.get("id").and_then(|v| v.as_str()).unwrap_or("?");
  49. let dl = m.get("downloads").and_then(|v| v.as_u64()).unwrap_or(0);
  50. let likes = m.get("likes").and_then(|v| v.as_u64()).unwrap_or(0);
  51. let upd = m
  52. .get("lastModified")
  53. .and_then(|v| v.as_str())
  54. .map(|s| s.get(0..10).unwrap_or(s))
  55. .unwrap_or("");
  56. println!("{id:<52} {dl:>9} {likes:>6} {upd}");
  57. }
  58. Ok(())
  59. }
  60. /// `sm121 hf card <repo> [--revision R]` — the card summary plus every
  61. /// revision (branch/tag) with its commit, so a recipe can pin one.
  62. pub async fn card(repo: &str, revision: Option<&str>, json: bool) -> Result<()> {
  63. let info = get_json(&format!(
  64. "https://huggingface.co/api/models/{repo}?full=true"
  65. ))
  66. .await?;
  67. let refs = get_json(&format!("https://huggingface.co/api/models/{repo}/refs")).await?;
  68. let files = match revision {
  69. Some(rev) => Some(
  70. get_json(&format!(
  71. "https://huggingface.co/api/models/{repo}/tree/{rev}?recursive=1"
  72. ))
  73. .await?,
  74. ),
  75. None => None,
  76. };
  77. if json {
  78. println!(
  79. "{}",
  80. serde_json::to_string_pretty(&serde_json::json!({
  81. "model": info, "refs": refs, "files": files,
  82. }))?
  83. );
  84. return Ok(());
  85. }
  86. let s = |k: &str| info.get(k).and_then(|v| v.as_str()).unwrap_or("");
  87. let n = |k: &str| info.get(k).and_then(|v| v.as_u64());
  88. println!(
  89. "{}",
  90. info.get("id").and_then(|v| v.as_str()).unwrap_or(repo)
  91. );
  92. if let Some(c) = info.get("cardData") {
  93. let lic = c.get("license").and_then(|v| v.as_str()).unwrap_or("—");
  94. let base = c
  95. .get("base_model")
  96. .and_then(|v| {
  97. v.as_str().map(str::to_string).or_else(|| {
  98. v.as_array().map(|a| {
  99. a.iter()
  100. .filter_map(|x| x.as_str())
  101. .collect::<Vec<_>>()
  102. .join(", ")
  103. })
  104. })
  105. })
  106. .unwrap_or_else(|| "—".into());
  107. println!(" licence: {lic} base: {base}");
  108. }
  109. if !s("pipeline_tag").is_empty() {
  110. println!(" task: {}", s("pipeline_tag"));
  111. }
  112. println!(
  113. " updated: {} downloads: {}",
  114. s("lastModified"),
  115. n("downloads").unwrap_or(0)
  116. );
  117. if info.get("gated").and_then(|v| v.as_bool()) == Some(true) {
  118. println!(" ⚠ gated repo — needs HF_TOKEN");
  119. }
  120. if let Some(tags) = info.get("tags").and_then(|v| v.as_array()) {
  121. let list: Vec<&str> = tags.iter().filter_map(|t| t.as_str()).take(12).collect();
  122. println!(" tags: {}", list.join(", "));
  123. }
  124. if let Some(siblings) = info.get("siblings").and_then(|v| v.as_array()) {
  125. println!(" files: {} (use --revision to list)", siblings.len());
  126. }
  127. if let Some(branches) = refs.get("branches").and_then(|v| v.as_array()) {
  128. println!("\nrevisions (branches):");
  129. for b in branches {
  130. println!(
  131. " {:<24} {}",
  132. b.get("name").and_then(|v| v.as_str()).unwrap_or("?"),
  133. b.get("targetCommit")
  134. .and_then(|v| v.as_str())
  135. .map(|c| c.get(0..12).unwrap_or(c))
  136. .unwrap_or("")
  137. );
  138. }
  139. }
  140. if let Some(tags) = refs.get("tags").and_then(|v| v.as_array()) {
  141. let named: Vec<String> = tags
  142. .iter()
  143. .filter_map(|t| t.get("name").and_then(|v| v.as_str()).map(str::to_string))
  144. .collect();
  145. if !named.is_empty() {
  146. println!("tags: {}", named.join(", "));
  147. }
  148. }
  149. if let Some(files) = files.and_then(|f| f.as_array().cloned()) {
  150. let mut total = 0u64;
  151. println!("\nfiles at {}:", revision.unwrap_or("main"));
  152. for f in files.iter().take(40) {
  153. let size = f.get("size").and_then(|v| v.as_u64()).unwrap_or(0);
  154. total += size;
  155. println!(
  156. " {:>10.2} GB {}",
  157. size as f64 / 1e9,
  158. f.get("path").and_then(|v| v.as_str()).unwrap_or("?")
  159. );
  160. }
  161. println!(" total ≈ {:.1} GB", total as f64 / 1e9);
  162. }
  163. Ok(())
  164. }
  165. fn urlencode(s: &str) -> String {
  166. s.bytes()
  167. .map(|b| match b {
  168. b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
  169. (b as char).to_string()
  170. }
  171. b' ' => "+".to_string(),
  172. _ => format!("%{b:02X}"),
  173. })
  174. .collect()
  175. }
  176. /// CLI dispatch.
  177. pub async fn dispatch(cmd: &crate::cli::HfCommands) -> Result<()> {
  178. use crate::cli::HfCommands as H;
  179. match cmd {
  180. H::Search { query, limit, json } => search(query, *limit, *json).await,
  181. H::Card {
  182. repo,
  183. revision,
  184. json,
  185. } => card(repo, revision.as_deref(), *json).await,
  186. }
  187. }