issue-620 user agent is required

crates/web/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "web"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = "1.0.94"
+mime = "0.3.17"
+regex = "1.11.1"
+reqwest = { version = "0.12.9", features = ["blocking", "json", "stream"] }
+select = "0.6.0"
+serde = { version = "1.0.215", features = ["derive"] }
+tempfile = "3.14.0"
+tokio = { version = "1.42.0", features = ["full"] }
+url = "2.5.4"

crates/web/src/lib.rs

@@ -0,0 +1,15 @@
+mod paginated;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_reverse_dependencies() -> Result<()> {
+        for dep in paginated::ReverseDependencies::of("serde")?.take(5) {
+            let dependency = dep?;
+            println!("{} depends on {}", dependency.id, dependency.crate_id);
+        }
+        Ok(())
+    }
+}

crates/web/src/paginated.rs

@@ -0,0 +1,79 @@
+use reqwest::Result;
+use reqwest::header::USER_AGENT;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+struct ApiResponse {
+    dependencies: Vec<Dependency>,
+    meta: Meta,
+}
+
+#[derive(Deserialize)]
+pub struct Dependency {
+    pub crate_id: String,
+    pub id: u32,
+}
+
+#[derive(Deserialize)]
+struct Meta {
+    total: u32,
+}
+
+pub struct ReverseDependencies {
+    crate_id: String,
+    dependencies: <Vec<Dependency> as IntoIterator>::IntoIter,
+    client: reqwest::blocking::Client,
+    page: u32,
+    per_page: u32,
+    total: u32,
+}
+
+impl ReverseDependencies {
+    pub fn of(crate_id: &str) -> Result<Self> {
+        Ok(ReverseDependencies {
+               crate_id: crate_id.to_owned(),
+               dependencies: vec![].into_iter(),
+               client: reqwest::blocking::Client::new(),
+               page: 0,
+               per_page: 100,
+               total: 0,
+           })
+    }
+
+    fn try_next(&mut self) -> Result<Option<Dependency>> {
+        if let Some(dep) = self.dependencies.next() {
+            return Ok(Some(dep));
+        }
+
+        if self.page > 0 && self.page * self.per_page >= self.total {
+            return Ok(None);
+        }
+
+        self.page += 1;
+        let url = format!("https://crates.io/api/v1/crates/{}/reverse_dependencies?page={}&per_page={}",
+                          self.crate_id,
+                          self.page,
+                          self.per_page);
+        println!("{}", url);
+
+        let response = self.client.get(&url).header(
+                   USER_AGENT,
+                   "cookbook agent",
+               ).send()?.json::<ApiResponse>()?;
+        self.dependencies = response.dependencies.into_iter();
+        self.total = response.meta.total;
+        Ok(self.dependencies.next())
+    }
+}
+
+impl Iterator for ReverseDependencies {
+    type Item = Result<Dependency>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.try_next() {
+            Ok(Some(dep)) => Some(Ok(dep)),
+            Ok(None) => None,
+            Err(err) => Some(Err(err)),
+        }
+    }
+}

src/web/clients/api/paginated.md

@@ -3,87 +3,22 @@
 [![reqwest-badge]][reqwest] [![serde-badge]][serde] [![cat-net-badge]][cat-net] [![cat-encoding-badge]][cat-encoding]
 
 Wraps a paginated web API in a convenient Rust iterator. The iterator lazily
-fetches the next page of results from the remote server as it arrives at the end
-of each page.
+fetches the next page of results from the remote server as it arrives at the end of each page.
 
-```rust,edition2018,no_run
-use reqwest::Result;
-use serde::Deserialize;
-
-#[derive(Deserialize)]
-struct ApiResponse {
-    dependencies: Vec<Dependency>,
-    meta: Meta,
-}
-
-#[derive(Deserialize)]
-struct Dependency {
-    crate_id: String,
-}
-
-#[derive(Deserialize)]
-struct Meta {
-    total: u32,
-}
-
-struct ReverseDependencies {
-    crate_id: String,
-    dependencies: <Vec<Dependency> as IntoIterator>::IntoIter,
-    client: reqwest::blocking::Client,
-    page: u32,
-    per_page: u32,
-    total: u32,
-}
-
-impl ReverseDependencies {
-    fn of(crate_id: &str) -> Result<Self> {
-        Ok(ReverseDependencies {
-               crate_id: crate_id.to_owned(),
-               dependencies: vec![].into_iter(),
-               client: reqwest::blocking::Client::new(),
-               page: 0,
-               per_page: 100,
-               total: 0,
-           })
-    }
-
-    fn try_next(&mut self) -> Result<Option<Dependency>> {
-        if let Some(dep) = self.dependencies.next() {
-            return Ok(Some(dep));
-        }
-
-        if self.page > 0 && self.page * self.per_page >= self.total {
-            return Ok(None);
-        }
-
-        self.page += 1;
-        let url = format!("https://crates.io/api/v1/crates/{}/reverse_dependencies?page={}&per_page={}",
-                          self.crate_id,
-                          self.page,
-                          self.per_page);
-
-        let response = self.client.get(&url).send()?.json::<ApiResponse>()?;
-        self.dependencies = response.dependencies.into_iter();
-        self.total = response.meta.total;
-        Ok(self.dependencies.next())
-    }
-}
+This file is named paginated.rs.
+```rust,edition2024,no_run
+{{#include ../../../../crates/web/src/paginated.rs}}
+```
 
-impl Iterator for ReverseDependencies {
-    type Item = Result<Dependency>;
+In this main we import the paginated file and use the `ReverseDependencies` iterator to fetch all the crates that depend on the `serde` crate.  This file is named main.rs.  The two files are in the same directory.
 
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.try_next() {
-            Ok(Some(dep)) => Some(Ok(dep)),
-            Ok(None) => None,
-            Err(err) => Some(Err(err)),
-        }
-    }
-}
+```rust,no_run
+mod paginated;
 
 fn main() -> Result<()> {
-    for dep in ReverseDependencies::of("serde")? {
-        println!("reverse dependency: {}", dep?.crate_id);
+    for dep in paginated::ReverseDependencies::of("serde")? {
+        let dependency = dep?;
+        println!("{} depends on {}", dependency.id, dependency.crate_id);
     }
     Ok(())
 }