🐛 Update old repl fetch function

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "replit-takeout"
-version = "1.7.8"
+version = "1.7.10"
 edition = "2021"
 authors = ["Ben Dixon <[email protected]>"]
 

examples/repls.rs

@@ -3,7 +3,7 @@ use std::collections::HashMap;
 use anyhow::Result;
 use dotenv::var;
 use log::error;
-use replit_takeout::replit::repls::Repl;
+use replit_takeout::{replit::repls::Repl, replit_graphql::ProfileRepls};
 
 #[tokio::main]
 async fn main() -> Result<()> {
@@ -12,18 +12,29 @@ async fn main() -> Result<()> {
 
     let token = var("REPLIT_TEST_TOKEN")?;
 
-    let repls = Repl::fetch(&token, None).await.expect("some repls");
-    error!("got {} repls", repls.len());
+    //#region New method
+    // {
+    //     let repls = Repl::fetch(&token, None).await?;
+    //     error!("got {} repls", repls.len());
 
-    let mut map: HashMap<String, Repl> = HashMap::new();
+    //     let mut map: HashMap<String, Repl> = HashMap::new();
 
-    for repl in repls {
-        if map.contains_key(&repl.id) {
-            log::error!("ALREADY CONTAINS {:?}", repl.clone());
-        }
+    //     for repl in repls {
+    //         if map.contains_key(&repl.id) {
+    //             log::error!("ALREADY CONTAINS {:?}", repl.clone());
+    //         }
 
-        map.insert(repl.id.clone(), repl);
+    //         map.insert(repl.id.clone(), repl);
+    //     }
+    // }
+    //#endregion
+
+    //#region Old, fixed method
+    {
+        let repls = ProfileRepls::fetch(&token, 222834, None).await?;
+        println!("{:#?}", repls.len());
     }
+    //#endregion
 
     Ok(())
 }

src/graphql/profilerepls-query.graphql

@@ -1,24 +1,24 @@
-query ProfileRepls($after: String, $id: Int!) {
-	user(id: $id) {
-		profileRepls(after: $after, count: 100) {
-			# Max per page is 25
-			items {
-				id
-				slug
-				title
-				url
-				description
-				isRenamed
-				isAlwaysOn
-				isProjectFork
-				likeCount
-				language
-				timeCreated
-			}
-			pageInfo {
-				hasNextPage
-				nextCursor
-			}
-		}
-	}
+query ProfileRepls($after: String, $user_id: Int!) {
+    user(id: $user_id) {
+        profileRepls(after: $after, count: 100) {
+            # Max per page is 25
+            items {
+                id
+                slug
+                title
+                url
+                description
+                isRenamed
+                isAlwaysOn
+                isProjectFork
+                likeCount
+                language
+                timeCreated
+            }
+            pageInfo {
+                hasNextPage
+                nextCursor
+            }
+        }
+    }
 }

src/replit_graphql.rs

@@ -10,7 +10,7 @@ use reqwest::{
 use std::sync::Arc;
 use std::time::Duration;
 use time::OffsetDateTime;
-use tokio::fs;
+use tokio::{fs, time::sleep};
 
 use serde::{Deserialize, Serialize};
 
@@ -118,58 +118,76 @@ type DateTime = String;
 )]
 pub struct ProfileRepls;
 impl ProfileRepls {
-    // /// Get one page of repls.
-    // #[deprecated]
-    // async fn fetch(
-    //     token: &String,
-    //     id: i64,
-    //     client_opt: Option<Client>,
-    //     after: Option<String>,
-    // ) -> Result<(
-    //     Vec<profile_repls::ProfileReplsUserProfileReplsItems>,
-    //     Option<String>,
-    // )> {
-    //     let client = create_client(token, client_opt)?;
-
-    //     let repls_query = ProfileRepls::build_query(profile_repls::Variables { id, after });
-
-    //     let repls_data: String = client
-    //         .post(REPLIT_GQL_URL)
-    //         .json(&repls_query)
-    //         .send()
-    //         .await?
-    //         .text()
-    //         .await?;
-    //     debug!(
-    //         "{}:{} Raw text repl data: {repls_data}",
-    //         std::line!(),
-    //         std::column!()
-    //     );
-
-    //     let repls_data_result =
-    //         match serde_json::from_str::<Response<profile_repls::ResponseData>>(&repls_data) {
-    //             Ok(data) => data.data,
-    //             Err(e) => {
-    //                 error!("Failed to deserialize JSON: {}", e);
-    //                 return Err(anyhow::Error::new(e));
-    //             }
-    //         };
-
-    //     let next_page = repls_data_result
-    //         .as_ref()
-    //         .and_then(|data| {
-    //             data.user
-    //                 .as_ref()
-    //                 .map(|user| user.profile_repls.page_info.next_cursor.clone())
-    //         })
-    //         .ok_or(anyhow::Error::msg("Page Info not found during download"))?;
-
-    //     let repls = repls_data_result
-    //         .and_then(|data| data.user.map(|user| user.profile_repls.items))
-    //         .ok_or(anyhow::Error::msg("Repls not found during download"))?;
-
-    //     Ok((repls, next_page))
-    // }
+    /// Get one page of repls.
+    #[deprecated]
+    pub async fn fetch(
+        token: &String,
+        user_id: i64,
+        client_opt: Option<Client>,
+    ) -> Result<Vec<profile_repls::ProfileReplsUserProfileReplsItems>> {
+        let mut all_repls = Vec::new();
+        let mut after = None;
+        let client = create_client(token, client_opt)?;
+
+        loop {
+            let (repls, next_page) = Self::fetch_page(&client, user_id, after.clone()).await?;
+            all_repls.extend(repls);
+
+            if let Some(next_cursor) = next_page {
+                after = Some(next_cursor);
+                // Add a small delay between requests to avoid rate limiting
+                sleep(Duration::from_millis(100)).await;
+            } else {
+                break;
+            }
+        }
+
+        info!("Fetched a total of {} repls", all_repls.len());
+        Ok(all_repls)
+    }
+
+    async fn fetch_page(
+        client: &Client,
+        user_id: i64,
+        after: Option<String>,
+    ) -> Result<(
+        Vec<profile_repls::ProfileReplsUserProfileReplsItems>,
+        Option<String>,
+    )> {
+        let repls_query = ProfileRepls::build_query(profile_repls::Variables { user_id, after });
+        let repls_data: String = client
+            .post(REPLIT_GQL_URL)
+            .json(&repls_query)
+            .send()
+            .await?
+            .text()
+            .await?;
+
+        debug!(
+            "{}:{} Raw text repl data: {repls_data}",
+            std::line!(),
+            std::column!()
+        );
+
+        let repls_data_result: Response<profile_repls::ResponseData> =
+            serde_json::from_str(&repls_data).map_err(|e| {
+                error!("Failed to deserialize JSON: {}", e);
+                anyhow::Error::new(e)
+            })?;
+
+        let data = repls_data_result
+            .data
+            .ok_or_else(|| anyhow::Error::msg("No data returned from API"))?;
+
+        let user = data
+            .user
+            .ok_or_else(|| anyhow::Error::msg("User data not found"))?;
+
+        let next_page = user.profile_repls.page_info.next_cursor;
+        let repls = user.profile_repls.items;
+
+        Ok((repls, next_page))
+    }
 
     pub async fn download(
         token: &String,