Better bevy version/license algorithm for asset generation

generate-assets/Cargo.toml

@@ -11,15 +11,14 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-toml = "0.5"
+toml = "0.7"
 serde = { version = "1", features = ["derive"] }
 rand = "0.8"
 regex = "1"
-cargo_toml = "0.11.5"
+cargo_toml = "0.15"
 url = "2.2.2"
 anyhow = "1.0.58"
 base64 = "0.13.0"
-cratesio-dbdump-lookup = "0.1.1"
 cratesio-dbdump-csvtab = "0.2.2"
 ureq = { version = "2.5.0", features = ["json"] }
 dotenv = "0.15.0"

generate-assets/src/bin/generate.rs

@@ -40,9 +40,12 @@ fn main() -> anyhow::Result<()> {
     let _ = fs::create_dir(content_dir.clone());
     let asset_root_section = parse_assets(
         &asset_dir,
-        Some(&db),
-        github_client.as_ref(),
-        gitlab_client.as_ref(),
+        MetadataSource {
+            crates_io_db: Some(&db),
+            github_client: github_client.as_ref(),
+            gitlab_client: gitlab_client.as_ref(),
+            ..Default::default()
+        },
     )?;
 
     asset_root_section

generate-assets/src/bin/validate.rs

@@ -15,7 +15,7 @@ fn main() -> Result<()> {
         .ok_or_else(|| anyhow!("Please specify the path to bevy-assets"))?;
 
     let asset_root_section =
-        parse_assets(&asset_dir, None, None, None).with_context(|| "Parsing assets")?;
+        parse_assets(&asset_dir, MetadataSource::default()).with_context(|| "Parsing assets")?;
 
     let results = asset_root_section.validate();
 

generate-assets/src/github_client.rs

@@ -1,77 +1,135 @@
-use anyhow::bail;
-use serde::Deserialize;
-
-const BASE_URL: &str = "https://api.github.com";
-
-#[derive(Deserialize)]
-struct GithubContentResponse {
-    encoding: String,
-    content: String,
-}
-
-#[derive(Deserialize)]
-struct GithubLicenseResponse {
-    license: GithubLicenseLicense,
-}
-
-#[derive(Deserialize)]
-struct GithubLicenseLicense {
-    spdx_id: String,
-}
-
-pub struct GithubClient {
-    agent: ureq::Agent,
-    token: String,
-}
-
-impl GithubClient {
-    pub fn new(token: String) -> Self {
-        let agent: ureq::Agent = ureq::AgentBuilder::new()
-            .user_agent("bevy-website-generate-assets")
-            .build();
-
-        Self { agent, token }
-    }
-
-    /// Gets the content of a file from a github repo
-    pub fn get_content(
-        &self,
-        username: &str,
-        repository_name: &str,
-        content_path: &str,
-    ) -> anyhow::Result<String> {
-        let response: GithubContentResponse = self
-            .agent
-            .get(&format!(
-                "{BASE_URL}/repos/{username}/{repository_name}/contents/{content_path}"
-            ))
-            .set("Accept", "application/json")
-            .set("Authorization", &format!("Bearer {}", self.token))
-            .call()?
-            .into_json()?;
-
-        if response.encoding == "base64" {
-            let data = base64::decode(response.content.replace('\n', "").trim())?;
-            Ok(String::from_utf8(data)?)
-        } else {
-            bail!("Content is not in base64");
-        }
-    }
-
-    /// Gets the license from a github repo
-    /// Technically, github supports multiple licenses, but the api only returns one
-    #[allow(unused)]
-    pub fn get_license(&self, username: &str, repository_name: &str) -> anyhow::Result<String> {
-        let response: GithubLicenseResponse = self
-            .agent
-            .get(&format!(
-                "{BASE_URL}/repos/{username}/{repository_name}/license"
-            ))
-            .set("Accept", "application/json")
-            .set("Authorization", &format!("Bearer {}", self.token))
-            .call()?
-            .into_json()?;
-
-        Ok(response.license.spdx_id)
-    }
-}
+use anyhow::bail;
+use serde::Deserialize;
+
+const BASE_URL: &str = "https://api.github.com";
+
+#[derive(Deserialize, Debug)]
+struct GithubContentResponse {
+    encoding: String,
+    content: String,
+}
+
+#[derive(Deserialize)]
+struct GithubLicenseResponse {
+    license: GithubLicenseLicense,
+}
+
+#[derive(Deserialize)]
+struct GithubLicenseLicense {
+    spdx_id: String,
+}
+
+#[derive(Deserialize, Debug)]
+struct GithubSearchFile {
+    total_count: u32,
+    incomplete_results: bool,
+    items: Vec<GithubSearchFileItem>,
+}
+
+#[derive(Deserialize, Debug)]
+struct GithubSearchFileItem {
+    path: std::path::PathBuf,
+}
+
+pub struct GithubClient {
+    agent: ureq::Agent,
+    token: String,
+}
+
+impl GithubClient {
+    pub fn new(token: String) -> Self {
+        let agent: ureq::Agent = ureq::AgentBuilder::new()
+            .user_agent("bevy-website-generate-assets")
+            .build();
+
+        Self { agent, token }
+    }
+
+    /// Gets the content of a file from a github repo
+    pub fn get_content(
+        &self,
+        username: &str,
+        repository_name: &str,
+        content_path: &str,
+    ) -> anyhow::Result<String> {
+        let response: GithubContentResponse = self
+            .agent
+            .get(&format!(
+                "{BASE_URL}/repos/{username}/{repository_name}/contents/{content_path}"
+            ))
+            .set("Accept", "application/json")
+            .set("Authorization", &format!("Bearer {}", self.token))
+            .call()?
+            .into_json()?;
+
+        if response.encoding == "base64" {
+            let data = base64::decode(response.content.replace('\n', "").trim())?;
+            Ok(String::from_utf8(data)?)
+        } else {
+            bail!("Content is not in base64");
+        }
+    }
+
+    /// Gets the license from a github repo
+    /// Technically, github supports multiple licenses, but the api only returns one
+    #[allow(unused)]
+    pub fn get_license(&self, username: &str, repository_name: &str) -> anyhow::Result<String> {
+        let response: GithubLicenseResponse = self
+            .agent
+            .get(&format!(
+                "{BASE_URL}/repos/{username}/{repository_name}/license"
+            ))
+            .set("Accept", "application/json")
+            .set("Authorization", &format!("Bearer {}", self.token))
+            .call()?
+            .into_json()?;
+
+        let license = response.license.spdx_id;
+
+        if license != "NOASSERTION" {
+            Ok(license)
+        } else {
+            bail!("No spdx license assertion")
+        }
+    }
+
+    /// Search file by name
+    pub fn search_file(
+        &self,
+        username: &str,
+        repository_name: &str,
+        file_name: &str,
+    ) -> anyhow::Result<Vec<String>> {
+        let response: GithubSearchFile = self
+            .agent
+            .get(&format!(
+                "{BASE_URL}/search/code?q=repo:{username}/{repository_name}+filename:{file_name}"
+            ))
+            .set("Accept", "application/json")
+            .set("Authorization", &format!("Bearer {}", self.token))
+            .call()?
+            .into_json()?;
+
+        if response.incomplete_results {
+            println!(
+                "Too many {} files in repository, checking only the first {} ones.",
+                file_name, response.total_count,
+            );
+        }
+
+        let paths = response
+            .items
+            .iter()
+            .filter_map(|i| {
+                if let Some(path_string) = i.path.to_str() {
+                    Some(path_string.to_string())
+                } else {
+                    println!("Path.to_str failed for {}", i.path.to_string_lossy());
+                    None
+                }
+            })
+            .collect();
+
+        Ok(paths)
+    }
+}