From d95f81ffd37e8223121e18dfda6a24c8005ff95e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9l=C3=A8ne=20Amanita?= Date: Wed, 14 Jun 2023 17:45:02 +0100 Subject: [PATCH 1/9] Better Bevy version algorithm for assets --- generate-assets/src/lib.rs | 160 +++++++++++++++++++++++++++++-------- 1 file changed, 127 insertions(+), 33 deletions(-) diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index b482782081..3fcaa84bda 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -10,6 +10,53 @@ pub mod gitlab_client; type CratesIoDb = cratesio_dbdump_csvtab::rusqlite::Connection; +const OFFICIAL_BEVY_CRATES: &'static [&'static str] = &[ + // Should be sorted + "bevy", + "bevy_a11y", + "bevy_animation", + "bevy_app", + "bevy_asset", + "bevy_audio", + "bevy_core", + "bevy_core_pipeline", + "bevy_derive", + "bevy_diagnostic", + "bevy_dylib", + "bevy_dynamic_plugin", + "bevy_ecs", + "bevy_ecs_compile_fail_tests", + "bevy_encase_derive", + "bevy_gilrs", + "bevy_gizmos", + "bevy_gltf", + "bevy_hierarchy", + "bevy_input", + "bevy_internal", + "bevy_log", + "bevy_macros_compile_fail_tests", + "bevy_macro_utils", + "bevy_math", + "bevy_mikktspace", + "bevy_pbr", + "bevy_ptr", + "bevy_reflect", + "bevy_reflect_compile_fail_tests", + "bevy_render", + "bevy_scene", + "bevy_sprite", + "bevy_tasks", + "bevy_text", + "bevy_time", + "bevy_transform", + "bevy_ui", + "bevy_utils", + "bevy_window", + "bevy_winit", +]; +const OFFICIAL_BEVY_CRATE_PREFIX_RANGE_START: &str = "bevy"; +const OFFICIAL_BEVY_CRATE_PREFIX_RANGE_END: &str = "bevz"; + #[derive(Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] pub struct Asset { @@ -268,28 +315,6 @@ fn get_metadata_from_gitlab( )) } -/// Gets the bevy version from the dependency list -/// Returns the version number if available. -/// If is is a git dependency, return either "main" or "git" for anything that isn't "main". -fn get_bevy_dependency_version(dep: &cargo_toml::Dependency) -> Option { - match dep { - cargo_toml::Dependency::Simple(version) => Some(version.to_string()), - cargo_toml::Dependency::Detailed(detail) => { - if let Some(version) = &detail.version { - Some(version.to_string()) - } else if detail.git.is_some() { - if detail.branch == Some(String::from("main")) { - Some(String::from("main")) - } else { - Some(String::from("git")) - } - } else { - None - } - } - } -} - /// Gets the license from a Cargo.toml file /// Tries to emulate crates.io behaviour fn get_license(cargo_manifest: &cargo_toml::Manifest) -> Option { @@ -310,19 +335,88 @@ fn get_license(cargo_manifest: &cargo_toml::Manifest) -> Option { } } -/// Find any dep that starts with bevy and get the version +/// Find any bevy dependency and get the corresponding bevy version from an asset's manifest /// This makes sure to handle all the bevy_* crates fn get_bevy_version(cargo_manifest: &cargo_toml::Manifest) -> Option { - cargo_manifest - .dependencies - .keys() - .find(|k| k.starts_with("bevy")) - .and_then(|key| { - cargo_manifest - .dependencies - .get(key) - .and_then(get_bevy_dependency_version) - }) + get_bevy_dependency(cargo_manifest) + .and_then(get_bevy_dependency_version) +} + +/// Find any bevy dependency from an asset's manifest +fn get_bevy_dependency(cargo_manifest: &cargo_toml::Manifest) -> Option<&cargo_toml::Dependency> { + let search_range = OFFICIAL_BEVY_CRATE_PREFIX_RANGE_START.to_owned()..OFFICIAL_BEVY_CRATE_PREFIX_RANGE_END.to_owned(); + + // Tries to find an official bevy crate from the asset's dependencies + let mut dependencies = cargo_manifest.dependencies.range(search_range.clone()); + let bevy_crates = OFFICIAL_BEVY_CRATES.iter(); + let mut bevy_dependency = search_bevy_in_dependencies(dependencies.clone(), bevy_crates.clone()); + + if bevy_dependency.is_none() { + // Tries to find an official bevy crate from the asset's dev dependencies + // An asset can indirectly depend on bevy through another crate, but would probably depend on bevy directly + // for its examples, benchmarks or tests, in its dev dependencies + let dev_dependencies = cargo_manifest.dev_dependencies.range(search_range); + bevy_dependency = search_bevy_in_dependencies(dev_dependencies, bevy_crates.clone()); + + if bevy_dependency.is_none() { + // If everything else fails, try to find any crate with a name starting with bevy + // This can happen if the asset depends only on past or future bevy sub-crates, + // or on third-party crates which name starts with bevy (might yield unaccurate results in that last case) + bevy_dependency = dependencies.next().map(|(_, d)| d); + } + } + + bevy_dependency +} + +/// Seach the first official bevy crate found in a collection of dependencies. +/// If it was a bit more generic, this function could be called find_first_intersect_in_sorted_iterators. +/// Both dependencies and bevy_crates are assumed to be sorted (by key for dependencies, they are in this context), +/// and we find the first element that intersect both of them using that knowledge +fn search_bevy_in_dependencies<'a>( + mut dependencies: std::collections::btree_map::Range<'a, String, cargo_toml::Dependency>, + mut bevy_crates: std::slice::Iter<&str> +) -> Option<&'a cargo_toml::Dependency> { + let mut dependency = dependencies.next(); + let mut bevy_crate = bevy_crates.next(); + + while !dependency.is_none() && !bevy_crate.is_none() { + let dependency_name = dependency.unwrap().0; + let bevy_crate_name = bevy_crate.unwrap(); + + if dependency_name.as_str() < *bevy_crate_name { + dependency = dependencies.next(); + } + else { + if dependency_name == bevy_crate_name { + return Some(dependency.unwrap().1); + } + bevy_crate = bevy_crates.next(); + } + } + return None; +} + +/// Gets the bevy version from the bevy dependency provided +/// Returns the version number if available. +/// If is is a git dependency, return either "main" or "git" for anything that isn't "main". +fn get_bevy_dependency_version(dep: &cargo_toml::Dependency) -> Option { + match dep { + cargo_toml::Dependency::Simple(version) => Some(version.to_string()), + cargo_toml::Dependency::Detailed(detail) => { + if let Some(version) = &detail.version { + Some(version.to_string()) + } else if detail.git.is_some() { + if detail.branch == Some(String::from("main")) { + Some(String::from("main")) + } else { + Some(String::from("git")) + } + } else { + None + } + } + } } /// Downloads the crates.io database dump and open a connection to the db From 384e6bb10e16eb09d9b9be0a1a7b03f5886d5946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9l=C3=A8ne=20Amanita?= Date: Wed, 14 Jun 2023 17:49:08 +0100 Subject: [PATCH 2/9] rustfmt --- generate-assets/src/lib.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index 3fcaa84bda..f2238adac9 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -338,23 +338,24 @@ fn get_license(cargo_manifest: &cargo_toml::Manifest) -> Option { /// Find any bevy dependency and get the corresponding bevy version from an asset's manifest /// This makes sure to handle all the bevy_* crates fn get_bevy_version(cargo_manifest: &cargo_toml::Manifest) -> Option { - get_bevy_dependency(cargo_manifest) - .and_then(get_bevy_dependency_version) + get_bevy_dependency(cargo_manifest).and_then(get_bevy_dependency_version) } /// Find any bevy dependency from an asset's manifest fn get_bevy_dependency(cargo_manifest: &cargo_toml::Manifest) -> Option<&cargo_toml::Dependency> { - let search_range = OFFICIAL_BEVY_CRATE_PREFIX_RANGE_START.to_owned()..OFFICIAL_BEVY_CRATE_PREFIX_RANGE_END.to_owned(); + let search_range = OFFICIAL_BEVY_CRATE_PREFIX_RANGE_START.to_owned() + ..OFFICIAL_BEVY_CRATE_PREFIX_RANGE_END.to_owned(); // Tries to find an official bevy crate from the asset's dependencies let mut dependencies = cargo_manifest.dependencies.range(search_range.clone()); let bevy_crates = OFFICIAL_BEVY_CRATES.iter(); - let mut bevy_dependency = search_bevy_in_dependencies(dependencies.clone(), bevy_crates.clone()); + let mut bevy_dependency = + search_bevy_in_dependencies(dependencies.clone(), bevy_crates.clone()); if bevy_dependency.is_none() { // Tries to find an official bevy crate from the asset's dev dependencies // An asset can indirectly depend on bevy through another crate, but would probably depend on bevy directly - // for its examples, benchmarks or tests, in its dev dependencies + // for its examples, benchmarks or tests, in its dev dependencies let dev_dependencies = cargo_manifest.dev_dependencies.range(search_range); bevy_dependency = search_bevy_in_dependencies(dev_dependencies, bevy_crates.clone()); @@ -375,7 +376,7 @@ fn get_bevy_dependency(cargo_manifest: &cargo_toml::Manifest) -> Option<&cargo_t /// and we find the first element that intersect both of them using that knowledge fn search_bevy_in_dependencies<'a>( mut dependencies: std::collections::btree_map::Range<'a, String, cargo_toml::Dependency>, - mut bevy_crates: std::slice::Iter<&str> + mut bevy_crates: std::slice::Iter<&str>, ) -> Option<&'a cargo_toml::Dependency> { let mut dependency = dependencies.next(); let mut bevy_crate = bevy_crates.next(); @@ -386,8 +387,7 @@ fn search_bevy_in_dependencies<'a>( if dependency_name.as_str() < *bevy_crate_name { dependency = dependencies.next(); - } - else { + } else { if dependency_name == bevy_crate_name { return Some(dependency.unwrap().1); } From dce0d550170c8da83db76974ad66edc99b2efc6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9l=C3=A8ne=20Amanita?= Date: Wed, 14 Jun 2023 22:24:44 +0100 Subject: [PATCH 3/9] Add tests and handle more cases --- generate-assets/Cargo.lock | 52 ++++++- generate-assets/Cargo.toml | 4 +- generate-assets/src/lib.rs | 272 +++++++++++++++++++++++++++++++++++-- 3 files changed, 306 insertions(+), 22 deletions(-) diff --git a/generate-assets/Cargo.lock b/generate-assets/Cargo.lock index 3f5151c063..91ce5249d8 100644 --- a/generate-assets/Cargo.lock +++ b/generate-assets/Cargo.lock @@ -137,12 +137,11 @@ dependencies = [ [[package]] name = "cargo_toml" -version = "0.11.8" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c3ff59e3b7d24630206bb63a73af65da4ed5df1f76ee84dfafb9fee2ba60e" +checksum = "599aa35200ffff8f04c1925aa1acc92fa2e08874379ef42e210a80e527e60838" dependencies = [ "serde", - "serde_derive", "toml", ] @@ -1088,6 +1087,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1267,11 +1275,36 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.10" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" +checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" dependencies = [ "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", ] [[package]] @@ -1588,6 +1621,15 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "winnow" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" diff --git a/generate-assets/Cargo.toml b/generate-assets/Cargo.toml index 1916dd682b..c5d56c4981 100644 --- a/generate-assets/Cargo.toml +++ b/generate-assets/Cargo.toml @@ -11,11 +11,11 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -toml = "0.5" +toml = "0.7" serde = { version = "1", features = ["derive"] } rand = "0.8" regex = "1" -cargo_toml = "0.11.5" +cargo_toml = "0.15" url = "2.2.2" anyhow = "1.0.58" base64 = "0.13.0" diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index f2238adac9..90cac35bb2 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -325,7 +325,7 @@ fn get_license(cargo_manifest: &cargo_toml::Manifest) -> Option { .. }) = &cargo_manifest.package { - if let Some(license) = license { + if let Some(cargo_toml::Inheritable::Set(license)) = license { Some(license.clone()) } else { license_file.as_ref().map(|_| String::from("non-standard")) @@ -338,11 +338,6 @@ fn get_license(cargo_manifest: &cargo_toml::Manifest) -> Option { /// Find any bevy dependency and get the corresponding bevy version from an asset's manifest /// This makes sure to handle all the bevy_* crates fn get_bevy_version(cargo_manifest: &cargo_toml::Manifest) -> Option { - get_bevy_dependency(cargo_manifest).and_then(get_bevy_dependency_version) -} - -/// Find any bevy dependency from an asset's manifest -fn get_bevy_dependency(cargo_manifest: &cargo_toml::Manifest) -> Option<&cargo_toml::Dependency> { let search_range = OFFICIAL_BEVY_CRATE_PREFIX_RANGE_START.to_owned() ..OFFICIAL_BEVY_CRATE_PREFIX_RANGE_END.to_owned(); @@ -356,28 +351,36 @@ fn get_bevy_dependency(cargo_manifest: &cargo_toml::Manifest) -> Option<&cargo_t // Tries to find an official bevy crate from the asset's dev dependencies // An asset can indirectly depend on bevy through another crate, but would probably depend on bevy directly // for its examples, benchmarks or tests, in its dev dependencies - let dev_dependencies = cargo_manifest.dev_dependencies.range(search_range); + let dev_dependencies = cargo_manifest.dev_dependencies.range(search_range.clone()); bevy_dependency = search_bevy_in_dependencies(dev_dependencies, bevy_crates.clone()); if bevy_dependency.is_none() { - // If everything else fails, try to find any crate with a name starting with bevy - // This can happen if the asset depends only on past or future bevy sub-crates, - // or on third-party crates which name starts with bevy (might yield unaccurate results in that last case) - bevy_dependency = dependencies.next().map(|(_, d)| d); + // Tries to find an official bevy crate from the asset's workspace dependencies + if let Some(ref workspace) = cargo_manifest.workspace { + let workspace_dependencies = workspace.dependencies.range(search_range); + bevy_dependency = search_bevy_in_dependencies(workspace_dependencies, bevy_crates); + } + + if bevy_dependency.is_none() { + // If everything else fails, try to find any crate with a name starting with bevy + // This can happen if the asset depends only on past or future bevy sub-crates, + // or on third-party crates which name starts with bevy (might yield unaccurate results in that last case) + bevy_dependency = dependencies.find_map(|(_, d)| get_bevy_dependency_version(d)); + } } } bevy_dependency } -/// Seach the first official bevy crate found in a collection of dependencies. +/// Seach the first official bevy crate found in a collection of dependencies and return its version. /// If it was a bit more generic, this function could be called find_first_intersect_in_sorted_iterators. /// Both dependencies and bevy_crates are assumed to be sorted (by key for dependencies, they are in this context), /// and we find the first element that intersect both of them using that knowledge fn search_bevy_in_dependencies<'a>( mut dependencies: std::collections::btree_map::Range<'a, String, cargo_toml::Dependency>, mut bevy_crates: std::slice::Iter<&str>, -) -> Option<&'a cargo_toml::Dependency> { +) -> Option { let mut dependency = dependencies.next(); let mut bevy_crate = bevy_crates.next(); @@ -389,9 +392,17 @@ fn search_bevy_in_dependencies<'a>( dependency = dependencies.next(); } else { if dependency_name == bevy_crate_name { - return Some(dependency.unwrap().1); + let dependency_version = get_bevy_dependency_version(dependency.unwrap().1); + if dependency_version.is_some() { + return dependency_version; + } else { + // In this case we found an official bevy crate but we couldn't get a version from it + dependency = dependencies.next(); + bevy_crate = bevy_crates.next(); + } + } else { + bevy_crate = bevy_crates.next(); } - bevy_crate = bevy_crates.next(); } } return None; @@ -416,6 +427,7 @@ fn get_bevy_dependency_version(dep: &cargo_toml::Dependency) -> Option { None } } + cargo_toml::Dependency::Inherited(_) => None, } } @@ -473,3 +485,233 @@ fn get_metadata_from_db_by_crate_name( bail!("Not found in crates.io db: {crate_name}") } } + +#[cfg(test)] +mod tests { + mod get_version { + use super::super::*; + + use cargo_toml::{Dependency, Manifest}; + use std::collections::BTreeMap; + + /* + fn get_manifest( + dependencies: Vec, + dependencies_separated: Vec, + dev_dependencies: Vec, + dev_dependencies_separated: Vec, + ) -> cargo_toml::Manifest { + toml::from_str::(&content)?; + } */ + + fn get_manifest( + dependencies: BTreeMap, + dev_dependencies: BTreeMap, + workspace_dependencies: BTreeMap, + ) -> Manifest { + #[allow(deprecated)] + Manifest { + package: Default::default(), + workspace: Some(cargo_toml::Workspace { + members: Default::default(), + package: Default::default(), + default_members: Default::default(), + exclude: Default::default(), + metadata: Default::default(), + resolver: Default::default(), + dependencies: workspace_dependencies, + }), + dependencies, + dev_dependencies, + build_dependencies: Default::default(), + target: Default::default(), + features: Default::default(), + replace: Default::default(), + patch: Default::default(), + lib: Default::default(), + profile: Default::default(), + badges: Default::default(), + bin: Default::default(), + bench: Default::default(), + test: Default::default(), + example: Default::default(), + } + } + + #[test] + fn from_no_dependency() { + let dependencies = BTreeMap::new(); + let dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, None); + } + + #[test] + fn from_other_dependencies() { + let mut dependencies = BTreeMap::new(); + let mut dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + dependencies.insert("other".to_string(), Dependency::Simple("0.10".to_string())); + dev_dependencies.insert("other".to_string(), Dependency::Simple("0.10".to_string())); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, None); + } + + #[test] + fn from_main_crate() { + let mut dependencies = BTreeMap::new(); + let dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, Some("0.10".to_string())); + } + + #[test] + fn from_sub_crate() { + let mut dependencies = BTreeMap::new(); + let dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + dependencies.insert( + "bevy_transform".to_string(), + Dependency::Simple("0.10".to_string()), + ); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, Some("0.10".to_string())); + } + + #[test] + fn from_dev_dependencies() { + let dependencies = BTreeMap::new(); + let mut dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + dev_dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, Some("0.10".to_string())); + } + + #[test] + fn from_workspace_dependencies() { + let dependencies = BTreeMap::new(); + let dev_dependencies = BTreeMap::new(); + let mut workspace_dependencies = BTreeMap::new(); + + workspace_dependencies + .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, Some("0.10".to_string())); + } + + #[test] + fn from_third_party() { + let mut dependencies = BTreeMap::new(); + let dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + dependencies.insert( + "bevy_third_party_crate_example".to_string(), + Dependency::Simple("0.5".to_string()), + ); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + // Note that this result is expected, but potentially wrong + assert_eq!(version, Some("0.5".to_string())); + } + + #[test] + fn from_dependencies_ignore_third_party() { + let mut dependencies = BTreeMap::new(); + let dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + dependencies.insert( + "bevy_third_party_crate_example".to_string(), + Dependency::Simple("0.5".to_string()), + ); + dependencies.insert( + "bevy_transform".to_string(), + Dependency::Simple("0.10".to_string()), + ); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, Some("0.10".to_string())); + } + + #[test] + fn from_dev_dependencies_ignore_third_party() { + let mut dependencies = BTreeMap::new(); + let mut dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + dependencies.insert( + "bevy_third_party_crate_example".to_string(), + Dependency::Simple("0.5".to_string()), + ); + dev_dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, Some("0.10".to_string())); + } + + #[test] + fn from_workspace_dependencies_ignore_third_party() { + let mut dependencies = BTreeMap::new(); + let dev_dependencies = BTreeMap::new(); + let mut workspace_dependencies = BTreeMap::new(); + + dependencies.insert( + "bevy_third_party_crate_example".to_string(), + Dependency::Simple("0.5".to_string()), + ); + workspace_dependencies + .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, Some("0.10".to_string())); + } + + #[test] + fn from_dev_dependencies_with_path_dependency() { + let mut dependencies = BTreeMap::new(); + let mut dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + dependencies.insert( + "bevy".to_string(), + Dependency::Detailed(cargo_toml::DependencyDetail { + path: Some("fake/path/to/crate".to_string()), + ..Default::default() + }), + ); + dev_dependencies.insert( + "bevy_transform".to_string(), + Dependency::Simple("0.10".to_string()), + ); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, Some("0.10".to_string())); + } + } +} From e8122afa646bed8e6fd148bb80f1a2e81c6d4a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9l=C3=A8ne=20Amanita?= Date: Wed, 14 Jun 2023 22:45:27 +0100 Subject: [PATCH 4/9] Add from_third_party_crate_with_path_dependency test --- generate-assets/src/lib.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index 90cac35bb2..c718834bb3 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -642,6 +642,7 @@ mod tests { let dev_dependencies = BTreeMap::new(); let workspace_dependencies = BTreeMap::new(); + // Alphabetical order could matter in this example, "third" < "transform" dependencies.insert( "bevy_third_party_crate_example".to_string(), Dependency::Simple("0.5".to_string()), @@ -713,5 +714,29 @@ mod tests { let version = get_bevy_version(&manifest); assert_eq!(version, Some("0.10".to_string())); } + + #[test] + fn from_third_party_crate_with_path_dependency() { + let mut dependencies = BTreeMap::new(); + let dev_dependencies = BTreeMap::new(); + let workspace_dependencies = BTreeMap::new(); + + // Alphabetical order could matter in this example, "first" < "second" + dependencies.insert( + "bevy_first_third_party_crate".to_string(), + Dependency::Detailed(cargo_toml::DependencyDetail { + path: Some("fake/path/to/crate".to_string()), + ..Default::default() + }), + ); + dependencies.insert( + "bevy_second_third_party_crate".to_string(), + Dependency::Simple("0.10".to_string()), + ); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest); + assert_eq!(version, Some("0.10".to_string())); + } } } From 0ea3c59204b3bd22f5dd5ddfcf86de9f7f981661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9l=C3=A8ne=20Amanita?= Date: Wed, 14 Jun 2023 23:26:05 +0100 Subject: [PATCH 5/9] Delete dev comment and modify from_other_dependencies --- generate-assets/src/lib.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index c718834bb3..ce3d53b896 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -494,16 +494,6 @@ mod tests { use cargo_toml::{Dependency, Manifest}; use std::collections::BTreeMap; - /* - fn get_manifest( - dependencies: Vec, - dependencies_separated: Vec, - dev_dependencies: Vec, - dev_dependencies_separated: Vec, - ) -> cargo_toml::Manifest { - toml::from_str::(&content)?; - } */ - fn get_manifest( dependencies: BTreeMap, dev_dependencies: BTreeMap, @@ -553,10 +543,20 @@ mod tests { fn from_other_dependencies() { let mut dependencies = BTreeMap::new(); let mut dev_dependencies = BTreeMap::new(); - let workspace_dependencies = BTreeMap::new(); + let mut workspace_dependencies = BTreeMap::new(); - dependencies.insert("other".to_string(), Dependency::Simple("0.10".to_string())); - dev_dependencies.insert("other".to_string(), Dependency::Simple("0.10".to_string())); + dependencies.insert( + "other_first".to_string(), + Dependency::Simple("0.10".to_string()), + ); + dev_dependencies.insert( + "other_second".to_string(), + Dependency::Simple("0.10".to_string()), + ); + workspace_dependencies.insert( + "other_third".to_string(), + Dependency::Simple("0.10".to_string()), + ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); let version = get_bevy_version(&manifest); From fcd9b3a9e698c23fff66de6206fcb4720f51a1b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9l=C3=A8ne=20Amanita?= Date: Thu, 15 Jun 2023 09:15:41 +0100 Subject: [PATCH 6/9] Remove static official bevy crates and pull it from crates.io db dump --- generate-assets/src/bin/generate.rs | 9 +- generate-assets/src/bin/validate.rs | 2 +- generate-assets/src/lib.rs | 233 +++++++++++++++++----------- 3 files changed, 146 insertions(+), 98 deletions(-) diff --git a/generate-assets/src/bin/generate.rs b/generate-assets/src/bin/generate.rs index 2211098a7f..d47d875f5c 100644 --- a/generate-assets/src/bin/generate.rs +++ b/generate-assets/src/bin/generate.rs @@ -40,9 +40,12 @@ fn main() -> anyhow::Result<()> { let _ = fs::create_dir(content_dir.clone()); let asset_root_section = parse_assets( &asset_dir, - Some(&db), - github_client.as_ref(), - gitlab_client.as_ref(), + MetadataSource { + crates_io_db: Some(&db), + github_client: github_client.as_ref(), + gitlab_client: gitlab_client.as_ref(), + ..Default::default() + }, )?; asset_root_section diff --git a/generate-assets/src/bin/validate.rs b/generate-assets/src/bin/validate.rs index b2f3eef9c8..5c0d179e93 100644 --- a/generate-assets/src/bin/validate.rs +++ b/generate-assets/src/bin/validate.rs @@ -15,7 +15,7 @@ fn main() -> Result<()> { .ok_or_else(|| anyhow!("Please specify the path to bevy-assets"))?; let asset_root_section = - parse_assets(&asset_dir, None, None, None).with_context(|| "Parsing assets")?; + parse_assets(&asset_dir, MetadataSource::default()).with_context(|| "Parsing assets")?; let results = asset_root_section.validate(); diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index ce3d53b896..9176ecb14b 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -10,50 +10,6 @@ pub mod gitlab_client; type CratesIoDb = cratesio_dbdump_csvtab::rusqlite::Connection; -const OFFICIAL_BEVY_CRATES: &'static [&'static str] = &[ - // Should be sorted - "bevy", - "bevy_a11y", - "bevy_animation", - "bevy_app", - "bevy_asset", - "bevy_audio", - "bevy_core", - "bevy_core_pipeline", - "bevy_derive", - "bevy_diagnostic", - "bevy_dylib", - "bevy_dynamic_plugin", - "bevy_ecs", - "bevy_ecs_compile_fail_tests", - "bevy_encase_derive", - "bevy_gilrs", - "bevy_gizmos", - "bevy_gltf", - "bevy_hierarchy", - "bevy_input", - "bevy_internal", - "bevy_log", - "bevy_macros_compile_fail_tests", - "bevy_macro_utils", - "bevy_math", - "bevy_mikktspace", - "bevy_pbr", - "bevy_ptr", - "bevy_reflect", - "bevy_reflect_compile_fail_tests", - "bevy_render", - "bevy_scene", - "bevy_sprite", - "bevy_tasks", - "bevy_text", - "bevy_time", - "bevy_transform", - "bevy_ui", - "bevy_utils", - "bevy_window", - "bevy_winit", -]; const OFFICIAL_BEVY_CRATE_PREFIX_RANGE_START: &str = "bevy"; const OFFICIAL_BEVY_CRATE_PREFIX_RANGE_END: &str = "bevz"; @@ -128,12 +84,18 @@ impl AssetNode { } } +#[derive(Default)] +pub struct MetadataSource<'a> { + pub crates_io_db: Option<&'a CratesIoDb>, + pub github_client: Option<&'a GithubClient>, + pub gitlab_client: Option<&'a GitlabClient>, + pub bevy_crates: Option>, +} + fn visit_dirs( dir: PathBuf, section: &mut Section, - crates_io_db: Option<&CratesIoDb>, - github_client: Option<&GithubClient>, - gitlab_client: Option<&GitlabClient>, + metadata_source: &MetadataSource, ) -> anyhow::Result<()> { if dir.is_file() { return Ok(()); @@ -172,13 +134,7 @@ fn visit_dirs( order, sort_order_reversed, }; - visit_dirs( - path.clone(), - &mut new_section, - crates_io_db, - github_client, - gitlab_client, - )?; + visit_dirs(path.clone(), &mut new_section, metadata_source)?; section.content.push(AssetNode::Section(new_section)); } else { if path.file_name().unwrap() == "_category.toml" @@ -190,9 +146,7 @@ fn visit_dirs( let mut asset: Asset = toml::from_str(&fs::read_to_string(&path).unwrap())?; asset.original_path = Some(path); - if let Err(err) = - get_extra_metadata(&mut asset, crates_io_db, github_client, gitlab_client) - { + if let Err(err) = get_extra_metadata(&mut asset, metadata_source) { // We don't want to stop execution here eprintln!("Failed to get metadata for {}", asset.name); eprintln!("ERROR: {err:?}"); @@ -205,11 +159,9 @@ fn visit_dirs( Ok(()) } -pub fn parse_assets( +pub fn parse_assets<'a>( asset_dir: &str, - crates_io_db: Option<&CratesIoDb>, - github_client: Option<&GithubClient>, - gitlab_client: Option<&GitlabClient>, + mut metadata_source: MetadataSource, ) -> anyhow::Result
{ let mut asset_root_section = Section { name: "Assets".to_string(), @@ -219,30 +171,33 @@ pub fn parse_assets( order: None, sort_order_reversed: false, }; + + if let Some(db) = metadata_source.crates_io_db { + if let Ok(bevy_crates) = get_official_bevy_crates_from_crates_io_db(db) { + metadata_source.bevy_crates = Some(bevy_crates); + } + } + visit_dirs( PathBuf::from_str(asset_dir).unwrap(), &mut asset_root_section, - crates_io_db, - github_client, - gitlab_client, + &metadata_source, + //crates_io_db, + //github_client, + //gitlab_client, )?; Ok(asset_root_section) } /// Tries to get bevy supported version and license information from various external sources -fn get_extra_metadata( - asset: &mut Asset, - crates_io_db: Option<&CratesIoDb>, - github_client: Option<&GithubClient>, - gitlab_client: Option<&GitlabClient>, -) -> anyhow::Result<()> { +fn get_extra_metadata(asset: &mut Asset, metadata_source: &MetadataSource) -> anyhow::Result<()> { println!("Getting extra metadata for {}", asset.name); let url = url::Url::parse(&asset.link)?; let segments = url.path_segments().map(|c| c.collect::>()).unwrap(); let metadata = match url.host_str() { - Some("crates.io") if crates_io_db.is_some() => { - if let Some(db) = crates_io_db { + Some("crates.io") => { + if let Some(db) = metadata_source.crates_io_db { let crate_name = segments[1]; Some(get_metadata_from_crates_io_db(db, crate_name)?) } else { @@ -250,18 +205,27 @@ fn get_extra_metadata( } } Some("github.com") => { - if let Some(client) = github_client { + if let Some(client) = metadata_source.github_client { let username = segments[0]; let repository_name = segments[1]; - Some(get_metadata_from_github(client, username, repository_name)?) + Some(get_metadata_from_github( + client, + username, + repository_name, + &metadata_source.bevy_crates, + )?) } else { None } } Some("gitlab.com") => { - if let Some(client) = gitlab_client { + if let Some(client) = metadata_source.gitlab_client { let repository_name = segments[1]; - Some(get_metadata_from_gitlab(client, repository_name)?) + Some(get_metadata_from_gitlab( + client, + repository_name, + &metadata_source.bevy_crates, + )?) } else { None } @@ -282,6 +246,7 @@ fn get_metadata_from_github( client: &GithubClient, username: &str, repository_name: &str, + bevy_crates: &Option>, ) -> anyhow::Result<(Option, Option)> { let content = client .get_content(username, repository_name, "Cargo.toml") @@ -290,13 +255,14 @@ fn get_metadata_from_github( let cargo_manifest = toml::from_str::(&content)?; Ok(( get_license(&cargo_manifest), - get_bevy_version(&cargo_manifest), + get_bevy_version(&cargo_manifest, bevy_crates), )) } fn get_metadata_from_gitlab( client: &GitlabClient, repository_name: &str, + bevy_crates: &Option>, ) -> anyhow::Result<(Option, Option)> { let search_result = client.search_project_by_name(repository_name)?; @@ -311,7 +277,7 @@ fn get_metadata_from_gitlab( let cargo_manifest = toml::from_str::(&content)?; Ok(( get_license(&cargo_manifest), - get_bevy_version(&cargo_manifest), + get_bevy_version(&cargo_manifest, bevy_crates), )) } @@ -337,13 +303,22 @@ fn get_license(cargo_manifest: &cargo_toml::Manifest) -> Option { /// Find any bevy dependency and get the corresponding bevy version from an asset's manifest /// This makes sure to handle all the bevy_* crates -fn get_bevy_version(cargo_manifest: &cargo_toml::Manifest) -> Option { +fn get_bevy_version( + cargo_manifest: &cargo_toml::Manifest, + bevy_crates: &Option>, +) -> Option { let search_range = OFFICIAL_BEVY_CRATE_PREFIX_RANGE_START.to_owned() ..OFFICIAL_BEVY_CRATE_PREFIX_RANGE_END.to_owned(); // Tries to find an official bevy crate from the asset's dependencies let mut dependencies = cargo_manifest.dependencies.range(search_range.clone()); - let bevy_crates = OFFICIAL_BEVY_CRATES.iter(); + let empty_vec = Vec::new(); + let bevy_crates = if let Some(bevy_crates) = bevy_crates { + bevy_crates.iter() + } else { + empty_vec.iter() + }; + let mut bevy_dependency = search_bevy_in_dependencies(dependencies.clone(), bevy_crates.clone()); @@ -379,7 +354,7 @@ fn get_bevy_version(cargo_manifest: &cargo_toml::Manifest) -> Option { /// and we find the first element that intersect both of them using that knowledge fn search_bevy_in_dependencies<'a>( mut dependencies: std::collections::btree_map::Range<'a, String, cargo_toml::Dependency>, - mut bevy_crates: std::slice::Iter<&str>, + mut bevy_crates: std::slice::Iter, ) -> Option { let mut dependency = dependencies.next(); let mut bevy_crate = bevy_crates.next(); @@ -388,7 +363,7 @@ fn search_bevy_in_dependencies<'a>( let dependency_name = dependency.unwrap().0; let bevy_crate_name = bevy_crate.unwrap(); - if dependency_name.as_str() < *bevy_crate_name { + if dependency_name < bevy_crate_name { dependency = dependencies.next(); } else { if dependency_name == bevy_crate_name { @@ -486,6 +461,37 @@ fn get_metadata_from_db_by_crate_name( } } +/// Gets at list of the official bevy crates from the crates.io database dump +fn get_official_bevy_crates_from_crates_io_db(db: &CratesIoDb) -> anyhow::Result> { + if let Ok(mut crates) = get_bevy_crates(db) { + crates.sort(); + Ok(crates) + } else { + bail!("Problem fetching official bevy crates from crates.io") + } +} + +fn get_bevy_crates( + db: &CratesIoDb, +) -> Result, cratesio_dbdump_csvtab::rusqlite::Error> { + let mut s = + db.prepare_cached("SELECT name FROM crates WHERE homepage = ? AND repository = ?")?; + let rows = s.query_and_then( + [ + "https://bevyengine.org", + "https://github.com/bevyengine/bevy", + ], + |r| -> Result { + Ok(r.get_unwrap::<_, String>(0)) + }, + )?; + let mut bevy_crates = Vec::new(); + for bevy_crate in rows { + bevy_crates.push(bevy_crate?); + } + Ok(bevy_crates) +} + #[cfg(test)] mod tests { mod get_version { @@ -528,6 +534,10 @@ mod tests { } } + fn get_bevy_crates() -> Option> { + Some(vec!["bevy".to_string(), "bevy_transform".to_string()]) + } + #[test] fn from_no_dependency() { let dependencies = BTreeMap::new(); @@ -535,7 +545,7 @@ mod tests { let workspace_dependencies = BTreeMap::new(); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, None); } @@ -559,7 +569,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, None); } @@ -572,7 +582,7 @@ mod tests { dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, Some("0.10".to_string())); } @@ -588,7 +598,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, Some("0.10".to_string())); } @@ -601,7 +611,7 @@ mod tests { dev_dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, Some("0.10".to_string())); } @@ -615,7 +625,7 @@ mod tests { .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, Some("0.10".to_string())); } @@ -631,7 +641,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); // Note that this result is expected, but potentially wrong assert_eq!(version, Some("0.5".to_string())); } @@ -653,7 +663,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, Some("0.10".to_string())); } @@ -670,7 +680,7 @@ mod tests { dev_dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, Some("0.10".to_string())); } @@ -688,7 +698,7 @@ mod tests { .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, Some("0.10".to_string())); } @@ -711,7 +721,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, Some("0.10".to_string())); } @@ -735,8 +745,43 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest); + let version = get_bevy_version(&manifest, &get_bevy_crates()); assert_eq!(version, Some("0.10".to_string())); } + + #[test] + fn from_third_party_with_no_official_bevy_crates() { + let mut dependencies = BTreeMap::new(); + let mut dev_dependencies = BTreeMap::new(); + let mut workspace_dependencies = BTreeMap::new(); + + dependencies.insert( + "bevy_third_party_crate_example".to_string(), + Dependency::Simple("0.5".to_string()), + ); + dev_dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); + workspace_dependencies + .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest, &Some(vec![])); + assert_eq!(version, Some("0.5".to_string())); + } + + #[test] + fn from_no_dependency_with_no_official_bevy_crates() { + let mut dependencies = BTreeMap::new(); + let mut dev_dependencies = BTreeMap::new(); + let mut workspace_dependencies = BTreeMap::new(); + + dependencies.insert("other".to_string(), Dependency::Simple("0.5".to_string())); + dev_dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); + workspace_dependencies + .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); + + let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); + let version = get_bevy_version(&manifest, &Some(vec![])); + assert_eq!(version, None); + } } } From e618f8ffa1b7f6f9d154c5d0bf64c3dc79b206d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9l=C3=A8ne=20Amanita?= Date: Thu, 15 Jun 2023 09:40:45 +0100 Subject: [PATCH 7/9] Fix comments --- generate-assets/src/lib.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index 9176ecb14b..543fa230d9 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -182,9 +182,6 @@ pub fn parse_assets<'a>( PathBuf::from_str(asset_dir).unwrap(), &mut asset_root_section, &metadata_source, - //crates_io_db, - //github_client, - //gitlab_client, )?; Ok(asset_root_section) } @@ -338,8 +335,8 @@ fn get_bevy_version( if bevy_dependency.is_none() { // If everything else fails, try to find any crate with a name starting with bevy - // This can happen if the asset depends only on past or future bevy sub-crates, - // or on third-party crates which name starts with bevy (might yield unaccurate results in that last case) + // This can happen if the asset depends only on third-party crates + // which name starts with bevy (might yield unaccurate results) bevy_dependency = dependencies.find_map(|(_, d)| get_bevy_dependency_version(d)); } } From 6c5b0ebb1adba04017851b26cc7899005158e0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9l=C3=A8ne=20Amanita?= Date: Tue, 27 Jun 2023 12:28:10 +0100 Subject: [PATCH 8/9] Better Bevy version algorithm for assets with crates.io --- generate-assets/src/lib.rs | 265 ++++++++++++++++++++++++------------- 1 file changed, 170 insertions(+), 95 deletions(-) diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index 543fa230d9..b5943280af 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -1,4 +1,5 @@ use anyhow::{bail, Context}; +use cratesio_dbdump_csvtab::rusqlite; use cratesio_dbdump_csvtab::CratesIODumpLoader; use github_client::GithubClient; use gitlab_client::GitlabClient; @@ -8,7 +9,7 @@ use std::{fs, path::PathBuf, str::FromStr}; pub mod github_client; pub mod gitlab_client; -type CratesIoDb = cratesio_dbdump_csvtab::rusqlite::Connection; +type CratesIoDb = rusqlite::Connection; const OFFICIAL_BEVY_CRATE_PREFIX_RANGE_START: &str = "bevy"; const OFFICIAL_BEVY_CRATE_PREFIX_RANGE_END: &str = "bevz"; @@ -89,13 +90,14 @@ pub struct MetadataSource<'a> { pub crates_io_db: Option<&'a CratesIoDb>, pub github_client: Option<&'a GithubClient>, pub gitlab_client: Option<&'a GitlabClient>, - pub bevy_crates: Option>, + pub bevy_crates_names: Option>, + pub get_metadata_from_cratesio_statement: Option>, } fn visit_dirs( dir: PathBuf, section: &mut Section, - metadata_source: &MetadataSource, + metadata_source: &mut MetadataSource, ) -> anyhow::Result<()> { if dir.is_file() { return Ok(()); @@ -173,30 +175,41 @@ pub fn parse_assets<'a>( }; if let Some(db) = metadata_source.crates_io_db { - if let Ok(bevy_crates) = get_official_bevy_crates_from_crates_io_db(db) { - metadata_source.bevy_crates = Some(bevy_crates); - } + let bevy_crates_ids = if let Ok((bevy_crates_names, bevy_crates_ids)) = + get_official_bevy_crates_from_crates_io_db(db) + { + metadata_source.bevy_crates_names = Some(bevy_crates_names); + Some(bevy_crates_ids) + } else { + None + }; + metadata_source.get_metadata_from_cratesio_statement = + Some(get_metadata_from_cratesio_statement(db, bevy_crates_ids)?); } visit_dirs( PathBuf::from_str(asset_dir).unwrap(), &mut asset_root_section, - &metadata_source, + &mut metadata_source, )?; Ok(asset_root_section) } /// Tries to get bevy supported version and license information from various external sources -fn get_extra_metadata(asset: &mut Asset, metadata_source: &MetadataSource) -> anyhow::Result<()> { +fn get_extra_metadata( + asset: &mut Asset, + metadata_source: &mut MetadataSource, +) -> anyhow::Result<()> { println!("Getting extra metadata for {}", asset.name); let url = url::Url::parse(&asset.link)?; let segments = url.path_segments().map(|c| c.collect::>()).unwrap(); + let metadata = match url.host_str() { Some("crates.io") => { - if let Some(db) = metadata_source.crates_io_db { + if let Some(ref mut statement) = metadata_source.get_metadata_from_cratesio_statement { let crate_name = segments[1]; - Some(get_metadata_from_crates_io_db(db, crate_name)?) + Some(get_metadata_from_crates_io_db(crate_name, statement)?) } else { None } @@ -209,7 +222,7 @@ fn get_extra_metadata(asset: &mut Asset, metadata_source: &MetadataSource) -> an client, username, repository_name, - &metadata_source.bevy_crates, + &metadata_source.bevy_crates_names, )?) } else { None @@ -221,7 +234,7 @@ fn get_extra_metadata(asset: &mut Asset, metadata_source: &MetadataSource) -> an Some(get_metadata_from_gitlab( client, repository_name, - &metadata_source.bevy_crates, + &metadata_source.bevy_crates_names, )?) } else { None @@ -252,7 +265,7 @@ fn get_metadata_from_github( let cargo_manifest = toml::from_str::(&content)?; Ok(( get_license(&cargo_manifest), - get_bevy_version(&cargo_manifest, bevy_crates), + get_bevy_version_from_manifest(&cargo_manifest, bevy_crates), )) } @@ -274,7 +287,7 @@ fn get_metadata_from_gitlab( let cargo_manifest = toml::from_str::(&content)?; Ok(( get_license(&cargo_manifest), - get_bevy_version(&cargo_manifest, bevy_crates), + get_bevy_version_from_manifest(&cargo_manifest, bevy_crates), )) } @@ -300,49 +313,43 @@ fn get_license(cargo_manifest: &cargo_toml::Manifest) -> Option { /// Find any bevy dependency and get the corresponding bevy version from an asset's manifest /// This makes sure to handle all the bevy_* crates -fn get_bevy_version( +fn get_bevy_version_from_manifest( cargo_manifest: &cargo_toml::Manifest, bevy_crates: &Option>, ) -> Option { let search_range = OFFICIAL_BEVY_CRATE_PREFIX_RANGE_START.to_owned() ..OFFICIAL_BEVY_CRATE_PREFIX_RANGE_END.to_owned(); - // Tries to find an official bevy crate from the asset's dependencies - let mut dependencies = cargo_manifest.dependencies.range(search_range.clone()); - let empty_vec = Vec::new(); - let bevy_crates = if let Some(bevy_crates) = bevy_crates { - bevy_crates.iter() - } else { - empty_vec.iter() - }; - - let mut bevy_dependency = - search_bevy_in_dependencies(dependencies.clone(), bevy_crates.clone()); + let dependencies = cargo_manifest.dependencies.range(search_range.clone()); + if let Some(bevy_crates) = bevy_crates { + let bevy_crates = bevy_crates.iter(); - if bevy_dependency.is_none() { - // Tries to find an official bevy crate from the asset's dev dependencies - // An asset can indirectly depend on bevy through another crate, but would probably depend on bevy directly - // for its examples, benchmarks or tests, in its dev dependencies - let dev_dependencies = cargo_manifest.dev_dependencies.range(search_range.clone()); - bevy_dependency = search_bevy_in_dependencies(dev_dependencies, bevy_crates.clone()); + // Tries to find an official bevy crate from the asset's dependencies. + let mut bevy_dependency = + search_bevy_in_dependencies(dependencies.clone(), bevy_crates.clone()); if bevy_dependency.is_none() { - // Tries to find an official bevy crate from the asset's workspace dependencies - if let Some(ref workspace) = cargo_manifest.workspace { - let workspace_dependencies = workspace.dependencies.range(search_range); - bevy_dependency = search_bevy_in_dependencies(workspace_dependencies, bevy_crates); - } + // Tries to find an official bevy crate from the asset's dev dependencies. + // An asset can indirectly depend on bevy through another crate, + // but would probably depend on bevy directly for its examples, + // benchmarks or tests, in its dev dependencies. + let dev_dependencies = cargo_manifest.dev_dependencies.range(search_range.clone()); + bevy_dependency = search_bevy_in_dependencies(dev_dependencies, bevy_crates.clone()); if bevy_dependency.is_none() { - // If everything else fails, try to find any crate with a name starting with bevy - // This can happen if the asset depends only on third-party crates - // which name starts with bevy (might yield unaccurate results) - bevy_dependency = dependencies.find_map(|(_, d)| get_bevy_dependency_version(d)); + // Tries to find an official bevy crate from the asset's workspace dependencies. + if let Some(ref workspace) = cargo_manifest.workspace { + let workspace_dependencies = workspace.dependencies.range(search_range); + bevy_dependency = + search_bevy_in_dependencies(workspace_dependencies, bevy_crates); + } } } - } - bevy_dependency + bevy_dependency + } else { + None + } } /// Seach the first official bevy crate found in a collection of dependencies and return its version. @@ -426,72 +433,140 @@ pub fn prepare_crates_db() -> anyhow::Result { /// Gets the required metadata from the crates.io database dump fn get_metadata_from_crates_io_db( - db: &CratesIoDb, crate_name: &str, + get_metadata_from_cratesio_statement: &mut rusqlite::Statement, ) -> anyhow::Result<(Option, Option)> { - if let Ok(metadata) = get_metadata_from_db_by_crate_name(db, crate_name) { - Ok(metadata) - } else if let Ok(metadata) = - get_metadata_from_db_by_crate_name(db, &crate_name.replace('_', "-")) + if let Ok(metadata) = + get_metadata_from_db_by_crate_name(crate_name, get_metadata_from_cratesio_statement) { Ok(metadata) + } else if let Ok(metadata) = get_metadata_from_db_by_crate_name( + &crate_name.replace('_', "-"), + get_metadata_from_cratesio_statement, + ) { + Ok(metadata) } else { bail!("Failed to get data from crates.io db for {crate_name}") } } fn get_metadata_from_db_by_crate_name( - db: &CratesIoDb, crate_name: &str, + get_metadata_from_cratesio_statement: &mut rusqlite::Statement, ) -> anyhow::Result<(Option, Option)> { - if let Some(Ok((_, _, license, _, deps))) = - &cratesio_dbdump_lookup::get_rev_dependency(db, crate_name, "bevy")?.first() + if let Ok((license, version)) = + get_metadata_from_cratesio(crate_name, get_metadata_from_cratesio_statement) { - let version = deps - .as_ref() - .ok() - .and_then(|deps| deps.first()) - .map(|(version, _)| version.clone()); - Ok((Some(license.clone()), version)) + let license = if license != "" { Some(license) } else { None }; + + Ok((license, version)) } else { bail!("Not found in crates.io db: {crate_name}") } } /// Gets at list of the official bevy crates from the crates.io database dump -fn get_official_bevy_crates_from_crates_io_db(db: &CratesIoDb) -> anyhow::Result> { - if let Ok(mut crates) = get_bevy_crates(db) { - crates.sort(); - Ok(crates) +fn get_official_bevy_crates_from_crates_io_db( + db: &CratesIoDb, +) -> anyhow::Result<(Vec, Vec)> { + if let Ok(mut bevy_crates) = get_bevy_crates(db) { + bevy_crates.sort_by(|(name_a, _), (name_b, _)| name_a.cmp(name_b)); + Ok(bevy_crates.into_iter().unzip()) } else { bail!("Problem fetching official bevy crates from crates.io") } } -fn get_bevy_crates( - db: &CratesIoDb, -) -> Result, cratesio_dbdump_csvtab::rusqlite::Error> { - let mut s = - db.prepare_cached("SELECT name FROM crates WHERE homepage = ? AND repository = ?")?; - let rows = s.query_and_then( - [ - "https://bevyengine.org", - "https://github.com/bevyengine/bevy", - ], - |r| -> Result { - Ok(r.get_unwrap::<_, String>(0)) - }, +// Get official bevy crates name and ids from crates.io db dump +fn get_bevy_crates(db: &CratesIoDb) -> Result, rusqlite::Error> { + let mut bevy_crates_statement = db.prepare( + "\ + SELECT name, id \ + FROM crates \ + WHERE homepage = ? \ + AND repository = ?\ + ", )?; - let mut bevy_crates = Vec::new(); - for bevy_crate in rows { - bevy_crates.push(bevy_crate?); - } - Ok(bevy_crates) + let bevy_crates = bevy_crates_statement + .query_and_then( + [ + "https://bevyengine.org", + "https://github.com/bevyengine/bevy", + ], + |r| -> Result<(String, String), rusqlite::Error> { + Ok((r.get_unwrap::<_, String>(0), r.get_unwrap::<_, String>(1))) + }, + )? + .collect(); + bevy_crates +} + +// Get a prepared statement to get license and version for a crate from crates.io +// To be used later by get_metadata_from_cratesio +pub fn get_metadata_from_cratesio_statement<'a>( + db: &'a CratesIoDb, + bevy_crates_ids: Option>, +) -> Result, rusqlite::Error> { + let bevy_crates_ids = bevy_crates_ids.unwrap_or_default(); + + Ok(db.prepare(&format!( + "\ + SELECT last_version.license, dep.req \ + FROM ( \ + SELECT version_id, license, major, \ + CAST(SUBSTR(minor_and_patch,0,second_point) AS INTEGER) minor, \ + SUBSTR(minor_and_patch,second_point+1) patch \ + FROM ( \ + SELECT version_id, license, major, minor_and_patch, \ + INSTR(minor_and_patch, '.') second_point \ + FROM ( \ + SELECT version_id, license, \ + CAST(SUBSTR(num,0,first_point) AS INTEGER) major, \ + SUBSTR(num,first_point+1) minor_and_patch \ + FROM ( \ + SELECT v.id version_id, v.license license, v.num num, \ + INSTR(v.num, '.') first_point \ + FROM crates c \ + INNER JOIN versions v ON c.id = v.crate_id \ + WHERE c.name = ? \ + ) \ + ) \ + ) \ + ORDER BY major DESC, minor DESC, patch DESC \ + LIMIT 1 \ + ) last_version \ + LEFT JOIN dependencies dep ON \ + ( \ + last_version.version_id = dep.version_id AND \ + dep.crate_id IN ({}) \ + ) \ + ORDER BY dep.kind \ + LIMIT 1\ + ", + bevy_crates_ids.join(",") + ))?) +} + +// Get license and version for a crate from crates.io +// Using the statement from get_metadata_from_cratesio_statement +pub fn get_metadata_from_cratesio( + crate_name: &str, + get_metadata_from_cratesio_statement: &mut rusqlite::Statement, +) -> Result<(String, Option), rusqlite::Error> { + Ok(get_metadata_from_cratesio_statement.query_row( + [crate_name], + |r| -> Result<(String, Option), rusqlite::Error> { + Ok(( + r.get_unwrap::<_, String>(0), + r.get_unwrap::<_, Option>(1), + )) + }, + )?) } #[cfg(test)] mod tests { - mod get_version { + mod get_bevy_version_from_manifest { use super::super::*; use cargo_toml::{Dependency, Manifest}; @@ -531,7 +606,7 @@ mod tests { } } - fn get_bevy_crates() -> Option> { + fn get_bevy_crates_names() -> Option> { Some(vec!["bevy".to_string(), "bevy_transform".to_string()]) } @@ -542,7 +617,7 @@ mod tests { let workspace_dependencies = BTreeMap::new(); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, None); } @@ -566,7 +641,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, None); } @@ -579,7 +654,7 @@ mod tests { dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, Some("0.10".to_string())); } @@ -595,7 +670,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, Some("0.10".to_string())); } @@ -608,7 +683,7 @@ mod tests { dev_dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, Some("0.10".to_string())); } @@ -622,7 +697,7 @@ mod tests { .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, Some("0.10".to_string())); } @@ -638,7 +713,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); // Note that this result is expected, but potentially wrong assert_eq!(version, Some("0.5".to_string())); } @@ -660,7 +735,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, Some("0.10".to_string())); } @@ -677,7 +752,7 @@ mod tests { dev_dependencies.insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, Some("0.10".to_string())); } @@ -695,7 +770,7 @@ mod tests { .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, Some("0.10".to_string())); } @@ -718,7 +793,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, Some("0.10".to_string())); } @@ -742,7 +817,7 @@ mod tests { ); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &get_bevy_crates()); + let version = get_bevy_version_from_manifest(&manifest, &get_bevy_crates_names()); assert_eq!(version, Some("0.10".to_string())); } @@ -761,7 +836,7 @@ mod tests { .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &Some(vec![])); + let version = get_bevy_version_from_manifest(&manifest, &Some(vec![])); assert_eq!(version, Some("0.5".to_string())); } @@ -777,7 +852,7 @@ mod tests { .insert("bevy".to_string(), Dependency::Simple("0.10".to_string())); let manifest = get_manifest(dependencies, dev_dependencies, workspace_dependencies); - let version = get_bevy_version(&manifest, &Some(vec![])); + let version = get_bevy_version_from_manifest(&manifest, &Some(vec![])); assert_eq!(version, None); } } From f8259197acc64210971d6db0fe214f3a7ed4f8b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9l=C3=A8ne=20Amanita?= Date: Wed, 28 Jun 2023 00:13:39 +0100 Subject: [PATCH 9/9] Check all cargo.toml files in github repository --- generate-assets/Cargo.lock | 25 ---- generate-assets/Cargo.toml | 1 - generate-assets/src/github_client.rs | 212 +++++++++++++++++---------- generate-assets/src/lib.rs | 112 +++++++++++++- 4 files changed, 246 insertions(+), 104 deletions(-) diff --git a/generate-assets/Cargo.lock b/generate-assets/Cargo.lock index ee2f6be63b..0c1e084022 100644 --- a/generate-assets/Cargo.lock +++ b/generate-assets/Cargo.lock @@ -207,18 +207,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "cratesio-dbdump-lookup" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0e240e659ad18b47646482ab1c67d1c0ea5deb0d1dfdbd8c839c389a105136b" -dependencies = [ - "rusqlite", - "serde", - "thiserror", - "tracing", -] - [[package]] name = "crc32fast" version = "1.3.2" @@ -430,7 +418,6 @@ dependencies = [ "base64", "cargo_toml", "cratesio-dbdump-csvtab", - "cratesio-dbdump-lookup", "dotenv", "rand", "regex", @@ -1320,21 +1307,9 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tracing-core" version = "0.1.30" diff --git a/generate-assets/Cargo.toml b/generate-assets/Cargo.toml index c5d56c4981..1e16957c43 100644 --- a/generate-assets/Cargo.toml +++ b/generate-assets/Cargo.toml @@ -19,7 +19,6 @@ cargo_toml = "0.15" url = "2.2.2" anyhow = "1.0.58" base64 = "0.13.0" -cratesio-dbdump-lookup = "0.1.1" cratesio-dbdump-csvtab = "0.2.2" ureq = { version = "2.5.0", features = ["json"] } dotenv = "0.15.0" diff --git a/generate-assets/src/github_client.rs b/generate-assets/src/github_client.rs index 55732a65ce..a9dab80b25 100644 --- a/generate-assets/src/github_client.rs +++ b/generate-assets/src/github_client.rs @@ -1,77 +1,135 @@ -use anyhow::bail; -use serde::Deserialize; - -const BASE_URL: &str = "https://api.github.com"; - -#[derive(Deserialize)] -struct GithubContentResponse { - encoding: String, - content: String, -} - -#[derive(Deserialize)] -struct GithubLicenseResponse { - license: GithubLicenseLicense, -} - -#[derive(Deserialize)] -struct GithubLicenseLicense { - spdx_id: String, -} - -pub struct GithubClient { - agent: ureq::Agent, - token: String, -} - -impl GithubClient { - pub fn new(token: String) -> Self { - let agent: ureq::Agent = ureq::AgentBuilder::new() - .user_agent("bevy-website-generate-assets") - .build(); - - Self { agent, token } - } - - /// Gets the content of a file from a github repo - pub fn get_content( - &self, - username: &str, - repository_name: &str, - content_path: &str, - ) -> anyhow::Result { - let response: GithubContentResponse = self - .agent - .get(&format!( - "{BASE_URL}/repos/{username}/{repository_name}/contents/{content_path}" - )) - .set("Accept", "application/json") - .set("Authorization", &format!("Bearer {}", self.token)) - .call()? - .into_json()?; - - if response.encoding == "base64" { - let data = base64::decode(response.content.replace('\n', "").trim())?; - Ok(String::from_utf8(data)?) - } else { - bail!("Content is not in base64"); - } - } - - /// Gets the license from a github repo - /// Technically, github supports multiple licenses, but the api only returns one - #[allow(unused)] - pub fn get_license(&self, username: &str, repository_name: &str) -> anyhow::Result { - let response: GithubLicenseResponse = self - .agent - .get(&format!( - "{BASE_URL}/repos/{username}/{repository_name}/license" - )) - .set("Accept", "application/json") - .set("Authorization", &format!("Bearer {}", self.token)) - .call()? - .into_json()?; - - Ok(response.license.spdx_id) - } -} +use anyhow::bail; +use serde::Deserialize; + +const BASE_URL: &str = "https://api.github.com"; + +#[derive(Deserialize, Debug)] +struct GithubContentResponse { + encoding: String, + content: String, +} + +#[derive(Deserialize)] +struct GithubLicenseResponse { + license: GithubLicenseLicense, +} + +#[derive(Deserialize)] +struct GithubLicenseLicense { + spdx_id: String, +} + +#[derive(Deserialize, Debug)] +struct GithubSearchFile { + total_count: u32, + incomplete_results: bool, + items: Vec, +} + +#[derive(Deserialize, Debug)] +struct GithubSearchFileItem { + path: std::path::PathBuf, +} + +pub struct GithubClient { + agent: ureq::Agent, + token: String, +} + +impl GithubClient { + pub fn new(token: String) -> Self { + let agent: ureq::Agent = ureq::AgentBuilder::new() + .user_agent("bevy-website-generate-assets") + .build(); + + Self { agent, token } + } + + /// Gets the content of a file from a github repo + pub fn get_content( + &self, + username: &str, + repository_name: &str, + content_path: &str, + ) -> anyhow::Result { + let response: GithubContentResponse = self + .agent + .get(&format!( + "{BASE_URL}/repos/{username}/{repository_name}/contents/{content_path}" + )) + .set("Accept", "application/json") + .set("Authorization", &format!("Bearer {}", self.token)) + .call()? + .into_json()?; + + if response.encoding == "base64" { + let data = base64::decode(response.content.replace('\n', "").trim())?; + Ok(String::from_utf8(data)?) + } else { + bail!("Content is not in base64"); + } + } + + /// Gets the license from a github repo + /// Technically, github supports multiple licenses, but the api only returns one + #[allow(unused)] + pub fn get_license(&self, username: &str, repository_name: &str) -> anyhow::Result { + let response: GithubLicenseResponse = self + .agent + .get(&format!( + "{BASE_URL}/repos/{username}/{repository_name}/license" + )) + .set("Accept", "application/json") + .set("Authorization", &format!("Bearer {}", self.token)) + .call()? + .into_json()?; + + let license = response.license.spdx_id; + + if license != "NOASSERTION" { + Ok(license) + } else { + bail!("No spdx license assertion") + } + } + + /// Search file by name + pub fn search_file( + &self, + username: &str, + repository_name: &str, + file_name: &str, + ) -> anyhow::Result> { + let response: GithubSearchFile = self + .agent + .get(&format!( + "{BASE_URL}/search/code?q=repo:{username}/{repository_name}+filename:{file_name}" + )) + .set("Accept", "application/json") + .set("Authorization", &format!("Bearer {}", self.token)) + .call()? + .into_json()?; + + if response.incomplete_results { + println!( + "Too many {} files in repository, checking only the first {} ones.", + file_name, response.total_count, + ); + } + + let paths = response + .items + .iter() + .filter_map(|i| { + if let Some(path_string) = i.path.to_str() { + Some(path_string.to_string()) + } else { + println!("Path.to_str failed for {}", i.path.to_string_lossy()); + None + } + }) + .collect(); + + Ok(paths) + } +} diff --git a/generate-assets/src/lib.rs b/generate-assets/src/lib.rs index b5943280af..90c46d6e3d 100644 --- a/generate-assets/src/lib.rs +++ b/generate-assets/src/lib.rs @@ -252,17 +252,127 @@ fn get_extra_metadata( Ok(()) } +// Merge two licenses, get the combination of both of them +fn merge_license(license1: Option, license2: Option) -> Option { + if license1.is_none() { + return license2; + } + if license2.is_none() { + return license1; + } + + let license1 = license1.unwrap(); + let license2 = license2.unwrap(); + if license1.contains(&license2) { + return Some(license1); + } + if license2.contains(&license1) { + return Some(license2); + } + + Some(license1 + " " + &license2) +} + +// Merge two versions, get the maximum of the two +// TODO: normalize versions to be able to compare them +// In the mean time this just returns version1 if it's Some +fn merge_version(version1: Option, version2: Option) -> Option { + if version1.is_some() { + return version1; + } + version2 +} + fn get_metadata_from_github( client: &GithubClient, username: &str, repository_name: &str, bevy_crates: &Option>, +) -> anyhow::Result<(Option, Option)> { + let result = get_metadata_from_github_manifest( + client, + username, + repository_name, + bevy_crates, + "Cargo.toml", + ); + + let (mut license, mut version) = match result { + Ok(lic_ver) => lic_ver, + Err(err) => { + println!( + "Error getting metadata from root cargo file from github: {}", + err + ); + (None, None) + } + }; + + if license.is_none() { + license = client.get_license(username, repository_name).ok(); + } + + if license == None || version == None { + let cargo_files = match client.search_file(username, repository_name, "Cargo.toml") { + Ok(cargo_files) => cargo_files, + Err(err) => { + println!("Error fetching cargo files from github: {:#}", err); + return Ok((license, version)); + } + }; + + let mut cargo_files = cargo_files + .iter() + //Exclude the root Cargo.toml, we already searched in it + .filter(|f| f != &"Cargo.toml"); + + let mut cargo_file = cargo_files.next(); + while (license == None || version == None) && cargo_file.is_some() { + let cargo_file_path = cargo_file.unwrap(); + + let result = get_metadata_from_github_manifest( + client, + username, + repository_name, + bevy_crates, + cargo_file_path, + ); + match result { + Ok((new_license, new_version)) => { + (license, version) = ( + merge_license(license, new_license), + merge_version(version, new_version), + ); + } + Err(err) => { + println!( + "Error getting metadata from other cargo file from github: {}", + err + ); + return Ok((license, version)); + } + } + + cargo_file = cargo_files.next(); + } + } + + Ok((license, version)) +} + +fn get_metadata_from_github_manifest( + client: &GithubClient, + username: &str, + repository_name: &str, + bevy_crates: &Option>, + path: &str, ) -> anyhow::Result<(Option, Option)> { let content = client - .get_content(username, repository_name, "Cargo.toml") + .get_content(username, repository_name, path) .context("Failed to get Cargo.toml from github")?; let cargo_manifest = toml::from_str::(&content)?; + Ok(( get_license(&cargo_manifest), get_bevy_version_from_manifest(&cargo_manifest, bevy_crates),