Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check status code of crate download #97

Merged
merged 3 commits into from
Oct 20, 2024
Merged

Check status code of crate download #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cec3605
Check status code of crate download
wcampbell0x2a Oct 16, 2024
c40992e
Download .crate files multithreaded
wcampbell0x2a Oct 20, 2024
beeaf41
Update tests
wcampbell0x2a Oct 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
mod build_std;
use build_std::prepare_build_std;
use guppy::errors::Error::CommandError;
use reqwest::StatusCode;

use std::path::{Path, PathBuf};
use std::{fs, iter};
Expand All @@ -15,7 +16,7 @@ use crate::git::{clone, pull};

mod git;

#[derive(PartialEq, Eq, PartialOrd, Ord)]
#[derive(PartialEq, Eq, PartialOrd, Ord, Debug)]
struct Crate {
name: String,
version: String,
Expand Down Expand Up @@ -107,10 +108,10 @@ fn get_deps(args: &Args) -> Option<Vec<(String, Vec<Crate>)>> {
let query = package_graph.query_forward(iter::once(id)).unwrap();
let package_set = query.resolve();
for package in package_set.packages(DependencyDirection::Forward) {
crates.push(Crate::new(
package.name().to_string(),
package.version().to_string(),
));
let c = Crate::new(package.name().to_string(), package.version().to_string());
if !crates.contains(&c) {
crates.push(c);
}
}
}
ret.push((workspace.clone(), crates));
Expand Down Expand Up @@ -163,27 +164,34 @@ fn download_and_save(mirror_path: &Path, vendors: Vec<(String, Vec<Crate>)>) ->
println!("[-] Vendoring: {workspace}");
let client = Client::new();
crates.sort();
for Crate { name, version } in crates {

crates.into_par_iter().for_each(|c| {
let Crate { name, version } = c;
let dir_crate_path = get_crate_path(mirror_path, &name, &version).unwrap();
let crate_path = dir_crate_path.join(format!("{name}-{version}.crate"));

// check if file already exists
while fs::metadata(crate_path.clone()).is_err() {
if !fs::exists(&crate_path).unwrap() {
// download
let url = format!("https://static.crates.io/crates/{name}/{name}-{version}.crate");
println!("[-] Downloading: {url}");
let Ok(response) = client.get(url).send() else {
break;
return;
};

if response.status() != StatusCode::OK {
println!("[-] Couldn't download {name}-{version}, not hosted on crates.io");
return;
}

let Ok(response) = response.bytes() else {
break;
return;
};

fs::create_dir_all(&dir_crate_path).unwrap();
fs::write(crate_path.clone(), response).unwrap();
}
}
})
});

Ok(())
Expand Down
20 changes: 20 additions & 0 deletions tests/test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ fn test_old_nightly_version() {
let tmp_dir_path = tmp_dir.into_path();
let output = cmd
.env("RUST_LOG", "none")
.env("RAYON_NUM_THREADS", "1") // deterministic ordering
.args([
tmp_dir_path.to_str().unwrap(),
"--skip-git-index",
Expand All @@ -37,24 +38,30 @@ fn test_old_nightly_version() {
r#"[-] Created {}
[-] Vendoring: {rustup_home}/toolchains/{nightly_ver}-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/test/Cargo.toml
[-] Downloading: https://static.crates.io/crates/alloc/alloc-0.0.0.crate
[-] Couldn't download alloc-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/allocator-api2/allocator-api2-0.2.18.crate
[-] Downloading: https://static.crates.io/crates/cfg-if/cfg-if-1.0.0.crate
[-] Downloading: https://static.crates.io/crates/compiler_builtins/compiler_builtins-0.1.109.crate
[-] Downloading: https://static.crates.io/crates/core/core-0.0.0.crate
[-] Couldn't download core-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/getopts/getopts-0.2.21.crate
[-] Downloading: https://static.crates.io/crates/hashbrown/hashbrown-0.14.5.crate
[-] Downloading: https://static.crates.io/crates/libc/libc-0.2.153.crate
[-] Downloading: https://static.crates.io/crates/panic_abort/panic_abort-0.0.0.crate
[-] Couldn't download panic_abort-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/panic_unwind/panic_unwind-0.0.0.crate
[-] Downloading: https://static.crates.io/crates/rustc-demangle/rustc-demangle-0.1.24.crate
[-] Downloading: https://static.crates.io/crates/rustc-std-workspace-alloc/rustc-std-workspace-alloc-1.0.0.crate
[-] Downloading: https://static.crates.io/crates/rustc-std-workspace-core/rustc-std-workspace-core-1.0.0.crate
[-] Downloading: https://static.crates.io/crates/rustc-std-workspace-std/rustc-std-workspace-std-1.0.1.crate
[-] Downloading: https://static.crates.io/crates/std/std-0.0.0.crate
[-] Couldn't download std-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/std_detect/std_detect-0.1.5.crate
[-] Downloading: https://static.crates.io/crates/test/test-0.0.0.crate
[-] Couldn't download test-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/unicode-width/unicode-width-0.1.12.crate
[-] Downloading: https://static.crates.io/crates/unwind/unwind-0.0.0.crate
[-] Couldn't download unwind-0.0.0, not hosted on crates.io
[-] Finished downloading crates
"#,
tmp_dir_path.to_str().unwrap()
Expand All @@ -74,6 +81,7 @@ fn test_new_nightly_version() {
let tmp_dir_path = tmp_dir.into_path();
let output = cmd
.env("RUST_LOG", "none")
.env("RAYON_NUM_THREADS", "1") // deterministic ordering
.args([
tmp_dir_path.to_str().unwrap(),
"--skip-git-index",
Expand All @@ -97,11 +105,13 @@ fn test_new_nightly_version() {
[-] Downloading: https://static.crates.io/crates/addr2line/addr2line-0.22.0.crate
[-] Downloading: https://static.crates.io/crates/adler/adler-1.0.2.crate
[-] Downloading: https://static.crates.io/crates/alloc/alloc-0.0.0.crate
[-] Couldn't download alloc-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/allocator-api2/allocator-api2-0.2.18.crate
[-] Downloading: https://static.crates.io/crates/cc/cc-1.1.22.crate
[-] Downloading: https://static.crates.io/crates/cfg-if/cfg-if-1.0.0.crate
[-] Downloading: https://static.crates.io/crates/compiler_builtins/compiler_builtins-0.1.133.crate
[-] Downloading: https://static.crates.io/crates/core/core-0.0.0.crate
[-] Couldn't download core-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/getopts/getopts-0.2.21.crate
[-] Downloading: https://static.crates.io/crates/gimli/gimli-0.29.0.crate
[-] Downloading: https://static.crates.io/crates/hashbrown/hashbrown-0.15.0.crate
Expand All @@ -110,23 +120,33 @@ fn test_new_nightly_version() {
[-] Downloading: https://static.crates.io/crates/miniz_oxide/miniz_oxide-0.7.4.crate
[-] Downloading: https://static.crates.io/crates/object/object-0.36.4.crate
[-] Downloading: https://static.crates.io/crates/panic_abort/panic_abort-0.0.0.crate
[-] Couldn't download panic_abort-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/panic_unwind/panic_unwind-0.0.0.crate
[-] Downloading: https://static.crates.io/crates/proc_macro/proc_macro-0.0.0.crate
[-] Couldn't download proc_macro-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/profiler_builtins/profiler_builtins-0.0.0.crate
[-] Couldn't download profiler_builtins-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/rand/rand-0.8.5.crate
[-] Downloading: https://static.crates.io/crates/rand_core/rand_core-0.6.4.crate
[-] Downloading: https://static.crates.io/crates/rand_xorshift/rand_xorshift-0.3.0.crate
[-] Downloading: https://static.crates.io/crates/rustc-demangle/rustc-demangle-0.1.24.crate
[-] Downloading: https://static.crates.io/crates/rustc-std-workspace-alloc/rustc-std-workspace-alloc-1.99.0.crate
[-] Couldn't download rustc-std-workspace-alloc-1.99.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/rustc-std-workspace-core/rustc-std-workspace-core-1.99.0.crate
[-] Couldn't download rustc-std-workspace-core-1.99.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/rustc-std-workspace-std/rustc-std-workspace-std-1.99.0.crate
[-] Couldn't download rustc-std-workspace-std-1.99.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/shlex/shlex-1.3.0.crate
[-] Downloading: https://static.crates.io/crates/std/std-0.0.0.crate
[-] Couldn't download std-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/std_detect/std_detect-0.1.5.crate
[-] Downloading: https://static.crates.io/crates/sysroot/sysroot-0.0.0.crate
[-] Couldn't download sysroot-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/test/test-0.0.0.crate
[-] Couldn't download test-0.0.0, not hosted on crates.io
[-] Downloading: https://static.crates.io/crates/unicode-width/unicode-width-0.1.14.crate
[-] Downloading: https://static.crates.io/crates/unwind/unwind-0.0.0.crate
[-] Couldn't download unwind-0.0.0, not hosted on crates.io
[-] Finished downloading crates
"#,
tmp_dir_path.to_str().unwrap()
Expand Down
Loading