From 77709a24d63fea4d496e4aebe667e08194a36445 Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Mon, 15 Mar 2021 19:51:45 -0700 Subject: [PATCH] Refactor a method to asynchronously download a file on native, with a simple progress bar. #523 Use it in the updater and all the importer tools. Only place it's not used yet is parking_mapper and the in-game updater. [rebuild] Also make the RunCommand state understand the control code for erasing the current line. It... sort of works. --- Cargo.lock | 4 +-- abstio/Cargo.toml | 3 ++ abstio/src/download.rs | 46 +++++++++++++++++++++++++++++ abstio/src/lib.rs | 5 ++++ importer/Cargo.toml | 2 +- importer/src/bin/one_step_import.rs | 9 ++---- importer/src/bin/pick_geofabrik.rs | 18 ++++------- map_gui/src/tools/command.rs | 9 +++++- updater/Cargo.toml | 1 - updater/src/main.rs | 42 ++++++-------------------- 10 files changed, 83 insertions(+), 56 deletions(-) create mode 100644 abstio/src/download.rs diff --git a/Cargo.lock b/Cargo.lock index a457be6cdb..89869eb056 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,7 @@ dependencies = [ "instant", "lazy_static", "log", + "reqwest", "serde", "serde_json", "web-sys", @@ -1807,10 +1808,10 @@ dependencies = [ "popdat", "rand", "rand_xorshift", - "reqwest", "serde", "serde_json", "sim", + "tokio", ] [[package]] @@ -4135,7 +4136,6 @@ dependencies = [ "flate2", "geom", "md5", - "reqwest", "tokio", "walkdir", ] diff --git a/abstio/Cargo.toml b/abstio/Cargo.toml index 8acda57ae0..4d95b9af65 100644 --- a/abstio/Cargo.toml +++ b/abstio/Cargo.toml @@ -14,6 +14,9 @@ log = "0.4.14" serde = "1.0.123" serde_json = "1.0.61" +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +reqwest = { version = "0.11.0", default-features=false, features=["rustls-tls"] } + [target.'cfg(target_arch = "wasm32")'.dependencies] include_dir = { git = "https://github.com/dabreegster/include_dir", branch = "union" } web-sys = { version = "0.3.47", features=["Storage", "Window"] } diff --git a/abstio/src/download.rs b/abstio/src/download.rs new file mode 100644 index 0000000000..a980d7366c --- /dev/null +++ b/abstio/src/download.rs @@ -0,0 +1,46 @@ +use std::io::Write; + +use anyhow::{Context, Result}; + +use abstutil::prettyprint_usize; + +/// Downloads bytes from a URL. If `quiet` is false, prints progress. This must be called with a +/// tokio runtime somewhere. +pub async fn download_bytes>(url: I, quiet: bool) -> Result> { + let url = url.as_ref(); + let mut resp = reqwest::get(url).await.unwrap(); + resp.error_for_status_ref() + .with_context(|| format!("downloading {}", url))?; + + let total_size = resp.content_length().map(|x| x as usize); + let mut bytes = Vec::new(); + while let Some(chunk) = resp.chunk().await.unwrap() { + if let Some(n) = total_size { + if !quiet { + abstutil::clear_current_line(); + print!( + "{:.2}% ({} / {} bytes)", + (bytes.len() as f64) / (n as f64) * 100.0, + prettyprint_usize(bytes.len()), + prettyprint_usize(n) + ); + } + } + + bytes.write_all(&chunk).unwrap(); + } + if !quiet { + println!(); + } + Ok(bytes) +} + +/// Downloads a file. If `quiet` is false, prints progress. This must be called with a tokio +/// runtime somewhere. +pub async fn download_to_file>(url: I, path: String, quiet: bool) -> Result<()> { + let bytes = download_bytes(url, quiet).await?; + std::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap())?; + let mut file = std::fs::File::create(&path)?; + file.write_all(&bytes)?; + Ok(()) +} diff --git a/abstio/src/lib.rs b/abstio/src/lib.rs index cb53c26d93..ecbcf32a76 100644 --- a/abstio/src/lib.rs +++ b/abstio/src/lib.rs @@ -16,6 +16,11 @@ mod io_web; #[cfg(target_arch = "wasm32")] pub use io_web::*; +#[cfg(not(target_arch = "wasm32"))] +mod download; +#[cfg(not(target_arch = "wasm32"))] +pub use download::*; + pub use abst_data::*; pub use abst_paths::*; diff --git a/importer/Cargo.toml b/importer/Cargo.toml index 10a9f2c0c0..fb6afb39ef 100644 --- a/importer/Cargo.toml +++ b/importer/Cargo.toml @@ -27,7 +27,7 @@ osmio = "0.3.0" popdat = { path = "../popdat" } rand = "0.8.3" rand_xorshift = "0.3.0" -reqwest = { version = "0.11.0", default-features=false, features=["blocking"] } serde = "1.0.123" serde_json = "1.0.61" sim = { path = "../sim" } +tokio = { version = "1.1.1", features = ["full"] } diff --git a/importer/src/bin/one_step_import.rs b/importer/src/bin/one_step_import.rs index 714bae721b..4a3fd6c69c 100644 --- a/importer/src/bin/one_step_import.rs +++ b/importer/src/bin/one_step_import.rs @@ -9,7 +9,8 @@ use abstutil::{must_run_cmd, CmdArgs}; /// Import a one-shot A/B Street map in a single command. Takes a GeoJSON file with a boundary as /// input. Automatically fetches the OSM data, clips it, and runs the importer. -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { let mut args = CmdArgs::new(); let geojson_path = args.required_free(); let drive_on_left = args.enabled("--drive_on_left"); @@ -66,11 +67,7 @@ fn main() -> Result<()> { // TODO This is timing out. Also, really could use progress bars. if !abstio::file_exists(&pbf) { println!("Downloading {}", url); - let resp = reqwest::blocking::get(&url)?; - assert!(resp.status().is_success()); - let bytes = resp.bytes()?; - let mut out = std::fs::File::create(&pbf)?; - out.write_all(&bytes)?; + abstio::download_to_file(url, pbf.clone(), false).await?; } // Clip it diff --git a/importer/src/bin/pick_geofabrik.rs b/importer/src/bin/pick_geofabrik.rs index 9709604cab..a40d65b9fe 100644 --- a/importer/src/bin/pick_geofabrik.rs +++ b/importer/src/bin/pick_geofabrik.rs @@ -20,7 +20,8 @@ use geom::LonLat; /// /// This is a useful tool when importing a new map, if you don't already know which geofabrik file /// you should use as your OSM input. -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { let mut args = CmdArgs::new(); let input = args.required_free(); args.done(); @@ -32,7 +33,8 @@ fn main() -> Result<()> { let geofabrik_idx = load_remote_geojson( abstio::path_shared_input("geofabrik-index.json"), "https://download.geofabrik.de/index-v1.json", - )?; + ) + .await?; let matches = find_matching_regions(geofabrik_idx, center); info!( "{} regions contain boundary center {}", @@ -49,18 +51,10 @@ fn main() -> Result<()> { Ok(()) } -fn load_remote_geojson(path: String, url: &str) -> Result { +async fn load_remote_geojson(path: String, url: &str) -> Result { if !abstio::file_exists(&path) { info!("Downloading {}", url); - let resp = reqwest::blocking::get(url)?; - if !resp.status().is_success() { - bail!("bad status: {:?}", resp.status()); - } - - std::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()) - .expect("Creating parent dir failed"); - let mut file = std::fs::File::create(&path)?; - file.write_all(&resp.bytes()?)?; + abstio::download_to_file(url, path.clone(), false).await?; } abstio::maybe_read_json(path, &mut Timer::throwaway()) } diff --git a/map_gui/src/tools/command.rs b/map_gui/src/tools/command.rs index c5f3e412de..b8fe34ed80 100644 --- a/map_gui/src/tools/command.rs +++ b/map_gui/src/tools/command.rs @@ -89,7 +89,14 @@ impl RunCommand { if self.lines.len() == self.max_capacity { self.lines.pop_front(); } - self.lines.push_back(line); + // Handle the "clear the current line" escape code + if line.contains("\r") { + self.lines.pop_front(); + self.lines + .push_back(line.split('\r').last().unwrap().to_string()); + } else { + self.lines.push_back(line); + } } } } diff --git a/updater/Cargo.toml b/updater/Cargo.toml index 2e64f2effe..364481be2b 100644 --- a/updater/Cargo.toml +++ b/updater/Cargo.toml @@ -11,6 +11,5 @@ anyhow = "1.0.38" flate2 = "1.0.20" geom = { path = "../geom" } md5 = "0.7.0" -reqwest = { version = "0.11.0", default-features=false, features=["rustls-tls"] } tokio = { version = "1.1.1", features = ["full"] } walkdir = "2.3.1" diff --git a/updater/src/main.rs b/updater/src/main.rs index 29c961e4c1..a0a773d972 100644 --- a/updater/src/main.rs +++ b/updater/src/main.rs @@ -1,14 +1,13 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs::File; -use std::io::{BufReader, Read, Write}; +use std::io::{BufReader, Read}; use std::process::Command; -use anyhow::{Context, Result}; +use anyhow::Result; use walkdir::WalkDir; use abstio::{DataPacks, Entry, Manifest}; use abstutil::{must_run_cmd, prettyprint_usize, CmdArgs, Parallelism, Timer}; -use geom::Percent; const MD5_BUF_READ_SIZE: usize = 4096; @@ -42,11 +41,11 @@ async fn main() { } else { let quiet = args.enabled("--quiet"); args.done(); - download(version, quiet).await; + download_updates(version, quiet).await; } } -async fn download(version: String, quiet: bool) { +async fn download_updates(version: String, quiet: bool) { let data_packs = DataPacks::load_or_create(); let local = generate_manifest(); let truth = Manifest::load().filter(data_packs); @@ -63,7 +62,7 @@ async fn download(version: String, quiet: bool) { for (path, entry) in truth.entries { if local.entries.get(&path).map(|x| &x.checksum) != Some(&entry.checksum) { std::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).unwrap(); - match curl(&version, &path, quiet).await { + match download_file(&version, &path, quiet).await { Ok(bytes) => { println!( "> decompress {}, which is {} bytes compressed", @@ -285,7 +284,7 @@ fn rm(path: &str) { } } -async fn curl(version: &str, path: &str, quiet: bool) -> Result> { +async fn download_file(version: &str, path: &str, quiet: bool) -> Result> { // Manually enable to "download" from my local copy if false { return abstio::slurp_file(format!( @@ -294,35 +293,12 @@ async fn curl(version: &str, path: &str, quiet: bool) -> Result> { )); } - let src = format!( + let url = format!( "http://abstreet.s3-website.us-east-2.amazonaws.com/{}/{}.gz", version, path ); - println!("> download {}", src); - - let mut resp = reqwest::get(&src).await.unwrap(); - resp.error_for_status_ref() - .with_context(|| format!("downloading {}", src))?; - - let total_size = resp.content_length().map(|x| x as usize); - let mut bytes = Vec::new(); - while let Some(chunk) = resp.chunk().await.unwrap() { - if let Some(n) = total_size { - if !quiet { - abstutil::clear_current_line(); - print!( - "{} ({} / {} bytes)", - Percent::of(bytes.len(), n), - prettyprint_usize(bytes.len()), - prettyprint_usize(n) - ); - } - } - - bytes.write_all(&chunk).unwrap(); - } - println!(); - Ok(bytes) + println!("> download {}", url); + abstio::download_bytes(url, quiet).await } // download() will remove stray files, but leave empty directories around. Since some runtime code