Refactor a method to asynchronously download a file on native, with a…

… simple progress bar. #523

Use it in the updater and all the importer tools. Only place it's not
used yet is parking_mapper and the in-game updater.  [rebuild]

Also make the RunCommand state understand the control code for erasing
the current line. It... sort of works.

abstio/Cargo.toml

@@ -14,6 +14,9 @@ log = "0.4.14"
 serde = "1.0.123"
 serde_json = "1.0.61"
 
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+reqwest = { version = "0.11.0", default-features=false, features=["rustls-tls"] }
+
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 include_dir = { git = "https://github.com/dabreegster/include_dir", branch = "union" }
 web-sys = { version = "0.3.47", features=["Storage", "Window"] }

abstio/src/download.rs

@@ -0,0 +1,46 @@
+use std::io::Write;
+
+use anyhow::{Context, Result};
+
+use abstutil::prettyprint_usize;
+
+/// Downloads bytes from a URL. If `quiet` is false, prints progress. This must be called with a
+/// tokio runtime somewhere.
+pub async fn download_bytes<I: AsRef<str>>(url: I, quiet: bool) -> Result<Vec<u8>> {
+    let url = url.as_ref();
+    let mut resp = reqwest::get(url).await.unwrap();
+    resp.error_for_status_ref()
+        .with_context(|| format!("downloading {}", url))?;
+
+    let total_size = resp.content_length().map(|x| x as usize);
+    let mut bytes = Vec::new();
+    while let Some(chunk) = resp.chunk().await.unwrap() {
+        if let Some(n) = total_size {
+            if !quiet {
+                abstutil::clear_current_line();
+                print!(
+                    "{:.2}% ({} / {} bytes)",
+                    (bytes.len() as f64) / (n as f64) * 100.0,
+                    prettyprint_usize(bytes.len()),
+                    prettyprint_usize(n)
+                );
+            }
+        }
+
+        bytes.write_all(&chunk).unwrap();
+    }
+    if !quiet {
+        println!();
+    }
+    Ok(bytes)
+}
+
+/// Downloads a file. If `quiet` is false, prints progress. This must be called with a tokio
+/// runtime somewhere.
+pub async fn download_to_file<I: AsRef<str>>(url: I, path: String, quiet: bool) -> Result<()> {
+    let bytes = download_bytes(url, quiet).await?;
+    std::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap())?;
+    let mut file = std::fs::File::create(&path)?;
+    file.write_all(&bytes)?;
+    Ok(())
+}

abstio/src/lib.rs

@@ -16,6 +16,11 @@ mod io_web;
 #[cfg(target_arch = "wasm32")]
 pub use io_web::*;
 
+#[cfg(not(target_arch = "wasm32"))]
+mod download;
+#[cfg(not(target_arch = "wasm32"))]
+pub use download::*;
+
 pub use abst_data::*;
 pub use abst_paths::*;
 

importer/Cargo.toml

@@ -27,7 +27,7 @@ osmio = "0.3.0"
 popdat = { path = "../popdat" }
 rand  = "0.8.3"
 rand_xorshift = "0.3.0"
-reqwest = { version = "0.11.0", default-features=false, features=["blocking"] }
 serde = "1.0.123"
 serde_json = "1.0.61"
 sim = { path = "../sim" }
+tokio = { version = "1.1.1", features = ["full"] }

importer/src/bin/one_step_import.rs

@@ -9,7 +9,8 @@ use abstutil::{must_run_cmd, CmdArgs};
 
 /// Import a one-shot A/B Street map in a single command. Takes a GeoJSON file with a boundary as
 /// input. Automatically fetches the OSM data, clips it, and runs the importer.
-fn main() -> Result<()> {
+#[tokio::main]
+async fn main() -> Result<()> {
     let mut args = CmdArgs::new();
     let geojson_path = args.required_free();
     let drive_on_left = args.enabled("--drive_on_left");
@@ -66,11 +67,7 @@ fn main() -> Result<()> {
     // TODO This is timing out. Also, really could use progress bars.
     if !abstio::file_exists(&pbf) {
         println!("Downloading {}", url);
-        let resp = reqwest::blocking::get(&url)?;
-        assert!(resp.status().is_success());
-        let bytes = resp.bytes()?;
-        let mut out = std::fs::File::create(&pbf)?;
-        out.write_all(&bytes)?;
+        abstio::download_to_file(url, pbf.clone(), false).await?;
     }
 
     // Clip it

importer/src/bin/pick_geofabrik.rs

@@ -20,7 +20,8 @@ use geom::LonLat;
 ///
 /// This is a useful tool when importing a new map, if you don't already know which geofabrik file
 /// you should use as your OSM input.
-fn main() -> Result<()> {
+#[tokio::main]
+async fn main() -> Result<()> {
     let mut args = CmdArgs::new();
     let input = args.required_free();
     args.done();
@@ -32,7 +33,8 @@ fn main() -> Result<()> {
     let geofabrik_idx = load_remote_geojson(
         abstio::path_shared_input("geofabrik-index.json"),
         "https://download.geofabrik.de/index-v1.json",
-    )?;
+    )
+    .await?;
     let matches = find_matching_regions(geofabrik_idx, center);
     info!(
         "{} regions contain boundary center {}",
@@ -49,18 +51,10 @@ fn main() -> Result<()> {
     Ok(())
 }
 
-fn load_remote_geojson(path: String, url: &str) -> Result<GeoJson> {
+async fn load_remote_geojson(path: String, url: &str) -> Result<GeoJson> {
     if !abstio::file_exists(&path) {
         info!("Downloading {}", url);
-        let resp = reqwest::blocking::get(url)?;
-        if !resp.status().is_success() {
-            bail!("bad status: {:?}", resp.status());
-        }
-
-        std::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap())
-            .expect("Creating parent dir failed");
-        let mut file = std::fs::File::create(&path)?;
-        file.write_all(&resp.bytes()?)?;
+        abstio::download_to_file(url, path.clone(), false).await?;
     }
     abstio::maybe_read_json(path, &mut Timer::throwaway())
 }

map_gui/src/tools/command.rs

@@ -89,7 +89,14 @@ impl<A: AppLike + 'static> RunCommand<A> {
                 if self.lines.len() == self.max_capacity {
                     self.lines.pop_front();
                 }
-                self.lines.push_back(line);
+                // Handle the "clear the current line" escape code
+                if line.contains("\r") {
+                    self.lines.pop_front();
+                    self.lines
+                        .push_back(line.split('\r').last().unwrap().to_string());
+                } else {
+                    self.lines.push_back(line);
+                }
             }
         }
     }

updater/Cargo.toml

@@ -11,6 +11,5 @@ anyhow = "1.0.38"
 flate2 = "1.0.20"
 geom = { path = "../geom" }
 md5 = "0.7.0"
-reqwest = { version = "0.11.0", default-features=false, features=["rustls-tls"] }
 tokio = { version = "1.1.1", features = ["full"] }
 walkdir = "2.3.1"

updater/src/main.rs

@@ -1,14 +1,13 @@
 use std::collections::{BTreeMap, BTreeSet};
 use std::fs::File;
-use std::io::{BufReader, Read, Write};
+use std::io::{BufReader, Read};
 use std::process::Command;
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 use walkdir::WalkDir;
 
 use abstio::{DataPacks, Entry, Manifest};
 use abstutil::{must_run_cmd, prettyprint_usize, CmdArgs, Parallelism, Timer};
-use geom::Percent;
 
 const MD5_BUF_READ_SIZE: usize = 4096;
 
@@ -42,11 +41,11 @@ async fn main() {
     } else {
         let quiet = args.enabled("--quiet");
         args.done();
-        download(version, quiet).await;
+        download_updates(version, quiet).await;
     }
 }
 
-async fn download(version: String, quiet: bool) {
+async fn download_updates(version: String, quiet: bool) {
     let data_packs = DataPacks::load_or_create();
     let local = generate_manifest();
     let truth = Manifest::load().filter(data_packs);
@@ -63,7 +62,7 @@ async fn download(version: String, quiet: bool) {
     for (path, entry) in truth.entries {
         if local.entries.get(&path).map(|x| &x.checksum) != Some(&entry.checksum) {
             std::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).unwrap();
-            match curl(&version, &path, quiet).await {
+            match download_file(&version, &path, quiet).await {
                 Ok(bytes) => {
                     println!(
                         "> decompress {}, which is {} bytes compressed",
@@ -285,7 +284,7 @@ fn rm(path: &str) {
     }
 }
 
-async fn curl(version: &str, path: &str, quiet: bool) -> Result<Vec<u8>> {
+async fn download_file(version: &str, path: &str, quiet: bool) -> Result<Vec<u8>> {
     // Manually enable to "download" from my local copy
     if false {
         return abstio::slurp_file(format!(
@@ -294,35 +293,12 @@ async fn curl(version: &str, path: &str, quiet: bool) -> Result<Vec<u8>> {
         ));
     }
 
-    let src = format!(
+    let url = format!(
         "http://abstreet.s3-website.us-east-2.amazonaws.com/{}/{}.gz",
         version, path
     );
-    println!("> download {}", src);
-
-    let mut resp = reqwest::get(&src).await.unwrap();
-    resp.error_for_status_ref()
-        .with_context(|| format!("downloading {}", src))?;
-
-    let total_size = resp.content_length().map(|x| x as usize);
-    let mut bytes = Vec::new();
-    while let Some(chunk) = resp.chunk().await.unwrap() {
-        if let Some(n) = total_size {
-            if !quiet {
-                abstutil::clear_current_line();
-                print!(
-                    "{} ({} / {} bytes)",
-                    Percent::of(bytes.len(), n),
-                    prettyprint_usize(bytes.len()),
-                    prettyprint_usize(n)
-                );
-            }
-        }
-
-        bytes.write_all(&chunk).unwrap();
-    }
-    println!();
-    Ok(bytes)
+    println!("> download {}", url);
+    abstio::download_bytes(url, quiet).await
 }
 
 // download() will remove stray files, but leave empty directories around. Since some runtime code