From fc4462e6ec881dc86d3059616b8c127a5832eab9 Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Fri, 6 Oct 2023 19:57:27 -0300 Subject: [PATCH] feat: `Download` trait for `Artifact` --- Cargo.lock | 1 + crates/fluvio-hub-util/Cargo.toml | 1 + crates/fluvio-hub-util/src/fvm/api.rs | 109 ---------- crates/fluvio-hub-util/src/fvm/api/client.rs | 190 ++++++++++++++++++ .../fluvio-hub-util/src/fvm/api/download.rs | 187 +++++++++++++++++ crates/fluvio-hub-util/src/fvm/api/mod.rs | 5 + crates/fluvio-hub-util/src/fvm/mod.rs | 2 +- 7 files changed, 385 insertions(+), 110 deletions(-) delete mode 100644 crates/fluvio-hub-util/src/fvm/api.rs create mode 100644 crates/fluvio-hub-util/src/fvm/api/client.rs create mode 100644 crates/fluvio-hub-util/src/fvm/api/download.rs create mode 100644 crates/fluvio-hub-util/src/fvm/api/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 084ebd31c87..4a16aa7745c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2893,6 +2893,7 @@ name = "fluvio-hub-util" version = "0.0.0" dependencies = [ "anyhow", + "async-std", "cargo_toml 0.16.3", "const_format", "dirs 5.0.1", diff --git a/crates/fluvio-hub-util/Cargo.toml b/crates/fluvio-hub-util/Cargo.toml index dd518c5a0a4..b93765aa4b4 100644 --- a/crates/fluvio-hub-util/Cargo.toml +++ b/crates/fluvio-hub-util/Cargo.toml @@ -41,4 +41,5 @@ fluvio-hub-protocol = { path = "../fluvio-hub-protocol" } fluvio-types = { workspace = true } [dev-dependencies] +async-std = { workspace = true, features = ["attributes"] } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } diff --git a/crates/fluvio-hub-util/src/fvm/api.rs b/crates/fluvio-hub-util/src/fvm/api.rs deleted file mode 100644 index 7687604b110..00000000000 --- a/crates/fluvio-hub-util/src/fvm/api.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! Hub FVM API Client - -use anyhow::{Error, Result}; -use surf::get; -use url::Url; - -use fluvio_hub_protocol::constants::HUB_REMOTE; - -use super::{Channel, PackageSet, RustTarget}; - -/// HTTP Client for interacting with the Hub FVM API -pub struct Client { - api_url: Url, -} - -impl Client { - /// Creates a new [`Client`] with the default Hub API URL - #[allow(unused)] - pub fn new() -> Result { - let api_url = Url::parse(HUB_REMOTE)?; - - Ok(Self { api_url }) - } - - /// Fetches a [`PackageSet`] from the Hub with the specific [`Channel`] - #[allow(unused)] - pub async fn fetch_package_set( - &self, - name: impl AsRef, - channel: &Channel, - arch: &RustTarget, - ) -> Result { - let url = self.make_fetch_package_set_url(name, channel, arch)?; - let mut res = get(url).await.map_err(|err| Error::msg(err.to_string()))?; - let pkg = res - .body_json::() - .await - .map_err(|err| Error::msg(err.to_string()))?; - - Ok(pkg) - } - - /// Builds the URL to the Hub API for fetching a [`PackageSet`] using the - /// [`Client`]'s `api_url`. - fn make_fetch_package_set_url( - &self, - name: impl AsRef, - channel: &Channel, - arch: &RustTarget, - ) -> Result { - let url = format!( - "{}hub/v1/fvm/pkgset/{name}/{channel}/{arch}", - self.api_url, - name = name.as_ref(), - channel = channel, - arch = arch - ); - - Ok(Url::parse(&url)?) - } -} - -#[cfg(test)] -mod tests { - use std::str::FromStr; - - use url::Url; - use semver::Version; - - use super::{Client, Channel, RustTarget}; - - #[test] - fn creates_a_default_client() { - let client = Client::new().unwrap(); - - assert_eq!( - client.api_url, - Url::parse("https://hub.infinyon.cloud").unwrap() - ); - } - - #[test] - fn builds_url_for_fetching_pkgsets() { - let client = Client::new().unwrap(); - let url = client - .make_fetch_package_set_url( - "fluvio", - &Channel::Stable, - &RustTarget::ArmUnknownLinuxGnueabihf, - ) - .unwrap(); - - assert_eq!(url.as_str(), "https://hub.infinyon.cloud/hub/v1/fvm/pkgset/fluvio/stable/arm-unknown-linux-gnueabihf"); - } - - #[test] - fn builds_url_for_fetching_pkgsets_on_version() { - let client = Client::new().unwrap(); - let url = client - .make_fetch_package_set_url( - "fluvio", - &Channel::Tag(Version::from_str("0.10.14-dev+123345abc").unwrap()), - &RustTarget::ArmUnknownLinuxGnueabihf, - ) - .unwrap(); - - assert_eq!(url.as_str(), "https://hub.infinyon.cloud/hub/v1/fvm/pkgset/fluvio/0.10.14-dev+123345abc/arm-unknown-linux-gnueabihf"); - } -} diff --git a/crates/fluvio-hub-util/src/fvm/api/client.rs b/crates/fluvio-hub-util/src/fvm/api/client.rs new file mode 100644 index 00000000000..ecc7f63792a --- /dev/null +++ b/crates/fluvio-hub-util/src/fvm/api/client.rs @@ -0,0 +1,190 @@ +//! Hub FVM API Client + +use std::path::PathBuf; +use std::fs::File; +use std::io::{Cursor, copy, Read}; + +use anyhow::{Error, Result}; +use sha2::{Digest, Sha256}; +use surf::{get, StatusCode}; +use url::Url; + +use crate::fvm::{Channel, PackageSet, Artifact}; + +/// HTTP Client for interacting with the Hub FVM API +pub struct Client { + api_url: Url, +} + +impl Client { + /// Creates a new [`Client`] with the default Hub API URL + pub fn new(url: &str) -> Result { + let api_url = url.parse::()?; + + Ok(Self { api_url }) + } + + /// Fetches a [`PackageSet`] from the Hub with the specific [`Channel`] + pub async fn fetch_package_set( + &self, + name: impl AsRef, + channel: &Channel, + arch: &str, + ) -> Result { + let url = self.make_fetch_package_set_url(name, channel, arch)?; + let mut res = get(url).await.map_err(|err| Error::msg(err.to_string()))?; + let pkg = res.body_json::().await.map_err(|err| { + Error::msg(format!( + "Server responded with status code {}", + err.status() + )) + })?; + + Ok(pkg) + } + + /// Downloads binaries from the Hub into the specified `target_dir`. + /// Returns the [`PackageSet`] that was downloaded. + pub async fn download_package_set( + &self, + name: impl AsRef, + channel: &Channel, + arch: &str, + target_dir: PathBuf, + ) -> Result { + let pkgset = self.fetch_package_set(name, channel, arch).await?; + + for artf in pkgset.artifacts.iter() { + let mut res = surf::get(&artf.download_url) + .await + .map_err(|err| Error::msg(err.to_string()))?; + + if res.status() == StatusCode::Ok { + let out_path = target_dir.join(&artf.name); + let mut file = File::create(&out_path)?; + let mut buf = Cursor::new( + res.body_bytes() + .await + .map_err(|err| Error::msg(err.to_string()))?, + ); + + copy(&mut buf, &mut file)?; + self.checksum_artifact(artf, &file).await?; + + tracing::debug!( + "Artifact downloaded: {} at {:?}", + artf.name, + out_path.display() + ); + + continue; + } + + tracing::warn!( + "Failed to download artifact {}@{} from {}", + artf.name, + artf.version, + artf.download_url + ); + } + + Ok(pkgset) + } + + /// Verifies downloaded artifacts checksums against the upstream checksums + async fn checksum_artifact(&self, artf: &Artifact, local: &File) -> Result<()> { + let local_file_shasum = Self::shasum256(local)?; + let upstream_shasum = surf::get(&artf.sha256_url) + .await + .map_err(|err| Error::msg(err.to_string()))? + .body_string() + .await + .map_err(|err| Error::msg(err.to_string()))?; + + if local_file_shasum != upstream_shasum { + return Err(Error::msg(format!( + "Artifact {} didnt matched upstream shasum. {} != {}", + artf.name, local_file_shasum, upstream_shasum + ))); + } + + Ok(()) + } + + /// Generates the Sha256 checksum for the specified file + fn shasum256(file: &File) -> Result { + let meta = file.metadata()?; + let mut file = file.clone(); + let mut hasher = Sha256::new(); + let mut buffer = vec![0; meta.len() as usize]; + + file.read_exact(&mut buffer)?; + hasher.update(buffer); + + let output = hasher.finalize(); + Ok(hex::encode(output)) + } + + /// Builds the URL to the Hub API for fetching a [`PackageSet`] using the + /// [`Client`]'s `api_url`. + fn make_fetch_package_set_url( + &self, + name: impl AsRef, + channel: &Channel, + arch: &str, + ) -> Result { + let url = format!( + "{}hub/v1/fvm/pkgset/{name}/{channel}/{arch}", + self.api_url, + name = name.as_ref(), + channel = channel, + arch = arch + ); + + Ok(Url::parse(&url)?) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use url::Url; + use semver::Version; + + use super::{Client, Channel}; + + #[test] + fn creates_a_default_client() { + let client = Client::new("https://hub.infinyon.cloud").unwrap(); + + assert_eq!( + client.api_url, + Url::parse("https://hub.infinyon.cloud").unwrap() + ); + } + + #[test] + fn builds_url_for_fetching_pkgsets() { + let client = Client::new("https://hub.infinyon.cloud").unwrap(); + let url = client + .make_fetch_package_set_url("fluvio", &Channel::Stable, "arm-unknown-linux-gnueabihf") + .unwrap(); + + assert_eq!(url.as_str(), "https://hub.infinyon.cloud/hub/v1/fvm/pkgset/fluvio/stable/arm-unknown-linux-gnueabihf"); + } + + #[test] + fn builds_url_for_fetching_pkgsets_on_version() { + let client = Client::new("https://hub.infinyon.cloud").unwrap(); + let url = client + .make_fetch_package_set_url( + "fluvio", + &Channel::Tag(Version::from_str("0.10.14-dev+123345abc").unwrap()), + "arm-unknown-linux-gnueabihf", + ) + .unwrap(); + + assert_eq!(url.as_str(), "https://hub.infinyon.cloud/hub/v1/fvm/pkgset/fluvio/0.10.14-dev+123345abc/arm-unknown-linux-gnueabihf"); + } +} diff --git a/crates/fluvio-hub-util/src/fvm/api/download.rs b/crates/fluvio-hub-util/src/fvm/api/download.rs new file mode 100644 index 00000000000..a33e2f72fbc --- /dev/null +++ b/crates/fluvio-hub-util/src/fvm/api/download.rs @@ -0,0 +1,187 @@ +//! Download API for downloading the artifacts from the server + +use std::path::{PathBuf}; +use std::io::{Cursor, copy, Read}; +use std::fs::File; + +use anyhow::{Error, Result}; +use http_client::async_trait; +use sha2::{Sha256, Digest}; +use surf::StatusCode; + +use crate::fvm::Artifact; + +/// Generates the Sha256 checksum for the specified file +fn shasum256(file: &File) -> Result { + let meta = file.metadata()?; + let mut file = file.clone(); + let mut hasher = Sha256::new(); + let mut buffer = vec![0; meta.len() as usize]; + + file.read_exact(&mut buffer)?; + hasher.update(buffer); + + let output = hasher.finalize(); + Ok(hex::encode(output)) +} + +/// Verifies downloaded artifact checksums against the upstream checksums +async fn checksum(artf: &Artifact, path: &PathBuf) -> Result<()> { + let file = File::open(path)?; + let local_file_shasum = shasum256(&file)?; + let upstream_shasum = surf::get(&artf.sha256_url) + .await + .map_err(|err| Error::msg(err.to_string()))? + .body_string() + .await + .map_err(|err| Error::msg(err.to_string()))?; + + if local_file_shasum != upstream_shasum { + return Err(Error::msg(format!( + "Artifact {} didnt matched upstream shasum. {} != {}", + artf.name, local_file_shasum, upstream_shasum + ))); + } + + Ok(()) +} + +#[async_trait] +pub trait Download { + /// Downloads the artifact to the specified directory + /// + /// Internally validates the checksum of the downloaded artifact + async fn download(&self, target_dir: PathBuf) -> Result<()>; +} + +#[async_trait] +impl Download for Artifact { + async fn download(&self, target_dir: PathBuf) -> Result<()> { + let mut res = surf::get(&self.download_url) + .await + .map_err(|err| Error::msg(err.to_string()))?; + + if res.status() == StatusCode::Ok { + let out_path = target_dir.join(&self.name); + let mut file = File::create(&out_path)?; + let mut buf = Cursor::new( + res.body_bytes() + .await + .map_err(|err| Error::msg(err.to_string()))?, + ); + + copy(&mut buf, &mut file)?; + checksum(self, &out_path).await?; + + tracing::debug!( + "Artifact downloaded: {} at {:?}", + self.name, + out_path.display() + ); + + return Ok(()); + } + + Err(Error::msg(format!( + "Server responded with Status Code {}", + res.status() + ))) + } +} + +#[cfg(test)] +mod test { + use tempfile::TempDir; + + use super::*; + + #[async_std::test] + async fn download_artifact() { + let target_dir = TempDir::new().unwrap().into_path().to_path_buf(); + let artifact = Artifact { + name: "fluvio".to_string(), + version: "0.10.15".parse().unwrap(), + download_url: "https://packages.fluvio.io/v1/packages/fluvio/fluvio/0.10.15/aarch64-apple-darwin/fluvio".parse().unwrap(), + sha256_url: "https://packages.fluvio.io/v1/packages/fluvio/fluvio/0.10.15/aarch64-apple-darwin/fluvio.sha256".parse().unwrap(), + }; + + artifact.download(target_dir.clone()).await.unwrap(); + assert!(target_dir.join("fluvio").exists()); + } + + #[async_std::test] + async fn downloaded_artifact_matches_upstream_checksum() { + let target_dir = TempDir::new().unwrap().into_path().to_path_buf(); + let artifact = Artifact { + name: "fluvio".to_string(), + version: "0.10.15".parse().unwrap(), + download_url: "https://packages.fluvio.io/v1/packages/fluvio/fluvio/0.10.15/aarch64-apple-darwin/fluvio".parse().unwrap(), + sha256_url: "https://packages.fluvio.io/v1/packages/fluvio/fluvio/0.10.15/aarch64-apple-darwin/fluvio.sha256".parse().unwrap(), + }; + + artifact.download(target_dir.clone()).await.unwrap(); + + let binary_path = target_dir.join("fluvio"); + let file = File::open(binary_path).unwrap(); + let downstream_shasum = shasum256(&file).unwrap(); + let upstream_shasum = surf::get(&artifact.sha256_url) + .await + .map_err(|err| Error::msg(err.to_string())) + .unwrap() + .body_string() + .await + .map_err(|err| Error::msg(err.to_string())) + .unwrap(); + + assert_eq!(downstream_shasum, upstream_shasum); + } + + #[test] + fn creates_shasum_digest() { + use std::fs::write; + + let tempdir = TempDir::new().unwrap().into_path().to_path_buf(); + let foo_path = tempdir.join("foo"); + + write(&foo_path, "foo").unwrap(); + + let foo_a_checksum = shasum256(&File::open(&foo_path).unwrap()).unwrap(); + + assert_eq!( + foo_a_checksum, + "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" + ); + } + + #[test] + fn checks_files_checksums_diff() { + use std::fs::write; + + let tempdir = TempDir::new().unwrap().into_path().to_path_buf(); + let foo_path = tempdir.join("foo"); + let bar_path = tempdir.join("bar"); + + write(&foo_path, "foo").unwrap(); + write(&bar_path, "bar").unwrap(); + + let foo_checksum = shasum256(&File::open(&foo_path).unwrap()).unwrap(); + let bar_checksum = shasum256(&File::open(&bar_path).unwrap()).unwrap(); + + assert_ne!(foo_checksum, bar_checksum); + } + + #[test] + fn checks_files_checksums_same() { + use std::fs::write; + + let tempdir = TempDir::new().unwrap().into_path().to_path_buf(); + let foo_path = tempdir.join("foo"); + + write(&foo_path, "foo").unwrap(); + + let foo_a_checksum = shasum256(&File::open(&foo_path).unwrap()).unwrap(); + let foo_b_checksum = shasum256(&File::open(&foo_path).unwrap()).unwrap(); + + assert_eq!(foo_a_checksum, foo_b_checksum); + } +} diff --git a/crates/fluvio-hub-util/src/fvm/api/mod.rs b/crates/fluvio-hub-util/src/fvm/api/mod.rs new file mode 100644 index 00000000000..b8a79f2b728 --- /dev/null +++ b/crates/fluvio-hub-util/src/fvm/api/mod.rs @@ -0,0 +1,5 @@ +mod client; +mod download; + +pub use client::Client; +pub use download::Download; diff --git a/crates/fluvio-hub-util/src/fvm/mod.rs b/crates/fluvio-hub-util/src/fvm/mod.rs index 6e5ae2645b6..e233238eacf 100644 --- a/crates/fluvio-hub-util/src/fvm/mod.rs +++ b/crates/fluvio-hub-util/src/fvm/mod.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use semver::Version; use url::Url; -pub use api::Client; +pub use api::{Client, Download}; pub const STABLE_VERSION_CHANNEL: &str = "stable"; pub const LATEST_VERSION_CHANNEL: &str = "latest";