feat: Download trait for Artifact (#3589)

Introduces download logic for `Artifact` resource in the Hub Util API.

Download logic is introduced through the `Download` trait which allow for decoupling
from the type definition.

crates/fluvio-hub-util/Cargo.toml

@@ -36,7 +36,7 @@ thiserror = { workspace = true }
 url = { workspace = true }
 wasmparser = { workspace = true }
 
-fluvio-future = { workspace = true, features = ["task"]}
+fluvio-future = { workspace = true, features = ["fixture", "task"] }
 fluvio-hub-protocol = { path = "../fluvio-hub-protocol" }
 fluvio-types = { workspace = true }
 

crates/fluvio-hub-util/src/fvm/api.rs → crates/fluvio-hub-util/src/fvm/api/client.rs

@@ -1,12 +1,9 @@
 //! Hub FVM API Client
 
 use anyhow::{Error, Result};
-use surf::get;
 use url::Url;
 
-use fluvio_hub_protocol::constants::HUB_REMOTE;
-
-use super::{Channel, PackageSet, RustTarget};
+use crate::fvm::{Channel, PackageSet};
 
 /// HTTP Client for interacting with the Hub FVM API
 pub struct Client {
@@ -15,27 +12,31 @@ pub struct Client {
 
 impl Client {
     /// Creates a new [`Client`] with the default Hub API URL
-    #[allow(unused)]
-    pub fn new() -> Result<Self> {
-        let api_url = Url::parse(HUB_REMOTE)?;
+    pub fn new(url: &str) -> Result<Self> {
+        let api_url = url.parse::<Url>()?;
 
         Ok(Self { api_url })
     }
 
     /// Fetches a [`PackageSet`] from the Hub with the specific [`Channel`]
-    #[allow(unused)]
     pub async fn fetch_package_set(
         &self,
         name: impl AsRef<str>,
         channel: &Channel,
-        arch: &RustTarget,
+        arch: &str,
     ) -> Result<PackageSet> {
         let url = self.make_fetch_package_set_url(name, channel, arch)?;
-        let mut res = get(url).await.map_err(|err| Error::msg(err.to_string()))?;
-        let pkg = res
-            .body_json::<PackageSet>()
+        let mut res = surf::get(url)
             .await
             .map_err(|err| Error::msg(err.to_string()))?;
+        let pkg = res.body_json::<PackageSet>().await.map_err(|err| {
+            Error::msg(format!(
+                "Server responded with status code {}",
+                err.status()
+            ))
+        })?;
+
+        tracing::info!(?pkg, "Found PackageSet");
 
         Ok(pkg)
     }
@@ -46,7 +47,7 @@ impl Client {
         &self,
         name: impl AsRef<str>,
         channel: &Channel,
-        arch: &RustTarget,
+        arch: &str,
     ) -> Result<Url> {
         let url = format!(
             "{}hub/v1/fvm/pkgset/{name}/{channel}/{arch}",
@@ -67,11 +68,11 @@ mod tests {
     use url::Url;
     use semver::Version;
 
-    use super::{Client, Channel, RustTarget};
+    use super::{Client, Channel};
 
     #[test]
     fn creates_a_default_client() {
-        let client = Client::new().unwrap();
+        let client = Client::new("https://hub.infinyon.cloud").unwrap();
 
         assert_eq!(
             client.api_url,
@@ -81,26 +82,22 @@ mod tests {
 
     #[test]
     fn builds_url_for_fetching_pkgsets() {
-        let client = Client::new().unwrap();
+        let client = Client::new("https://hub.infinyon.cloud").unwrap();
         let url = client
-            .make_fetch_package_set_url(
-                "fluvio",
-                &Channel::Stable,
-                &RustTarget::ArmUnknownLinuxGnueabihf,
-            )
+            .make_fetch_package_set_url("fluvio", &Channel::Stable, "arm-unknown-linux-gnueabihf")
             .unwrap();
 
         assert_eq!(url.as_str(), "https://hub.infinyon.cloud/hub/v1/fvm/pkgset/fluvio/stable/arm-unknown-linux-gnueabihf");
     }
 
     #[test]
     fn builds_url_for_fetching_pkgsets_on_version() {
-        let client = Client::new().unwrap();
+        let client = Client::new("https://hub.infinyon.cloud").unwrap();
         let url = client
             .make_fetch_package_set_url(
                 "fluvio",
                 &Channel::Tag(Version::from_str("0.10.14-dev+123345abc").unwrap()),
-                &RustTarget::ArmUnknownLinuxGnueabihf,
+                "arm-unknown-linux-gnueabihf",
             )
             .unwrap();
 

crates/fluvio-hub-util/src/fvm/api/download.rs

@@ -0,0 +1,132 @@
+//! Download API for downloading the artifacts from the server
+
+use std::path::PathBuf;
+use std::io::{Cursor, copy};
+use std::fs::File;
+
+use anyhow::{Error, Result};
+use http_client::async_trait;
+use surf::StatusCode;
+use tracing::instrument;
+
+use crate::fvm::Artifact;
+use crate::utils::sha256_digest;
+
+/// Verifies downloaded artifact checksums against the upstream checksums
+async fn checksum(artf: &Artifact, path: &PathBuf) -> Result<()> {
+    let local_file_shasum = sha256_digest(path)?;
+    let upstream_shasum = surf::get(&artf.sha256_url)
+        .await
+        .map_err(|err| Error::msg(err.to_string()))?
+        .body_string()
+        .await
+        .map_err(|err| Error::msg(err.to_string()))?;
+
+    if local_file_shasum != upstream_shasum {
+        return Err(Error::msg(format!(
+            "Artifact {} didnt matched upstream shasum. {} != {}",
+            artf.name, local_file_shasum, upstream_shasum
+        )));
+    }
+
+    Ok(())
+}
+
+#[async_trait]
+pub trait Download {
+    /// Downloads the artifact to the specified directory
+    ///
+    /// Internally validates the checksum of the downloaded artifact
+    async fn download(&self, target_dir: PathBuf) -> Result<()>;
+}
+
+#[async_trait]
+impl Download for Artifact {
+    #[instrument(skip(self, target_dir))]
+    async fn download(&self, target_dir: PathBuf) -> Result<()> {
+        tracing::info!(
+            name = self.name,
+            download_url = ?self.download_url,
+            "Downloading artifact"
+        );
+
+        let mut res = surf::get(&self.download_url)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+
+        if res.status() == StatusCode::Ok {
+            let out_path = target_dir.join(&self.name);
+            let mut file = File::create(&out_path)?;
+            let mut buf = Cursor::new(
+                res.body_bytes()
+                    .await
+                    .map_err(|err| Error::msg(err.to_string()))?,
+            );
+
+            copy(&mut buf, &mut file)?;
+            checksum(self, &out_path).await?;
+
+            tracing::debug!(
+                name = self.name,
+                out_path = ?out_path.display(),
+                "Artifact downloaded",
+            );
+
+            return Ok(());
+        }
+
+        Err(Error::msg(format!(
+            "Server responded with Status Code {}",
+            res.status()
+        )))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use tempfile::TempDir;
+
+    use super::*;
+
+    #[ignore]
+    #[fluvio_future::test]
+    async fn download_artifact() {
+        let target_dir = TempDir::new().unwrap().into_path().to_path_buf();
+        let artifact = Artifact {
+            name: "fluvio".to_string(),
+            version: "0.10.15".parse().unwrap(),
+            download_url: "https://packages.fluvio.io/v1/packages/fluvio/fluvio/0.10.15/aarch64-apple-darwin/fluvio".parse().unwrap(),
+            sha256_url: "https://packages.fluvio.io/v1/packages/fluvio/fluvio/0.10.15/aarch64-apple-darwin/fluvio.sha256".parse().unwrap(),
+        };
+
+        artifact.download(target_dir.clone()).await.unwrap();
+        assert!(target_dir.join("fluvio").exists());
+    }
+
+    #[ignore]
+    #[fluvio_future::test]
+    async fn downloaded_artifact_matches_upstream_checksum() {
+        let target_dir = TempDir::new().unwrap().into_path().to_path_buf();
+        let artifact = Artifact {
+            name: "fluvio".to_string(),
+            version: "0.10.15".parse().unwrap(),
+            download_url: "https://packages.fluvio.io/v1/packages/fluvio/fluvio/0.10.15/aarch64-apple-darwin/fluvio".parse().unwrap(),
+            sha256_url: "https://packages.fluvio.io/v1/packages/fluvio/fluvio/0.10.15/aarch64-apple-darwin/fluvio.sha256".parse().unwrap(),
+        };
+
+        artifact.download(target_dir.clone()).await.unwrap();
+
+        let binary_path = target_dir.join("fluvio");
+        let downstream_shasum = sha256_digest(&binary_path).unwrap();
+        let upstream_shasum = surf::get(&artifact.sha256_url)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))
+            .unwrap()
+            .body_string()
+            .await
+            .map_err(|err| Error::msg(err.to_string()))
+            .unwrap();
+
+        assert_eq!(downstream_shasum, upstream_shasum);
+    }
+}

crates/fluvio-hub-util/src/fvm/api/mod.rs

@@ -0,0 +1,5 @@
+mod client;
+mod download;
+
+pub use client::Client;
+pub use download::Download;

crates/fluvio-hub-util/src/fvm/mod.rs

@@ -2,30 +2,23 @@
 
 mod api;
 
-use std::{fmt::Display, cmp::Ordering};
+use std::fmt::Display;
+use std::cmp::Ordering;
 use std::str::FromStr;
 
 use thiserror::Error;
 use serde::{Deserialize, Serialize};
 use semver::Version;
 use url::Url;
 
-pub use api::Client;
+pub use api::{Client, Download};
 
 pub const STABLE_VERSION_CHANNEL: &str = "stable";
 pub const LATEST_VERSION_CHANNEL: &str = "latest";
 pub const DEFAULT_PKGSET: &str = "default";
 
-pub const ARM_UNKNOWN_LINUX_GNUEABIHF: &str = "arm-unknown-linux-gnueabihf";
-pub const ARMV7_UNKNOWN_LINUX_GNUEABIHF: &str = "armv7-unknown-linux-gnueabihf";
-pub const X86_64_APPLE_DARWIN: &str = "x86_64-apple-darwin";
-pub const AARCH64_APPLE_DARWIN: &str = "aarch64-apple-darwin";
-pub const X86_64_PC_WINDOWS_GNU: &str = "x86_64-pc-windows-gnu";
-
 #[derive(Clone, Debug, Error)]
 pub enum Error {
-    #[error("The provided Rust Target Triple \"{0}\" is not supported")]
-    RustTripleNotSupported(String),
     #[error("Invalid Fluvio Channel \"{0}\"")]
     InvalidChannel(String),
 }
@@ -104,50 +97,6 @@ impl FromStr for Channel {
     }
 }
 
-/// Available Rust Targets for Fluvio.
-///
-/// Refer: https://github.com/infinyon/fluvio/blob/f2c49e126c771d58d24d5f5cb0282a6aaa6b23ca/.github/workflows/ci.yml#L141
-#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
-pub enum RustTarget {
-    /// arm-unknown-linux-gnueabihf
-    ArmUnknownLinuxGnueabihf,
-    /// armv7-unknown-linux-gnueabihf
-    Armv7UnknownLinuxGnueabihf,
-    /// x86_64-apple-darwin
-    X86_64AppleDarwin,
-    /// aarch64-apple-darwin
-    Aarch64AppleDarwin,
-    /// x86_64-pc-windows-gnu
-    X86_64PcWindowsGnu,
-}
-
-impl Display for RustTarget {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::ArmUnknownLinuxGnueabihf => write!(f, "{}", ARM_UNKNOWN_LINUX_GNUEABIHF),
-            Self::Armv7UnknownLinuxGnueabihf => write!(f, "{}", ARMV7_UNKNOWN_LINUX_GNUEABIHF),
-            Self::X86_64AppleDarwin => write!(f, "{}", X86_64_APPLE_DARWIN),
-            Self::Aarch64AppleDarwin => write!(f, "{}", AARCH64_APPLE_DARWIN),
-            Self::X86_64PcWindowsGnu => write!(f, "{}", X86_64_PC_WINDOWS_GNU),
-        }
-    }
-}
-
-impl FromStr for RustTarget {
-    type Err = Error;
-
-    fn from_str(v: &str) -> Result<Self, Self::Err> {
-        match v {
-            ARM_UNKNOWN_LINUX_GNUEABIHF => Ok(Self::ArmUnknownLinuxGnueabihf),
-            ARMV7_UNKNOWN_LINUX_GNUEABIHF => Ok(Self::Armv7UnknownLinuxGnueabihf),
-            X86_64_APPLE_DARWIN => Ok(Self::X86_64AppleDarwin),
-            AARCH64_APPLE_DARWIN => Ok(Self::Aarch64AppleDarwin),
-            X86_64_PC_WINDOWS_GNU => Ok(Self::X86_64PcWindowsGnu),
-            _ => Err(Error::RustTripleNotSupported(v.to_string())),
-        }
-    }
-}
-
 /// Artifact download URL
 #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
 pub struct Artifact {
@@ -161,7 +110,7 @@ pub struct Artifact {
 #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
 pub struct PackageSet {
     pub version: Version,
-    pub arch: RustTarget,
+    pub arch: String,
     pub artifacts: Vec<Artifact>,
 }
 

crates/fluvio-hub-util/src/package.rs

@@ -556,16 +556,6 @@ mod tests {
 
     const PKG_SIGN_PUBKEY: &str = "tests/hubutil_package_sign-pubkey.pem";
 
-    fn rust_log_init() {
-        let trs = tracing_subscriber::fmt().with_max_level(tracing::Level::INFO);
-        if std::env::var("RUST_LOG").is_ok() {
-            trs.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
-                .init();
-        } else {
-            trs.init();
-        }
-    }
-
     #[test]
     fn hubutil_serialize_package_meta() {
         let pm = PackageMeta {
@@ -617,7 +607,6 @@ mod tests {
 
     #[test]
     fn hubutil_package_assemble() {
-        rust_log_init();
         let testfile: &str = "tests/apackage/package-meta.yaml";
         let res = package_assemble(testfile, "tests", None);
         assert!(res.is_ok());