Download objects from remote store without authentication

crates/sui-storage/Cargo.toml

@@ -21,6 +21,7 @@ tempfile.workspace = true
 tap.workspace = true
 reqwest = "0.11.20"
 percent-encoding = "2.3.0"
+chrono.workspace = true
 object_store.workspace = true
 backoff.workspace = true
 bytes.workspace = true

crates/sui-storage/src/object_store/downloader/gcs.rs

@@ -0,0 +1,70 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::object_store::downloader::{get, Downloader, DEFAULT_USER_AGENT};
+use anyhow::Result;
+use async_trait::async_trait;
+use bytes::Bytes;
+use object_store::path::Path;
+use object_store::GetResult;
+use percent_encoding::{percent_encode, utf8_percent_encode, NON_ALPHANUMERIC};
+use reqwest::Client;
+use reqwest::ClientBuilder;
+use std::sync::Arc;
+
+#[derive(Debug)]
+struct GoogleCloudStorageClient {
+    client: Client,
+    bucket_name_encoded: String,
+}
+
+impl GoogleCloudStorageClient {
+    pub fn new(bucket: &str) -> Result<Self> {
+        let mut builder = ClientBuilder::new();
+        builder = builder.user_agent(DEFAULT_USER_AGENT);
+        let client = builder.https_only(false).build()?;
+        let bucket_name_encoded = percent_encode(bucket.as_bytes(), NON_ALPHANUMERIC).to_string();
+
+        Ok(Self {
+            client,
+            bucket_name_encoded,
+        })
+    }
+
+    async fn get(&self, path: &Path) -> Result<GetResult> {
+        let url = self.object_url(path);
+        get(&url, "gcs", path, &self.client).await
+    }
+
+    fn object_url(&self, path: &Path) -> String {
+        let encoded = utf8_percent_encode(path.as_ref(), NON_ALPHANUMERIC);
+        format!(
+            "https://storage.googleapis.com/{}/{}",
+            self.bucket_name_encoded, encoded
+        )
+    }
+}
+
+/// Interface for [Google Cloud Storage](https://cloud.google.com/storage/).
+#[derive(Debug)]
+pub struct GoogleCloudStorage {
+    client: Arc<GoogleCloudStorageClient>,
+}
+
+impl GoogleCloudStorage {
+    pub fn new(bucket: &str) -> Result<Self> {
+        let gcs_client = GoogleCloudStorageClient::new(bucket)?;
+        Ok(GoogleCloudStorage {
+            client: Arc::new(gcs_client),
+        })
+    }
+}
+
+#[async_trait]
+impl Downloader for GoogleCloudStorage {
+    async fn get(&self, location: &Path) -> Result<Bytes> {
+        let result = self.client.get(location).await?;
+        let bytes = result.bytes().await?;
+        Ok(bytes)
+    }
+}

crates/sui-storage/src/object_store/downloader/local.rs

@@ -0,0 +1,44 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::object_store::downloader::Downloader;
+use crate::object_store::util::path_to_filesystem;
+use anyhow::{anyhow, Context, Result};
+use async_trait::async_trait;
+use bytes::Bytes;
+use object_store::path::Path;
+use std::fs;
+use std::fs::File;
+use std::io::Read;
+use std::path::PathBuf;
+
+pub struct LocalStorage {
+    root: PathBuf,
+}
+
+impl LocalStorage {
+    pub fn new(directory: &std::path::Path) -> Result<Self> {
+        let path = fs::canonicalize(directory).context(anyhow!("Unable to canonicalize"))?;
+        fs::create_dir_all(&path).context(anyhow!(
+            "Failed to create local directory: {}",
+            path.display()
+        ))?;
+        Ok(LocalStorage { root: path })
+    }
+}
+
+#[async_trait]
+impl Downloader for LocalStorage {
+    async fn get(&self, location: &Path) -> Result<Bytes> {
+        let path_to_filesystem = path_to_filesystem(self.root.clone(), location)?;
+        let handle = tokio::task::spawn_blocking(move || {
+            let mut f = File::open(path_to_filesystem)
+                .map_err(|e| anyhow!("Failed to open file with error: {}", e.to_string()))?;
+            let mut buf = vec![];
+            f.read_to_end(&mut buf)
+                .context(anyhow!("Failed to read file"))?;
+            Ok(buf.into())
+        });
+        handle.await?
+    }
+}