-
Notifications
You must be signed in to change notification settings - Fork 11.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Download objects from remote store without authentication
- Loading branch information
1 parent
c3a04c2
commit 532d5fb
Showing
8 changed files
with
341 additions
and
148 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
use crate::object_store::downloader::{get, Downloader, DEFAULT_USER_AGENT}; | ||
use anyhow::Result; | ||
use async_trait::async_trait; | ||
use bytes::Bytes; | ||
use object_store::path::Path; | ||
use object_store::GetResult; | ||
use percent_encoding::{percent_encode, utf8_percent_encode, NON_ALPHANUMERIC}; | ||
use reqwest::Client; | ||
use reqwest::ClientBuilder; | ||
use std::sync::Arc; | ||
|
||
#[derive(Debug)] | ||
struct GoogleCloudStorageClient { | ||
client: Client, | ||
bucket_name_encoded: String, | ||
} | ||
|
||
impl GoogleCloudStorageClient { | ||
pub fn new(bucket: &str) -> Result<Self> { | ||
let mut builder = ClientBuilder::new(); | ||
builder = builder.user_agent(DEFAULT_USER_AGENT); | ||
let client = builder.https_only(false).build()?; | ||
let bucket_name_encoded = percent_encode(bucket.as_bytes(), NON_ALPHANUMERIC).to_string(); | ||
|
||
Ok(Self { | ||
client, | ||
bucket_name_encoded, | ||
}) | ||
} | ||
|
||
async fn get(&self, path: &Path) -> Result<GetResult> { | ||
let url = self.object_url(path); | ||
get(&url, "gcs", path, &self.client).await | ||
} | ||
|
||
fn object_url(&self, path: &Path) -> String { | ||
let encoded = utf8_percent_encode(path.as_ref(), NON_ALPHANUMERIC); | ||
format!( | ||
"https://storage.googleapis.com/{}/{}", | ||
self.bucket_name_encoded, encoded | ||
) | ||
} | ||
} | ||
|
||
/// Interface for [Google Cloud Storage](https://cloud.google.com/storage/). | ||
#[derive(Debug)] | ||
pub struct GoogleCloudStorage { | ||
client: Arc<GoogleCloudStorageClient>, | ||
} | ||
|
||
impl GoogleCloudStorage { | ||
pub fn new(bucket: &str) -> Result<Self> { | ||
let gcs_client = GoogleCloudStorageClient::new(bucket)?; | ||
Ok(GoogleCloudStorage { | ||
client: Arc::new(gcs_client), | ||
}) | ||
} | ||
} | ||
|
||
#[async_trait] | ||
impl Downloader for GoogleCloudStorage { | ||
async fn get(&self, location: &Path) -> Result<Bytes> { | ||
let result = self.client.get(location).await?; | ||
let bytes = result.bytes().await?; | ||
Ok(bytes) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
use crate::object_store::downloader::Downloader; | ||
use crate::object_store::util::path_to_filesystem; | ||
use anyhow::{anyhow, Context, Result}; | ||
use async_trait::async_trait; | ||
use bytes::Bytes; | ||
use object_store::path::Path; | ||
use std::fs; | ||
use std::fs::File; | ||
use std::io::Read; | ||
use std::path::PathBuf; | ||
|
||
pub struct LocalStorage { | ||
root: PathBuf, | ||
} | ||
|
||
impl LocalStorage { | ||
pub fn new(directory: &std::path::Path) -> Result<Self> { | ||
let path = fs::canonicalize(directory).context(anyhow!("Unable to canonicalize"))?; | ||
fs::create_dir_all(&path).context(anyhow!( | ||
"Failed to create local directory: {}", | ||
path.display() | ||
))?; | ||
Ok(LocalStorage { root: path }) | ||
} | ||
} | ||
|
||
#[async_trait] | ||
impl Downloader for LocalStorage { | ||
async fn get(&self, location: &Path) -> Result<Bytes> { | ||
let path_to_filesystem = path_to_filesystem(self.root.clone(), location)?; | ||
let handle = tokio::task::spawn_blocking(move || { | ||
let mut f = File::open(path_to_filesystem) | ||
.map_err(|e| anyhow!("Failed to open file with error: {}", e.to_string()))?; | ||
let mut buf = vec![]; | ||
f.read_to_end(&mut buf) | ||
.context(anyhow!("Failed to read file"))?; | ||
Ok(buf.into()) | ||
}); | ||
handle.await? | ||
} | ||
} |
Oops, something went wrong.