Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: apache/opendal
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: v0.2.4
Choose a base ref
...
head repository: apache/opendal
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: v0.2.5
Choose a head ref
  • 8 commits
  • 7 files changed
  • 1 contributor

Commits on Mar 18, 2022

  1. docs: Fix wrong URL in README (#162)

    Signed-off-by: Xuanwo <[email protected]>
    Xuanwo authored Mar 18, 2022
    Copy the full SHA
    8227a5d View commit details

Commits on Mar 21, 2022

  1. feat: Adopt quick_xml to parse xml (#164)

    Signed-off-by: Xuanwo <[email protected]>
    Xuanwo authored Mar 21, 2022
    Copy the full SHA
    1674f77 View commit details
  2. test: Add behavior test for not exist object (#166)

    Signed-off-by: Xuanwo <[email protected]>
    Xuanwo authored Mar 21, 2022
    Copy the full SHA
    ed5ffae View commit details
  3. feat: Allow user input region (#168)

    Signed-off-by: Xuanwo <[email protected]>
    Xuanwo authored Mar 21, 2022
    Copy the full SHA
    0d86ae3 View commit details
  4. feat: Improve error handle for s3 service (#169)

    Signed-off-by: Xuanwo <[email protected]>
    Xuanwo authored Mar 21, 2022
    Copy the full SHA
    a7728c9 View commit details
  5. feat: Read error response for better debugging (#170)

    * feat: Read error response for better debugging
    
    Signed-off-by: Xuanwo <[email protected]>
    
    * Format code
    
    Signed-off-by: Xuanwo <[email protected]>
    
    * Simplify logic
    
    Signed-off-by: Xuanwo <[email protected]>
    
    * Refactor error handle logic
    
    Signed-off-by: Xuanwo <[email protected]>
    Xuanwo authored Mar 21, 2022
    Copy the full SHA
    bd45177 View commit details
  6. examples: Improve examples for s3 (#171)

    * examples: Improve examples for s3
    
    Signed-off-by: Xuanwo <[email protected]>
    
    * Make clippy happy
    
    Signed-off-by: Xuanwo <[email protected]>
    Xuanwo authored Mar 21, 2022
    Copy the full SHA
    15f8aaa View commit details

Commits on Mar 22, 2022

  1. Bump to version 0.2.5 (#173)

    Signed-off-by: Xuanwo <[email protected]>
    Xuanwo authored Mar 22, 2022
    Copy the full SHA
    6806cb4 View commit details
Showing with 364 additions and 220 deletions.
  1. +15 −0 CHANGELOG.md
  2. +4 −2 Cargo.toml
  3. +1 −1 README.md
  4. +69 −6 examples/s3.rs
  5. +194 −99 src/services/s3/backend.rs
  6. +65 −110 src/services/s3/object_stream.rs
  7. +16 −2 tests/behavior/behavior.rs
15 changes: 15 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/)
and this project adheres to [Semantic Versioning](https://semver.org/).

## [v0.2.5] - 2022-03-22

### Added

- feat: Adopt quick_xml to parse xml (#164)
- test: Add behavior test for not exist object (#166)
- feat: Allow user input region (#168)

## Changed

- feat: Improve error handle for s3 service (#169)
- feat: Read error response for better debugging (#170)
- examples: Improve examples for s3 (#171)

## [v0.2.4] - 2022-03-18

### Added
@@ -167,6 +181,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/).

Hello, OpenDAL!

[v0.2.5]: https://github.com/datafuselabs/opendal/compare/v0.2.4...v0.2.5
[v0.2.4]: https://github.com/datafuselabs/opendal/compare/v0.2.3...v0.2.4
[v0.2.3]: https://github.com/datafuselabs/opendal/compare/v0.2.2...v0.2.3
[v0.2.2]: https://github.com/datafuselabs/opendal/compare/v0.2.1...v0.2.2
6 changes: 4 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ keywords = ["storage", "data", "s3", "fs"]
license = "Apache-2.0"
name = "opendal"
repository = "https://github.com/datafuselabs/opendal"
version = "0.2.4"
version = "0.2.5"

[workspace]
members = ["opendal_test"]
@@ -33,13 +33,15 @@ log = "0.4"
metrics = "0.18"
once_cell = "1"
pin-project = "1"
quick-xml = { version = "0.22.0", features = ["serialize"] }
reqsign = "0.0.1"
reqwest = { version = "0.11", features = ["stream"] }
roxmltree = "0.14"
serde = { version = "1.0.136", features = ["derive"] }
thiserror = "1"
time = "0.3.7"
tokio = { version = "1.17", features = ["full"] }
tower = "0.4"
time = "0.3.7"

[dev-dependencies]
anyhow = "1.0"
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# OpenDAL &emsp; [![Build Status]][actions] [![Latest Version]][crates.io]

[Build Status]: https://img.shields.io/github/workflow/status/datafuselabs/opendal/CI/main
[actions]: https://github.com/datafuselabs/opendal/actions?query=branch%3Amaster
[actions]: https://github.com/datafuselabs/opendal/actions?query=branch%3Amain
[Latest Version]: https://img.shields.io/crates/v/opendal.svg
[crates.io]: https://crates.io/crates/opendal

75 changes: 69 additions & 6 deletions examples/s3.rs
Original file line number Diff line number Diff line change
@@ -13,26 +13,53 @@
// limitations under the License.

//! Example for initiating a s3 backend.
use std::env;
use std::sync::Arc;

use anyhow::Result;
use futures::AsyncReadExt;
use log::info;
use opendal::credential::Credential;
use opendal::services::s3;
use opendal::services::s3::Builder;
use opendal::Accessor;
use opendal::Object;
use opendal::Operator;

#[tokio::main]
async fn main() -> Result<()> {
if env::var("RUST_LOG").is_err() {
env::set_var("RUST_LOG", "debug");
}
env_logger::init();

println!(
r#"OpenDAL S3 Example.
Available Environment Values:
RUST_LOG: log level
OPENDAL_S3_ROOT: root path, default: /
OPENDAL_S3_BUCKET: bukcet name, required.
OPENDAL_S3_ENDPOINT: endpoint of s3 service, default: https://s3.amazonaws.com
OPENDAL_S3_REGION: region of s3 service, could be auto detected.
OPENDAL_S3_ACCESS_KEY_ID: access key id of s3 service, could be auto detected.
OPENDAL_S3_SECRET_ACCESS_KEY: secret access key of s3 service, could be auto detected.
Examples:
OPENDAL_S3_BUCKET=opendal OPENDAL_S3_ACCESS_KEY_ID=minioadmin OPENDAL_S3_SECRET_ACCESS_KEY=minioadminx OPENDAL_S3_ENDPOINT=http://127.0.0.1:9900 OPENDAL_S3_REGION=test cargo run --example s3
"#
);

// Create s3 backend builder.
let mut builder: Builder = s3::Backend::build();
// Set the root for s3, all operations will happen under this root.
//
// NOTE: the root must be absolute path.
builder.root("/path/to/dir");
builder.root(&env::var("OPENDAL_S3_ROOT").unwrap_or_default());
// Set the bucket name, this is required.
builder.bucket("bucket_name");
builder.bucket(&env::var("OPENDAL_S3_BUCKET").expect("env OPENDAL_S3_BUCKET not set"));
// Set the endpoint.
//
// For examples:
@@ -42,21 +69,57 @@ async fn main() -> Result<()> {
// - "https://cos.ap-seoul.myqcloud.com"
//
// Default to "https://s3.amazonaws.com"
builder.endpoint("https://s3.amazonaws.com");
builder.endpoint(
&env::var("OPENDAL_S3_ENDPOINT").unwrap_or_else(|_| "https://s3.amazonaws.com".to_string()),
);
// Set the region in we have this env.
if let Ok(region) = env::var("OPENDAL_S3_REGION") {
builder.region(&region);
}
// Set the credential.
//
// OpenDAL will try load credential from the env.
// If credential not set and no valid credential in env, OpenDAL will
// send request without signing like anonymous user.
builder.credential(Credential::hmac("access_key_id", "secret_access_key"));
builder.credential(Credential::hmac(
&env::var("OPENDAL_S3_ACCESS_KEY_ID").expect("env OPENDAL_S3_ACCESS_KEY_ID not set"),
&env::var("OPENDAL_S3_SECRET_ACCESS_KEY")
.expect("env OPENDAL_S3_SECRET_ACCESS_KEY not set"),
));
// Build the `Accessor`.
let accessor: Arc<dyn Accessor> = builder.finish().await?;

// `Accessor` provides the low level APIs, we will use `Operator` normally.
let op: Operator = Operator::new(accessor);

let path = uuid::Uuid::new_v4().to_string();

// Create an object handle to start operation on object.
let _: Object = op.object("test_file");
info!("try to write file: {}", &path);
op.object(&path)
.writer()
.write_bytes(String::from("Hello, world!").into_bytes())
.await?;
info!("write file successful!");

info!("try to read file: {}", &path);
let mut content = String::new();
op.object(&path)
.reader()
.read_to_string(&mut content)
.await?;
info!("read file successful, content: {}", content);

info!("try to get file metadata: {}", &path);
let meta = op.object(&path).metadata().await?;
info!(
"get file metadata successful, size: {}B",
meta.content_length()
);

info!("try to delete file: {}", &path);
let _ = op.object(&path).delete().await?;
info!("delete file successful");

Ok(())
}
Loading