chore(zarrs_opendal): bump opendal to 0.51 (#107)

LDeakin · Dec 23, 2024 · 81e4a29 · 81e4a29
1 parent 9a4c540
commit 81e4a29
Show file tree

Hide file tree

Showing 7 changed files with 79 additions and 33 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -47,7 +47,7 @@ version = "0.3.0"
 path = "zarrs_object_store"
 
 [workspace.dependencies.zarrs_opendal]
-version = "0.4.0"
+version = "0.5.0"
 path = "zarrs_opendal"
 
 [workspace.dependencies.zarrs_zip]
@@ -58,7 +58,7 @@ path = "zarrs_zip"
 version = "0.11"
 
 [workspace.dependencies.opendal]
-version = "0.50.0"
+version = "0.51.0"
 
 [workspace.dependencies.zip]
 version = "2.1.3"
diff --git a/zarrs_opendal/CHANGELOG.md b/zarrs_opendal/CHANGELOG.md
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+ - **Breaking**: Bump `opendal` to 0.51
+
 ## [0.4.0] - 2024-11-15
 
 ### Added

diff --git a/zarrs_opendal/Cargo.toml b/zarrs_opendal/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "zarrs_opendal"
-version = "0.4.0"
+version = "0.5.0"
 authors = ["Lachlan Deakin <[email protected]>"]
 edition = "2021"
 rust-version = "1.77"
@@ -18,7 +18,7 @@ workspace = true
 [dependencies]
 async-trait = "0.1.74"
 futures = "0.3.29"
-opendal = { version = ">=0.46,<0.51", default-features = false }
+opendal = { version = ">=0.51,<0.52", default-features = false }
 zarrs_storage = { workspace = true, features = ["async"] }
 
 [dev-dependencies]

diff --git a/zarrs_opendal/README.md b/zarrs_opendal/README.md
@@ -1,7 +1,7 @@
 # zarrs_opendal
 
 [![Latest Version](https://img.shields.io/crates/v/zarrs_opendal.svg)](https://crates.io/crates/zarrs_opendal)
-[![opendal 0.50](https://img.shields.io/badge/opendal-0.50-blue)](https://crates.io/crates/opendal)
+[![opendal 0.51](https://img.shields.io/badge/opendal-0.51-blue)](https://crates.io/crates/opendal)
 [![zarrs_opendal documentation](https://docs.rs/zarrs_opendal/badge.svg)](https://docs.rs/zarrs_opendal)
 ![msrv](https://img.shields.io/crates/msrv/zarrs_opendal)
 [![build](https://github.com/LDeakin/zarrs/actions/workflows/ci.yml/badge.svg)](https://github.com/LDeakin/zarrs/actions/workflows/ci.yml)

diff --git a/zarrs_opendal/doc/version_compatibility_matrix.md b/zarrs_opendal/doc/version_compatibility_matrix.md
@@ -1,7 +1,8 @@
 | [zarrs_opendal] | [opendal] | [zarrs] ([zarrs_storage]) |
 | --------------- | --------- | ------------------------- |
-| 0.3             | 0.46-0.50 | 0.17 (0.2.0)              |
-| 0.4             | 0.46-0.50 | 0.18 (0.3.0)              |
+| 0.5             | 0.51-0.51 | 0.18.x (0.3.x)            |
+| 0.4             | 0.46-0.50 | 0.18.x (0.3.x)            |
+| 0.3             | 0.46-0.50 | 0.17.x (0.2.x)            |
 
 [zarrs_opendal]: https://crates.io/crates/zarrs_opendal
 [opendal]: https://crates.io/crates/opendal

diff --git a/zarrs_opendal/src/async.rs b/zarrs_opendal/src/async.rs
@@ -83,7 +83,7 @@ impl AsyncWritableStorageTraits for AsyncOpendalStore {
     }
 
     async fn erase(&self, key: &StoreKey) -> Result<(), StorageError> {
-        handle_result(self.operator.remove(vec![key.to_string()]).await)
+        handle_result(self.operator.delete(key.as_str()).await)
     }
 
     async fn erase_prefix(&self, prefix: &StorePrefix) -> Result<(), StorageError> {
@@ -157,23 +157,44 @@ impl AsyncListableStorageTraits for AsyncOpendalStore {
     }
 
     async fn size_prefix(&self, prefix: &StorePrefix) -> Result<u64, StorageError> {
-        handle_result_notfound(
+        let Some(files) = handle_result_notfound(
             self.operator
                 .list_with(prefix.as_str())
                 .recursive(true)
-                .metakey(opendal::Metakey::ContentLength)
                 .await,
         )?
-        .map_or_else(
-            || Ok(0),
-            |list| {
-                let size = list
-                    .into_iter()
-                    .map(|entry| entry.metadata().content_length())
-                    .sum::<u64>();
-                Ok(size)
-            },
-        )
+        else {
+            return Ok(0);
+        };
+
+        if self
+            .operator
+            .info()
+            .full_capability()
+            .list_has_content_length
+        {
+            let size = files
+                .into_iter()
+                .filter_map(|entry| {
+                    if entry.metadata().is_file() {
+                        Some(entry.metadata().content_length())
+                    } else {
+                        None
+                    }
+                })
+                .sum::<u64>();
+            Ok(size)
+        } else {
+            // TODO: concurrent
+            let mut size = 0;
+            for entry in files {
+                let meta = handle_result(self.operator.stat(entry.path()).await)?;
+                if meta.is_file() {
+                    size += meta.content_length();
+                }
+            }
+            Ok(size)
+        }
     }
 }
 

diff --git a/zarrs_opendal/src/sync.rs b/zarrs_opendal/src/sync.rs
@@ -71,7 +71,7 @@ impl WritableStorageTraits for OpendalStore {
     }
 
     fn erase(&self, key: &StoreKey) -> Result<(), StorageError> {
-        handle_result(self.operator.remove(vec![key.to_string()]))
+        handle_result(self.operator.delete(key.as_str()))
     }
 
     fn erase_prefix(&self, prefix: &StorePrefix) -> Result<(), StorageError> {
@@ -145,23 +145,44 @@ impl ListableStorageTraits for OpendalStore {
     }
 
     fn size_prefix(&self, prefix: &StorePrefix) -> Result<u64, StorageError> {
-        handle_result_notfound(
+        let Some(files) = handle_result_notfound(
             self.operator
                 .list_with(prefix.as_str())
                 .recursive(true)
-                .metakey(opendal::Metakey::ContentLength)
                 .call(),
         )?
-        .map_or_else(
-            || Ok(0),
-            |list| {
-                let size = list
-                    .into_iter()
-                    .map(|entry| entry.metadata().content_length())
-                    .sum::<u64>();
-                Ok(size)
-            },
-        )
+        else {
+            return Ok(0);
+        };
+
+        if self
+            .operator
+            .info()
+            .full_capability()
+            .list_has_content_length
+        {
+            let size = files
+                .into_iter()
+                .filter_map(|entry| {
+                    if entry.metadata().is_file() {
+                        Some(entry.metadata().content_length())
+                    } else {
+                        None
+                    }
+                })
+                .sum::<u64>();
+            Ok(size)
+        } else {
+            // TODO: concurrent
+            let mut size = 0;
+            for entry in files {
+                let meta = handle_result(self.operator.stat(entry.path()))?;
+                if meta.is_file() {
+                    size += meta.content_length();
+                }
+            }
+            Ok(size)
+        }
     }
 }