feat(core)!: Make config data object (#4915)

Signed-off-by: tison <[email protected]>

.github/services/swift/ceph_rados_swift/action.yml

@@ -44,6 +44,7 @@ runs:
       shell: bash
       run: |
         cat << EOF >> $GITHUB_ENV
+        OPENDAL_DISABLE_RANDOM_ROOT=true
         OPENDAL_SWIFT_CONTAINER=testing
         OPENDAL_SWIFT_ROOT=/
         EOF

.github/services/swift/swift/action.yml

@@ -41,6 +41,7 @@ runs:
       shell: bash
       run: |
         cat << EOF >> $GITHUB_ENV
+        OPENDAL_DISABLE_RANDOM_ROOT=true
         OPENDAL_SWIFT_CONTAINER=testing
         OPENDAL_SWIFT_ROOT=/
         EOF

core/src/layers/retry.rs

@@ -776,6 +776,12 @@ mod tests {
         const SCHEME: Scheme = Scheme::Custom("mock");
         type Accessor = MockService;
 
+        type Config = ();
+
+        fn from_config(_: Self::Config) -> Self {
+            Self::default()
+        }
+
         fn from_map(_: HashMap<String, String>) -> Self {
             Self::default()
         }

core/src/raw/serde_util.rs

@@ -45,7 +45,7 @@ pub fn new_json_deserialize_error(e: serde_json::Error) -> Error {
 
 /// ConfigDeserializer is used to deserialize given configs from `HashMap<String, String>`.
 ///
-/// This is only used by our services config.
+/// This is only used by our services' config.
 pub struct ConfigDeserializer(MapDeserializer<'static, Pairs, de::value::Error>);
 
 impl ConfigDeserializer {

core/src/services/aliyun_drive/backend.rs

@@ -26,7 +26,7 @@ use http::Request;
 use http::Response;
 use http::StatusCode;
 use log::debug;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use tokio::sync::Mutex;
 
 use super::core::*;
@@ -38,7 +38,7 @@ use crate::raw::*;
 use crate::*;
 
 /// Aliyun Drive services support.
-#[derive(Default, Deserialize)]
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
 #[serde(default)]
 #[non_exhaustive]
 pub struct AliyunDriveConfig {
@@ -170,12 +170,11 @@ impl Builder for AliyunDriveBuilder {
 
     type Accessor = AliyunDriveBackend;
 
-    fn from_map(map: std::collections::HashMap<String, String>) -> Self {
-        let config = AliyunDriveConfig::deserialize(ConfigDeserializer::new(map))
-            .expect("config deserialize must succeed");
+    type Config = AliyunDriveConfig;
+
+    fn from_config(config: Self::Config) -> Self {
         AliyunDriveBuilder {
             config,
-
             http_client: None,
         }
     }

core/src/services/alluxio/backend.rs

@@ -15,14 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::HashMap;
 use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::sync::Arc;
 
 use http::Response;
 use log::debug;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
 use super::core::AlluxioCore;
 use super::error::parse_error;
@@ -33,7 +32,7 @@ use crate::raw::*;
 use crate::*;
 
 /// Config for alluxio services support.
-#[derive(Default, Deserialize)]
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
 #[serde(default)]
 #[non_exhaustive]
 pub struct AlluxioConfig {
@@ -119,22 +118,9 @@ impl AlluxioBuilder {
 impl Builder for AlluxioBuilder {
     const SCHEME: Scheme = Scheme::Alluxio;
     type Accessor = AlluxioBackend;
+    type Config = AlluxioConfig;
 
-    /// Converts a HashMap into an AlluxioBuilder instance.
-    ///
-    /// # Arguments
-    ///
-    /// * `map` - A HashMap containing the configuration values.
-    ///
-    /// # Returns
-    ///
-    /// Returns an instance of AlluxioBuilder.
-    fn from_map(map: HashMap<String, String>) -> Self {
-        // Deserialize the configuration from the HashMap.
-        let config = AlluxioConfig::deserialize(ConfigDeserializer::new(map))
-            .expect("config deserialize must succeed");
-
-        // Create an AlluxioBuilder instance with the deserialized config.
+    fn from_config(config: Self::Config) -> Self {
         AlluxioBuilder {
             config,
             http_client: None,
@@ -265,6 +251,7 @@ impl Access for AlluxioBackend {
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::collections::HashMap;
 
     #[test]
     fn test_builder_from_map() {

core/src/services/atomicserver/backend.rs

@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::HashMap;
 use std::fmt::Debug;
 use std::fmt::Formatter;
 
@@ -36,7 +35,7 @@ use crate::*;
 /// Atomicserver service support.
 
 /// Config for Atomicserver services support
-#[derive(Default, Deserialize, Clone)]
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
 #[serde(default)]
 #[non_exhaustive]
 pub struct AtomicserverConfig {
@@ -115,14 +114,10 @@ impl AtomicserverBuilder {
 impl Builder for AtomicserverBuilder {
     const SCHEME: Scheme = Scheme::Atomicserver;
     type Accessor = AtomicserverBackend;
+    type Config = AtomicserverConfig;
 
-    fn from_map(map: HashMap<String, String>) -> Self {
-        // Deserialize the configuration from the HashMap.
-        let config = AtomicserverConfig::deserialize(ConfigDeserializer::new(map))
-            .expect("config deserialize must succeed");
-
-        // Create an AtomicserverBuilder instance with the deserialized config.
-        AtomicserverBuilder { config }
+    fn from_config(config: Self::Config) -> Self {
+        Self { config }
     }
 
     fn build(&mut self) -> Result<Self::Accessor> {

core/src/services/azblob/backend.rs

@@ -30,7 +30,7 @@ use log::debug;
 use reqsign::AzureStorageConfig;
 use reqsign::AzureStorageLoader;
 use reqsign::AzureStorageSigner;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use sha2::Digest;
 use sha2::Sha256;
 
@@ -55,7 +55,7 @@ const KNOWN_AZBLOB_ENDPOINT_SUFFIX: &[&str] = &[
 const AZBLOB_BATCH_LIMIT: usize = 256;
 
 /// Azure Storage Blob services support.
-#[derive(Default, Deserialize, Clone)]
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct AzblobConfig {
     /// The root of Azblob service backend.
     ///
@@ -396,12 +396,10 @@ impl AzblobBuilder {
 impl Builder for AzblobBuilder {
     const SCHEME: Scheme = Scheme::Azblob;
     type Accessor = AzblobBackend;
+    type Config = AzblobConfig;
 
-    fn from_map(map: HashMap<String, String>) -> Self {
-        let config = AzblobConfig::deserialize(ConfigDeserializer::new(map))
-            .expect("config deserialize must succeed");
-
-        AzblobBuilder {
+    fn from_config(config: Self::Config) -> Self {
+        Self {
             config,
             http_client: None,
         }

core/src/services/azdls/backend.rs

@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::HashMap;
 use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::sync::Arc;
@@ -26,7 +25,7 @@ use log::debug;
 use reqsign::AzureStorageConfig;
 use reqsign::AzureStorageLoader;
 use reqsign::AzureStorageSigner;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
 use super::core::AzdlsCore;
 use super::error::parse_error;
@@ -47,13 +46,18 @@ const KNOWN_AZDLS_ENDPOINT_SUFFIX: &[&str] = &[
 ];
 
 /// Azure Data Lake Storage Gen2 Support.
-#[derive(Default, Deserialize, Clone)]
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct AzdlsConfig {
-    root: Option<String>,
-    filesystem: String,
-    endpoint: Option<String>,
-    account_name: Option<String>,
-    account_key: Option<String>,
+    /// Root of this backend.
+    pub root: Option<String>,
+    /// Filesystem name of this backend.
+    pub filesystem: String,
+    /// Endpoint of this backend.
+    pub endpoint: Option<String>,
+    /// Account name of this backend.
+    pub account_name: Option<String>,
+    /// Account key of this backend.
+    pub account_key: Option<String>,
 }
 
 impl Debug for AzdlsConfig {
@@ -164,8 +168,16 @@ impl AzdlsBuilder {
 }
 
 impl Builder for AzdlsBuilder {
-    type Accessor = AzdlsBackend;
     const SCHEME: Scheme = Scheme::Azdls;
+    type Accessor = AzdlsBackend;
+    type Config = AzdlsConfig;
+
+    fn from_config(config: Self::Config) -> Self {
+        AzdlsBuilder {
+            config,
+            http_client: None,
+        }
+    }
 
     fn build(&mut self) -> Result<Self::Accessor> {
         debug!("backend build started: {:?}", &self);
@@ -225,16 +237,6 @@ impl Builder for AzdlsBuilder {
             }),
         })
     }
-
-    fn from_map(map: HashMap<String, String>) -> Self {
-        let config = AzdlsConfig::deserialize(ConfigDeserializer::new(map))
-            .expect("config deserialize must succeed");
-
-        AzdlsBuilder {
-            config,
-            http_client: None,
-        }
-    }
 }
 
 /// Backend for azblob services.

core/src/services/azdls/mod.rs

@@ -17,6 +17,7 @@
 
 mod backend;
 pub use backend::AzdlsBuilder as Azdls;
+pub use backend::AzdlsConfig;
 
 mod core;
 mod error;

core/src/services/azfile/backend.rs

@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::HashMap;
 use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::sync::Arc;
@@ -26,7 +25,7 @@ use log::debug;
 use reqsign::AzureStorageConfig;
 use reqsign::AzureStorageLoader;
 use reqsign::AzureStorageSigner;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
 use super::core::AzfileCore;
 use super::error::parse_error;
@@ -40,14 +39,20 @@ use crate::*;
 const DEFAULT_AZFILE_ENDPOINT_SUFFIX: &str = "file.core.windows.net";
 
 /// Azure File services support.
-#[derive(Default, Deserialize, Clone)]
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct AzfileConfig {
-    root: Option<String>,
-    endpoint: Option<String>,
-    share_name: String,
-    account_name: Option<String>,
-    account_key: Option<String>,
-    sas_token: Option<String>,
+    /// The root path for azfile.
+    pub root: Option<String>,
+    /// The endpoint for azfile.
+    pub endpoint: Option<String>,
+    /// The share name for azfile.
+    pub share_name: String,
+    /// The account name for azfile.
+    pub account_name: Option<String>,
+    /// The account key for azfile.
+    pub account_key: Option<String>,
+    /// The sas token for azfile.
+    pub sas_token: Option<String>,
 }
 
 impl Debug for AzfileConfig {
@@ -163,12 +168,10 @@ impl AzfileBuilder {
 impl Builder for AzfileBuilder {
     const SCHEME: Scheme = Scheme::Azfile;
     type Accessor = AzfileBackend;
+    type Config = AzfileConfig;
 
-    fn from_map(map: HashMap<String, String>) -> Self {
-        let config = AzfileConfig::deserialize(ConfigDeserializer::new(map))
-            .expect("config deserialize must succeed");
-
-        AzfileBuilder {
+    fn from_config(config: Self::Config) -> Self {
+        Self {
             config,
             http_client: None,
         }

core/src/services/azfile/mod.rs

@@ -16,6 +16,7 @@
 // under the License.
 
 pub use backend::AzfileBuilder as Azfile;
+pub use backend::AzfileConfig;
 
 mod backend;
 mod core;