From aa6781a405d0e49671b01b662b2b24e6eec158f0 Mon Sep 17 00:00:00 2001 From: tison Date: Fri, 19 Jul 2024 21:55:51 -0700 Subject: [PATCH] feat(core)!: Make config data object (#4915) Signed-off-by: tison --- .../swift/ceph_rados_swift/action.yml | 1 + .github/services/swift/swift/action.yml | 1 + core/src/layers/retry.rs | 6 + core/src/raw/serde_util.rs | 2 +- core/src/services/aliyun_drive/backend.rs | 11 +- core/src/services/alluxio/backend.rs | 23 +-- core/src/services/atomicserver/backend.rs | 13 +- core/src/services/azblob/backend.rs | 12 +- core/src/services/azdls/backend.rs | 40 ++--- core/src/services/azdls/mod.rs | 1 + core/src/services/azfile/backend.rs | 31 ++-- core/src/services/azfile/mod.rs | 1 + core/src/services/b2/backend.rs | 22 +-- core/src/services/cacache/backend.rs | 14 +- core/src/services/cacache/mod.rs | 1 + core/src/services/chainsafe/backend.rs | 22 +-- core/src/services/cloudflare_kv/backend.rs | 21 ++- core/src/services/cloudflare_kv/mod.rs | 1 + core/src/services/compfs/backend.rs | 37 +++-- core/src/services/compfs/mod.rs | 5 +- core/src/services/cos/backend.rs | 29 ++-- core/src/services/cos/mod.rs | 1 + core/src/services/d1/backend.rs | 13 +- core/src/services/dashmap/backend.rs | 15 +- core/src/services/dashmap/mod.rs | 1 + core/src/services/dbfs/backend.rs | 20 +-- core/src/services/dbfs/mod.rs | 3 +- core/src/services/dropbox/builder.rs | 13 +- core/src/services/etcd/backend.rs | 13 +- core/src/services/foundationdb/backend.rs | 11 +- core/src/services/fs/backend.rs | 12 +- core/src/services/ftp/backend.rs | 17 +- core/src/services/gcs/backend.rs | 34 ++-- core/src/services/gdrive/builder.rs | 14 +- core/src/services/gdrive/mod.rs | 1 + core/src/services/ghac/backend.rs | 54 ++++--- core/src/services/ghac/mod.rs | 1 + core/src/services/github/backend.rs | 31 +--- core/src/services/gridfs/backend.rs | 76 +++++---- core/src/services/gridfs/mod.rs | 1 + core/src/services/hdfs/backend.rs | 13 +- core/src/services/hdfs_native/backend.rs | 15 +- core/src/services/http/backend.rs | 11 +- core/src/services/huggingface/backend.rs | 13 +- core/src/services/icloud/backend.rs | 11 +- core/src/services/icloud/mod.rs | 1 + core/src/services/ipfs/backend.rs | 45 +++--- core/src/services/ipfs/mod.rs | 1 + core/src/services/ipmfs/builder.rs | 40 +++-- core/src/services/ipmfs/mod.rs | 1 + core/src/services/koofr/backend.rs | 22 +-- core/src/services/libsql/backend.rs | 31 ++-- core/src/services/memcached/backend.rs | 20 ++- core/src/services/memory/backend.rs | 16 +- core/src/services/mini_moka/backend.rs | 58 ++++--- core/src/services/mini_moka/mod.rs | 1 + core/src/services/mod.rs | 50 +++++- core/src/services/moka/backend.rs | 14 +- core/src/services/mongodb/backend.rs | 11 +- core/src/services/monoiofs/backend.rs | 10 +- core/src/services/mysql/backend.rs | 28 ++-- core/src/services/obs/backend.rs | 101 +++++++----- core/src/services/obs/mod.rs | 1 + core/src/services/onedrive/builder.rs | 15 +- core/src/services/oss/backend.rs | 150 ++++++++++-------- core/src/services/oss/mod.rs | 1 + core/src/services/pcloud/backend.rs | 22 +-- core/src/services/persy/backend.rs | 47 +++--- core/src/services/persy/mod.rs | 1 + core/src/services/postgresql/backend.rs | 19 +-- core/src/services/redb/backend.rs | 45 +++--- core/src/services/redb/mod.rs | 1 + core/src/services/redis/backend.rs | 25 ++- core/src/services/rocksdb/backend.rs | 14 +- core/src/services/s3/backend.rs | 10 +- core/src/services/seafile/backend.rs | 22 +-- core/src/services/sftp/backend.rs | 17 +- core/src/services/sled/backend.rs | 21 ++- core/src/services/sqlite/backend.rs | 11 +- core/src/services/supabase/backend.rs | 80 ++++++---- core/src/services/supabase/mod.rs | 1 + core/src/services/surrealdb/backend.rs | 39 +++-- core/src/services/swift/backend.rs | 78 +++++---- core/src/services/swift/mod.rs | 1 + core/src/services/tikv/backend.rs | 12 +- core/src/services/upyun/backend.rs | 22 +-- core/src/services/vercel_artifacts/builder.rs | 49 ++++-- core/src/services/vercel_artifacts/mod.rs | 1 + core/src/services/vercel_blob/backend.rs | 22 +-- core/src/services/webdav/backend.rs | 11 +- core/src/services/webhdfs/backend.rs | 88 +++++----- core/src/services/webhdfs/mod.rs | 1 + core/src/services/yandex_disk/backend.rs | 27 +--- core/src/types/builder.rs | 20 ++- 94 files changed, 1014 insertions(+), 998 deletions(-) diff --git a/.github/services/swift/ceph_rados_swift/action.yml b/.github/services/swift/ceph_rados_swift/action.yml index a4665212229d..649ce2bdda1d 100644 --- a/.github/services/swift/ceph_rados_swift/action.yml +++ b/.github/services/swift/ceph_rados_swift/action.yml @@ -44,6 +44,7 @@ runs: shell: bash run: | cat << EOF >> $GITHUB_ENV + OPENDAL_DISABLE_RANDOM_ROOT=true OPENDAL_SWIFT_CONTAINER=testing OPENDAL_SWIFT_ROOT=/ EOF diff --git a/.github/services/swift/swift/action.yml b/.github/services/swift/swift/action.yml index 7dc7e4ff1e1f..e7cfb4b2856a 100644 --- a/.github/services/swift/swift/action.yml +++ b/.github/services/swift/swift/action.yml @@ -41,6 +41,7 @@ runs: shell: bash run: | cat << EOF >> $GITHUB_ENV + OPENDAL_DISABLE_RANDOM_ROOT=true OPENDAL_SWIFT_CONTAINER=testing OPENDAL_SWIFT_ROOT=/ EOF diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs index af0fdc214dee..be11eb1abaae 100644 --- a/core/src/layers/retry.rs +++ b/core/src/layers/retry.rs @@ -776,6 +776,12 @@ mod tests { const SCHEME: Scheme = Scheme::Custom("mock"); type Accessor = MockService; + type Config = (); + + fn from_config(_: Self::Config) -> Self { + Self::default() + } + fn from_map(_: HashMap) -> Self { Self::default() } diff --git a/core/src/raw/serde_util.rs b/core/src/raw/serde_util.rs index 6507d3612af9..f34833a47d1c 100644 --- a/core/src/raw/serde_util.rs +++ b/core/src/raw/serde_util.rs @@ -45,7 +45,7 @@ pub fn new_json_deserialize_error(e: serde_json::Error) -> Error { /// ConfigDeserializer is used to deserialize given configs from `HashMap`. /// -/// This is only used by our services config. +/// This is only used by our services' config. pub struct ConfigDeserializer(MapDeserializer<'static, Pairs, de::value::Error>); impl ConfigDeserializer { diff --git a/core/src/services/aliyun_drive/backend.rs b/core/src/services/aliyun_drive/backend.rs index 7662d8f912df..fe7e2b2bf08c 100644 --- a/core/src/services/aliyun_drive/backend.rs +++ b/core/src/services/aliyun_drive/backend.rs @@ -26,7 +26,7 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; use super::core::*; @@ -38,7 +38,7 @@ use crate::raw::*; use crate::*; /// Aliyun Drive services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct AliyunDriveConfig { @@ -170,12 +170,11 @@ impl Builder for AliyunDriveBuilder { type Accessor = AliyunDriveBackend; - fn from_map(map: std::collections::HashMap) -> Self { - let config = AliyunDriveConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); + type Config = AliyunDriveConfig; + + fn from_config(config: Self::Config) -> Self { AliyunDriveBuilder { config, - http_client: None, } } diff --git a/core/src/services/alluxio/backend.rs b/core/src/services/alluxio/backend.rs index 018c138ed56a..59c927fb540a 100644 --- a/core/src/services/alluxio/backend.rs +++ b/core/src/services/alluxio/backend.rs @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; use http::Response; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::AlluxioCore; use super::error::parse_error; @@ -33,7 +32,7 @@ use crate::raw::*; use crate::*; /// Config for alluxio services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct AlluxioConfig { @@ -119,22 +118,9 @@ impl AlluxioBuilder { impl Builder for AlluxioBuilder { const SCHEME: Scheme = Scheme::Alluxio; type Accessor = AlluxioBackend; + type Config = AlluxioConfig; - /// Converts a HashMap into an AlluxioBuilder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of AlluxioBuilder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = AlluxioConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an AlluxioBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { AlluxioBuilder { config, http_client: None, @@ -265,6 +251,7 @@ impl Access for AlluxioBackend { #[cfg(test)] mod test { use super::*; + use std::collections::HashMap; #[test] fn test_builder_from_map() { diff --git a/core/src/services/atomicserver/backend.rs b/core/src/services/atomicserver/backend.rs index 56f9089930ef..463056fe8a27 100644 --- a/core/src/services/atomicserver/backend.rs +++ b/core/src/services/atomicserver/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; @@ -36,7 +35,7 @@ use crate::*; /// Atomicserver service support. /// Config for Atomicserver services support -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct AtomicserverConfig { @@ -115,14 +114,10 @@ impl AtomicserverBuilder { impl Builder for AtomicserverBuilder { const SCHEME: Scheme = Scheme::Atomicserver; type Accessor = AtomicserverBackend; + type Config = AtomicserverConfig; - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = AtomicserverConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an AtomicserverBuilder instance with the deserialized config. - AtomicserverBuilder { config } + fn from_config(config: Self::Config) -> Self { + Self { config } } fn build(&mut self) -> Result { diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs index eb7c2f024192..a9146f319dad 100644 --- a/core/src/services/azblob/backend.rs +++ b/core/src/services/azblob/backend.rs @@ -30,7 +30,7 @@ use log::debug; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use sha2::Digest; use sha2::Sha256; @@ -55,7 +55,7 @@ const KNOWN_AZBLOB_ENDPOINT_SUFFIX: &[&str] = &[ const AZBLOB_BATCH_LIMIT: usize = 256; /// Azure Storage Blob services support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct AzblobConfig { /// The root of Azblob service backend. /// @@ -396,12 +396,10 @@ impl AzblobBuilder { impl Builder for AzblobBuilder { const SCHEME: Scheme = Scheme::Azblob; type Accessor = AzblobBackend; + type Config = AzblobConfig; - fn from_map(map: HashMap) -> Self { - let config = AzblobConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - AzblobBuilder { + fn from_config(config: Self::Config) -> Self { + Self { config, http_client: None, } diff --git a/core/src/services/azdls/backend.rs b/core/src/services/azdls/backend.rs index 4774383c81c4..438ca3b8ec17 100644 --- a/core/src/services/azdls/backend.rs +++ b/core/src/services/azdls/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -26,7 +25,7 @@ use log::debug; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::AzdlsCore; use super::error::parse_error; @@ -47,13 +46,18 @@ const KNOWN_AZDLS_ENDPOINT_SUFFIX: &[&str] = &[ ]; /// Azure Data Lake Storage Gen2 Support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct AzdlsConfig { - root: Option, - filesystem: String, - endpoint: Option, - account_name: Option, - account_key: Option, + /// Root of this backend. + pub root: Option, + /// Filesystem name of this backend. + pub filesystem: String, + /// Endpoint of this backend. + pub endpoint: Option, + /// Account name of this backend. + pub account_name: Option, + /// Account key of this backend. + pub account_key: Option, } impl Debug for AzdlsConfig { @@ -164,8 +168,16 @@ impl AzdlsBuilder { } impl Builder for AzdlsBuilder { - type Accessor = AzdlsBackend; const SCHEME: Scheme = Scheme::Azdls; + type Accessor = AzdlsBackend; + type Config = AzdlsConfig; + + fn from_config(config: Self::Config) -> Self { + AzdlsBuilder { + config, + http_client: None, + } + } fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); @@ -225,16 +237,6 @@ impl Builder for AzdlsBuilder { }), }) } - - fn from_map(map: HashMap) -> Self { - let config = AzdlsConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - AzdlsBuilder { - config, - http_client: None, - } - } } /// Backend for azblob services. diff --git a/core/src/services/azdls/mod.rs b/core/src/services/azdls/mod.rs index 4284e4595ed0..911cbb1ec934 100644 --- a/core/src/services/azdls/mod.rs +++ b/core/src/services/azdls/mod.rs @@ -17,6 +17,7 @@ mod backend; pub use backend::AzdlsBuilder as Azdls; +pub use backend::AzdlsConfig; mod core; mod error; diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs index c47abf0ca5dc..7f2fb15bceac 100644 --- a/core/src/services/azfile/backend.rs +++ b/core/src/services/azfile/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -26,7 +25,7 @@ use log::debug; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::AzfileCore; use super::error::parse_error; @@ -40,14 +39,20 @@ use crate::*; const DEFAULT_AZFILE_ENDPOINT_SUFFIX: &str = "file.core.windows.net"; /// Azure File services support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct AzfileConfig { - root: Option, - endpoint: Option, - share_name: String, - account_name: Option, - account_key: Option, - sas_token: Option, + /// The root path for azfile. + pub root: Option, + /// The endpoint for azfile. + pub endpoint: Option, + /// The share name for azfile. + pub share_name: String, + /// The account name for azfile. + pub account_name: Option, + /// The account key for azfile. + pub account_key: Option, + /// The sas token for azfile. + pub sas_token: Option, } impl Debug for AzfileConfig { @@ -163,12 +168,10 @@ impl AzfileBuilder { impl Builder for AzfileBuilder { const SCHEME: Scheme = Scheme::Azfile; type Accessor = AzfileBackend; + type Config = AzfileConfig; - fn from_map(map: HashMap) -> Self { - let config = AzfileConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - AzfileBuilder { + fn from_config(config: Self::Config) -> Self { + Self { config, http_client: None, } diff --git a/core/src/services/azfile/mod.rs b/core/src/services/azfile/mod.rs index 0ae9da72918f..1047102572f1 100644 --- a/core/src/services/azfile/mod.rs +++ b/core/src/services/azfile/mod.rs @@ -16,6 +16,7 @@ // under the License. pub use backend::AzfileBuilder as Azfile; +pub use backend::AzfileConfig; mod backend; mod core; diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs index e210f3e65467..8f8c7d2c9d08 100644 --- a/core/src/services/b2/backend.rs +++ b/core/src/services/b2/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -25,7 +24,7 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use super::core::constants; @@ -41,7 +40,7 @@ use crate::services::b2::core::ListFileNamesResponse; use crate::*; /// Config for backblaze b2 services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct B2Config { @@ -167,22 +166,9 @@ impl B2Builder { impl Builder for B2Builder { const SCHEME: Scheme = Scheme::B2; type Accessor = B2Backend; + type Config = B2Config; - /// Converts a HashMap into an B2Builder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of B2Builder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = B2Config::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an B2Builder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { B2Builder { config, http_client: None, diff --git a/core/src/services/cacache/backend.rs b/core/src/services/cacache/backend.rs index fe3208c28d51..a76772a4b7a6 100644 --- a/core/src/services/cacache/backend.rs +++ b/core/src/services/cacache/backend.rs @@ -15,16 +15,14 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::str; use cacache; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::raw::adapters::kv; -use crate::raw::ConfigDeserializer; use crate::Builder; use crate::Error; use crate::ErrorKind; @@ -32,10 +30,10 @@ use crate::Scheme; use crate::*; /// cacache service support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct CacacheConfig { /// That path to the cacache data directory. - datadir: Option, + pub datadir: Option, } /// cacache service support. @@ -56,11 +54,9 @@ impl CacacheBuilder { impl Builder for CacacheBuilder { const SCHEME: Scheme = Scheme::Cacache; type Accessor = CacacheBackend; + type Config = CacacheConfig; - fn from_map(map: HashMap) -> Self { - let config = CacacheConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { Self { config } } diff --git a/core/src/services/cacache/mod.rs b/core/src/services/cacache/mod.rs index e24897ca6124..b48d7f2b7633 100644 --- a/core/src/services/cacache/mod.rs +++ b/core/src/services/cacache/mod.rs @@ -18,3 +18,4 @@ mod backend; pub use backend::CacacheBuilder as Cacache; +pub use backend::CacacheConfig; diff --git a/core/src/services/chainsafe/backend.rs b/core/src/services/chainsafe/backend.rs index a961ec35a997..6702f25a53fc 100644 --- a/core/src/services/chainsafe/backend.rs +++ b/core/src/services/chainsafe/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -24,7 +23,7 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::parse_info; use super::core::ChainsafeCore; @@ -37,7 +36,7 @@ use crate::raw::*; use crate::*; /// Config for backblaze Chainsafe services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct ChainsafeConfig { @@ -131,22 +130,9 @@ impl ChainsafeBuilder { impl Builder for ChainsafeBuilder { const SCHEME: Scheme = Scheme::Chainsafe; type Accessor = ChainsafeBackend; + type Config = ChainsafeConfig; - /// Converts a HashMap into an ChainsafeBuilder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of ChainsafeBuilder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = ChainsafeConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an ChainsafeBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { ChainsafeBuilder { config, http_client: None, diff --git a/core/src/services/cloudflare_kv/backend.rs b/core/src/services/cloudflare_kv/backend.rs index 14c361cc077d..43b18efb2343 100644 --- a/core/src/services/cloudflare_kv/backend.rs +++ b/core/src/services/cloudflare_kv/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; @@ -23,7 +22,7 @@ use bytes::Buf; use http::header; use http::Request; use http::StatusCode; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::error::parse_error; use crate::raw::adapters::kv; @@ -31,18 +30,18 @@ use crate::raw::*; use crate::ErrorKind; use crate::*; -/// Cloudflare Kv Service Support. -#[derive(Default, Deserialize, Clone)] +/// Cloudflare KV Service Support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct CloudflareKvConfig { /// The token used to authenticate with CloudFlare. - token: Option, + pub token: Option, /// The account ID used to authenticate with CloudFlare. Used as URI path parameter. - account_id: Option, + pub account_id: Option, /// The namespace ID. Used as URI path parameter. - namespace_id: Option, + pub namespace_id: Option, /// Root within this backend. - root: Option, + pub root: Option, } impl Debug for CloudflareKvConfig { @@ -116,11 +115,9 @@ impl Builder for CloudflareKvBuilder { const SCHEME: Scheme = Scheme::CloudflareKv; type Accessor = CloudflareKvBackend; + type Config = CloudflareKvConfig; - fn from_map(map: HashMap) -> Self { - let config = CloudflareKvConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { Self { config, http_client: None, diff --git a/core/src/services/cloudflare_kv/mod.rs b/core/src/services/cloudflare_kv/mod.rs index fa09aa3d0024..85011ee1e2e9 100644 --- a/core/src/services/cloudflare_kv/mod.rs +++ b/core/src/services/cloudflare_kv/mod.rs @@ -19,3 +19,4 @@ mod backend; mod error; pub use backend::CloudflareKvBuilder as CloudflareKv; +pub use backend::CloudflareKvConfig; diff --git a/core/src/services/compfs/backend.rs b/core/src/services/compfs/backend.rs index fb218b096e46..2396f3d01a36 100644 --- a/core/src/services/compfs/backend.rs +++ b/core/src/services/compfs/backend.rs @@ -15,28 +15,38 @@ // specific language governing permissions and limitations // under the License. -use compio::{dispatcher::Dispatcher, fs::OpenOptions}; +use std::{io::Cursor, sync::Arc}; -use super::{core::CompfsCore, lister::CompfsLister, reader::CompfsReader, writer::CompfsWriter}; +use compio::{dispatcher::Dispatcher, fs::OpenOptions}; +use serde::{Deserialize, Serialize}; use crate::raw::*; use crate::*; -use std::{collections::HashMap, io::Cursor, path::PathBuf, sync::Arc}; +use super::{core::CompfsCore, lister::CompfsLister, reader::CompfsReader, writer::CompfsWriter}; + +/// compio-based file system support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct CompfsConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, +} /// [`compio`]-based file system support. #[derive(Debug, Clone, Default)] pub struct CompfsBuilder { - root: Option, + config: CompfsConfig, } impl CompfsBuilder { /// Set root for Compfs pub fn root(&mut self, root: &str) -> &mut Self { - self.root = if root.is_empty() { + self.config.root = if root.is_empty() { None } else { - Some(PathBuf::from(root)) + Some(root.to_string()) }; self @@ -46,17 +56,14 @@ impl CompfsBuilder { impl Builder for CompfsBuilder { const SCHEME: Scheme = Scheme::Compfs; type Accessor = CompfsBackend; + type Config = CompfsConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = CompfsBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - - builder + fn from_config(config: Self::Config) -> Self { + Self { config } } fn build(&mut self) -> Result { - let root = match self.root.take() { + let root = match self.config.root.take() { Some(root) => Ok(root), None => Err(Error::new( ErrorKind::ConfigInvalid, @@ -70,7 +77,7 @@ impl Builder for CompfsBuilder { std::fs::create_dir_all(&root).map_err(|e| { Error::new(ErrorKind::Unexpected, "create root dir failed") .with_operation("Builder::build") - .with_context("root", root.to_string_lossy()) + .with_context("root", root.as_str()) .set_source(e) })?; } @@ -83,7 +90,7 @@ impl Builder for CompfsBuilder { ) })?; let core = CompfsCore { - root, + root: root.into(), dispatcher, buf_pool: oio::PooledBuf::new(16), }; diff --git a/core/src/services/compfs/mod.rs b/core/src/services/compfs/mod.rs index f22a4c7af74b..0fd86a7e3cfc 100644 --- a/core/src/services/compfs/mod.rs +++ b/core/src/services/compfs/mod.rs @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. +pub use backend::CompfsBuilder as Compfs; +pub use backend::CompfsConfig; + mod backend; mod core; mod lister; mod reader; mod writer; - -pub use backend::CompfsBuilder as Compfs; diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 11478b0e8c00..5c6c19d6be39 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; @@ -26,7 +25,7 @@ use log::debug; use reqsign::TencentCosConfig; use reqsign::TencentCosCredentialLoader; use reqsign::TencentCosSigner; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::*; use super::error::parse_error; @@ -37,15 +36,21 @@ use crate::services::cos::writer::CosWriters; use crate::*; /// Tencent-Cloud COS services support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] pub struct CosConfig { - root: Option, - endpoint: Option, - secret_id: Option, - secret_key: Option, - bucket: Option, - disable_config_load: bool, + /// Root of this backend. + pub root: Option, + /// Endpoint of this backend. + pub endpoint: Option, + /// Secret ID of this backend. + pub secret_id: Option, + /// Secret key of this backend. + pub secret_key: Option, + /// Bucket of this backend. + pub bucket: Option, + /// Disable config load so that opendal will not load config from + pub disable_config_load: bool, } impl Debug for CosConfig { @@ -161,11 +166,9 @@ impl CosBuilder { impl Builder for CosBuilder { const SCHEME: Scheme = Scheme::Cos; type Accessor = CosBackend; + type Config = CosConfig; - fn from_map(map: HashMap) -> Self { - let config = CosConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { Self { config, http_client: None, diff --git a/core/src/services/cos/mod.rs b/core/src/services/cos/mod.rs index 780c2c4796cf..e035b71abccc 100644 --- a/core/src/services/cos/mod.rs +++ b/core/src/services/cos/mod.rs @@ -17,6 +17,7 @@ mod backend; pub use backend::CosBuilder as Cos; +pub use backend::CosConfig; mod core; mod error; diff --git a/core/src/services/d1/backend.rs b/core/src/services/d1/backend.rs index 8a1d95d0e0d0..768bf92aed19 100644 --- a/core/src/services/d1/backend.rs +++ b/core/src/services/d1/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; @@ -23,7 +22,7 @@ use bytes::Buf; use http::header; use http::Request; use http::StatusCode; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; use super::error::parse_error; @@ -34,7 +33,7 @@ use crate::ErrorKind; use crate::*; /// Config for [Cloudflare D1](https://developers.cloudflare.com/d1) backend support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct D1Config { @@ -159,12 +158,12 @@ impl D1Builder { impl Builder for D1Builder { const SCHEME: Scheme = Scheme::D1; type Accessor = D1Backend; + type Config = D1Config; - fn from_map(map: HashMap) -> Self { + fn from_config(config: Self::Config) -> Self { Self { - config: D1Config::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"), - ..Default::default() + config, + http_client: None, } } diff --git a/core/src/services/dashmap/backend.rs b/core/src/services/dashmap/backend.rs index 8f4bb5cceda4..54cf44b19d25 100644 --- a/core/src/services/dashmap/backend.rs +++ b/core/src/services/dashmap/backend.rs @@ -15,21 +15,20 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use dashmap::DashMap; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::raw::adapters::typed_kv; -use crate::raw::ConfigDeserializer; use crate::*; /// [dashmap](https://github.com/xacrimon/dashmap) backend support. -#[derive(Default, Deserialize, Clone, Debug)] +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct DashmapConfig { - root: Option, + /// The root path for dashmap. + pub root: Option, } /// [dashmap](https://github.com/xacrimon/dashmap) backend support. @@ -50,11 +49,9 @@ impl DashmapBuilder { impl Builder for DashmapBuilder { const SCHEME: Scheme = Scheme::Dashmap; type Accessor = DashmapBackend; + type Config = DashmapConfig; - fn from_map(map: HashMap) -> Self { - let config = DashmapConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { Self { config } } diff --git a/core/src/services/dashmap/mod.rs b/core/src/services/dashmap/mod.rs index 1ca0e1ae0f5b..0cd7e387fa24 100644 --- a/core/src/services/dashmap/mod.rs +++ b/core/src/services/dashmap/mod.rs @@ -17,3 +17,4 @@ mod backend; pub use backend::DashmapBuilder as Dashmap; +pub use backend::DashmapConfig; diff --git a/core/src/services/dbfs/backend.rs b/core/src/services/dbfs/backend.rs index 5da63fe8991c..fe42708b03d6 100644 --- a/core/src/services/dbfs/backend.rs +++ b/core/src/services/dbfs/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -23,7 +22,7 @@ use std::sync::Arc; use bytes::Buf; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::DbfsCore; use super::error::parse_error; @@ -33,11 +32,14 @@ use crate::raw::*; use crate::*; /// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct DbfsConfig { - root: Option, - endpoint: Option, - token: Option, + /// The root for dbfs. + pub root: Option, + /// The endpoint for dbfs. + pub endpoint: Option, + /// The token for dbfs. + pub token: Option, } impl Debug for DbfsConfig { @@ -111,11 +113,9 @@ impl DbfsBuilder { impl Builder for DbfsBuilder { const SCHEME: Scheme = Scheme::Dbfs; type Accessor = DbfsBackend; + type Config = DbfsConfig; - fn from_map(map: HashMap) -> Self { - let config = DbfsConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { Self { config } } diff --git a/core/src/services/dbfs/mod.rs b/core/src/services/dbfs/mod.rs index c3437eceaec7..98ac4f40d306 100644 --- a/core/src/services/dbfs/mod.rs +++ b/core/src/services/dbfs/mod.rs @@ -15,9 +15,10 @@ // specific language governing permissions and limitations // under the License. -mod backend; pub use backend::DbfsBuilder as Dbfs; +pub use backend::DbfsConfig; +mod backend; mod core; mod error; mod lister; diff --git a/core/src/services/dropbox/builder.rs b/core/src/services/dropbox/builder.rs index 2bdc7726e4a3..521d41310abc 100644 --- a/core/src/services/dropbox/builder.rs +++ b/core/src/services/dropbox/builder.rs @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; use chrono::DateTime; use chrono::Utc; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; use super::backend::DropboxBackend; @@ -32,7 +31,7 @@ use crate::raw::*; use crate::*; /// Config for [Dropbox](https://www.dropbox.com/) backend support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct DropboxConfig { @@ -134,12 +133,12 @@ impl DropboxBuilder { impl Builder for DropboxBuilder { const SCHEME: Scheme = Scheme::Dropbox; type Accessor = DropboxBackend; + type Config = DropboxConfig; - fn from_map(map: HashMap) -> Self { + fn from_config(config: Self::Config) -> Self { Self { - config: DropboxConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"), - ..Default::default() + config, + http_client: None, } } diff --git a/core/src/services/etcd/backend.rs b/core/src/services/etcd/backend.rs index baad96b58d53..45bc2514b16e 100644 --- a/core/src/services/etcd/backend.rs +++ b/core/src/services/etcd/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; @@ -28,7 +27,7 @@ use etcd_client::Error as EtcdError; use etcd_client::GetOptions; use etcd_client::Identity; use etcd_client::TlsOptions; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::OnceCell; use crate::raw::adapters::kv; @@ -38,7 +37,7 @@ use crate::*; const DEFAULT_ETCD_ENDPOINTS: &str = "http://127.0.0.1:2379"; /// Config for Etcd services support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct EtcdConfig { @@ -192,12 +191,10 @@ impl EtcdBuilder { impl Builder for EtcdBuilder { const SCHEME: Scheme = Scheme::Etcd; type Accessor = EtcdBackend; + type Config = EtcdConfig; - fn from_map(map: HashMap) -> Self { - EtcdBuilder { - config: EtcdConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"), - } + fn from_config(config: Self::Config) -> Self { + Self { config } } fn build(&mut self) -> Result { diff --git a/core/src/services/foundationdb/backend.rs b/core/src/services/foundationdb/backend.rs index 83e8e62fb9fa..70b2ed8a8bd2 100644 --- a/core/src/services/foundationdb/backend.rs +++ b/core/src/services/foundationdb/backend.rs @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; use foundationdb::api::NetworkAutoStop; use foundationdb::Database; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::raw::adapters::kv; use crate::raw::*; @@ -34,7 +33,7 @@ use crate::*; /// [foundationdb](https://www.foundationdb.org/) service support. ///Config for FoundationDB. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct FoundationConfig { @@ -78,11 +77,9 @@ impl FoundationdbBuilder { impl Builder for FoundationdbBuilder { const SCHEME: Scheme = Scheme::Foundationdb; type Accessor = FoundationdbBackend; + type Config = FoundationConfig; - fn from_map(map: HashMap) -> Self { - let config = FoundationConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { Self { config } } diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index eb338bf55957..e03c55c9a227 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::io::SeekFrom; use std::path::Path; use std::path::PathBuf; @@ -23,7 +22,7 @@ use std::sync::Arc; use chrono::DateTime; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::raw::*; use crate::*; @@ -35,7 +34,7 @@ use super::writer::FsWriter; use super::writer::FsWriters; /// config for file system -#[derive(Default, Deserialize, Debug)] +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct FsConfig { @@ -81,11 +80,10 @@ impl FsBuilder { impl Builder for FsBuilder { const SCHEME: Scheme = Scheme::Fs; type Accessor = FsBackend; + type Config = FsConfig; - fn from_map(map: HashMap) -> Self { - let config = FsConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - FsBuilder { config } + fn from_config(config: Self::Config) -> Self { + Self { config } } fn build(&mut self) -> Result { diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs index 17c0fd54056f..8fb347539901 100644 --- a/core/src/services/ftp/backend.rs +++ b/core/src/services/ftp/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::str; @@ -27,7 +26,7 @@ use bb8::PooledConnection; use bb8::RunError; use http::Uri; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use suppaftp::list::File; use suppaftp::types::FileType; use suppaftp::types::Response; @@ -47,7 +46,7 @@ use crate::raw::*; use crate::*; /// Config for Ftpservices support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct FtpConfig { @@ -134,6 +133,11 @@ impl FtpBuilder { impl Builder for FtpBuilder { const SCHEME: Scheme = Scheme::Ftp; type Accessor = FtpBackend; + type Config = FtpConfig; + + fn from_config(config: Self::Config) -> Self { + FtpBuilder { config } + } fn build(&mut self) -> Result { debug!("ftp backend build started: {:?}", &self); @@ -194,13 +198,6 @@ impl Builder for FtpBuilder { pool: OnceCell::new(), }) } - - fn from_map(map: HashMap) -> Self { - FtpBuilder { - config: FtpConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"), - } - } } pub struct Manager { diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index ecc74a9731c0..ca327bdb561e 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -28,7 +27,7 @@ use reqsign::GoogleCredentialLoader; use reqsign::GoogleSigner; use reqsign::GoogleTokenLoad; use reqsign::GoogleTokenLoader; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json; use super::core::*; @@ -43,29 +42,29 @@ const DEFAULT_GCS_ENDPOINT: &str = "https://storage.googleapis.com"; const DEFAULT_GCS_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_write"; /// [Google Cloud Storage](https://cloud.google.com/storage) services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct GcsConfig { /// root URI, all operations happens under `root` - root: Option, + pub root: Option, /// bucket name - bucket: String, + pub bucket: String, /// endpoint URI of GCS service, /// default is `https://storage.googleapis.com` - endpoint: Option, + pub endpoint: Option, /// Scope for gcs. - scope: Option, + pub scope: Option, /// Service Account for gcs. - service_account: Option, + pub service_account: Option, /// Credentials string for GCS service OAuth2 authentication. - credential: Option, + pub credential: Option, /// Local path to credentials file for GCS service OAuth2 authentication. - credential_path: Option, + pub credential_path: Option, /// The predefined acl for GCS. - predefined_acl: Option, + pub predefined_acl: Option, /// The default storage class used by gcs. - default_storage_class: Option, + pub default_storage_class: Option, } impl Debug for GcsConfig { @@ -229,14 +228,13 @@ impl GcsBuilder { impl Builder for GcsBuilder { const SCHEME: Scheme = Scheme::Gcs; type Accessor = GcsBackend; + type Config = GcsConfig; - fn from_map(map: HashMap) -> Self { - let config = GcsConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - GcsBuilder { + fn from_config(config: Self::Config) -> Self { + Self { config, - ..GcsBuilder::default() + http_client: None, + customized_token_loader: None, } } diff --git a/core/src/services/gdrive/builder.rs b/core/src/services/gdrive/builder.rs index 5ee80ab4c644..014d158e449a 100644 --- a/core/src/services/gdrive/builder.rs +++ b/core/src/services/gdrive/builder.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -23,12 +22,11 @@ use std::sync::Arc; use chrono::DateTime; use chrono::Utc; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; use super::backend::GdriveBackend; use crate::raw::normalize_root; -use crate::raw::ConfigDeserializer; use crate::raw::HttpClient; use crate::raw::PathCacher; use crate::services::gdrive::core::GdriveCore; @@ -38,7 +36,7 @@ use crate::Scheme; use crate::*; /// [GoogleDrive](https://drive.google.com/) configuration. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct GdriveConfig { @@ -141,16 +139,12 @@ impl GdriveBuilder { impl Builder for GdriveBuilder { const SCHEME: Scheme = Scheme::Gdrive; - type Accessor = GdriveBackend; + type Config = GdriveConfig; - fn from_map(map: HashMap) -> Self { - let config = GdriveConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { Self { config, - http_client: None, } } diff --git a/core/src/services/gdrive/mod.rs b/core/src/services/gdrive/mod.rs index 015f177b28e0..7b37822ee0e5 100644 --- a/core/src/services/gdrive/mod.rs +++ b/core/src/services/gdrive/mod.rs @@ -21,5 +21,6 @@ mod core; mod error; pub use builder::GdriveBuilder as Gdrive; +pub use builder::GdriveConfig; mod lister; mod writer; diff --git a/core/src/services/ghac/backend.rs b/core/src/services/ghac/backend.rs index 9b3932c15c1a..6cd382c5896a 100644 --- a/core/src/services/ghac/backend.rs +++ b/core/src/services/ghac/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::env; use std::sync::Arc; @@ -82,15 +81,26 @@ fn value_or_env( }) } +/// Config for GitHub Action Cache Services support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct GhacConfig { + /// The root path for ghac. + pub root: Option, + /// The version that used by cache. + pub version: Option, + /// The endpoint for ghac service. + pub endpoint: Option, + /// The runtime token for ghac service. + pub runtime_token: Option, +} + /// GitHub Action Cache Services support. #[doc = include_str!("docs.md")] #[derive(Debug, Default)] pub struct GhacBuilder { - root: Option, - version: Option, - endpoint: Option, - runtime_token: Option, - + config: GhacConfig, http_client: Option, } @@ -98,7 +108,7 @@ impl GhacBuilder { /// set the working directory root of backend pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -112,7 +122,7 @@ impl GhacBuilder { /// If not set, we will use `opendal` as default. pub fn version(&mut self, version: &str) -> &mut Self { if !version.is_empty() { - self.version = Some(version.to_string()) + self.config.version = Some(version.to_string()) } self @@ -125,7 +135,7 @@ impl GhacBuilder { /// Default: the value of the `ACTIONS_CACHE_URL` environment variable. pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { - self.endpoint = Some(endpoint.to_string()) + self.config.endpoint = Some(endpoint.to_string()) } self } @@ -138,7 +148,7 @@ impl GhacBuilder { /// Default: the value of the `ACTIONS_RUNTIME_TOKEN` environment variable. pub fn runtime_token(&mut self, runtime_token: &str) -> &mut Self { if !runtime_token.is_empty() { - self.runtime_token = Some(runtime_token.to_string()) + self.config.runtime_token = Some(runtime_token.to_string()) } self } @@ -158,20 +168,19 @@ impl GhacBuilder { impl Builder for GhacBuilder { const SCHEME: Scheme = Scheme::Ghac; type Accessor = GhacBackend; + type Config = GhacConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = GhacBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("version").map(|v| builder.version(v)); - - builder + fn from_config(config: Self::Config) -> Self { + GhacBuilder { + config, + http_client: None, + } } fn build(&mut self) -> Result { debug!("backend build started: {:?}", self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); let client = if let Some(client) = self.http_client.take() { @@ -186,13 +195,18 @@ impl Builder for GhacBuilder { let backend = GhacBackend { root, - cache_url: value_or_env(self.endpoint.take(), ACTIONS_CACHE_URL, "Builder::build")?, + cache_url: value_or_env( + self.config.endpoint.take(), + ACTIONS_CACHE_URL, + "Builder::build", + )?, catch_token: value_or_env( - self.runtime_token.take(), + self.config.runtime_token.take(), ACTIONS_RUNTIME_TOKEN, "Builder::build", )?, version: self + .config .version .clone() .unwrap_or_else(|| "opendal".to_string()), diff --git a/core/src/services/ghac/mod.rs b/core/src/services/ghac/mod.rs index b819c409dd78..2a87b2258399 100644 --- a/core/src/services/ghac/mod.rs +++ b/core/src/services/ghac/mod.rs @@ -17,6 +17,7 @@ mod backend; pub use backend::GhacBuilder as Ghac; +pub use backend::GhacConfig; mod error; mod writer; diff --git a/core/src/services/github/backend.rs b/core/src/services/github/backend.rs index 714ed6c4fc0d..3cb3c9623969 100644 --- a/core/src/services/github/backend.rs +++ b/core/src/services/github/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -24,7 +23,7 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::Entry; use super::core::GithubCore; @@ -35,8 +34,8 @@ use super::writer::GithubWriters; use crate::raw::*; use crate::*; -/// Config for backblaze Github services support. -#[derive(Default, Deserialize)] +/// Config for backblaze GitHub services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct GithubConfig { @@ -44,17 +43,17 @@ pub struct GithubConfig { /// /// All operations will happen under this root. pub root: Option, - /// Github access_token. + /// GitHub access_token. /// /// optional. /// If not provided, the backend will only support read operations for public repositories. /// And rate limit will be limited to 60 requests per hour. pub token: Option, - /// Github repo owner. + /// GitHub repo owner. /// /// required. pub owner: String, - /// Github repo name. + /// GitHub repo name. /// /// required. pub repo: String, @@ -77,7 +76,6 @@ impl Debug for GithubConfig { #[derive(Default)] pub struct GithubBuilder { config: GithubConfig, - http_client: Option, } @@ -142,22 +140,9 @@ impl GithubBuilder { impl Builder for GithubBuilder { const SCHEME: Scheme = Scheme::Github; type Accessor = GithubBackend; + type Config = GithubConfig; - /// Converts a HashMap into an GithubBuilder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of GithubBuilder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = GithubConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an GithubBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { GithubBuilder { config, http_client: None, diff --git a/core/src/services/gridfs/backend.rs b/core/src/services/gridfs/backend.rs index c00724c77b8b..26772ca870a6 100644 --- a/core/src/services/gridfs/backend.rs +++ b/core/src/services/gridfs/backend.rs @@ -25,25 +25,33 @@ use mongodb::options::ClientOptions; use mongodb::options::GridFsBucketOptions; use mongodb::options::GridFsFindOptions; use mongodb::GridFsBucket; +use serde::{Deserialize, Serialize}; use tokio::sync::OnceCell; use crate::raw::adapters::kv; use crate::raw::new_std_io_error; use crate::*; -#[doc = include_str!("docs.md")] -#[derive(Default)] -pub struct GridFsBuilder { - connection_string: Option, - database: Option, - bucket: Option, - chunk_size: Option, - root: Option, +/// Config for Grid file system support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct GridFsConfig { + /// The connection string of the MongoDB service. + pub connection_string: Option, + /// The database name of the MongoDB GridFs service to read/write. + pub database: Option, + /// The bucket name of the MongoDB GridFs service to read/write. + pub bucket: Option, + /// The chunk size of the MongoDB GridFs service used to break the user file into chunks. + pub chunk_size: Option, + /// The working directory, all operations will be performed under it. + pub root: Option, } -impl Debug for GridFsBuilder { +impl Debug for GridFsConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("GridFsBuilder") + f.debug_struct("GridFsConfig") .field("database", &self.database) .field("bucket", &self.bucket) .field("chunk_size", &self.chunk_size) @@ -52,6 +60,20 @@ impl Debug for GridFsBuilder { } } +#[doc = include_str!("docs.md")] +#[derive(Default)] +pub struct GridFsBuilder { + config: GridFsConfig, +} + +impl Debug for GridFsBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("GridFsBuilder"); + d.field("config", &self.config); + d.finish_non_exhaustive() + } +} + impl GridFsBuilder { /// Set the connection_string of the MongoDB service. /// @@ -75,7 +97,7 @@ impl GridFsBuilder { /// For more information, please refer to [MongoDB Connection String URI Format](https://docs.mongodb.com/manual/reference/connection-string/). pub fn connection_string(&mut self, v: &str) -> &mut Self { if !v.is_empty() { - self.connection_string = Some(v.to_string()); + self.config.connection_string = Some(v.to_string()); } self } @@ -85,7 +107,7 @@ impl GridFsBuilder { /// default: "/" pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_owned()); + self.config.root = Some(root.to_owned()); } self } @@ -93,7 +115,7 @@ impl GridFsBuilder { /// Set the database name of the MongoDB GridFs service to read/write. pub fn database(&mut self, database: &str) -> &mut Self { if !database.is_empty() { - self.database = Some(database.to_string()); + self.config.database = Some(database.to_string()); } self } @@ -103,7 +125,7 @@ impl GridFsBuilder { /// Default to `fs` if not specified. pub fn bucket(&mut self, bucket: &str) -> &mut Self { if !bucket.is_empty() { - self.bucket = Some(bucket.to_string()); + self.config.bucket = Some(bucket.to_string()); } self } @@ -113,7 +135,7 @@ impl GridFsBuilder { /// Default to `255 KiB` if not specified. pub fn chunk_size(&mut self, chunk_size: u32) -> &mut Self { if chunk_size > 0 { - self.chunk_size = Some(chunk_size); + self.config.chunk_size = Some(chunk_size); } self } @@ -121,25 +143,15 @@ impl GridFsBuilder { impl Builder for GridFsBuilder { const SCHEME: Scheme = Scheme::Mongodb; - type Accessor = GridFsBackend; + type Config = GridFsConfig; - fn from_map(map: std::collections::HashMap) -> Self { - let mut builder = Self::default(); - - map.get("connection_string") - .map(|v| builder.connection_string(v)); - map.get("database").map(|v| builder.database(v)); - map.get("bucket").map(|v| builder.bucket(v)); - map.get("chunk_size") - .map(|v| builder.chunk_size(v.parse::().unwrap_or_default())); - map.get("root").map(|v| builder.root(v)); - - builder + fn from_config(config: Self::Config) -> Self { + GridFsBuilder { config } } fn build(&mut self) -> Result { - let conn = match &self.connection_string.clone() { + let conn = match &self.config.connection_string.clone() { Some(v) => v.clone(), None => { return Err( @@ -148,18 +160,18 @@ impl Builder for GridFsBuilder { ) } }; - let database = match &self.database.clone() { + let database = match &self.config.database.clone() { Some(v) => v.clone(), None => { return Err(Error::new(ErrorKind::ConfigInvalid, "database is required") .with_context("service", Scheme::Gridfs)) } }; - let bucket = match &self.bucket.clone() { + let bucket = match &self.config.bucket.clone() { Some(v) => v.clone(), None => "fs".to_string(), }; - let chunk_size = self.chunk_size.unwrap_or(255); + let chunk_size = self.config.chunk_size.unwrap_or(255); Ok(GridFsBackend::new(Adapter { connection_string: conn, diff --git a/core/src/services/gridfs/mod.rs b/core/src/services/gridfs/mod.rs index 4623f8cb09bc..676a4d128d10 100644 --- a/core/src/services/gridfs/mod.rs +++ b/core/src/services/gridfs/mod.rs @@ -17,3 +17,4 @@ mod backend; pub use backend::GridFsBuilder as Gridfs; +pub use backend::GridFsConfig; diff --git a/core/src/services/hdfs/backend.rs b/core/src/services/hdfs/backend.rs index f56353e9eb1d..cd67a3a8944d 100644 --- a/core/src/services/hdfs/backend.rs +++ b/core/src/services/hdfs/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::io; @@ -25,7 +24,7 @@ use std::sync::Arc; use futures::AsyncWriteExt; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use uuid::Uuid; use super::lister::HdfsLister; @@ -37,7 +36,7 @@ use crate::*; /// [Hadoop Distributed File System (HDFS™)](https://hadoop.apache.org/) support. /// /// Config for Hdfs services support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct HdfsConfig { @@ -159,13 +158,9 @@ impl HdfsBuilder { impl Builder for HdfsBuilder { const SCHEME: Scheme = Scheme::Hdfs; type Accessor = HdfsBackend; + type Config = HdfsConfig; - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = HdfsConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an HdfsBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { HdfsBuilder { config } } diff --git a/core/src/services/hdfs_native/backend.rs b/core/src/services/hdfs_native/backend.rs index fab3e56c1be0..93507db5d3e0 100644 --- a/core/src/services/hdfs_native/backend.rs +++ b/core/src/services/hdfs_native/backend.rs @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; use hdfs_native::WriteOptions; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; // use uuid::Uuid; use super::error::parse_hdfs_error; @@ -36,7 +35,7 @@ use crate::*; /// Using [Native Rust HDFS client](https://github.com/Kimahriman/hdfs-native). /// Config for HdfsNative services support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct HdfsNativeConfig { @@ -113,14 +112,10 @@ impl HdfsNativeBuilder { impl Builder for HdfsNativeBuilder { const SCHEME: Scheme = Scheme::HdfsNative; type Accessor = HdfsNativeBackend; + type Config = HdfsNativeConfig; - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = HdfsNativeConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an HdfsNativeBuilder instance with the deserialized config. - HdfsNativeBuilder { config } + fn from_config(config: Self::Config) -> Self { + Self { config } } fn build(&mut self) -> Result { diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs index 920a86734569..c4aa157dcecd 100644 --- a/core/src/services/http/backend.rs +++ b/core/src/services/http/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -27,14 +26,14 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::error::parse_error; use crate::raw::*; use crate::*; /// Config for Http service support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct HttpConfig { @@ -146,11 +145,9 @@ impl HttpBuilder { impl Builder for HttpBuilder { const SCHEME: Scheme = Scheme::Http; type Accessor = HttpBackend; + type Config = HttpConfig; - fn from_map(map: HashMap) -> Self { - let config = HttpConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { HttpBuilder { config, http_client: None, diff --git a/core/src/services/huggingface/backend.rs b/core/src/services/huggingface/backend.rs index c97befbae59c..3ed4ff1a92b4 100644 --- a/core/src/services/huggingface/backend.rs +++ b/core/src/services/huggingface/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -24,7 +23,7 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::HuggingfaceCore; use super::core::HuggingfaceStatus; @@ -34,7 +33,7 @@ use crate::raw::*; use crate::*; /// Configuration for Huggingface service support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct HuggingfaceConfig { @@ -172,12 +171,10 @@ impl HuggingfaceBuilder { impl Builder for HuggingfaceBuilder { const SCHEME: Scheme = Scheme::Huggingface; type Accessor = HuggingfaceBackend; + type Config = HuggingfaceConfig; - fn from_map(map: HashMap) -> Self { - let config = HuggingfaceConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - HuggingfaceBuilder { config } + fn from_config(config: Self::Config) -> Self { + Self { config } } /// Build a HuggingfaceBackend. diff --git a/core/src/services/icloud/backend.rs b/core/src/services/icloud/backend.rs index e7957b8dd9d2..2592b60577fc 100644 --- a/core/src/services/icloud/backend.rs +++ b/core/src/services/icloud/backend.rs @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; use http::Response; use http::StatusCode; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; use super::core::*; @@ -30,7 +29,7 @@ use crate::raw::*; use crate::*; /// Config for icloud services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct IcloudConfig { @@ -53,6 +52,7 @@ pub struct IcloudConfig { /// /// token must be valid. pub trust_token: Option, + /// ds_web_auth_token must be set in Session pub ds_web_auth_token: Option, /// enable the china origin /// China region `origin` Header needs to be set to "https://www.icloud.com.cn". @@ -184,10 +184,9 @@ impl IcloudBuilder { impl Builder for IcloudBuilder { const SCHEME: Scheme = Scheme::Icloud; type Accessor = IcloudBackend; + type Config = IcloudConfig; - fn from_map(map: HashMap) -> Self { - let config = IcloudConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); + fn from_config(config: Self::Config) -> Self { IcloudBuilder { config, http_client: None, diff --git a/core/src/services/icloud/mod.rs b/core/src/services/icloud/mod.rs index d76c934ddde2..fd9b28633cb3 100644 --- a/core/src/services/icloud/mod.rs +++ b/core/src/services/icloud/mod.rs @@ -17,5 +17,6 @@ mod backend; pub use backend::IcloudBuilder as Icloud; +pub use backend::IcloudConfig; mod core; diff --git a/core/src/services/ipfs/backend.rs b/core/src/services/ipfs/backend.rs index b1d1e67397f1..ca6b400e8674 100644 --- a/core/src/services/ipfs/backend.rs +++ b/core/src/services/ipfs/backend.rs @@ -15,28 +15,37 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; +use super::error::parse_error; +use super::ipld::PBNode; +use crate::raw::*; +use crate::*; use http::Request; use http::Response; use http::StatusCode; use log::debug; use prost::Message; - -use super::error::parse_error; -use super::ipld::PBNode; -use crate::raw::*; -use crate::*; +use serde::{Deserialize, Serialize}; + +/// Config for IPFS file system support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct IpfsConfig { + /// IPFS gateway endpoint. + pub endpoint: Option, + /// IPFS root. + pub root: Option, +} /// IPFS file system support based on [IPFS HTTP Gateway](https://docs.ipfs.tech/concepts/ipfs-gateway/). #[doc = include_str!("docs.md")] #[derive(Default, Clone, Debug)] pub struct IpfsBuilder { - endpoint: Option, - root: Option, + config: IpfsConfig, http_client: Option, } @@ -50,7 +59,7 @@ impl IpfsBuilder { /// - `/ipns/opendal.apache.org/` (IPNS) pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -70,7 +79,7 @@ impl IpfsBuilder { pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/` - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); } self @@ -91,19 +100,19 @@ impl IpfsBuilder { impl Builder for IpfsBuilder { const SCHEME: Scheme = Scheme::Ipfs; type Accessor = IpfsBackend; - fn from_map(map: HashMap) -> Self { - let mut builder = IpfsBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); + type Config = IpfsConfig; - builder + fn from_config(config: Self::Config) -> Self { + IpfsBuilder { + config, + http_client: None, + } } fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); if !root.starts_with("/ipfs/") && !root.starts_with("/ipns/") { return Err(Error::new( ErrorKind::ConfigInvalid, @@ -114,7 +123,7 @@ impl Builder for IpfsBuilder { } debug!("backend use root {}", root); - let endpoint = match &self.endpoint { + let endpoint = match &self.config.endpoint { Some(endpoint) => Ok(endpoint.clone()), None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_context("service", Scheme::Ipfs) diff --git a/core/src/services/ipfs/mod.rs b/core/src/services/ipfs/mod.rs index 4bd46822fbb3..dfc259740a1e 100644 --- a/core/src/services/ipfs/mod.rs +++ b/core/src/services/ipfs/mod.rs @@ -17,6 +17,7 @@ mod backend; pub use backend::IpfsBuilder as Ipfs; +pub use backend::IpfsConfig; mod error; mod ipld; diff --git a/core/src/services/ipmfs/builder.rs b/core/src/services/ipmfs/builder.rs index d432b24bbfc6..924b18b06ab0 100644 --- a/core/src/services/ipmfs/builder.rs +++ b/core/src/services/ipmfs/builder.rs @@ -15,13 +15,22 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; - -use log::debug; - use super::backend::IpmfsBackend; use crate::raw::*; use crate::*; +use log::debug; +use serde::{Deserialize, Serialize}; + +/// Config for IPFS MFS support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct IpmfsConfig { + /// Root for ipfs. + pub root: Option, + /// Endpoint for ipfs. + pub endpoint: Option, +} /// IPFS file system support based on [IPFS MFS](https://docs.ipfs.tech/concepts/file-systems/) API. /// @@ -66,15 +75,14 @@ use crate::*; /// ``` #[derive(Default, Debug)] pub struct IpmfsBuilder { - root: Option, - endpoint: Option, + config: IpmfsConfig, http_client: Option, } impl IpmfsBuilder { /// Set root for ipfs. pub fn root(&mut self, root: &str) -> &mut Self { - self.root = if root.is_empty() { + self.config.root = if root.is_empty() { None } else { Some(root.to_string()) @@ -87,7 +95,7 @@ impl IpmfsBuilder { /// /// Default: http://localhost:5001 pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { - self.endpoint = if endpoint.is_empty() { + self.config.endpoint = if endpoint.is_empty() { None } else { Some(endpoint.to_string()) @@ -110,21 +118,21 @@ impl IpmfsBuilder { impl Builder for IpmfsBuilder { const SCHEME: Scheme = Scheme::Ipmfs; type Accessor = IpmfsBackend; + type Config = IpmfsConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = IpmfsBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - - builder + fn from_config(config: Self::Config) -> Self { + IpmfsBuilder { + config, + http_client: None, + } } fn build(&mut self) -> Result { - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); let endpoint = self + .config .endpoint .clone() .unwrap_or_else(|| "http://localhost:5001".to_string()); diff --git a/core/src/services/ipmfs/mod.rs b/core/src/services/ipmfs/mod.rs index 2a7ce5c0ad71..a054047875eb 100644 --- a/core/src/services/ipmfs/mod.rs +++ b/core/src/services/ipmfs/mod.rs @@ -18,6 +18,7 @@ mod backend; mod builder; pub use builder::IpmfsBuilder as Ipmfs; +pub use builder::IpmfsConfig; mod error; mod lister; diff --git a/core/src/services/koofr/backend.rs b/core/src/services/koofr/backend.rs index 98008187e814..b68eaa169db1 100644 --- a/core/src/services/koofr/backend.rs +++ b/core/src/services/koofr/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -24,7 +23,7 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; use tokio::sync::OnceCell; @@ -39,7 +38,7 @@ use crate::raw::*; use crate::*; /// Config for backblaze Koofr services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct KoofrConfig { @@ -151,22 +150,9 @@ impl KoofrBuilder { impl Builder for KoofrBuilder { const SCHEME: Scheme = Scheme::Koofr; type Accessor = KoofrBackend; + type Config = KoofrConfig; - /// Converts a HashMap into an KoofrBuilder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of KoofrBuilder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = KoofrConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an KoofrBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { KoofrBuilder { config, http_client: None, diff --git a/core/src/services/libsql/backend.rs b/core/src/services/libsql/backend.rs index a74e5dab57bf..52f0ebcef924 100644 --- a/core/src/services/libsql/backend.rs +++ b/core/src/services/libsql/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::str; @@ -36,7 +35,7 @@ use hrana_client_proto::StmtResult; use hrana_client_proto::Value; use http::Request; use http::Uri; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::error::parse_error; use crate::raw::adapters::kv; @@ -44,17 +43,23 @@ use crate::raw::*; use crate::*; /// Config for Libsqlservices support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct LibsqlConfig { - connection_string: Option, - auth_token: Option, - - table: Option, - key_field: Option, - value_field: Option, - root: Option, + /// Connection string for libsql service. + pub connection_string: Option, + /// Authentication token for libsql service. + pub auth_token: Option, + + /// Table name for libsql service. + pub table: Option, + /// Key field name for libsql service. + pub key_field: Option, + /// Value field name for libsql service. + pub value_field: Option, + /// Root for libsql service. + pub root: Option, } impl Debug for LibsqlConfig { @@ -162,11 +167,9 @@ impl LibsqlBuilder { impl Builder for LibsqlBuilder { const SCHEME: Scheme = Scheme::Libsql; type Accessor = LibsqlBackend; + type Config = LibsqlConfig; - fn from_map(map: HashMap) -> Self { - let config = LibsqlConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { LibsqlBuilder { config } } diff --git a/core/src/services/memcached/backend.rs b/core/src/services/memcached/backend.rs index b4d4a1e6b486..cdb46a488cd0 100644 --- a/core/src/services/memcached/backend.rs +++ b/core/src/services/memcached/backend.rs @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::time::Duration; use bb8::RunError; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::net::TcpStream; use tokio::sync::OnceCell; @@ -29,24 +28,24 @@ use crate::raw::*; use crate::*; /// Config for MemCached services support -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct MemcachedConfig { /// network address of the memcached service. /// /// For example: "tcp://localhost:11211" - endpoint: Option, + pub endpoint: Option, /// the working directory of the service. Can be "/path/to/dir" /// /// default is "/" - root: Option, + pub root: Option, /// Memcached username, optional. - username: Option, + pub username: Option, /// Memcached password, optional. - password: Option, + pub password: Option, /// The default ttl for put operations. - default_ttl: Option, + pub default_ttl: Option, } /// [Memcached](https://memcached.org/) service support. @@ -99,10 +98,9 @@ impl MemcachedBuilder { impl Builder for MemcachedBuilder { const SCHEME: Scheme = Scheme::Memcached; type Accessor = MemcachedBackend; + type Config = MemcachedConfig; - fn from_map(map: HashMap) -> Self { - let config = MemcachedConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); + fn from_config(config: Self::Config) -> Self { MemcachedBuilder { config } } diff --git a/core/src/services/memory/backend.rs b/core/src/services/memory/backend.rs index c2ed8ce45ab0..49cbfb25e2e3 100644 --- a/core/src/services/memory/backend.rs +++ b/core/src/services/memory/backend.rs @@ -16,23 +16,21 @@ // under the License. use std::collections::BTreeMap; -use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; use std::sync::Mutex; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; -use self::raw::ConfigDeserializer; use crate::raw::adapters::typed_kv; use crate::*; ///Config for memory. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct MemoryConfig { - ///root of the backend. + /// root of the backend. pub root: Option, } @@ -54,12 +52,10 @@ impl MemoryBuilder { impl Builder for MemoryBuilder { const SCHEME: Scheme = Scheme::Memory; type Accessor = MemoryBackend; + type Config = MemoryConfig; - fn from_map(map: HashMap) -> Self { - MemoryBuilder { - config: MemoryConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"), - } + fn from_config(config: Self::Config) -> Self { + MemoryBuilder { config } } fn build(&mut self) -> Result { diff --git a/core/src/services/mini_moka/backend.rs b/core/src/services/mini_moka/backend.rs index ce05cb3e43aa..71374cb8dd52 100644 --- a/core/src/services/mini_moka/backend.rs +++ b/core/src/services/mini_moka/backend.rs @@ -15,33 +15,40 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::time::Duration; +use crate::raw::adapters::typed_kv; +use crate::*; use log::debug; use mini_moka::sync::Cache; use mini_moka::sync::CacheBuilder; +use serde::{Deserialize, Serialize}; -use crate::raw::adapters::typed_kv; -use crate::*; - -/// [mini-moka](https://github.com/moka-rs/mini-moka) backend support. -#[doc = include_str!("docs.md")] -#[derive(Default, Debug)] -pub struct MiniMokaBuilder { +/// Config for mini-moka support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct MiniMokaConfig { /// Sets the max capacity of the cache. /// /// Refer to [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity) - max_capacity: Option, + pub max_capacity: Option, /// Sets the time to live of the cache. /// /// Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live) - time_to_live: Option, + pub time_to_live: Option, /// Sets the time to idle of the cache. /// /// Refer to [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle) - time_to_idle: Option, + pub time_to_idle: Option, +} + +/// [mini-moka](https://github.com/moka-rs/mini-moka) backend support. +#[doc = include_str!("docs.md")] +#[derive(Default, Debug)] +pub struct MiniMokaBuilder { + config: MiniMokaConfig, } impl MiniMokaBuilder { @@ -50,7 +57,7 @@ impl MiniMokaBuilder { /// Refer to [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity) pub fn max_capacity(&mut self, v: u64) -> &mut Self { if v != 0 { - self.max_capacity = Some(v); + self.config.max_capacity = Some(v); } self } @@ -60,7 +67,7 @@ impl MiniMokaBuilder { /// Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live) pub fn time_to_live(&mut self, v: Duration) -> &mut Self { if !v.is_zero() { - self.time_to_live = Some(v); + self.config.time_to_live = Some(v); } self } @@ -70,7 +77,7 @@ impl MiniMokaBuilder { /// Refer to [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle) pub fn time_to_idle(&mut self, v: Duration) -> &mut Self { if !v.is_zero() { - self.time_to_idle = Some(v); + self.config.time_to_idle = Some(v); } self } @@ -79,21 +86,10 @@ impl MiniMokaBuilder { impl Builder for MiniMokaBuilder { const SCHEME: Scheme = Scheme::MiniMoka; type Accessor = MiniMokaBackend; + type Config = MiniMokaConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = MiniMokaBuilder::default(); - - map.get("max_capacity") - .map(|v| v.parse::().map(|v| builder.max_capacity(v))); - map.get("time_to_live").map(|v| { - v.parse::() - .map(|v| builder.time_to_live(Duration::from_secs(v))) - }); - map.get("time_to_idle").map(|v| { - v.parse::() - .map(|v| builder.time_to_idle(Duration::from_secs(v))) - }); - builder + fn from_config(config: Self::Config) -> Self { + Self { config } } fn build(&mut self) -> Result { @@ -102,13 +98,13 @@ impl Builder for MiniMokaBuilder { let mut builder: CacheBuilder = Cache::builder(); // Use entries' bytes as capacity weigher. builder = builder.weigher(|k, v| (k.len() + v.size()) as u32); - if let Some(v) = self.max_capacity { + if let Some(v) = self.config.max_capacity { builder = builder.max_capacity(v) } - if let Some(v) = self.time_to_live { + if let Some(v) = self.config.time_to_live { builder = builder.time_to_live(v) } - if let Some(v) = self.time_to_idle { + if let Some(v) = self.config.time_to_idle { builder = builder.time_to_idle(v) } diff --git a/core/src/services/mini_moka/mod.rs b/core/src/services/mini_moka/mod.rs index d32c035042d0..cef6a0630b07 100644 --- a/core/src/services/mini_moka/mod.rs +++ b/core/src/services/mini_moka/mod.rs @@ -17,3 +17,4 @@ mod backend; pub use backend::MiniMokaBuilder as MiniMoka; +pub use backend::MiniMokaConfig; diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index 8f5fc8e58e96..bc7c9de65e3a 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -37,21 +37,29 @@ pub use azblob::AzblobConfig; mod azdls; #[cfg(feature = "services-azdls")] pub use azdls::Azdls; +#[cfg(feature = "services-azdls")] +pub use azdls::AzdlsConfig; #[cfg(feature = "services-cloudflare-kv")] mod cloudflare_kv; #[cfg(feature = "services-cloudflare-kv")] pub use self::cloudflare_kv::CloudflareKv; +#[cfg(feature = "services-cloudflare-kv")] +pub use self::cloudflare_kv::CloudflareKvConfig; #[cfg(feature = "services-cos")] mod cos; #[cfg(feature = "services-cos")] pub use cos::Cos; +#[cfg(feature = "services-cos")] +pub use cos::CosConfig; #[cfg(feature = "services-dashmap")] mod dashmap; #[cfg(feature = "services-dashmap")] pub use self::dashmap::Dashmap; +#[cfg(feature = "services-dashmap")] +pub use self::dashmap::DashmapConfig; #[cfg(feature = "services-etcd")] mod etcd; @@ -85,10 +93,14 @@ pub use gcs::GcsConfig; mod ghac; #[cfg(feature = "services-ghac")] pub use ghac::Ghac; +#[cfg(feature = "services-ghac")] +pub use ghac::GhacConfig; #[cfg(feature = "services-gridfs")] mod gridfs; #[cfg(feature = "services-gridfs")] +pub use gridfs::GridFsConfig; +#[cfg(feature = "services-gridfs")] pub use gridfs::Gridfs; #[cfg(feature = "services-hdfs")] @@ -116,16 +128,22 @@ pub use huggingface::HuggingfaceConfig; mod ipfs; #[cfg(feature = "services-ipfs")] pub use self::ipfs::Ipfs; +#[cfg(feature = "services-ipfs")] +pub use self::ipfs::IpfsConfig; #[cfg(feature = "services-ipmfs")] mod ipmfs; #[cfg(feature = "services-ipmfs")] pub use ipmfs::Ipmfs; +#[cfg(feature = "services-ipmfs")] +pub use ipmfs::IpmfsConfig; #[cfg(feature = "services-icloud")] mod icloud; #[cfg(feature = "services-icloud")] pub use icloud::Icloud; +#[cfg(feature = "services-icloud")] +pub use icloud::IcloudConfig; #[cfg(feature = "services-libsql")] mod libsql; @@ -152,6 +170,8 @@ pub use self::memory::MemoryConfig; mod mini_moka; #[cfg(feature = "services-mini-moka")] pub use self::mini_moka::MiniMoka; +#[cfg(feature = "services-mini-moka")] +pub use self::mini_moka::MiniMokaConfig; #[cfg(feature = "services-moka")] mod moka; @@ -164,21 +184,29 @@ pub use self::moka::MokaConfig; mod obs; #[cfg(feature = "services-obs")] pub use obs::Obs; +#[cfg(feature = "services-obs")] +pub use obs::ObsConfig; #[cfg(feature = "services-oss")] mod oss; #[cfg(feature = "services-oss")] pub use oss::Oss; +#[cfg(feature = "services-oss")] +pub use oss::OssConfig; #[cfg(feature = "services-cacache")] mod cacache; #[cfg(feature = "services-cacache")] pub use self::cacache::Cacache; +#[cfg(feature = "services-cacache")] +pub use self::cacache::CacacheConfig; #[cfg(feature = "services-persy")] mod persy; #[cfg(feature = "services-persy")] pub use self::persy::Persy; +#[cfg(feature = "services-persy")] +pub use self::persy::PersyConfig; #[cfg(feature = "services-redis")] mod redis; @@ -219,6 +247,8 @@ pub use self::sled::SledConfig; mod supabase; #[cfg(feature = "services-supabase")] pub use supabase::Supabase; +#[cfg(feature = "services-supabase")] +pub use supabase::SupabaseConfig; #[cfg(feature = "services-webdav")] mod webdav; @@ -229,6 +259,10 @@ pub use webdav::WebdavConfig; #[cfg(feature = "services-webhdfs")] mod webhdfs; +#[cfg(feature = "services-webhdfs")] +pub use webhdfs::Webhdfs; +#[cfg(feature = "services-webhdfs")] +pub use webhdfs::WebhdfsConfig; #[cfg(feature = "services-onedrive")] mod onedrive; @@ -241,6 +275,8 @@ pub use onedrive::OnedriveConfig; mod gdrive; #[cfg(feature = "services-gdrive")] pub use gdrive::Gdrive; +#[cfg(feature = "services-gdrive")] +pub use gdrive::GdriveConfig; #[cfg(feature = "services-github")] mod github; @@ -255,18 +291,20 @@ mod dropbox; pub use dropbox::Dropbox; #[cfg(feature = "services-dropbox")] pub use dropbox::DropboxConfig; -#[cfg(feature = "services-webhdfs")] -pub use webhdfs::Webhdfs; #[cfg(feature = "services-vercel-artifacts")] mod vercel_artifacts; #[cfg(feature = "services-vercel-artifacts")] pub use vercel_artifacts::VercelArtifacts; +#[cfg(feature = "services-vercel-artifacts")] +pub use vercel_artifacts::VercelArtifactsConfig; #[cfg(feature = "services-redb")] mod redb; #[cfg(feature = "services-redb")] pub use self::redb::Redb; +#[cfg(feature = "services-redb")] +pub use self::redb::RedbConfig; #[cfg(feature = "services-tikv")] mod tikv; @@ -321,6 +359,8 @@ pub use self::d1::D1; mod azfile; #[cfg(feature = "services-azfile")] pub use self::azfile::Azfile; +#[cfg(feature = "services-azfile")] +pub use self::azfile::AzfileConfig; #[cfg(feature = "services-mongodb")] mod mongodb; @@ -333,11 +373,15 @@ pub use self::mongodb::MongodbConfig; mod dbfs; #[cfg(feature = "services-dbfs")] pub use self::dbfs::Dbfs; +#[cfg(feature = "services-dbfs")] +pub use self::dbfs::DbfsConfig; #[cfg(feature = "services-swift")] mod swift; #[cfg(feature = "services-swift")] pub use self::swift::Swift; +#[cfg(feature = "services-swift")] +pub use self::swift::SwiftConfig; #[cfg(feature = "services-alluxio")] mod alluxio; @@ -420,6 +464,8 @@ pub use surrealdb::SurrealdbConfig; mod compfs; #[cfg(feature = "services-compfs")] pub use compfs::Compfs; +#[cfg(feature = "services-compfs")] +pub use compfs::CompfsConfig; #[cfg(feature = "services-monoiofs")] mod monoiofs; diff --git a/core/src/services/moka/backend.rs b/core/src/services/moka/backend.rs index 8beba86cdbe0..8ef470be5ea0 100644 --- a/core/src/services/moka/backend.rs +++ b/core/src/services/moka/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::time::Duration; @@ -23,14 +22,13 @@ use std::time::Duration; use log::debug; use moka::sync::CacheBuilder; use moka::sync::SegmentedCache; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::raw::adapters::typed_kv; -use crate::raw::ConfigDeserializer; use crate::*; /// Config for Mokaservices support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct MokaConfig { @@ -125,12 +123,10 @@ impl MokaBuilder { impl Builder for MokaBuilder { const SCHEME: Scheme = Scheme::Moka; type Accessor = MokaBackend; + type Config = MokaConfig; - fn from_map(map: HashMap) -> Self { - MokaBuilder { - config: MokaConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"), - } + fn from_config(config: Self::Config) -> Self { + MokaBuilder { config } } fn build(&mut self) -> Result { diff --git a/core/src/services/mongodb/backend.rs b/core/src/services/mongodb/backend.rs index 438731021b8c..75cdd2dc2248 100644 --- a/core/src/services/mongodb/backend.rs +++ b/core/src/services/mongodb/backend.rs @@ -23,15 +23,14 @@ use mongodb::bson::Binary; use mongodb::bson::Document; use mongodb::options::ClientOptions; use mongodb::options::UpdateOptions; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::OnceCell; use crate::raw::adapters::kv; -use crate::raw::ConfigDeserializer; use crate::*; /// Config for Mongodb service support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct MongodbConfig { @@ -152,12 +151,10 @@ impl MongodbBuilder { impl Builder for MongodbBuilder { const SCHEME: Scheme = Scheme::Mongodb; - type Accessor = MongodbBackend; + type Config = MongodbConfig; - fn from_map(map: std::collections::HashMap) -> Self { - let config = MongodbConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); + fn from_config(config: Self::Config) -> Self { MongodbBuilder { config } } diff --git a/core/src/services/monoiofs/backend.rs b/core/src/services/monoiofs/backend.rs index 93d52ef568cb..c4bb92dfa97f 100644 --- a/core/src/services/monoiofs/backend.rs +++ b/core/src/services/monoiofs/backend.rs @@ -20,14 +20,14 @@ use std::io; use std::path::PathBuf; use std::sync::Arc; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::MonoiofsCore; use crate::raw::*; use crate::*; /// Config for monoiofs services support. -#[derive(Default, Deserialize, Debug)] +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct MonoiofsConfig { @@ -62,12 +62,10 @@ impl MonoiofsBuilder { impl Builder for MonoiofsBuilder { const SCHEME: Scheme = Scheme::Monoiofs; - type Accessor = MonoiofsBackend; + type Config = MonoiofsConfig; - fn from_map(map: std::collections::HashMap) -> Self { - let config = MonoiofsConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize should success"); + fn from_config(config: Self::Config) -> Self { MonoiofsBuilder { config } } diff --git a/core/src/services/mysql/backend.rs b/core/src/services/mysql/backend.rs index 06c43d365942..cc0f1b591da3 100644 --- a/core/src/services/mysql/backend.rs +++ b/core/src/services/mysql/backend.rs @@ -15,29 +15,33 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use mysql_async::prelude::*; use mysql_async::Opts; use mysql_async::Pool; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::raw::adapters::kv; use crate::raw::*; use crate::*; /// Config for Mysql services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct MysqlConfig { - connection_string: Option, - - table: Option, - key_field: Option, - value_field: Option, - root: Option, + /// The connection string for mysql. + pub connection_string: Option, + + /// The table name for mysql. + pub table: Option, + /// The key field name for mysql. + pub key_field: Option, + /// The value field name for mysql. + pub value_field: Option, + /// The root for mysql. + pub root: Option, } impl Debug for MysqlConfig { @@ -134,11 +138,9 @@ impl MysqlBuilder { impl Builder for MysqlBuilder { const SCHEME: Scheme = Scheme::Mysql; type Accessor = MySqlBackend; + type Config = MysqlConfig; - fn from_map(map: HashMap) -> Self { - let config = MysqlConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { MysqlBuilder { config } } diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 09e68c235bef..742f4a22c3e7 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -15,10 +15,16 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; -use std::fmt::Debug; +use std::fmt::{Debug, Formatter}; use std::sync::Arc; +use super::core::ObsCore; +use super::error::parse_error; +use super::lister::ObsLister; +use super::writer::ObsWriter; +use crate::raw::*; +use crate::services::obs::writer::ObsWriters; +use crate::*; use http::Response; use http::StatusCode; use http::Uri; @@ -26,36 +32,50 @@ use log::debug; use reqsign::HuaweicloudObsConfig; use reqsign::HuaweicloudObsCredentialLoader; use reqsign::HuaweicloudObsSigner; +use serde::{Deserialize, Serialize}; + +/// Config for Huawei-Cloud Object Storage Service (OBS) support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct ObsConfig { + /// Root for obs. + pub root: Option, + /// Endpoint for obs. + pub endpoint: Option, + /// Access key id for obs. + pub access_key_id: Option, + /// Secret access key for obs. + pub secret_access_key: Option, + /// Bucket for obs. + pub bucket: Option, +} -use super::core::ObsCore; -use super::error::parse_error; -use super::lister::ObsLister; -use super::writer::ObsWriter; -use crate::raw::*; -use crate::services::obs::writer::ObsWriters; -use crate::*; +impl Debug for ObsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ObsConfig") + .field("root", &self.root) + .field("endpoint", &self.endpoint) + .field("access_key_id", &"") + .field("secret_access_key", &"") + .field("bucket", &self.bucket) + .finish() + } +} /// Huawei-Cloud Object Storage Service (OBS) support #[doc = include_str!("docs.md")] #[derive(Default, Clone)] pub struct ObsBuilder { - root: Option, - endpoint: Option, - access_key_id: Option, - secret_access_key: Option, - bucket: Option, + config: ObsConfig, http_client: Option, } impl Debug for ObsBuilder { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Builder") - .field("root", &self.root) - .field("endpoint", &self.endpoint) - .field("access_key_id", &"") - .field("secret_access_key", &"") - .field("bucket", &self.bucket) - .finish() + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("ObsBuilder"); + d.field("config", &self.config); + d.finish_non_exhaustive() } } @@ -65,7 +85,7 @@ impl ObsBuilder { /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -81,7 +101,7 @@ impl ObsBuilder { /// - `https://custom.obs.com` (port should not be set) pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); } self @@ -92,7 +112,7 @@ impl ObsBuilder { /// - If not, we will try to load it from environment. pub fn access_key_id(&mut self, access_key_id: &str) -> &mut Self { if !access_key_id.is_empty() { - self.access_key_id = Some(access_key_id.to_string()); + self.config.access_key_id = Some(access_key_id.to_string()); } self @@ -103,7 +123,7 @@ impl ObsBuilder { /// - If not, we will try to load it from environment. pub fn secret_access_key(&mut self, secret_access_key: &str) -> &mut Self { if !secret_access_key.is_empty() { - self.secret_access_key = Some(secret_access_key.to_string()); + self.config.secret_access_key = Some(secret_access_key.to_string()); } self @@ -113,7 +133,7 @@ impl ObsBuilder { /// The param is required. pub fn bucket(&mut self, bucket: &str) -> &mut Self { if !bucket.is_empty() { - self.bucket = Some(bucket.to_string()); + self.config.bucket = Some(bucket.to_string()); } self @@ -134,27 +154,22 @@ impl ObsBuilder { impl Builder for ObsBuilder { const SCHEME: Scheme = Scheme::Obs; type Accessor = ObsBackend; + type Config = ObsConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = ObsBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("bucket").map(|v| builder.bucket(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("access_key_id").map(|v| builder.access_key_id(v)); - map.get("secret_access_key") - .map(|v| builder.secret_access_key(v)); - - builder + fn from_config(config: Self::Config) -> Self { + ObsBuilder { + config, + http_client: None, + } } fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); - let bucket = match &self.bucket { + let bucket = match &self.config.bucket { Some(bucket) => Ok(bucket.to_string()), None => Err( Error::new(ErrorKind::ConfigInvalid, "The bucket is misconfigured") @@ -163,7 +178,7 @@ impl Builder for ObsBuilder { }?; debug!("backend use bucket {}", &bucket); - let uri = match &self.endpoint { + let uri = match &self.config.endpoint { Some(endpoint) => endpoint.parse::().map_err(|err| { Error::new(ErrorKind::ConfigInvalid, "endpoint is invalid") .with_context("service", Scheme::Obs) @@ -201,11 +216,11 @@ impl Builder for ObsBuilder { // Load cfg from env first. cfg = cfg.from_env(); - if let Some(v) = self.access_key_id.take() { + if let Some(v) = self.config.access_key_id.take() { cfg.access_key_id = Some(v); } - if let Some(v) = self.secret_access_key.take() { + if let Some(v) = self.config.secret_access_key.take() { cfg.secret_access_key = Some(v); } diff --git a/core/src/services/obs/mod.rs b/core/src/services/obs/mod.rs index f8a78290f0b5..daa275138171 100644 --- a/core/src/services/obs/mod.rs +++ b/core/src/services/obs/mod.rs @@ -17,6 +17,7 @@ mod backend; pub use backend::ObsBuilder as Obs; +pub use backend::ObsConfig; mod core; mod error; diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index bd98985ae875..7c536531ee9f 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -15,22 +15,20 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::backend::OnedriveBackend; use crate::raw::normalize_root; -use crate::raw::ConfigDeserializer; use crate::raw::HttpClient; use crate::Scheme; use crate::*; /// Config for [OneDrive](https://onedrive.com) backend support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct OnedriveConfig { @@ -93,15 +91,10 @@ impl OnedriveBuilder { impl Builder for OnedriveBuilder { const SCHEME: Scheme = Scheme::Onedrive; - type Accessor = OnedriveBackend; + type Config = OnedriveConfig; - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = OnedriveConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an OnedriveBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { OnedriveBuilder { config, http_client: None, diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index b49d6989656c..e24e404b7c74 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -15,12 +15,18 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; +use super::core::*; +use super::error::parse_error; +use super::lister::OssLister; +use super::writer::OssWriter; +use crate::raw::*; +use crate::services::oss::writer::OssWriters; +use crate::*; use bytes::Buf; use http::Response; use http::StatusCode; @@ -29,42 +35,43 @@ use log::debug; use reqsign::AliyunConfig; use reqsign::AliyunLoader; use reqsign::AliyunOssSigner; - -use super::core::*; -use super::error::parse_error; -use super::lister::OssLister; -use super::writer::OssWriter; -use crate::raw::*; -use crate::services::oss::writer::OssWriters; -use crate::*; +use serde::{Deserialize, Serialize}; const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000; -/// Aliyun Object Storage Service (OSS) support -#[doc = include_str!("docs.md")] -#[derive(Default)] -pub struct OssBuilder { - root: Option, - - endpoint: Option, - presign_endpoint: Option, - bucket: String, +/// Config for Aliyun Object Storage Service (OSS) support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct OssConfig { + /// Root for oss. + pub root: Option, + + /// Endpoint for oss. + pub endpoint: Option, + /// Presign endpoint for oss. + pub presign_endpoint: Option, + /// Bucket for oss. + pub bucket: String, // OSS features - server_side_encryption: Option, - server_side_encryption_key_id: Option, - allow_anonymous: bool, + /// Server side encryption for oss. + pub server_side_encryption: Option, + /// Server side encryption key id for oss. + pub server_side_encryption_key_id: Option, + /// Allow anonymous for oss. + pub allow_anonymous: bool, // authenticate options - access_key_id: Option, - access_key_secret: Option, - - http_client: Option, + /// Access key id for oss. + pub access_key_id: Option, + /// Access key secret for oss. + pub access_key_secret: Option, /// batch_max_operations - batch_max_operations: Option, + pub batch_max_operations: Option, } -impl Debug for OssBuilder { +impl Debug for OssConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { let mut d = f.debug_struct("Builder"); d.field("root", &self.root) @@ -76,12 +83,29 @@ impl Debug for OssBuilder { } } +/// Aliyun Object Storage Service (OSS) support +#[doc = include_str!("docs.md")] +#[derive(Default)] +pub struct OssBuilder { + config: OssConfig, + http_client: Option, +} + +impl Debug for OssBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("OssBuilder"); + + d.field("config", &self.config); + d.finish_non_exhaustive() + } +} + impl OssBuilder { /// Set root of this backend. /// /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { - self.root = if root.is_empty() { + self.config.root = if root.is_empty() { None } else { Some(root.to_string()) @@ -92,7 +116,7 @@ impl OssBuilder { /// Set bucket name of this backend. pub fn bucket(&mut self, bucket: &str) -> &mut Self { - self.bucket = bucket.to_string(); + self.config.bucket = bucket.to_string(); self } @@ -101,7 +125,7 @@ impl OssBuilder { pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/` - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()) + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()) } self @@ -118,7 +142,7 @@ impl OssBuilder { pub fn presign_endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/` - self.presign_endpoint = Some(endpoint.trim_end_matches('/').to_string()) + self.config.presign_endpoint = Some(endpoint.trim_end_matches('/').to_string()) } self @@ -130,7 +154,7 @@ impl OssBuilder { /// - If not, we will try to load it from environment. pub fn access_key_id(&mut self, v: &str) -> &mut Self { if !v.is_empty() { - self.access_key_id = Some(v.to_string()) + self.config.access_key_id = Some(v.to_string()) } self @@ -142,7 +166,7 @@ impl OssBuilder { /// - If not, we will try to load it from environment. pub fn access_key_secret(&mut self, v: &str) -> &mut Self { if !v.is_empty() { - self.access_key_secret = Some(v.to_string()) + self.config.access_key_secret = Some(v.to_string()) } self @@ -220,7 +244,7 @@ impl OssBuilder { /// 4. If a specific CMK ID is specified, include the `x-oss-server-side-encryption-key-id` parameter in the request, and set its value to the specified CMK ID. pub fn server_side_encryption(&mut self, v: &str) -> &mut Self { if !v.is_empty() { - self.server_side_encryption = Some(v.to_string()) + self.config.server_side_encryption = Some(v.to_string()) } self } @@ -232,14 +256,14 @@ impl OssBuilder { /// This option only takes effect when server_side_encryption equals to KMS. pub fn server_side_encryption_key_id(&mut self, v: &str) -> &mut Self { if !v.is_empty() { - self.server_side_encryption_key_id = Some(v.to_string()) + self.config.server_side_encryption_key_id = Some(v.to_string()) } self } /// Set maximum batch operations of this backend. pub fn batch_max_operations(&mut self, batch_max_operations: usize) -> &mut Self { - self.batch_max_operations = Some(batch_max_operations); + self.config.batch_max_operations = Some(batch_max_operations); self } @@ -247,7 +271,7 @@ impl OssBuilder { /// Allow anonymous will allow opendal to send request without signing /// when credential is not loaded. pub fn allow_anonymous(&mut self) -> &mut Self { - self.allow_anonymous = true; + self.config.allow_anonymous = true; self } } @@ -255,40 +279,24 @@ impl OssBuilder { impl Builder for OssBuilder { const SCHEME: Scheme = Scheme::Oss; type Accessor = OssBackend; + type Config = OssConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = OssBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("bucket").map(|v| builder.bucket(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("presign_endpoint") - .map(|v| builder.presign_endpoint(v)); - map.get("access_key_id").map(|v| builder.access_key_id(v)); - map.get("access_key_secret") - .map(|v| builder.access_key_secret(v)); - map.get("server_side_encryption") - .map(|v| builder.server_side_encryption(v)); - map.get("server_side_encryption_key_id") - .map(|v| builder.server_side_encryption_key_id(v)); - map.get("batch_max_operations") - .map(|v| builder.batch_max_operations(v.parse::().unwrap())); - map.get("allow_anonymous") - .filter(|v| *v == "on" || *v == "true") - .map(|_| builder.allow_anonymous()); - - builder + fn from_config(config: Self::Config) -> Self { + OssBuilder { + config, + http_client: None, + } } fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.clone().unwrap_or_default()); + let root = normalize_root(&self.config.root.clone().unwrap_or_default()); debug!("backend use root {}", &root); // Handle endpoint, region and bucket name. - let bucket = match self.bucket.is_empty() { - false => Ok(&self.bucket), + let bucket = match self.config.bucket.is_empty() { + false => Ok(&self.config.bucket), true => Err( Error::new(ErrorKind::ConfigInvalid, "The bucket is misconfigured") .with_context("service", Scheme::Oss), @@ -306,17 +314,18 @@ impl Builder for OssBuilder { // Retrieve endpoint and host by parsing the endpoint option and bucket. If presign_endpoint is not // set, take endpoint as default presign_endpoint. - let (endpoint, host) = self.parse_endpoint(&self.endpoint, bucket)?; + let (endpoint, host) = self.parse_endpoint(&self.config.endpoint, bucket)?; debug!("backend use bucket {}, endpoint: {}", &bucket, &endpoint); - let presign_endpoint = if self.presign_endpoint.is_some() { - self.parse_endpoint(&self.presign_endpoint, bucket)?.0 + let presign_endpoint = if self.config.presign_endpoint.is_some() { + self.parse_endpoint(&self.config.presign_endpoint, bucket)? + .0 } else { endpoint.clone() }; debug!("backend use presign_endpoint: {}", &presign_endpoint); - let server_side_encryption = match &self.server_side_encryption { + let server_side_encryption = match &self.config.server_side_encryption { None => None, Some(v) => Some( build_header_value(v) @@ -324,7 +333,7 @@ impl Builder for OssBuilder { ), }; - let server_side_encryption_key_id = match &self.server_side_encryption_key_id { + let server_side_encryption_key_id = match &self.config.server_side_encryption_key_id { None => None, Some(v) => Some( build_header_value(v) @@ -336,11 +345,11 @@ impl Builder for OssBuilder { // Load cfg from env first. cfg = cfg.from_env(); - if let Some(v) = self.access_key_id.take() { + if let Some(v) = self.config.access_key_id.take() { cfg.access_key_id = Some(v); } - if let Some(v) = self.access_key_secret.take() { + if let Some(v) = self.config.access_key_secret.take() { cfg.access_key_secret = Some(v); } @@ -349,6 +358,7 @@ impl Builder for OssBuilder { let signer = AliyunOssSigner::new(bucket); let batch_max_operations = self + .config .batch_max_operations .unwrap_or(DEFAULT_BATCH_MAX_OPERATIONS); debug!("Backend build finished"); @@ -360,7 +370,7 @@ impl Builder for OssBuilder { endpoint, host, presign_endpoint, - allow_anonymous: self.allow_anonymous, + allow_anonymous: self.config.allow_anonymous, signer, loader, client, diff --git a/core/src/services/oss/mod.rs b/core/src/services/oss/mod.rs index 3954a9885c0d..4839cff2d37f 100644 --- a/core/src/services/oss/mod.rs +++ b/core/src/services/oss/mod.rs @@ -17,6 +17,7 @@ mod backend; pub use backend::OssBuilder as Oss; +pub use backend::OssConfig; mod core; mod error; diff --git a/core/src/services/pcloud/backend.rs b/core/src/services/pcloud/backend.rs index 688230aa10a7..35bf1022ce74 100644 --- a/core/src/services/pcloud/backend.rs +++ b/core/src/services/pcloud/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -24,7 +23,7 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::*; use super::error::parse_error; @@ -36,7 +35,7 @@ use crate::raw::*; use crate::*; /// Config for backblaze Pcloud services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct PcloudConfig { @@ -148,22 +147,9 @@ impl PcloudBuilder { impl Builder for PcloudBuilder { const SCHEME: Scheme = Scheme::Pcloud; type Accessor = PcloudBackend; + type Config = PcloudConfig; - /// Converts a HashMap into an PcloudBuilder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of PcloudBuilder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = PcloudConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an PcloudBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { PcloudBuilder { config, http_client: None, diff --git a/core/src/services/persy/backend.rs b/core/src/services/persy/backend.rs index 950f4c5ef995..5af2ef7e5abc 100644 --- a/core/src/services/persy/backend.rs +++ b/core/src/services/persy/backend.rs @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::str; use persy; +use serde::{Deserialize, Serialize}; use tokio::task; use crate::raw::adapters::kv; @@ -31,34 +31,42 @@ use crate::ErrorKind; use crate::Scheme; use crate::*; -/// persy service support. -#[doc = include_str!("docs.md")] -#[derive(Default)] -pub struct PersyBuilder { +/// Config for persy service support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct PersyConfig { /// That path to the persy data file. The directory in the path must already exist. - datafile: Option, + pub datafile: Option, /// That name of the persy segment. - segment: Option, + pub segment: Option, /// That name of the persy index. - index: Option, + pub index: Option, +} + +/// persy service support. +#[doc = include_str!("docs.md")] +#[derive(Default, Debug)] +pub struct PersyBuilder { + config: PersyConfig, } impl PersyBuilder { /// Set the path to the persy data directory. Will create if not exists. pub fn datafile(&mut self, path: &str) -> &mut Self { - self.datafile = Some(path.into()); + self.config.datafile = Some(path.into()); self } /// Set the name of the persy segment. Will create if not exists. pub fn segment(&mut self, path: &str) -> &mut Self { - self.segment = Some(path.into()); + self.config.segment = Some(path.into()); self } /// Set the name of the persy index. Will create if not exists. pub fn index(&mut self, path: &str) -> &mut Self { - self.index = Some(path.into()); + self.config.index = Some(path.into()); self } } @@ -66,31 +74,26 @@ impl PersyBuilder { impl Builder for PersyBuilder { const SCHEME: Scheme = Scheme::Persy; type Accessor = PersyBackend; + type Config = PersyConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = PersyBuilder::default(); - - map.get("datafile").map(|v| builder.datafile(v)); - map.get("segment").map(|v| builder.segment(v)); - map.get("index").map(|v| builder.index(v)); - - builder + fn from_config(config: Self::Config) -> Self { + Self { config } } fn build(&mut self) -> Result { - let datafile_path = self.datafile.take().ok_or_else(|| { + let datafile_path = self.config.datafile.take().ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "datafile is required but not set") .with_context("service", Scheme::Persy) })?; - let segment_name = self.segment.take().ok_or_else(|| { + let segment_name = self.config.segment.take().ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "segment is required but not set") .with_context("service", Scheme::Persy) })?; let segment = segment_name.clone(); - let index_name = self.index.take().ok_or_else(|| { + let index_name = self.config.index.take().ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "index is required but not set") .with_context("service", Scheme::Persy) })?; diff --git a/core/src/services/persy/mod.rs b/core/src/services/persy/mod.rs index b192b6635c13..a7456fc87be9 100644 --- a/core/src/services/persy/mod.rs +++ b/core/src/services/persy/mod.rs @@ -18,3 +18,4 @@ mod backend; pub use backend::PersyBuilder as Persy; +pub use backend::PersyConfig; diff --git a/core/src/services/postgresql/backend.rs b/core/src/services/postgresql/backend.rs index 6529ffe4242c..4a112424cc02 100644 --- a/core/src/services/postgresql/backend.rs +++ b/core/src/services/postgresql/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::str::FromStr; @@ -23,7 +22,7 @@ use std::sync::Arc; use bb8::Pool; use bb8_postgres::PostgresConnectionManager; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::OnceCell; use tokio_postgres::Config; @@ -32,7 +31,7 @@ use crate::raw::*; use crate::*; /// Config for PostgreSQL services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct PostgresqlConfig { @@ -43,13 +42,13 @@ pub struct PostgresqlConfig { /// Default to `/` if not set. pub root: Option, /// the connection string of postgres server - connection_string: Option, + pub connection_string: Option, /// the table of postgresql - table: Option, + pub table: Option, /// the key field of postgresql - key_field: Option, + pub key_field: Option, /// the value field of postgresql - value_field: Option, + pub value_field: Option, } impl Debug for PostgresqlConfig { @@ -164,11 +163,9 @@ impl PostgresqlBuilder { impl Builder for PostgresqlBuilder { const SCHEME: Scheme = Scheme::Postgresql; type Accessor = PostgresqlBackend; + type Config = PostgresqlConfig; - fn from_map(map: HashMap) -> Self { - let config = PostgresqlConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { PostgresqlBuilder { config } } diff --git a/core/src/services/redb/backend.rs b/core/src/services/redb/backend.rs index 140e9a555fe3..8a782937ac2f 100644 --- a/core/src/services/redb/backend.rs +++ b/core/src/services/redb/backend.rs @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; use redb::ReadableTable; +use serde::{Deserialize, Serialize}; use tokio::task; use crate::raw::adapters::kv; @@ -31,32 +31,42 @@ use crate::ErrorKind; use crate::Scheme; use crate::*; +/// Config for redb service support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct RedbConfig { + /// path to the redb data directory. + pub datadir: Option, + /// The root for redb. + pub root: Option, + /// The table name for redb. + pub table: Option, +} + /// Redb service support. #[doc = include_str!("docs.md")] -#[derive(Default)] +#[derive(Default, Debug)] pub struct RedbBuilder { - /// That path to the redb data directory. - datadir: Option, - root: Option, - table: Option, + config: RedbConfig, } impl RedbBuilder { /// Set the path to the redb data directory. Will create if not exists. pub fn datadir(&mut self, path: &str) -> &mut Self { - self.datadir = Some(path.into()); + self.config.datadir = Some(path.into()); self } /// Set the table name for Redb. pub fn table(&mut self, table: &str) -> &mut Self { - self.table = Some(table.into()); + self.config.table = Some(table.into()); self } /// Set the root for Redb. pub fn root(&mut self, path: &str) -> &mut Self { - self.root = Some(path.into()); + self.config.root = Some(path.into()); self } } @@ -64,24 +74,19 @@ impl RedbBuilder { impl Builder for RedbBuilder { const SCHEME: Scheme = Scheme::Redb; type Accessor = RedbBackend; + type Config = RedbConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = RedbBuilder::default(); - - map.get("datadir").map(|v| builder.datadir(v)); - map.get("table").map(|v| builder.table(v)); - map.get("root").map(|v| builder.root(v)); - - builder + fn from_config(config: Self::Config) -> Self { + Self { config } } fn build(&mut self) -> Result { - let datadir_path = self.datadir.take().ok_or_else(|| { + let datadir_path = self.config.datadir.take().ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set") .with_context("service", Scheme::Redb) })?; - let table_name = self.table.take().ok_or_else(|| { + let table_name = self.config.table.take().ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "table is required but not set") .with_context("service", Scheme::Redb) })?; @@ -95,7 +100,7 @@ impl Builder for RedbBuilder { table: table_name, db, }) - .with_root(self.root.as_deref().unwrap_or_default())) + .with_root(self.config.root.as_deref().unwrap_or_default())) } } diff --git a/core/src/services/redb/mod.rs b/core/src/services/redb/mod.rs index f53ba5722078..f75e57fb2f0f 100644 --- a/core/src/services/redb/mod.rs +++ b/core/src/services/redb/mod.rs @@ -18,3 +18,4 @@ mod backend; pub use backend::RedbBuilder as Redb; +pub use backend::RedbConfig; diff --git a/core/src/services/redis/backend.rs b/core/src/services/redis/backend.rs index 64fc2f9837e1..84375c66fc68 100644 --- a/core/src/services/redis/backend.rs +++ b/core/src/services/redis/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::path::PathBuf; @@ -32,7 +31,7 @@ use redis::ConnectionAddr; use redis::ConnectionInfo; use redis::RedisConnectionInfo; use redis::RedisError; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::OnceCell; use crate::raw::adapters::kv; @@ -43,36 +42,36 @@ const DEFAULT_REDIS_ENDPOINT: &str = "tcp://127.0.0.1:6379"; const DEFAULT_REDIS_PORT: u16 = 6379; /// Config for Redis services support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct RedisConfig { /// network address of the Redis service. Can be "tcp://127.0.0.1:6379", e.g. /// /// default is "tcp://127.0.0.1:6379" - endpoint: Option, + pub endpoint: Option, /// network address of the Redis cluster service. Can be "tcp://127.0.0.1:6379,tcp://127.0.0.1:6380,tcp://127.0.0.1:6381", e.g. /// /// default is None - cluster_endpoints: Option, + pub cluster_endpoints: Option, /// the username to connect redis service. /// /// default is None - username: Option, + pub username: Option, /// the password for authentication /// /// default is None - password: Option, + pub password: Option, /// the working directory of the Redis service. Can be "/path/to/dir" /// /// default is "/" - root: Option, + pub root: Option, /// the number of DBs redis can take is unlimited /// /// default is db 0 - db: i64, + pub db: i64, /// The default ttl for put operations. - default_ttl: Option, + pub default_ttl: Option, } impl Debug for RedisConfig { @@ -194,11 +193,9 @@ impl RedisBuilder { impl Builder for RedisBuilder { const SCHEME: Scheme = Scheme::Redis; type Accessor = RedisBackend; + type Config = RedisConfig; - fn from_map(map: HashMap) -> Self { - let config = RedisConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { RedisBuilder { config } } diff --git a/core/src/services/rocksdb/backend.rs b/core/src/services/rocksdb/backend.rs index 7f88820f1577..ec14a47e71ed 100644 --- a/core/src/services/rocksdb/backend.rs +++ b/core/src/services/rocksdb/backend.rs @@ -15,13 +15,12 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; use rocksdb::DB; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::task; use crate::raw::adapters::kv; @@ -29,17 +28,17 @@ use crate::raw::*; use crate::Result; use crate::*; -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] /// Config for Rocksdb Service. pub struct RocksdbConfig { /// The path to the rocksdb data directory. - datadir: Option, + pub datadir: Option, /// the working directory of the service. Can be "/path/to/dir" /// /// default is "/" - root: Option, + pub root: Option, } /// RocksDB service support. @@ -70,10 +69,9 @@ impl RocksdbBuilder { impl Builder for RocksdbBuilder { const SCHEME: Scheme = Scheme::Rocksdb; type Accessor = RocksdbBackend; + type Config = RocksdbConfig; - fn from_map(map: HashMap) -> Self { - let config = RocksdbConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); + fn from_config(config: Self::Config) -> Self { RocksdbBuilder { config } } diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index b2d7c9ca0705..9113c3da4812 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -39,7 +39,7 @@ use reqsign::AwsCredentialLoad; use reqsign::AwsDefaultLoader; use reqsign::AwsV4Signer; use reqwest::Url; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::*; use super::error::parse_error; @@ -64,7 +64,7 @@ static ENDPOINT_TEMPLATES: Lazy> = Lazy::new const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000; /// Config for Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct S3Config { @@ -804,11 +804,9 @@ impl S3Builder { impl Builder for S3Builder { const SCHEME: Scheme = Scheme::S3; type Accessor = S3Backend; + type Config = S3Config; - fn from_map(map: HashMap) -> Self { - let config = S3Config::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { S3Builder { config, customized_credential_load: None, diff --git a/core/src/services/seafile/backend.rs b/core/src/services/seafile/backend.rs index 18b9ace630ca..eb13f98fa2d6 100644 --- a/core/src/services/seafile/backend.rs +++ b/core/src/services/seafile/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -23,7 +22,7 @@ use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use super::core::parse_dir_detail; @@ -38,7 +37,7 @@ use crate::services::seafile::core::SeafileSigner; use crate::*; /// Config for backblaze seafile services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct SeafileConfig { @@ -166,22 +165,9 @@ impl SeafileBuilder { impl Builder for SeafileBuilder { const SCHEME: Scheme = Scheme::Seafile; type Accessor = SeafileBackend; + type Config = SeafileConfig; - /// Converts a HashMap into an SeafileBuilder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of SeafileBuilder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = SeafileConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an SeafileBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { SeafileBuilder { config, http_client: None, diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs index 37691effd7e6..76a26a1475b5 100644 --- a/core/src/services/sftp/backend.rs +++ b/core/src/services/sftp/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::io::SeekFrom; @@ -30,7 +29,7 @@ use openssh::KnownHosts; use openssh::SessionBuilder; use openssh_sftp_client::Sftp; use openssh_sftp_client::SftpOptions; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::io::AsyncSeekExt; use tokio::sync::OnceCell; @@ -45,7 +44,7 @@ use crate::raw::*; use crate::*; /// Config for Sftp Service support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct SftpConfig { @@ -169,6 +168,11 @@ impl SftpBuilder { impl Builder for SftpBuilder { const SCHEME: Scheme = Scheme::Sftp; type Accessor = SftpBackend; + type Config = SftpConfig; + + fn from_config(config: Self::Config) -> Self { + SftpBuilder { config } + } fn build(&mut self) -> Result { debug!("sftp backend build started: {:?}", &self); @@ -218,13 +222,6 @@ impl Builder for SftpBuilder { client: OnceCell::new(), }) } - - fn from_map(map: HashMap) -> Self { - SftpBuilder { - config: SftpConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"), - } - } } /// Backend is used to serve `Accessor` support for sftp. diff --git a/core/src/services/sled/backend.rs b/core/src/services/sled/backend.rs index c099900c7855..26e4fee5de28 100644 --- a/core/src/services/sled/backend.rs +++ b/core/src/services/sled/backend.rs @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::str; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::task; use crate::raw::adapters::kv; @@ -35,14 +34,16 @@ use crate::*; const DEFAULT_TREE_ID: &str = r#"__sled__default"#; /// Config for Sled services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct SledConfig { /// That path to the sled data directory. - datadir: Option, - root: Option, - tree: Option, + pub datadir: Option, + /// The root for sled. + pub root: Option, + /// The tree for sled. + pub tree: Option, } impl Debug for SledConfig { @@ -93,12 +94,10 @@ impl SledBuilder { impl Builder for SledBuilder { const SCHEME: Scheme = Scheme::Sled; type Accessor = SledBackend; + type Config = SledConfig; - fn from_map(map: HashMap) -> Self { - SledBuilder { - config: SledConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"), - } + fn from_config(config: Self::Config) -> Self { + SledBuilder { config } } fn build(&mut self) -> Result { diff --git a/core/src/services/sqlite/backend.rs b/core/src/services/sqlite/backend.rs index d91e210807a0..06fdddc10ee2 100644 --- a/core/src/services/sqlite/backend.rs +++ b/core/src/services/sqlite/backend.rs @@ -15,13 +15,12 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use rusqlite::params; use rusqlite::Connection; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::task; use crate::raw::adapters::kv; @@ -29,7 +28,7 @@ use crate::raw::*; use crate::*; /// Config for Sqlite support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct SqliteConfig { @@ -152,11 +151,9 @@ impl SqliteBuilder { impl Builder for SqliteBuilder { const SCHEME: Scheme = Scheme::Sqlite; type Accessor = SqliteBackend; + type Config = SqliteConfig; - fn from_map(map: HashMap) -> Self { - let config = SqliteConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { SqliteBuilder { config } } diff --git a/core/src/services/supabase/backend.rs b/core/src/services/supabase/backend.rs index 21ef418972cc..d14690df4faa 100644 --- a/core/src/services/supabase/backend.rs +++ b/core/src/services/supabase/backend.rs @@ -15,39 +15,38 @@ // specific language governing permissions and limitations // under the License. -use std::fmt::Debug; +use std::fmt::{Debug, Formatter}; use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; +use serde::{Deserialize, Serialize}; + +use crate::raw::*; +use crate::*; use super::core::*; use super::error::parse_error; use super::writer::*; -use crate::raw::*; -use crate::*; -/// [Supabase](https://supabase.com/) service support -#[doc = include_str!("docs.md")] -#[derive(Default)] -pub struct SupabaseBuilder { +/// Config for supabase service support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct SupabaseConfig { root: Option, - bucket: String, endpoint: Option, - key: Option, - - // todo: optional public, currently true always - // todo: optional file_size_limit, currently 0 - // todo: optional allowed_mime_types, currently only string - http_client: Option, + // TODO(1) optional public, currently true always + // TODO(2) optional file_size_limit, currently 0 + // TODO(3) optional allowed_mime_types, currently only string } -impl Debug for SupabaseBuilder { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SupabaseBuilder") +impl Debug for SupabaseConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SupabaseConfig") .field("root", &self.root) .field("bucket", &self.bucket) .field("endpoint", &self.endpoint) @@ -55,12 +54,28 @@ impl Debug for SupabaseBuilder { } } +/// [Supabase](https://supabase.com/) service support +#[doc = include_str!("docs.md")] +#[derive(Default)] +pub struct SupabaseBuilder { + config: SupabaseConfig, + http_client: Option, +} + +impl Debug for SupabaseBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("SupabaseBuilder"); + d.field("config", &self.config); + d.finish_non_exhaustive() + } +} + impl SupabaseBuilder { /// Set root of this backend. /// /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { - self.root = if root.is_empty() { + self.config.root = if root.is_empty() { None } else { Some(root.to_string()) @@ -71,7 +86,7 @@ impl SupabaseBuilder { /// Set bucket name of this backend. pub fn bucket(&mut self, bucket: &str) -> &mut Self { - self.bucket = bucket.to_string(); + self.config.bucket = bucket.to_string(); self } @@ -79,7 +94,7 @@ impl SupabaseBuilder { /// /// Endpoint must be full uri pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { - self.endpoint = if endpoint.is_empty() { + self.config.endpoint = if endpoint.is_empty() { None } else { Some(endpoint.trim_end_matches('/').to_string()) @@ -91,7 +106,7 @@ impl SupabaseBuilder { /// Set the authorization key for this backend /// Do not set this key if you want to read public bucket pub fn key(&mut self, key: &str) -> &mut Self { - self.key = Some(key.to_string()); + self.config.key = Some(key.to_string()); self } @@ -110,25 +125,22 @@ impl SupabaseBuilder { impl Builder for SupabaseBuilder { const SCHEME: Scheme = Scheme::Supabase; type Accessor = SupabaseBackend; + type Config = SupabaseConfig; - fn from_map(map: std::collections::HashMap) -> Self { - let mut builder = SupabaseBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("bucket").map(|v| builder.bucket(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("key").map(|v| builder.key(v)); - - builder + fn from_config(config: Self::Config) -> Self { + SupabaseBuilder { + config, + http_client: None, + } } fn build(&mut self) -> Result { - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", &root); - let bucket = &self.bucket; + let bucket = &self.config.bucket; - let endpoint = self.endpoint.take().unwrap_or_default(); + let endpoint = self.config.endpoint.take().unwrap_or_default(); let http_client = if let Some(client) = self.http_client.take() { client @@ -139,7 +151,7 @@ impl Builder for SupabaseBuilder { })? }; - let key = self.key.as_ref().map(|k| k.to_owned()); + let key = self.config.key.as_ref().map(|k| k.to_owned()); let core = SupabaseCore::new(&root, bucket, &endpoint, key, http_client); diff --git a/core/src/services/supabase/mod.rs b/core/src/services/supabase/mod.rs index 89aeada88983..af59bfcc57f6 100644 --- a/core/src/services/supabase/mod.rs +++ b/core/src/services/supabase/mod.rs @@ -17,6 +17,7 @@ mod backend; pub use backend::SupabaseBuilder as Supabase; +pub use backend::SupabaseConfig; mod core; mod error; mod writer; diff --git a/core/src/services/surrealdb/backend.rs b/core/src/services/surrealdb/backend.rs index f59ea4abe40e..b9e2066e206f 100644 --- a/core/src/services/surrealdb/backend.rs +++ b/core/src/services/surrealdb/backend.rs @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use surrealdb::engine::any::Any; use surrealdb::opt::auth::Database; use surrealdb::Surreal; @@ -28,7 +27,6 @@ use tokio::sync::OnceCell; use crate::raw::adapters::kv; use crate::raw::normalize_root; -use crate::raw::ConfigDeserializer; use crate::Buffer; use crate::Builder; use crate::Capability; @@ -37,19 +35,28 @@ use crate::ErrorKind; use crate::Scheme; /// Config for Surrealdb services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct SurrealdbConfig { - connection_string: Option, - username: Option, - password: Option, - namespace: Option, - database: Option, - table: Option, - key_field: Option, - value_field: Option, - root: Option, + /// The connection string for surrealdb. + pub connection_string: Option, + /// The username for surrealdb. + pub username: Option, + /// The password for surrealdb. + pub password: Option, + /// The namespace for surrealdb. + pub namespace: Option, + /// The database for surrealdb. + pub database: Option, + /// The table for surrealdb. + pub table: Option, + /// The key field for surrealdb. + pub key_field: Option, + /// The value field for surrealdb. + pub value_field: Option, + /// The root for surrealdb. + pub root: Option, } impl Debug for SurrealdbConfig { @@ -174,11 +181,9 @@ impl SurrealdbBuilder { impl Builder for SurrealdbBuilder { const SCHEME: Scheme = Scheme::Surrealdb; type Accessor = SurrealdbBackend; + type Config = SurrealdbConfig; - fn from_map(map: HashMap) -> Self { - let config = SurrealdbConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { SurrealdbBuilder { config } } diff --git a/core/src/services/swift/backend.rs b/core/src/services/swift/backend.rs index bdebfd4ba1c1..037c3895c438 100644 --- a/core/src/services/swift/backend.rs +++ b/core/src/services/swift/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -23,29 +22,34 @@ use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; +use serde::{Deserialize, Serialize}; + +use crate::raw::*; +use crate::*; use super::core::*; use super::error::parse_error; use super::lister::SwiftLister; use super::writer::SwiftWriter; -use crate::raw::*; -use crate::*; -/// [OpenStack Swift](https://docs.openstack.org/api-ref/object-store/#)'s REST API support. -/// For more information about swift-compatible services, refer to [Compatible Services](#compatible-services). -#[doc = include_str!("docs.md")] -#[doc = include_str!("compatible_services.md")] -#[derive(Default, Clone)] -pub struct SwiftBuilder { - endpoint: Option, - container: Option, - root: Option, - token: Option, +/// Config for OpenStack Swift support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct SwiftConfig { + /// The endpoint for Swift. + pub endpoint: Option, + /// The container for Swift. + pub container: Option, + /// The root for Swift. + pub root: Option, + /// The token for Swift. + pub token: Option, } -impl Debug for SwiftBuilder { +impl Debug for SwiftConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Builder"); + let mut ds = f.debug_struct("SwiftConfig"); ds.field("root", &self.root); ds.field("endpoint", &self.endpoint); @@ -59,6 +63,23 @@ impl Debug for SwiftBuilder { } } +/// [OpenStack Swift](https://docs.openstack.org/api-ref/object-store/#)'s REST API support. +/// For more information about swift-compatible services, refer to [Compatible Services](#compatible-services). +#[doc = include_str!("docs.md")] +#[doc = include_str!("compatible_services.md")] +#[derive(Default, Clone)] +pub struct SwiftBuilder { + config: SwiftConfig, +} + +impl Debug for SwiftBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("SwiftBuilder"); + d.field("config", &self.config); + d.finish_non_exhaustive() + } +} + impl SwiftBuilder { /// Set the remote address of this backend /// @@ -71,7 +92,7 @@ impl SwiftBuilder { /// If user inputs endpoint without scheme, we will /// prepend `https://` to it. pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { - self.endpoint = if endpoint.is_empty() { + self.config.endpoint = if endpoint.is_empty() { None } else { Some(endpoint.trim_end_matches('/').to_string()) @@ -83,7 +104,7 @@ impl SwiftBuilder { /// /// All operations will happen under this container. It is required. e.g. `snapshots` pub fn container(&mut self, container: &str) -> &mut Self { - self.container = if container.is_empty() { + self.config.container = if container.is_empty() { None } else { Some(container.trim_end_matches('/').to_string()) @@ -96,7 +117,7 @@ impl SwiftBuilder { /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -107,7 +128,7 @@ impl SwiftBuilder { /// Default to empty string. pub fn token(&mut self, token: &str) -> &mut Self { if !token.is_empty() { - self.token = Some(token.to_string()); + self.config.token = Some(token.to_string()); } self } @@ -116,25 +137,20 @@ impl SwiftBuilder { impl Builder for SwiftBuilder { const SCHEME: Scheme = Scheme::Swift; type Accessor = SwiftBackend; + type Config = SwiftConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = SwiftBuilder::default(); - - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("container").map(|v| builder.container(v)); - map.get("token").map(|v| builder.token(v)); - - builder + fn from_config(config: Self::Config) -> Self { + SwiftBuilder { config } } /// Build a SwiftBackend. fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); - let endpoint = match self.endpoint.take() { + let endpoint = match self.config.endpoint.take() { Some(endpoint) => { if endpoint.starts_with("http") { endpoint @@ -151,7 +167,7 @@ impl Builder for SwiftBuilder { }; debug!("backend use endpoint: {}", &endpoint); - let container = match self.container.take() { + let container = match self.config.container.take() { Some(container) => container, None => { return Err(Error::new( @@ -162,7 +178,7 @@ impl Builder for SwiftBuilder { }; debug!("backend use container: {}", &container); - let token = self.token.take().unwrap_or_default(); + let token = self.config.token.take().unwrap_or_default(); let client = HttpClient::new()?; diff --git a/core/src/services/swift/mod.rs b/core/src/services/swift/mod.rs index 9cfeb92e07a0..e15b8efd02ed 100644 --- a/core/src/services/swift/mod.rs +++ b/core/src/services/swift/mod.rs @@ -17,6 +17,7 @@ mod backend; pub use backend::SwiftBuilder as Swift; +pub use backend::SwiftConfig; mod core; mod error; diff --git a/core/src/services/tikv/backend.rs b/core/src/services/tikv/backend.rs index f3a3eb1cb679..f707719799f8 100644 --- a/core/src/services/tikv/backend.rs +++ b/core/src/services/tikv/backend.rs @@ -15,17 +15,15 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tikv_client::Config; use tikv_client::RawClient; use tokio::sync::OnceCell; use crate::raw::adapters::kv; -use crate::raw::*; use crate::Builder; use crate::Capability; use crate::Error; @@ -34,7 +32,7 @@ use crate::Scheme; use crate::*; /// Config for Tikv services support. -#[derive(Default, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct TikvConfig { @@ -122,11 +120,9 @@ impl TikvBuilder { impl Builder for TikvBuilder { const SCHEME: Scheme = Scheme::Tikv; type Accessor = TikvBackend; + type Config = TikvConfig; - fn from_map(map: HashMap) -> Self { - let config = TikvConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { TikvBuilder { config } } diff --git a/core/src/services/upyun/backend.rs b/core/src/services/upyun/backend.rs index 5359fef60974..afa3ac397848 100644 --- a/core/src/services/upyun/backend.rs +++ b/core/src/services/upyun/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -23,7 +22,7 @@ use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::parse_info; use super::core::UpyunCore; @@ -36,7 +35,7 @@ use crate::services::upyun::core::UpyunSigner; use crate::*; /// Config for backblaze upyun services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct UpyunConfig { @@ -146,22 +145,9 @@ impl UpyunBuilder { impl Builder for UpyunBuilder { const SCHEME: Scheme = Scheme::Upyun; type Accessor = UpyunBackend; + type Config = UpyunConfig; - /// Converts a HashMap into an UpyunBuilder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of UpyunBuilder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = UpyunConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an UpyunBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { UpyunBuilder { config, http_client: None, diff --git a/core/src/services/vercel_artifacts/builder.rs b/core/src/services/vercel_artifacts/builder.rs index 2390c09fd6a6..614c09df9717 100644 --- a/core/src/services/vercel_artifacts/builder.rs +++ b/core/src/services/vercel_artifacts/builder.rs @@ -15,27 +15,55 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; + +use serde::{Deserialize, Serialize}; -use super::backend::VercelArtifactsBackend; use crate::raw::HttpClient; use crate::Scheme; use crate::*; +use super::backend::VercelArtifactsBackend; + +/// Config for Vercel Cache support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct VercelArtifactsConfig { + /// The access token for Vercel. + pub access_token: Option, +} + +impl Debug for VercelArtifactsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VercelArtifactsConfig") + .field("access_token", &"") + .finish() + } +} + /// [Vercel Cache](https://vercel.com/docs/concepts/monorepos/remote-caching) backend support. #[doc = include_str!("docs.md")] #[derive(Default)] pub struct VercelArtifactsBuilder { - access_token: Option, + config: VercelArtifactsConfig, http_client: Option, } +impl Debug for VercelArtifactsBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("VercelArtifactsBuilder"); + d.field("config", &self.config); + d.finish_non_exhaustive() + } +} + impl VercelArtifactsBuilder { /// set the bearer access token for Vercel /// /// default: no access token, which leads to failure pub fn access_token(&mut self, access_token: &str) -> &mut Self { - self.access_token = Some(access_token.to_string()); + self.config.access_token = Some(access_token.to_string()); self } @@ -53,13 +81,14 @@ impl VercelArtifactsBuilder { impl Builder for VercelArtifactsBuilder { const SCHEME: Scheme = Scheme::VercelArtifacts; - type Accessor = VercelArtifactsBackend; + type Config = VercelArtifactsConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = Self::default(); - map.get("access_token").map(|v| builder.access_token(v)); - builder + fn from_config(config: Self::Config) -> Self { + Self { + config, + http_client: None, + } } fn build(&mut self) -> Result { @@ -72,7 +101,7 @@ impl Builder for VercelArtifactsBuilder { })? }; - match self.access_token.clone() { + match self.config.access_token.clone() { Some(access_token) => Ok(VercelArtifactsBackend { access_token, client, diff --git a/core/src/services/vercel_artifacts/mod.rs b/core/src/services/vercel_artifacts/mod.rs index 3bfddabca7af..eac0f6e43a7f 100644 --- a/core/src/services/vercel_artifacts/mod.rs +++ b/core/src/services/vercel_artifacts/mod.rs @@ -21,3 +21,4 @@ mod error; mod writer; pub use builder::VercelArtifactsBuilder as VercelArtifacts; +pub use builder::VercelArtifactsConfig; diff --git a/core/src/services/vercel_blob/backend.rs b/core/src/services/vercel_blob/backend.rs index b8da7729c21d..befdf06601ff 100644 --- a/core/src/services/vercel_blob/backend.rs +++ b/core/src/services/vercel_blob/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -24,7 +23,7 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::parse_blob; use super::core::Blob; @@ -37,7 +36,7 @@ use crate::raw::*; use crate::*; /// Config for backblaze VercelBlob services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct VercelBlobConfig { @@ -116,22 +115,9 @@ impl VercelBlobBuilder { impl Builder for VercelBlobBuilder { const SCHEME: Scheme = Scheme::VercelBlob; type Accessor = VercelBlobBackend; + type Config = VercelBlobConfig; - /// Converts a HashMap into an VercelBlobBuilder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of VercelBlobBuilder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = VercelBlobConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an VercelBlobBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { VercelBlobBuilder { config, http_client: None, diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index f1f37694a055..5730b26713f0 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::str::FromStr; @@ -24,7 +23,7 @@ use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use super::core::*; use super::error::parse_error; @@ -34,7 +33,7 @@ use crate::raw::*; use crate::*; /// Config for [WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) backend support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct WebdavConfig { @@ -152,11 +151,9 @@ impl WebdavBuilder { impl Builder for WebdavBuilder { const SCHEME: Scheme = Scheme::Webdav; type Accessor = WebdavBackend; + type Config = WebdavConfig; - fn from_map(map: HashMap) -> Self { - let config = WebdavConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - + fn from_config(config: Self::Config) -> Self { WebdavBuilder { config, http_client: None, diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs index 2fa89ad8f677..65e606101629 100644 --- a/core/src/services/webhdfs/backend.rs +++ b/core/src/services/webhdfs/backend.rs @@ -16,7 +16,7 @@ // under the License. use core::fmt::Debug; -use std::collections::HashMap; +use std::fmt::Formatter; use std::sync::Arc; use bytes::Buf; @@ -26,9 +26,12 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::sync::OnceCell; +use crate::raw::*; +use crate::*; + use super::error::parse_error; use super::lister::WebhdfsLister; use super::message::BooleanResp; @@ -36,26 +39,29 @@ use super::message::FileStatusType; use super::message::FileStatusWrapper; use super::writer::WebhdfsWriter; use super::writer::WebhdfsWriters; -use crate::raw::*; -use crate::*; const WEBHDFS_DEFAULT_ENDPOINT: &str = "http://127.0.0.1:9870"; -/// [WebHDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html)'s REST API support. -#[doc = include_str!("docs.md")] -#[derive(Default, Clone)] -pub struct WebhdfsBuilder { - root: Option, - endpoint: Option, - delegation: Option, - disable_list_batch: bool, +/// Config for WebHDFS support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct WebhdfsConfig { + /// Root for webhdfs. + pub root: Option, + /// Endpoint for webhdfs. + pub endpoint: Option, + /// Delegation token for webhdfs. + pub delegation: Option, + /// Disable batch listing + pub disable_list_batch: bool, /// atomic_write_dir of this backend pub atomic_write_dir: Option, } -impl Debug for WebhdfsBuilder { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Builder") +impl Debug for WebhdfsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WebhdfsConfig") .field("root", &self.root) .field("endpoint", &self.endpoint) .field("atomic_write_dir", &self.atomic_write_dir) @@ -63,6 +69,21 @@ impl Debug for WebhdfsBuilder { } } +/// [WebHDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html)'s REST API support. +#[doc = include_str!("docs.md")] +#[derive(Default, Clone)] +pub struct WebhdfsBuilder { + config: WebhdfsConfig, +} + +impl Debug for WebhdfsBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("WebhdfsBuilder"); + d.field("config", &self.config); + d.finish_non_exhaustive() + } +} + impl WebhdfsBuilder { /// Set the working directory of this backend /// @@ -73,7 +94,7 @@ impl WebhdfsBuilder { /// The root will be automatically created if not exists. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -92,7 +113,7 @@ impl WebhdfsBuilder { pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { // trim tailing slash so we can accept `http://127.0.0.1:9870/` - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); } self } @@ -105,7 +126,7 @@ impl WebhdfsBuilder { /// If both are set, delegation token will be used. pub fn delegation(&mut self, delegation: &str) -> &mut Self { if !delegation.is_empty() { - self.delegation = Some(delegation.to_string()); + self.config.delegation = Some(delegation.to_string()); } self } @@ -115,9 +136,9 @@ impl WebhdfsBuilder { /// # Note /// /// When listing a directory, the backend will default to use batch listing. - /// If disable, the backend will list all files/directories in one request. + /// If disabled, the backend will list all files/directories in one request. pub fn disable_list_batch(&mut self) -> &mut Self { - self.disable_list_batch = true; + self.config.disable_list_batch = true; self } @@ -127,7 +148,7 @@ impl WebhdfsBuilder { /// /// If not set, write multi not support, eg: `.opendal_tmp/`. pub fn atomic_write_dir(&mut self, dir: &str) -> &mut Self { - self.atomic_write_dir = if dir.is_empty() { + self.config.atomic_write_dir = if dir.is_empty() { None } else { Some(String::from(dir)) @@ -139,20 +160,10 @@ impl WebhdfsBuilder { impl Builder for WebhdfsBuilder { const SCHEME: Scheme = Scheme::Webhdfs; type Accessor = WebhdfsBackend; + type Config = WebhdfsConfig; - fn from_map(map: HashMap) -> Self { - let mut builder = WebhdfsBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("delegation").map(|v| builder.delegation(v)); - map.get("disable_list_batch") - .filter(|v| v == &"true") - .map(|_| builder.disable_list_batch()); - map.get("atomic_write_dir") - .map(|v| builder.atomic_write_dir(v)); - - builder + fn from_config(config: Self::Config) -> Self { + Self { config } } /// build the backend @@ -165,11 +176,11 @@ impl Builder for WebhdfsBuilder { fn build(&mut self) -> Result { debug!("start building backend: {:?}", self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {root}"); // check scheme - let endpoint = match self.endpoint.take() { + let endpoint = match self.config.endpoint.take() { Some(endpoint) => { if endpoint.starts_with("http") { endpoint @@ -181,9 +192,10 @@ impl Builder for WebhdfsBuilder { }; debug!("backend use endpoint {}", endpoint); - let atomic_write_dir = self.atomic_write_dir.take(); + let atomic_write_dir = self.config.atomic_write_dir.take(); let auth = self + .config .delegation .take() .map(|dt| format!("delegation_token={dt}")); @@ -197,7 +209,7 @@ impl Builder for WebhdfsBuilder { client, root_checker: OnceCell::new(), atomic_write_dir, - disable_list_batch: self.disable_list_batch, + disable_list_batch: self.config.disable_list_batch, }; Ok(backend) diff --git a/core/src/services/webhdfs/mod.rs b/core/src/services/webhdfs/mod.rs index 3bfbbc77dd14..30dda8b28f23 100644 --- a/core/src/services/webhdfs/mod.rs +++ b/core/src/services/webhdfs/mod.rs @@ -17,6 +17,7 @@ mod backend; pub use backend::WebhdfsBuilder as Webhdfs; +pub use backend::WebhdfsConfig; mod error; mod lister; diff --git a/core/src/services/yandex_disk/backend.rs b/core/src/services/yandex_disk/backend.rs index a25d0a1926f1..710c7006b5ad 100644 --- a/core/src/services/yandex_disk/backend.rs +++ b/core/src/services/yandex_disk/backend.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -26,18 +25,19 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; + +use crate::raw::*; +use crate::*; use super::core::*; use super::error::parse_error; use super::lister::YandexDiskLister; use super::writer::YandexDiskWriter; use super::writer::YandexDiskWriters; -use crate::raw::*; -use crate::*; /// Config for backblaze YandexDisk services support. -#[derive(Default, Deserialize)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] pub struct YandexDiskConfig { @@ -116,22 +116,9 @@ impl YandexDiskBuilder { impl Builder for YandexDiskBuilder { const SCHEME: Scheme = Scheme::YandexDisk; type Accessor = YandexDiskBackend; + type Config = YandexDiskConfig; - /// Converts a HashMap into an YandexDiskBuilder instance. - /// - /// # Arguments - /// - /// * `map` - A HashMap containing the configuration values. - /// - /// # Returns - /// - /// Returns an instance of YandexDiskBuilder. - fn from_map(map: HashMap) -> Self { - // Deserialize the configuration from the HashMap. - let config = YandexDiskConfig::deserialize(ConfigDeserializer::new(map)) - .expect("config deserialize must succeed"); - - // Create an YandexDiskBuilder instance with the deserialized config. + fn from_config(config: Self::Config) -> Self { YandexDiskBuilder { config, http_client: None, diff --git a/core/src/types/builder.rs b/core/src/types/builder.rs index 52255f51522f..b70759161807 100644 --- a/core/src/types/builder.rs +++ b/core/src/types/builder.rs @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; - use crate::raw::*; use crate::*; +use serde::de::DeserializeOwned; +use serde::Deserialize; +use std::collections::HashMap; /// Builder is used to set up a real underlying service, i.e. storage accessor. /// @@ -40,9 +41,18 @@ pub trait Builder: Default { const SCHEME: Scheme; /// The accessor that built by this builder. type Accessor: Access; + /// The config for this builder. + type Config: DeserializeOwned; + + /// Construct a builder from given config. + fn from_config(config: Self::Config) -> Self; /// Construct a builder from given map which contains several parameters needed by underlying service. - fn from_map(map: HashMap) -> Self; + fn from_map(map: HashMap) -> Self { + Self::Config::deserialize(ConfigDeserializer::new(map)) + .map(Self::from_config) + .expect("config deserialize must succeed") + } /// Consume the accessor builder to build a service. fn build(&mut self) -> Result; @@ -54,6 +64,10 @@ impl Builder for () { type Accessor = (); + type Config = (); + + fn from_config(_: Self::Config) -> Self {} + fn from_map(_: HashMap) -> Self {} fn build(&mut self) -> Result {