From 8fc710590f065efb78b1465581a3d9f5587b19fb Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Fri, 14 Jul 2023 06:27:07 +0900 Subject: [PATCH] Add Digital Ocean storage flavor --- docs/configuration/node-config.md | 7 +- quickwit/quickwit-config/src/lib.rs | 2 +- .../quickwit-config/src/storage_config.rs | 73 ++++++++++++++++++- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/docs/configuration/node-config.md b/docs/configuration/node-config.md index 319d6ac24ae..fb8fd694ca8 100644 --- a/docs/configuration/node-config.md +++ b/docs/configuration/node-config.md @@ -75,7 +75,7 @@ storage: | Property | Description | Default value | | --- | --- | --- | -| `flavor` | The optional storage flavor to use. Available flavors are `garage`, `gcs`, and `minio`. | | +| `flavor` | The optional storage flavor to use. Available flavors are `digital_ocean`, `garage`, `gcs`, and `minio`. | | | `access_key_id` | The AWS access key ID. | | | `secret_access_key` | The AWS secret access key. | | | `region` | The AWS region to send requests to. | `us-east-1` (SDK default) | @@ -91,10 +91,15 @@ Hardcoding credentials into configuration files is not secure and strongly disco **Storage flavors** Storage flavors ensure that Quickwit works correctly with storage providers that deviate from the S3 API by automatically configuring the appropriate settings. The available flavors are: +- `digital_ocean` - `garage` - `gcs` - `minio` +*Digital Ocean* + +The Digital Ocean flavor (`digital_ocean`) forces path-style access and turns off multi-object delete requests. + *Garage flavor* The Garage flavor (`garage`) overrides the `region` parameter to `garage` and forces path-style access. diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs index 303dab7b5da..c77bc7fcf8e 100644 --- a/quickwit/quickwit-config/src/lib.rs +++ b/quickwit/quickwit-config/src/lib.rs @@ -70,7 +70,7 @@ pub use crate::quickwit_config::{ use crate::source_config::serialize::{SourceConfigV0_6, VersionedSourceConfig}; pub use crate::storage_config::{ AzureStorageConfig, FileStorageConfig, RamStorageConfig, S3StorageConfig, StorageBackend, - StorageConfig, StorageConfigs, + StorageBackendFlavor, StorageConfig, StorageConfigs, }; #[derive(utoipa::OpenApi)] diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index ce1bb74e463..6f594cb192b 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -42,12 +42,16 @@ pub enum StorageBackend { #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum StorageBackendFlavor { + /// Digital Ocean Spaces + #[serde(alias = "do")] + DigitalOcean, /// Garage Garage, /// Google Cloud Storage #[serde(alias = "gcp", alias = "google")] Gcs, /// MinIO + #[serde(rename = "minio")] MinIO, } @@ -307,6 +311,10 @@ pub struct S3StorageConfig { impl S3StorageConfig { fn apply_flavor(&mut self) { match self.flavor { + Some(StorageBackendFlavor::DigitalOcean) => { + self.force_path_style_access = true; + self.disable_multi_object_delete = true; + } Some(StorageBackendFlavor::Garage) => { self.region = Some("garage".to_string()); self.force_path_style_access = true; @@ -402,6 +410,16 @@ mod tests { #[test] fn test_storage_configs_apply_flavors() { let mut storage_configs = StorageConfigs(vec![ + S3StorageConfig { + flavor: Some(StorageBackendFlavor::DigitalOcean), + ..Default::default() + } + .into(), + S3StorageConfig { + flavor: Some(StorageBackendFlavor::Garage), + ..Default::default() + } + .into(), S3StorageConfig { flavor: Some(StorageBackendFlavor::Gcs), ..Default::default() @@ -415,11 +433,19 @@ mod tests { ]); storage_configs.apply_flavors(); - let gcs_storage_config = storage_configs[0].as_s3().unwrap(); + let do_storage_config = storage_configs[0].as_s3().unwrap(); + assert!(do_storage_config.force_path_style_access); + assert!(do_storage_config.disable_multi_object_delete); + + let garage_storage_config = storage_configs[1].as_s3().unwrap(); + assert_eq!(garage_storage_config.region, Some("garage".to_string())); + assert!(garage_storage_config.force_path_style_access); + + let gcs_storage_config = storage_configs[2].as_s3().unwrap(); assert!(gcs_storage_config.disable_multi_object_delete); assert!(gcs_storage_config.disable_multipart_upload); - let minio_storage_config = storage_configs[1].as_s3().unwrap(); + let minio_storage_config = storage_configs[3].as_s3().unwrap(); assert!(minio_storage_config.force_path_style_access); } @@ -544,4 +570,47 @@ mod tests { assert_eq!(s3_storage_config, expected_s3_config); } } + + #[test] + fn test_storage_s3_config_flavor_serde() { + { + let s3_storage_config_yaml = r#" + flavor: digital_ocean + "#; + let s3_storage_config: S3StorageConfig = + serde_yaml::from_str(s3_storage_config_yaml).unwrap(); + + assert_eq!( + s3_storage_config.flavor, + Some(StorageBackendFlavor::DigitalOcean) + ); + } + { + let s3_storage_config_yaml = r#" + flavor: garage + "#; + let s3_storage_config: S3StorageConfig = + serde_yaml::from_str(s3_storage_config_yaml).unwrap(); + + assert_eq!(s3_storage_config.flavor, Some(StorageBackendFlavor::Garage)); + } + { + let s3_storage_config_yaml = r#" + flavor: gcs + "#; + let s3_storage_config: S3StorageConfig = + serde_yaml::from_str(s3_storage_config_yaml).unwrap(); + + assert_eq!(s3_storage_config.flavor, Some(StorageBackendFlavor::Gcs)); + } + { + let s3_storage_config_yaml = r#" + flavor: minio + "#; + let s3_storage_config: S3StorageConfig = + serde_yaml::from_str(s3_storage_config_yaml).unwrap(); + + assert_eq!(s3_storage_config.flavor, Some(StorageBackendFlavor::MinIO)); + } + } }