Skip to content

Commit

Permalink
Add Digital Ocean storage flavor
Browse files Browse the repository at this point in the history
  • Loading branch information
guilload committed Jul 17, 2023
1 parent 43cf4ac commit 8fc7105
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 4 deletions.
7 changes: 6 additions & 1 deletion docs/configuration/node-config.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ storage:
| Property | Description | Default value |
| --- | --- | --- |
| `flavor` | The optional storage flavor to use. Available flavors are `garage`, `gcs`, and `minio`. | |
| `flavor` | The optional storage flavor to use. Available flavors are `digital_ocean`, `garage`, `gcs`, and `minio`. | |
| `access_key_id` | The AWS access key ID. | |
| `secret_access_key` | The AWS secret access key. | |
| `region` | The AWS region to send requests to. | `us-east-1` (SDK default) |
Expand All @@ -91,10 +91,15 @@ Hardcoding credentials into configuration files is not secure and strongly disco
**Storage flavors**

Storage flavors ensure that Quickwit works correctly with storage providers that deviate from the S3 API by automatically configuring the appropriate settings. The available flavors are:
- `digital_ocean`
- `garage`
- `gcs`
- `minio`

*Digital Ocean*

The Digital Ocean flavor (`digital_ocean`) forces path-style access and turns off multi-object delete requests.

*Garage flavor*

The Garage flavor (`garage`) overrides the `region` parameter to `garage` and forces path-style access.
Expand Down
2 changes: 1 addition & 1 deletion quickwit/quickwit-config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ pub use crate::quickwit_config::{
use crate::source_config::serialize::{SourceConfigV0_6, VersionedSourceConfig};
pub use crate::storage_config::{
AzureStorageConfig, FileStorageConfig, RamStorageConfig, S3StorageConfig, StorageBackend,
StorageConfig, StorageConfigs,
StorageBackendFlavor, StorageConfig, StorageConfigs,
};

#[derive(utoipa::OpenApi)]
Expand Down
73 changes: 71 additions & 2 deletions quickwit/quickwit-config/src/storage_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,16 @@ pub enum StorageBackend {
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StorageBackendFlavor {
/// Digital Ocean Spaces
#[serde(alias = "do")]
DigitalOcean,
/// Garage
Garage,
/// Google Cloud Storage
#[serde(alias = "gcp", alias = "google")]
Gcs,
/// MinIO
#[serde(rename = "minio")]
MinIO,
}

Expand Down Expand Up @@ -307,6 +311,10 @@ pub struct S3StorageConfig {
impl S3StorageConfig {
fn apply_flavor(&mut self) {
match self.flavor {
Some(StorageBackendFlavor::DigitalOcean) => {
self.force_path_style_access = true;
self.disable_multi_object_delete = true;
}
Some(StorageBackendFlavor::Garage) => {
self.region = Some("garage".to_string());
self.force_path_style_access = true;
Expand Down Expand Up @@ -402,6 +410,16 @@ mod tests {
#[test]
fn test_storage_configs_apply_flavors() {
let mut storage_configs = StorageConfigs(vec![
S3StorageConfig {
flavor: Some(StorageBackendFlavor::DigitalOcean),
..Default::default()
}
.into(),
S3StorageConfig {
flavor: Some(StorageBackendFlavor::Garage),
..Default::default()
}
.into(),
S3StorageConfig {
flavor: Some(StorageBackendFlavor::Gcs),
..Default::default()
Expand All @@ -415,11 +433,19 @@ mod tests {
]);
storage_configs.apply_flavors();

let gcs_storage_config = storage_configs[0].as_s3().unwrap();
let do_storage_config = storage_configs[0].as_s3().unwrap();
assert!(do_storage_config.force_path_style_access);
assert!(do_storage_config.disable_multi_object_delete);

let garage_storage_config = storage_configs[1].as_s3().unwrap();
assert_eq!(garage_storage_config.region, Some("garage".to_string()));
assert!(garage_storage_config.force_path_style_access);

let gcs_storage_config = storage_configs[2].as_s3().unwrap();
assert!(gcs_storage_config.disable_multi_object_delete);
assert!(gcs_storage_config.disable_multipart_upload);

let minio_storage_config = storage_configs[1].as_s3().unwrap();
let minio_storage_config = storage_configs[3].as_s3().unwrap();
assert!(minio_storage_config.force_path_style_access);
}

Expand Down Expand Up @@ -544,4 +570,47 @@ mod tests {
assert_eq!(s3_storage_config, expected_s3_config);
}
}

#[test]
fn test_storage_s3_config_flavor_serde() {
{
let s3_storage_config_yaml = r#"
flavor: digital_ocean
"#;
let s3_storage_config: S3StorageConfig =
serde_yaml::from_str(s3_storage_config_yaml).unwrap();

assert_eq!(
s3_storage_config.flavor,
Some(StorageBackendFlavor::DigitalOcean)
);
}
{
let s3_storage_config_yaml = r#"
flavor: garage
"#;
let s3_storage_config: S3StorageConfig =
serde_yaml::from_str(s3_storage_config_yaml).unwrap();

assert_eq!(s3_storage_config.flavor, Some(StorageBackendFlavor::Garage));
}
{
let s3_storage_config_yaml = r#"
flavor: gcs
"#;
let s3_storage_config: S3StorageConfig =
serde_yaml::from_str(s3_storage_config_yaml).unwrap();

assert_eq!(s3_storage_config.flavor, Some(StorageBackendFlavor::Gcs));
}
{
let s3_storage_config_yaml = r#"
flavor: minio
"#;
let s3_storage_config: S3StorageConfig =
serde_yaml::from_str(s3_storage_config_yaml).unwrap();

assert_eq!(s3_storage_config.flavor, Some(StorageBackendFlavor::MinIO));
}
}
}

0 comments on commit 8fc7105

Please sign in to comment.