Skip to content

Commit

Permalink
fix(config): Copy Alibaba and IBM object storage configuration from c…
Browse files Browse the repository at this point in the history 
…ommon (grafana#14297)
  • Loading branch information
@xperimental @jeschkies
xperimental authored and jeschkies committed Oct 1, 2024
1 parent 5da89df commit d2712ed
Show file tree
Hide file tree
Showing 2 changed files with 138 additions and 0 deletions.
20 changes: 20 additions & 0 deletions pkg/loki/config_wrapper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,26 @@ func applyStorageConfig(cfg, defaults *ConfigWrapper) error {
}
}

if !reflect.DeepEqual(cfg.Common.Storage.AlibabaCloud, defaults.StorageConfig.AlibabaStorageConfig) {
configsFound++

applyConfig = func(r *ConfigWrapper) {
r.Ruler.StoreConfig.Type = "alibaba"
r.Ruler.StoreConfig.AlibabaCloud = r.Common.Storage.AlibabaCloud
r.StorageConfig.AlibabaStorageConfig = r.Common.Storage.AlibabaCloud
}
}

if !reflect.DeepEqual(cfg.Common.Storage.COS, defaults.StorageConfig.COSConfig) {
configsFound++

applyConfig = func(r *ConfigWrapper) {
r.Ruler.StoreConfig.Type = "cos"
r.Ruler.StoreConfig.COS = r.Common.Storage.COS
r.StorageConfig.COSConfig = r.Common.Storage.COS
}
}

if !reflect.DeepEqual(cfg.Common.Storage.CongestionControl, defaults.StorageConfig.CongestionControl) {
applyConfig = func(r *ConfigWrapper) {
r.StorageConfig.CongestionControl = r.Common.Storage.CongestionControl
Expand Down
118 changes: 118 additions & 0 deletions pkg/loki/config_wrapper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,12 +219,16 @@ memberlist:
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig, config.StorageConfig.AWSStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
})

t.Run("when multiple configs are provided, an error is returned", func(t *testing.T) {
Expand Down Expand Up @@ -296,12 +300,17 @@ memberlist:
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)

// should remain empty
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
})

t.Run("when common s3 storage config is provided (with session token), ruler and storage config are defaulted to use it", func(t *testing.T) {
Expand Down Expand Up @@ -356,12 +365,17 @@ memberlist:
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)

// should remain empty
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
})

t.Run("when common gcs storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
Expand Down Expand Up @@ -392,12 +406,17 @@ memberlist:
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)

// should remain empty
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
})

t.Run("when common azure storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
Expand Down Expand Up @@ -444,13 +463,17 @@ memberlist:
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)

// should remain empty
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
})

t.Run("when common bos storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
Expand Down Expand Up @@ -482,13 +505,17 @@ memberlist:
assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)

// should remain empty
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
})

t.Run("when common swift storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
Expand Down Expand Up @@ -549,12 +576,103 @@ memberlist:
assert.EqualValues(t, defaults.Ruler.StoreConfig.Azure, config.Ruler.StoreConfig.Azure)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)

// should remain empty
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
})

t.Run("when common alibaba storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
configInput := `common:
storage:
alibabacloud:
bucket: testbucket
endpoint: https://example.com
access_key_id: abc123
secret_access_key: def789`

config, defaults := testContext(configInput, nil)

assert.Equal(t, "alibaba", config.Ruler.StoreConfig.Type)

for _, actual := range []alibaba.OssConfig{
config.Ruler.StoreConfig.AlibabaCloud,
config.StorageConfig.AlibabaStorageConfig,
} {
assert.Equal(t, "testbucket", actual.Bucket)
assert.Equal(t, "https://example.com", actual.Endpoint)
assert.Equal(t, "abc123", actual.AccessKeyID)
assert.Equal(t, "def789", actual.SecretAccessKey)
}

// should remain empty
assert.EqualValues(t, defaults.Ruler.StoreConfig.GCS, config.Ruler.StoreConfig.GCS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Azure, config.Ruler.StoreConfig.Azure)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)

// should remain empty
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
})

t.Run("when common cos storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
configInput := `common:
storage:
cos:
bucketnames: testbucket
endpoint: https://example.com
region: test-region
access_key_id: abc123
secret_access_key: def789`

config, defaults := testContext(configInput, nil)

assert.Equal(t, "cos", config.Ruler.StoreConfig.Type)

for _, actual := range []ibmcloud.COSConfig{
config.Ruler.StoreConfig.COS,
config.StorageConfig.COSConfig,
} {
assert.Equal(t, "testbucket", actual.BucketNames)
assert.Equal(t, "https://example.com", actual.Endpoint)
assert.Equal(t, "test-region", actual.Region)
assert.Equal(t, "abc123", actual.AccessKeyID)
assert.Equal(t, flagext.SecretWithValue("def789"), actual.SecretAccessKey)
}

// should remain empty
assert.EqualValues(t, defaults.Ruler.StoreConfig.GCS, config.Ruler.StoreConfig.GCS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Azure, config.Ruler.StoreConfig.Azure)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)

// should remain empty
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
})

t.Run("when common filesystem/local config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
Expand Down

0 comments on commit d2712ed

Please sign in to comment.