From c71a801702be02414fe691048baaca8277721444 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Thu, 21 Jul 2022 17:51:32 +0800 Subject: [PATCH] add UT Signed-off-by: Lyndon-Li --- changelogs/unreleased/5142-lyndon | 4 + pkg/repository/repoconfig/aws.go | 32 +- pkg/repository/repoconfig/config.go | 36 +- pkg/repository/repoprovider/repo_provider.go | 26 ++ pkg/repository/repoprovider/unified_repo.go | 117 +++-- .../repoprovider/unified_repo_test.go | 414 ++++++++++++++++++ pkg/repository/udmrepo/repo-option-consts.go | 95 ++-- pkg/repository/udmrepo/repo.go | 28 ++ pkg/util/ownership/backup_owner.go | 43 +- 9 files changed, 673 insertions(+), 122 deletions(-) create mode 100644 changelogs/unreleased/5142-lyndon create mode 100644 pkg/repository/repoprovider/unified_repo_test.go diff --git a/changelogs/unreleased/5142-lyndon b/changelogs/unreleased/5142-lyndon new file mode 100644 index 00000000000..10286cf0bcf --- /dev/null +++ b/changelogs/unreleased/5142-lyndon @@ -0,0 +1,4 @@ +Kopia Integration: Add the Unified Repository Interface definition. +Kopia Integration: Add the changes for Unified Repository storage config. + +Related Issues; #5076, #5080 \ No newline at end of file diff --git a/pkg/repository/repoconfig/aws.go b/pkg/repository/repoconfig/aws.go index 3349df6e364..6bf71fff965 100644 --- a/pkg/repository/repoconfig/aws.go +++ b/pkg/repository/repoconfig/aws.go @@ -17,10 +17,14 @@ limitations under the License. package repoconfig import ( - "errors" + "context" "os" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/endpoints" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/pkg/errors" ) const ( @@ -67,3 +71,29 @@ func GetS3Credentials(config map[string]string) (credentials.Value, error) { return credValue, nil } + +// GetAWSBucketRegion returns the AWS region that a bucket is in, or an error +// if the region cannot be determined. +func GetAWSBucketRegion(bucket string) (string, error) { + var region string + + sess, err := session.NewSession() + if err != nil { + return "", errors.WithStack(err) + } + + for _, partition := range endpoints.DefaultPartitions() { + for regionHint := range partition.Regions() { + region, _ = s3manager.GetBucketRegion(context.Background(), sess, bucket, regionHint) + + // we only need to try a single region hint per partition, so break after the first + break + } + + if region != "" { + return region, nil + } + } + + return "", errors.New("unable to determine bucket's region") +} diff --git a/pkg/repository/repoconfig/config.go b/pkg/repository/repoconfig/config.go index 27b56e6d6d1..b928386ce54 100644 --- a/pkg/repository/repoconfig/config.go +++ b/pkg/repository/repoconfig/config.go @@ -17,14 +17,10 @@ limitations under the License. package repoconfig import ( - "context" "fmt" "path" "strings" - "github.com/aws/aws-sdk-go/aws/endpoints" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -78,7 +74,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error) var err error region := location.Spec.Config["region"] if region == "" { - region, err = GetAWSBucketRegion(bucket) + region, err = getAWSBucketRegion(bucket) } if err != nil { return "", errors.Wrapf(err, "failed to detect the region via bucket: %s", bucket) @@ -104,6 +100,10 @@ func GetBackendType(provider string) BackendType { return BackendType(provider) } +func IsBackendTypeValid(backendType BackendType) bool { + return (backendType == AWSBackend || backendType == AzureBackend || backendType == GCPBackend || backendType == FSBackend) +} + // GetRepoIdentifier returns the string to be used as the value of the --repo flag in // restic commands for the given repository. func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string) (string, error) { @@ -114,29 +114,3 @@ func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string) return fmt.Sprintf("%s/%s", strings.TrimSuffix(prefix, "/"), name), nil } - -// GetBucketRegion returns the AWS region that a bucket is in, or an error -// if the region cannot be determined. -func GetAWSBucketRegion(bucket string) (string, error) { - var region string - - sess, err := session.NewSession() - if err != nil { - return "", errors.WithStack(err) - } - - for _, partition := range endpoints.DefaultPartitions() { - for regionHint := range partition.Regions() { - region, _ = s3manager.GetBucketRegion(context.Background(), sess, bucket, regionHint) - - // we only need to try a single region hint per partition, so break after the first - break - } - - if region != "" { - return region, nil - } - } - - return "", errors.New("unable to determine bucket's region") -} diff --git a/pkg/repository/repoprovider/repo_provider.go b/pkg/repository/repoprovider/repo_provider.go index 57a11360ac9..4cca64c6396 100644 --- a/pkg/repository/repoprovider/repo_provider.go +++ b/pkg/repository/repoprovider/repo_provider.go @@ -1,19 +1,45 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repoprovider import "context" type RepositoryProvider interface { + //InitRepo is to initialize a repository from a new storage place InitRepo(ctx context.Context, bsl string) error + //ConnectToRepo is to establish the connection to a + //storage place that a repository is already initialized ConnectToRepo(ctx context.Context, bsl string) error + //PrepareRepo is a combination of InitRepo and ConnectToRepo, + //it may do initializing + connecting, connecting only if the repository + //is already initialized, or do nothing if the repository is already connected PrepareRepo(ctx context.Context, bsl string) error + //PruneRepo does a full prune/maintenance of the repository PruneRepo(ctx context.Context, bsl string) error + //PruneRepoQuick does a quick prune/maintenance of the repository if available PruneRepoQuick(ctx context.Context, bsl string) error + //EnsureUnlockRepo esures to remove any stale file locks in the storage EnsureUnlockRepo(ctx context.Context, bsl string) error + //Forget is to delete a snapshot from the repository Forget(ctx context.Context, snapshotID, bsl string) error } diff --git a/pkg/repository/repoprovider/unified_repo.go b/pkg/repository/repoprovider/unified_repo.go index 80c1f88b2e0..aaed03e3dca 100644 --- a/pkg/repository/repoprovider/unified_repo.go +++ b/pkg/repository/repoprovider/unified_repo.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repoprovider import ( @@ -6,7 +22,9 @@ import ( "path" "strings" + "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/repository/repoconfig" @@ -25,6 +43,14 @@ type unifiedRepoProvider struct { log *logrus.Logger } +// this func is assigned to a package-level variable so it can be +// replaced when unit-testing +var getAzureCredentials = repoconfig.GetAzureCredentials +var getS3Credentials = repoconfig.GetS3Credentials +var getGCPCredentials = repoconfig.GetGCPCredentials +var getS3BucketRegion = repoconfig.GetAWSBucketRegion +var getAzureStorageDomain = repoconfig.GetAzureStorageDomain + func NewUnifiedRepoProvider( ctx context.Context, credentialFileStore credentials.FileStore, @@ -71,9 +97,10 @@ func (urp *unifiedRepoProvider) InitRepo(ctx context.Context, backupLocation str err = urp.repoService.Init(ctx, repoOption, true) if err != nil { log.WithError(err).Error("Failed to init backup repo") + return err } - return err + return nil } func (urp *unifiedRepoProvider) ConnectToRepo(ctx context.Context, backupLocation string) error { @@ -122,11 +149,12 @@ func (urp *unifiedRepoProvider) getRepoOption() (udmrepo.RepoOptions, error) { StorageType: getStorageType(urp.backupLocation), RepoPassword: urp.repoPassword, ConfigFilePath: urp.configPath, - StorageOptions: make(map[string]string), - GeneralOptions: map[string]string{ - udmrepo.UNFIED_REPO_GEN_OPTION_OWNER_DOMAIN: ownership.GetBackupOwner().DomainName, - udmrepo.UNIFIED_REPO_GEN_OPTION_OWNER_NAME: ownership.GetBackupOwner().Username, + Ownership: udmrepo.OwnershipOptions{ + Username: ownership.GetRepositoryOwner().Username, + DomainName: ownership.GetRepositoryOwner().DomainName, }, + StorageOptions: make(map[string]string), + GeneralOptions: make(map[string]string), } storeVar, err := getStorageVariables(urp.backupLocation, urp.repoName) @@ -139,7 +167,7 @@ func (urp *unifiedRepoProvider) getRepoOption() (udmrepo.RepoOptions, error) { repoOption.StorageOptions[k] = v } - storeCred, err := getStorageCredentials(urp.backupLocation, urp.credentialsFileStore, urp.log) + storeCred, err := getStorageCredentials(urp.backupLocation, urp.credentialsFileStore) if err != nil { log.WithError(err).Error("Failed to get repo credential env") return repoOption, err @@ -157,22 +185,26 @@ func getStorageType(backupLocation *velerov1api.BackupStorageLocation) string { switch backendType { case repoconfig.AWSBackend: - return udmrepo.UNIFIED_REPO_OPTION_STORAGE_TYPE_S3 + return udmrepo.StorageTypeS3 case repoconfig.AzureBackend: - return udmrepo.UNIFIED_REPO_OPTION_STORAGE_TYPE_AZURE + return udmrepo.StorageTypeAzure case repoconfig.GCPBackend: - return udmrepo.UNIFIED_REPO_OPTION_STORAGE_TYPE_GCS + return udmrepo.StorageTypeGcs case repoconfig.FSBackend: - return udmrepo.UNIFIED_REPO_OPTION_STORAGE_TYPE_FS + return udmrepo.StorageTypeFs default: return "" } } -func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, credentialsFileStore credentials.FileStore, log *logrus.Logger) (map[string]string, error) { +func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, credentialsFileStore credentials.FileStore) (map[string]string, error) { result := make(map[string]string) var err error + backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider) + if !repoconfig.IsBackendTypeValid(backendType) { + return map[string]string{}, errors.New("invalid storage provider") + } config := backupLocation.Spec.Config if config == nil { @@ -182,34 +214,31 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr if backupLocation.Spec.Credential != nil { config[repoconfig.CredentialsFileKey], err = credentialsFileStore.Path(backupLocation.Spec.Credential) if err != nil { - log.WithError(err).Error("Failed to get credential file in BSL") - return map[string]string{}, err + return map[string]string{}, errors.Wrap(err, "error get credential file in bsl") } } switch backendType { case repoconfig.AWSBackend: - credValue, err := repoconfig.GetS3Credentials(config) + credValue, err := getS3Credentials(config) if err != nil { - log.WithError(err).Error("Failed to get S3 credentials") - return map[string]string{}, err + return map[string]string{}, errors.Wrap(err, "error get s3 credentials") } - result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_KEY_ID] = credValue.AccessKeyID - result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_PROVIDER] = credValue.ProviderName - result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_SECRET_KEY] = credValue.SecretAccessKey - result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_TOKEN] = credValue.SessionToken + result[udmrepo.StoreOptionS3KeyId] = credValue.AccessKeyID + result[udmrepo.StoreOptionS3Provider] = credValue.ProviderName + result[udmrepo.StoreOptionS3SecretKey] = credValue.SecretAccessKey + result[udmrepo.StoreOptionS3Token] = credValue.SessionToken case repoconfig.AzureBackend: - storageAccount, accountKey, err := repoconfig.GetAzureCredentials(config) + storageAccount, accountKey, err := getAzureCredentials(config) if err != nil { - log.WithError(err).Error("Failed to get Azure credentials") - return map[string]string{}, err + return map[string]string{}, errors.Wrap(err, "error get azure credentials") } - result[udmrepo.UNIFIED_REPO_STORE_OPTION_AZ_STORAGE_ACCOUNT] = storageAccount - result[udmrepo.UNIFIED_REPO_STORE_OPTION_AZ_KEY] = accountKey + result[udmrepo.StoreOptionAzureStorageAccount] = storageAccount + result[udmrepo.StoreOptionAzureKey] = accountKey case repoconfig.GCPBackend: - result[udmrepo.UNIFIED_REPO_STORE_OPTION_CRED_FILE] = repoconfig.GetGCPCredentials(config) + result[udmrepo.StoreOptionCredentialFile] = getGCPCredentials(config) } return result, nil @@ -217,7 +246,11 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repoName string) (map[string]string, error) { result := make(map[string]string) + backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider) + if !repoconfig.IsBackendTypeValid(backendType) { + return map[string]string{}, errors.New("invalid storage provider") + } config := backupLocation.Spec.Config if config == nil { @@ -231,33 +264,33 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo prefix = strings.Trim(backupLocation.Spec.ObjectStorage.Prefix, "/") } - prefix = path.Join(prefix, udmrepo.UNIFIED_REPO_STORE_OPTION_PREFIX_NAME, repoName) + "/" + prefix = path.Join(prefix, udmrepo.StoreOptionPrefixName, repoName) + "/" - s3Url := config["s3Url"] region := config["region"] - var err error if backendType == repoconfig.AWSBackend { - if s3Url == "" && region == "" { - region, err = repoconfig.GetAWSBucketRegion(bucket) + s3Url := config["s3Url"] + + var err error + if s3Url == "" { + region, err = getS3BucketRegion(bucket) if err != nil { - return map[string]string{}, err + return map[string]string{}, errors.Wrap(err, "error get s3 bucket region") } s3Url = fmt.Sprintf("s3-%s.amazonaws.com", region) } - } - - result[udmrepo.UNIFIED_REPO_STORE_OPTION_OSS_BUCKET] = bucket - result[udmrepo.UNIFIED_REPO_STORE_OPTION_PREFIX] = prefix - result[udmrepo.UNIFIED_REPO_STORE_OPTION_OSS_REGION] = strings.Trim(region, "/") - result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_ENDPOINT] = strings.Trim(s3Url, "/") - result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_DISABLE_TLS_VERIFY] = config["insecureSkipTLSVerify"] - - result[udmrepo.UNIFIED_REPO_STORE_OPTION_AZ_DOMAIN] = repoconfig.GetAzureStorageDomain(config) + result[udmrepo.StoreOptionS3Endpoint] = strings.Trim(s3Url, "/") + result[udmrepo.StoreOptionS3DisableTlsVerify] = config["insecureSkipTLSVerify"] + } else if backendType == repoconfig.AzureBackend { + result[udmrepo.StoreOptionAzureDomain] = getAzureStorageDomain(config) + } - result[udmrepo.UNIFIED_REPO_STORE_OPTION_FS_PATH] = config["fspath"] + result[udmrepo.StoreOptionOssBucket] = bucket + result[udmrepo.StoreOptionPrefix] = prefix + result[udmrepo.StoreOptionOssRegion] = strings.Trim(region, "/") + result[udmrepo.StoreOptionFsPath] = config["fspath"] return result, nil } diff --git a/pkg/repository/repoprovider/unified_repo_test.go b/pkg/repository/repoprovider/unified_repo_test.go new file mode 100644 index 00000000000..681fa1e984b --- /dev/null +++ b/pkg/repository/repoprovider/unified_repo_test.go @@ -0,0 +1,414 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repoprovider + +import ( + "errors" + "testing" + + awscredentials "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + + filecredentials "github.com/vmware-tanzu/velero/internal/credentials" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +func TestGetStorageCredentials(t *testing.T) { + testCases := []struct { + name string + backupLocation velerov1api.BackupStorageLocation + credFileStore filecredentials.FileStore + getAzureCredentials func(map[string]string) (string, string, error) + getS3Credentials func(map[string]string) (awscredentials.Value, error) + getGCPCredentials func(map[string]string) string + expected map[string]string + expectedErr string + }{ + { + name: "invalid provider", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "invalid-provider", + }, + }, + expected: map[string]string{}, + expectedErr: "invalid storage provider", + }, + { + name: "credential section exists in BSL, file store fail", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "aws", + Credential: &corev1api.SecretKeySelector{}, + }, + }, + credFileStore: velerotest.NewFakeCredentialsFileStore("", errors.New("fake error")), + expected: map[string]string{}, + expectedErr: "error get credential file in bsl: fake error", + }, + { + name: "aws, Credential section not exists in BSL", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/aws", + Config: map[string]string{ + "credentialsFile": "credentials-from-config-map", + }, + }, + }, + getS3Credentials: func(config map[string]string) (awscredentials.Value, error) { + return awscredentials.Value{ + AccessKeyID: "from: " + config["credentialsFile"], + }, nil + }, + + expected: map[string]string{ + "accessKeyID": "from: credentials-from-config-map", + "providerName": "", + "secretAccessKey": "", + "sessionToken": "", + }, + }, + { + name: "aws, Credential section exists in BSL", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/aws", + Config: map[string]string{ + "credentialsFile": "credentials-from-config-map", + }, + Credential: &corev1api.SecretKeySelector{}, + }, + }, + credFileStore: velerotest.NewFakeCredentialsFileStore("credentials-from-credential-key", nil), + getS3Credentials: func(config map[string]string) (awscredentials.Value, error) { + return awscredentials.Value{ + AccessKeyID: "from: " + config["credentialsFile"], + }, nil + }, + + expected: map[string]string{ + "accessKeyID": "from: credentials-from-credential-key", + "providerName": "", + "secretAccessKey": "", + "sessionToken": "", + }, + }, + { + name: "aws, get credentials fail", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/aws", + Config: map[string]string{ + "credentialsFile": "credentials-from-config-map", + }, + }, + }, + credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), + getS3Credentials: func(config map[string]string) (awscredentials.Value, error) { + return awscredentials.Value{}, errors.New("fake error") + }, + expected: map[string]string{}, + expectedErr: "error get s3 credentials: fake error", + }, + { + name: "azure, Credential section exists in BSL", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/azure", + Config: map[string]string{ + "credentialsFile": "credentials-from-config-map", + }, + Credential: &corev1api.SecretKeySelector{}, + }, + }, + credFileStore: velerotest.NewFakeCredentialsFileStore("credentials-from-credential-key", nil), + getAzureCredentials: func(config map[string]string) (string, string, error) { + return "storage account from: " + config["credentialsFile"], "", nil + }, + + expected: map[string]string{ + "storageAccount": "storage account from: credentials-from-credential-key", + "storageKey": "", + }, + }, + { + name: "azure, get azure credentials fails", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/azure", + Config: map[string]string{ + "credentialsFile": "credentials-from-config-map", + }, + }, + }, + getAzureCredentials: func(config map[string]string) (string, string, error) { + return "", "", errors.New("fake error") + }, + + expected: map[string]string{}, + expectedErr: "error get azure credentials: fake error", + }, + { + name: "gcp, Credential section not exists in BSL", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/gcp", + Config: map[string]string{ + "credentialsFile": "credentials-from-config-map", + }, + }, + }, + getGCPCredentials: func(config map[string]string) string { + return "credentials-from-config-map" + }, + + expected: map[string]string{ + "credFile": "credentials-from-config-map", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + getAzureCredentials = tc.getAzureCredentials + getS3Credentials = tc.getS3Credentials + getGCPCredentials = tc.getGCPCredentials + + actual, err := getStorageCredentials(&tc.backupLocation, tc.credFileStore) + + require.Equal(t, tc.expected, actual) + + if tc.expectedErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.expectedErr) + } + }) + } +} + +func TestGetStorageVariables(t *testing.T) { + testCases := []struct { + name string + backupLocation velerov1api.BackupStorageLocation + repoName string + getS3BucketRegion func(string) (string, error) + getAzureStorageDomain func(map[string]string) string + expected map[string]string + expectedErr string + }{ + { + name: "invalid provider", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "invalid-provider", + }, + }, + expected: map[string]string{}, + expectedErr: "invalid storage provider", + }, + { + name: "aws, ObjectStorage section not exists in BSL, s3Url exist", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/aws", + Config: map[string]string{ + "bucket": "fake-bucket", + "prefix": "fake-prefix", + "region": "fake-region/", + "s3Url": "fake-url", + "insecureSkipTLSVerify": "true", + }, + }, + }, + expected: map[string]string{ + "bucket": "fake-bucket", + "prefix": "fake-prefix/unified-repo/", + "region": "fake-region", + "fspath": "", + "endpoint": "fake-url", + "skipTLSVerify": "true", + }, + }, + { + name: "aws, ObjectStorage section not exists in BSL, s3Url not exist", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/aws", + Config: map[string]string{ + "bucket": "fake-bucket", + "prefix": "fake-prefix", + "insecureSkipTLSVerify": "false", + }, + }, + }, + getS3BucketRegion: func(bucket string) (string, error) { + return "region from bucket: " + bucket, nil + }, + expected: map[string]string{ + "bucket": "fake-bucket", + "prefix": "fake-prefix/unified-repo/", + "region": "region from bucket: fake-bucket", + "fspath": "", + "endpoint": "s3-region from bucket: fake-bucket.amazonaws.com", + "skipTLSVerify": "false", + }, + }, + { + name: "aws, ObjectStorage section not exists in BSL, s3Url not exist, get region fail", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/aws", + Config: map[string]string{}, + }, + }, + getS3BucketRegion: func(bucket string) (string, error) { + return "", errors.New("fake error") + }, + expected: map[string]string{}, + expectedErr: "error get s3 bucket region: fake error", + }, + { + name: "aws, ObjectStorage section exists in BSL, s3Url exist", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/aws", + Config: map[string]string{ + "bucket": "fake-bucket-config", + "prefix": "fake-prefix-config", + "region": "fake-region", + "s3Url": "fake-url", + "insecureSkipTLSVerify": "false", + }, + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ + Bucket: "fake-bucket-object-store", + Prefix: "fake-prefix-object-store", + }, + }, + }, + }, + getS3BucketRegion: func(bucket string) (string, error) { + return "region from bucket: " + bucket, nil + }, + expected: map[string]string{ + "bucket": "fake-bucket-object-store", + "prefix": "fake-prefix-object-store/unified-repo/", + "region": "fake-region", + "fspath": "", + "endpoint": "fake-url", + "skipTLSVerify": "false", + }, + }, + { + name: "azure, ObjectStorage section exists in BSL", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/azure", + Config: map[string]string{ + "bucket": "fake-bucket-config", + "prefix": "fake-prefix-config", + "region": "fake-region", + "fspath": "", + "storageDomain": "fake-domain", + }, + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ + Bucket: "fake-bucket-object-store", + Prefix: "fake-prefix-object-store", + }, + }, + }, + }, + getAzureStorageDomain: func(config map[string]string) string { + return config["storageDomain"] + }, + expected: map[string]string{ + "bucket": "fake-bucket-object-store", + "prefix": "fake-prefix-object-store/unified-repo/", + "region": "fake-region", + "fspath": "", + "storageDomain": "fake-domain", + }, + }, + { + name: "azure, ObjectStorage section not exists in BSL, repo name exists", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/azure", + Config: map[string]string{ + "bucket": "fake-bucket", + "prefix": "fake-prefix", + "region": "fake-region", + "fspath": "", + "storageDomain": "fake-domain", + }, + }, + }, + repoName: "//fake-name//", + getAzureStorageDomain: func(config map[string]string) string { + return config["storageDomain"] + }, + expected: map[string]string{ + "bucket": "fake-bucket", + "prefix": "fake-prefix/unified-repo/fake-name/", + "region": "fake-region", + "fspath": "", + "storageDomain": "fake-domain", + }, + }, + { + name: "fs", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/fs", + Config: map[string]string{ + "fspath": "fake-path", + "prefix": "fake-prefix", + }, + }, + }, + expected: map[string]string{ + "fspath": "fake-path", + "bucket": "", + "prefix": "fake-prefix/unified-repo/", + "region": "", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + getS3BucketRegion = tc.getS3BucketRegion + getAzureStorageDomain = tc.getAzureStorageDomain + + actual, err := getStorageVariables(&tc.backupLocation, tc.repoName) + + require.Equal(t, tc.expected, actual) + + if tc.expectedErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.expectedErr) + } + }) + } +} diff --git a/pkg/repository/udmrepo/repo-option-consts.go b/pkg/repository/udmrepo/repo-option-consts.go index 47ef1787f75..1c46ed70f4a 100644 --- a/pkg/repository/udmrepo/repo-option-consts.go +++ b/pkg/repository/udmrepo/repo-option-consts.go @@ -1,45 +1,58 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package udmrepo const ( - UNIFIED_REPO_OPTION_STORAGE_TYPE_S3 = "s3" - UNIFIED_REPO_OPTION_STORAGE_TYPE_AZURE = "azure" - UNIFIED_REPO_OPTION_STORAGE_TYPE_FS = "filesystem" - UNIFIED_REPO_OPTION_STORAGE_TYPE_GCS = "gcs" - - UNIFIED_REPO_GEN_OPTION_MAINTAIN_MODE = "mode" - UNIFIED_REPO_GEN_OPTION_MAINTAIN_FULL = "full" - UNIFIED_REPO_GEN_OPTION_MAINTAIN_QUICK = "quick" - - UNIFIED_REPO_GEN_OPTION_OWNER_NAME = "username" - UNFIED_REPO_GEN_OPTION_OWNER_DOMAIN = "domainname" - - UNIFIED_REPO_STORE_OPTION_S3_KEY_ID = "accessKeyID" - UNIFIED_REPO_STORE_OPTION_S3_PROVIDER = "providerName" - UNIFIED_REPO_STORE_OPTION_S3_SECRET_KEY = "secretAccessKey" - UNIFIED_REPO_STORE_OPTION_S3_TOKEN = "sessionToken" - UNIFIED_REPO_STORE_OPTION_S3_ENDPOINT = "endpoint" - UNIFIED_REPO_STORE_OPTION_S3_DISABLE_TLS = "doNotUseTLS" - UNIFIED_REPO_STORE_OPTION_S3_DISABLE_TLS_VERIFY = "skipTLSVerify" - - UNIFIED_REPO_STORE_OPTION_AZ_KEY = "storageKey" - UNIFIED_REPO_STORE_OPTION_AZ_DOMAIN = "storageDomain" - UNIFIED_REPO_STORE_OPTION_AZ_STORAGE_ACCOUNT = "storageAccount" - UNIFIED_REPO_STORE_OPTION_AZ_TOKEN = "sasToken" - - UNIFIED_REPO_STORE_OPTION_FS_PATH = "fspath" - - UNIFIED_REPO_STORE_OPTION_GCP_READONLY = "readonly" - - UNIFIED_REPO_STORE_OPTION_OSS_BUCKET = "bucket" - UNIFIED_REPO_STORE_OPTION_OSS_REGION = "region" - - UNIFIED_REPO_STORE_OPTION_CRED_FILE = "credFile" - UNIFIED_REPO_STORE_OPTION_PREFIX = "prefix" - UNIFIED_REPO_STORE_OPTION_PREFIX_NAME = "unified-repo" - - UNIFIED_REPO_THROTTLE_OPTION_READ_OPS = "readOPS" - UNIFIED_REPO_THROTTLE_OPTION_WRITE_OPS = "writeOPS" - UNIFIED_REPO_THROTTLE_OPTION_LIST_OPS = "listOPS" - UNIFIED_REPO_THROTTLE_OPTION_UPLOAD_BYTES = "uploadBytes" - UNIFIED_REPO_THROTTLE_OPTION_DOWNLOAD_BYTES = "downloadBytes" + StorageTypeS3 = "s3" + StorageTypeAzure = "azure" + StorageTypeFs = "filesystem" + StorageTypeGcs = "gcs" + + GenOptionMaintainMode = "mode" + GenOptionMaintainFull = "full" + GenOptionMaintainQuick = "quick" + + StoreOptionS3KeyId = "accessKeyID" + StoreOptionS3Provider = "providerName" + StoreOptionS3SecretKey = "secretAccessKey" + StoreOptionS3Token = "sessionToken" + StoreOptionS3Endpoint = "endpoint" + StoreOptionS3DisableTls = "doNotUseTLS" + StoreOptionS3DisableTlsVerify = "skipTLSVerify" + + StoreOptionAzureKey = "storageKey" + StoreOptionAzureDomain = "storageDomain" + StoreOptionAzureStorageAccount = "storageAccount" + StoreOptionAzureToken = "sasToken" + + StoreOptionFsPath = "fspath" + + StoreOptionGcsReadonly = "readonly" + + StoreOptionOssBucket = "bucket" + StoreOptionOssRegion = "region" + + StoreOptionCredentialFile = "credFile" + StoreOptionPrefix = "prefix" + StoreOptionPrefixName = "unified-repo" + + ThrottleOptionReadOps = "readOPS" + ThrottleOptionWriteOps = "writeOPS" + ThrottleOptionListOps = "listOPS" + ThrottleOptionUploadBytes = "uploadBytes" + ThrottleOptionDownloadBytes = "downloadBytes" ) diff --git a/pkg/repository/udmrepo/repo.go b/pkg/repository/udmrepo/repo.go index 5950961dfca..0790de7e0e6 100644 --- a/pkg/repository/udmrepo/repo.go +++ b/pkg/repository/udmrepo/repo.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package udmrepo import ( @@ -54,6 +70,16 @@ type ObjectWriteOptions struct { BackupMode int ///OBJECT_DATA_BACKUP_* } +// OwnershipOptions is used to add some access control to the unified repository. +// For example, some privileged operations of the unified repository can be done by the +// repository owner only; the data of a backup may be manipulated by the backup owner +// who created it only. It is optional for a backup repository to support this ownership control. +type OwnershipOptions struct { + Username string + DomainName string + FullQualified string +} + type RepoOptions struct { ///A repository specific string to identify a backup storage, i.e., "s3", "filesystem" StorageType string @@ -61,6 +87,8 @@ type RepoOptions struct { RepoPassword string ///A custom path to save the repository's configuration, if any ConfigFilePath string + ///The ownership for the current repository operation + Ownership OwnershipOptions ///Other repository specific options GeneralOptions map[string]string ///Storage specific options diff --git a/pkg/util/ownership/backup_owner.go b/pkg/util/ownership/backup_owner.go index 94dfe4b2da6..1d3b9d94455 100644 --- a/pkg/util/ownership/backup_owner.go +++ b/pkg/util/ownership/backup_owner.go @@ -1,13 +1,42 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package ownership -type Ownership struct { - Username string - DomainName string +import "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + +const ( + defaultOwnerUsername = "default" + defaultOwnerDomain = "default" +) + +// GetBackupOwner returns the owner used by uploaders when saving a snapshot or +// opening the unified repository. At present, use the default owner only +func GetBackupOwner() udmrepo.OwnershipOptions { + return udmrepo.OwnershipOptions{ + Username: defaultOwnerUsername, + DomainName: defaultOwnerDomain, + } } -func GetBackupOwner() Ownership { - return Ownership{ - Username: "default", - DomainName: "default", +// GetBackupOwner returns the owner used to create/connect the unified repository. +//At present, use the default owner only +func GetRepositoryOwner() udmrepo.OwnershipOptions { + return udmrepo.OwnershipOptions{ + Username: defaultOwnerUsername, + DomainName: defaultOwnerDomain, } }