add UT

Signed-off-by: Lyndon-Li <[email protected]>

changelogs/unreleased/5142-lyndon

@@ -0,0 +1,4 @@
+Kopia Integration: Add the Unified Repository Interface definition.
+Kopia Integration: Add the changes for Unified Repository storage config.
+
+Related Issues; #5076, #5080

pkg/repository/repoconfig/aws.go

@@ -17,10 +17,14 @@ limitations under the License.
 package repoconfig
 
 import (
-	"errors"
+	"context"
 	"os"
 
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -67,3 +71,29 @@ func GetS3Credentials(config map[string]string) (credentials.Value, error) {
 
 	return credValue, nil
 }
+
+// GetAWSBucketRegion returns the AWS region that a bucket is in, or an error
+// if the region cannot be determined.
+func GetAWSBucketRegion(bucket string) (string, error) {
+	var region string
+
+	sess, err := session.NewSession()
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	for _, partition := range endpoints.DefaultPartitions() {
+		for regionHint := range partition.Regions() {
+			region, _ = s3manager.GetBucketRegion(context.Background(), sess, bucket, regionHint)
+
+			// we only need to try a single region hint per partition, so break after the first
+			break
+		}
+
+		if region != "" {
+			return region, nil
+		}
+	}
+
+	return "", errors.New("unable to determine bucket's region")
+}

pkg/repository/repoconfig/config.go

@@ -17,14 +17,10 @@ limitations under the License.
 package repoconfig
 
 import (
-	"context"
 	"fmt"
 	"path"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/aws/endpoints"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/pkg/errors"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -78,7 +74,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 			var err error
 			region := location.Spec.Config["region"]
 			if region == "" {
-				region, err = GetAWSBucketRegion(bucket)
+				region, err = getAWSBucketRegion(bucket)
 			}
 			if err != nil {
 				return "", errors.Wrapf(err, "failed to detect the region via bucket: %s", bucket)
@@ -104,6 +100,10 @@ func GetBackendType(provider string) BackendType {
 	return BackendType(provider)
 }
 
+func IsBackendTypeValid(backendType BackendType) bool {
+	return (backendType == AWSBackend || backendType == AzureBackend || backendType == GCPBackend || backendType == FSBackend)
+}
+
 // GetRepoIdentifier returns the string to be used as the value of the --repo flag in
 // restic commands for the given repository.
 func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string) (string, error) {
@@ -114,29 +114,3 @@ func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string)
 
 	return fmt.Sprintf("%s/%s", strings.TrimSuffix(prefix, "/"), name), nil
 }
-
-// GetBucketRegion returns the AWS region that a bucket is in, or an error
-// if the region cannot be determined.
-func GetAWSBucketRegion(bucket string) (string, error) {
-	var region string
-
-	sess, err := session.NewSession()
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	for _, partition := range endpoints.DefaultPartitions() {
-		for regionHint := range partition.Regions() {
-			region, _ = s3manager.GetBucketRegion(context.Background(), sess, bucket, regionHint)
-
-			// we only need to try a single region hint per partition, so break after the first
-			break
-		}
-
-		if region != "" {
-			return region, nil
-		}
-	}
-
-	return "", errors.New("unable to determine bucket's region")
-}

pkg/repository/repoprovider/repo_provider.go

@@ -1,19 +1,45 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package repoprovider
 
 import "context"
 
 type RepositoryProvider interface {
+	//InitRepo is to initialize a repository from a new storage place
 	InitRepo(ctx context.Context, bsl string) error
 
+	//ConnectToRepo is to establish the connection to a
+	//storage place that a repository is already initialized
 	ConnectToRepo(ctx context.Context, bsl string) error
 
+	//PrepareRepo is a combination of InitRepo and ConnectToRepo,
+	//it may do initializing + connecting, connecting only if the repository
+	//is already initialized, or do nothing if the repository is already connected
 	PrepareRepo(ctx context.Context, bsl string) error
 
+	//PruneRepo does a full prune/maintenance of the repository
 	PruneRepo(ctx context.Context, bsl string) error
 
+	//PruneRepoQuick does a quick prune/maintenance of the repository if available
 	PruneRepoQuick(ctx context.Context, bsl string) error
 
+	//EnsureUnlockRepo esures to remove any stale file locks in the storage
 	EnsureUnlockRepo(ctx context.Context, bsl string) error
 
+	//Forget is to delete a snapshot from the repository
 	Forget(ctx context.Context, snapshotID, bsl string) error
 }

pkg/repository/repoprovider/unified_repo.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package repoprovider
 
 import (
@@ -6,7 +22,9 @@ import (
 	"path"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+
 	"github.com/vmware-tanzu/velero/internal/credentials"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/repository/repoconfig"
@@ -25,6 +43,14 @@ type unifiedRepoProvider struct {
 	log                  *logrus.Logger
 }
 
+// this func is assigned to a package-level variable so it can be
+// replaced when unit-testing
+var getAzureCredentials = repoconfig.GetAzureCredentials
+var getS3Credentials = repoconfig.GetS3Credentials
+var getGCPCredentials = repoconfig.GetGCPCredentials
+var getS3BucketRegion = repoconfig.GetAWSBucketRegion
+var getAzureStorageDomain = repoconfig.GetAzureStorageDomain
+
 func NewUnifiedRepoProvider(
 	ctx context.Context,
 	credentialFileStore credentials.FileStore,
@@ -71,9 +97,10 @@ func (urp *unifiedRepoProvider) InitRepo(ctx context.Context, backupLocation str
 	err = urp.repoService.Init(ctx, repoOption, true)
 	if err != nil {
 		log.WithError(err).Error("Failed to init backup repo")
+		return err
 	}
 
-	return err
+	return nil
 }
 
 func (urp *unifiedRepoProvider) ConnectToRepo(ctx context.Context, backupLocation string) error {
@@ -122,11 +149,12 @@ func (urp *unifiedRepoProvider) getRepoOption() (udmrepo.RepoOptions, error) {
 		StorageType:    getStorageType(urp.backupLocation),
 		RepoPassword:   urp.repoPassword,
 		ConfigFilePath: urp.configPath,
-		StorageOptions: make(map[string]string),
-		GeneralOptions: map[string]string{
-			udmrepo.UNFIED_REPO_GEN_OPTION_OWNER_DOMAIN: ownership.GetBackupOwner().DomainName,
-			udmrepo.UNIFIED_REPO_GEN_OPTION_OWNER_NAME:  ownership.GetBackupOwner().Username,
+		Ownership: udmrepo.OwnershipOptions{
+			Username:   ownership.GetRepositoryOwner().Username,
+			DomainName: ownership.GetRepositoryOwner().DomainName,
 		},
+		StorageOptions: make(map[string]string),
+		GeneralOptions: make(map[string]string),
 	}
 
 	storeVar, err := getStorageVariables(urp.backupLocation, urp.repoName)
@@ -139,7 +167,7 @@ func (urp *unifiedRepoProvider) getRepoOption() (udmrepo.RepoOptions, error) {
 		repoOption.StorageOptions[k] = v
 	}
 
-	storeCred, err := getStorageCredentials(urp.backupLocation, urp.credentialsFileStore, urp.log)
+	storeCred, err := getStorageCredentials(urp.backupLocation, urp.credentialsFileStore)
 	if err != nil {
 		log.WithError(err).Error("Failed to get repo credential env")
 		return repoOption, err
@@ -157,22 +185,26 @@ func getStorageType(backupLocation *velerov1api.BackupStorageLocation) string {
 
 	switch backendType {
 	case repoconfig.AWSBackend:
-		return udmrepo.UNIFIED_REPO_OPTION_STORAGE_TYPE_S3
+		return udmrepo.StorageTypeS3
 	case repoconfig.AzureBackend:
-		return udmrepo.UNIFIED_REPO_OPTION_STORAGE_TYPE_AZURE
+		return udmrepo.StorageTypeAzure
 	case repoconfig.GCPBackend:
-		return udmrepo.UNIFIED_REPO_OPTION_STORAGE_TYPE_GCS
+		return udmrepo.StorageTypeGcs
 	case repoconfig.FSBackend:
-		return udmrepo.UNIFIED_REPO_OPTION_STORAGE_TYPE_FS
+		return udmrepo.StorageTypeFs
 	default:
 		return ""
 	}
 }
 
-func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, credentialsFileStore credentials.FileStore, log *logrus.Logger) (map[string]string, error) {
+func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, credentialsFileStore credentials.FileStore) (map[string]string, error) {
 	result := make(map[string]string)
 	var err error
+
 	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider)
+	if !repoconfig.IsBackendTypeValid(backendType) {
+		return map[string]string{}, errors.New("invalid storage provider")
+	}
 
 	config := backupLocation.Spec.Config
 	if config == nil {
@@ -182,42 +214,43 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr
 	if backupLocation.Spec.Credential != nil {
 		config[repoconfig.CredentialsFileKey], err = credentialsFileStore.Path(backupLocation.Spec.Credential)
 		if err != nil {
-			log.WithError(err).Error("Failed to get credential file in BSL")
-			return map[string]string{}, err
+			return map[string]string{}, errors.Wrap(err, "error get credential file in bsl")
 		}
 	}
 
 	switch backendType {
 	case repoconfig.AWSBackend:
-		credValue, err := repoconfig.GetS3Credentials(config)
+		credValue, err := getS3Credentials(config)
 		if err != nil {
-			log.WithError(err).Error("Failed to get S3 credentials")
-			return map[string]string{}, err
+			return map[string]string{}, errors.Wrap(err, "error get s3 credentials")
 		}
-		result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_KEY_ID] = credValue.AccessKeyID
-		result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_PROVIDER] = credValue.ProviderName
-		result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_SECRET_KEY] = credValue.SecretAccessKey
-		result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_TOKEN] = credValue.SessionToken
+		result[udmrepo.StoreOptionS3KeyId] = credValue.AccessKeyID
+		result[udmrepo.StoreOptionS3Provider] = credValue.ProviderName
+		result[udmrepo.StoreOptionS3SecretKey] = credValue.SecretAccessKey
+		result[udmrepo.StoreOptionS3Token] = credValue.SessionToken
 
 	case repoconfig.AzureBackend:
-		storageAccount, accountKey, err := repoconfig.GetAzureCredentials(config)
+		storageAccount, accountKey, err := getAzureCredentials(config)
 		if err != nil {
-			log.WithError(err).Error("Failed to get Azure credentials")
-			return map[string]string{}, err
+			return map[string]string{}, errors.Wrap(err, "error get azure credentials")
 		}
-		result[udmrepo.UNIFIED_REPO_STORE_OPTION_AZ_STORAGE_ACCOUNT] = storageAccount
-		result[udmrepo.UNIFIED_REPO_STORE_OPTION_AZ_KEY] = accountKey
+		result[udmrepo.StoreOptionAzureStorageAccount] = storageAccount
+		result[udmrepo.StoreOptionAzureKey] = accountKey
 
 	case repoconfig.GCPBackend:
-		result[udmrepo.UNIFIED_REPO_STORE_OPTION_CRED_FILE] = repoconfig.GetGCPCredentials(config)
+		result[udmrepo.StoreOptionCredentialFile] = getGCPCredentials(config)
 	}
 
 	return result, nil
 }
 
 func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repoName string) (map[string]string, error) {
 	result := make(map[string]string)
+
 	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider)
+	if !repoconfig.IsBackendTypeValid(backendType) {
+		return map[string]string{}, errors.New("invalid storage provider")
+	}
 
 	config := backupLocation.Spec.Config
 	if config == nil {
@@ -231,33 +264,33 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo
 		prefix = strings.Trim(backupLocation.Spec.ObjectStorage.Prefix, "/")
 	}
 
-	prefix = path.Join(prefix, udmrepo.UNIFIED_REPO_STORE_OPTION_PREFIX_NAME, repoName) + "/"
+	prefix = path.Join(prefix, udmrepo.StoreOptionPrefixName, repoName) + "/"
 
-	s3Url := config["s3Url"]
 	region := config["region"]
-	var err error
 
 	if backendType == repoconfig.AWSBackend {
-		if s3Url == "" && region == "" {
-			region, err = repoconfig.GetAWSBucketRegion(bucket)
+		s3Url := config["s3Url"]
+
+		var err error
+		if s3Url == "" {
+			region, err = getS3BucketRegion(bucket)
 			if err != nil {
-				return map[string]string{}, err
+				return map[string]string{}, errors.Wrap(err, "error get s3 bucket region")
 			}
 
 			s3Url = fmt.Sprintf("s3-%s.amazonaws.com", region)
 		}
-	}
-
-	result[udmrepo.UNIFIED_REPO_STORE_OPTION_OSS_BUCKET] = bucket
-	result[udmrepo.UNIFIED_REPO_STORE_OPTION_PREFIX] = prefix
-	result[udmrepo.UNIFIED_REPO_STORE_OPTION_OSS_REGION] = strings.Trim(region, "/")
 
-	result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_ENDPOINT] = strings.Trim(s3Url, "/")
-	result[udmrepo.UNIFIED_REPO_STORE_OPTION_S3_DISABLE_TLS_VERIFY] = config["insecureSkipTLSVerify"]
-
-	result[udmrepo.UNIFIED_REPO_STORE_OPTION_AZ_DOMAIN] = repoconfig.GetAzureStorageDomain(config)
+		result[udmrepo.StoreOptionS3Endpoint] = strings.Trim(s3Url, "/")
+		result[udmrepo.StoreOptionS3DisableTlsVerify] = config["insecureSkipTLSVerify"]
+	} else if backendType == repoconfig.AzureBackend {
+		result[udmrepo.StoreOptionAzureDomain] = getAzureStorageDomain(config)
+	}
 
-	result[udmrepo.UNIFIED_REPO_STORE_OPTION_FS_PATH] = config["fspath"]
+	result[udmrepo.StoreOptionOssBucket] = bucket
+	result[udmrepo.StoreOptionPrefix] = prefix
+	result[udmrepo.StoreOptionOssRegion] = strings.Trim(region, "/")
+	result[udmrepo.StoreOptionFsPath] = config["fspath"]
 
 	return result, nil
 }