unified repo storge config

Signed-off-by: Lyndon-Li <[email protected]>

pkg/controller/restic_repository_controller.go

@@ -31,6 +31,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/repository/repoconfig"
 	"github.com/vmware-tanzu/velero/pkg/restic"
 	"github.com/vmware-tanzu/velero/pkg/util/kube"
 )
@@ -127,7 +128,7 @@ func (r *ResticRepoReconciler) initializeRepo(ctx context.Context, req *velerov1
 		return r.patchResticRepository(ctx, req, repoNotReady(err.Error()))
 	}
 
-	repoIdentifier, err := restic.GetRepoIdentifier(loc, req.Spec.VolumeNamespace)
+	repoIdentifier, err := repoconfig.GetRepoIdentifier(loc, req.Spec.VolumeNamespace)
 	if err != nil {
 		return r.patchResticRepository(ctx, req, func(rr *velerov1api.ResticRepository) {
 			rr.Status.Message = err.Error()

pkg/restic/aws.go → pkg/repository/repoconfig/aws.go

@@ -14,7 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
+
+import (
+	"errors"
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws/credentials"
+)
 
 const (
 	// AWS specific environment variable
@@ -23,13 +30,13 @@ const (
 	awsCredentialsFileEnvVar = "AWS_SHARED_CREDENTIALS_FILE"
 )
 
-// getS3ResticEnvVars gets the environment variables that restic
+// GetS3ResticEnvVars gets the environment variables that restic
 // relies on (AWS_PROFILE) based on info in the provided object
 // storage location config map.
-func getS3ResticEnvVars(config map[string]string) (map[string]string, error) {
+func GetS3ResticEnvVars(config map[string]string) (map[string]string, error) {
 	result := make(map[string]string)
 
-	if credentialsFile, ok := config[credentialsFileKey]; ok {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
 		result[awsCredentialsFileEnvVar] = credentialsFile
 	}
 
@@ -39,3 +46,24 @@ func getS3ResticEnvVars(config map[string]string) (map[string]string, error) {
 
 	return result, nil
 }
+
+// GetS3Credentials gets the S3 credential values according to the information
+// of the provided config or the system's environment variables
+func GetS3Credentials(config map[string]string) (credentials.Value, error) {
+	credentialsFile := config[CredentialsFileKey]
+	if credentialsFile == "" {
+		credentialsFile = os.Getenv("AWS_SHARED_CREDENTIALS_FILE")
+	}
+
+	if credentialsFile == "" {
+		return credentials.Value{}, errors.New("missing credential file")
+	}
+
+	creds := credentials.NewSharedCredentials(credentialsFile, "")
+	credValue, err := creds.Get()
+	if err != nil {
+		return credValue, err
+	}
+
+	return credValue, nil
+}

pkg/restic/aws_test.go → pkg/repository/repoconfig/aws_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"testing"
@@ -55,7 +55,7 @@ func TestGetS3ResticEnvVars(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			actual, err := getS3ResticEnvVars(tc.config)
+			actual, err := GetS3ResticEnvVars(tc.config)
 
 			require.NoError(t, err)
 

pkg/restic/azure.go → pkg/repository/repoconfig/azure.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"context"
@@ -37,6 +37,7 @@ const (
 	storageAccountConfigKey          = "storageAccount"
 	storageAccountKeyEnvVarConfigKey = "storageAccountKeyEnvVar"
 	subscriptionIDConfigKey          = "subscriptionId"
+	storageDomainConfigKey           = "storageDomain"
 )
 
 // getSubscriptionID gets the subscription ID from the 'config' map if it contains
@@ -131,10 +132,10 @@ func mapLookup(data map[string]string) func(string) string {
 	}
 }
 
-// getAzureResticEnvVars gets the environment variables that restic
+// GetAzureResticEnvVars gets the environment variables that restic
 // relies on (AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY) based
 // on info in the provided object storage location config map.
-func getAzureResticEnvVars(config map[string]string) (map[string]string, error) {
+func GetAzureResticEnvVars(config map[string]string) (map[string]string, error) {
 	storageAccountKey, _, err := getStorageAccountKey(config)
 	if err != nil {
 		return nil, err
@@ -158,7 +159,7 @@ func credentialsFileFromEnv() string {
 // selectCredentialsFile selects the Azure credentials file to use, retrieving it
 // from the given config or falling back to retrieving it from the environment.
 func selectCredentialsFile(config map[string]string) string {
-	if credentialsFile, ok := config[credentialsFileKey]; ok {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
 		return credentialsFile
 	}
 
@@ -208,3 +209,22 @@ func getRequiredValues(getValue func(string) string, keys ...string) (map[string
 
 	return results, nil
 }
+
+// GetAzureStorageDomain gets the Azure storage domain required by a Azure blob connection,
+// if the provided config doean't have the value, get it from system's environment variables
+func GetAzureStorageDomain(config map[string]string) string {
+	if domain, exist := config[storageDomainConfigKey]; exist {
+		return domain
+	} else {
+		return os.Getenv(cloudNameEnvVar)
+	}
+}
+
+func GetAzureCredentials(config map[string]string) (string, string, error) {
+	storageAccountKey, _, err := getStorageAccountKey(config)
+	if err != nil {
+		return "", "", err
+	}
+
+	return config[storageAccountConfigKey], storageAccountKey, nil
+}

pkg/restic/azure_test.go → pkg/repository/repoconfig/azure_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"os"

pkg/restic/config.go → pkg/repository/repoconfig/config.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"context"
@@ -37,11 +37,18 @@ const (
 	AWSBackend   BackendType = "velero.io/aws"
 	AzureBackend BackendType = "velero.io/azure"
 	GCPBackend   BackendType = "velero.io/gcp"
+	FSBackend    BackendType = "velero.io/fs"
+)
+
+const (
+	// CredentialsFileKey is the key within a BSL config that is checked to see if
+	// the BSL is using its own credentials, rather than those in the environment
+	CredentialsFileKey = "credentialsFile"
 )
 
 // this func is assigned to a package-level variable so it can be
 // replaced when unit-testing
-var getAWSBucketRegion = getBucketRegion
+var getAWSBucketRegion = GetAWSBucketRegion
 
 // getRepoPrefix returns the prefix of the value of the --repo flag for
 // restic commands, i.e. everything except the "/<repo-name>".
@@ -55,7 +62,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 		prefix = layout.GetResticDir()
 	}
 
-	backendType := getBackendType(location.Spec.Provider)
+	backendType := GetBackendType(location.Spec.Provider)
 
 	if repoPrefix := location.Spec.Config["resticRepoPrefix"]; repoPrefix != "" {
 		return repoPrefix, nil
@@ -71,7 +78,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 			var err error
 			region := location.Spec.Config["region"]
 			if region == "" {
-				region, err = getAWSBucketRegion(bucket)
+				region, err = GetAWSBucketRegion(bucket)
 			}
 			if err != nil {
 				return "", errors.Wrapf(err, "failed to detect the region via bucket: %s", bucket)
@@ -89,7 +96,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 	return "", errors.New("restic repository prefix (resticRepoPrefix) not specified in backup storage location's config")
 }
 
-func getBackendType(provider string) BackendType {
+func GetBackendType(provider string) BackendType {
 	if !strings.Contains(provider, "/") {
 		provider = "velero.io/" + provider
 	}
@@ -108,9 +115,9 @@ func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string)
 	return fmt.Sprintf("%s/%s", strings.TrimSuffix(prefix, "/"), name), nil
 }
 
-// getBucketRegion returns the AWS region that a bucket is in, or an error
+// GetBucketRegion returns the AWS region that a bucket is in, or an error
 // if the region cannot be determined.
-func getBucketRegion(bucket string) (string, error) {
+func GetAWSBucketRegion(bucket string) (string, error) {
 	var region string
 
 	sess, err := session.NewSession()

pkg/restic/config_test.go → pkg/repository/repoconfig/config_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"testing"

pkg/restic/gcp.go → pkg/repository/repoconfig/gcp.go

@@ -14,21 +14,33 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
+
+import "os"
 
 const (
 	// GCP specific environment variable
 	gcpCredentialsFileEnvVar = "GOOGLE_APPLICATION_CREDENTIALS"
 )
 
-// getGCPResticEnvVars gets the environment variables that restic relies
+// GetGCPResticEnvVars gets the environment variables that restic relies
 // on based on info in the provided object storage location config map.
-func getGCPResticEnvVars(config map[string]string) (map[string]string, error) {
+func GetGCPResticEnvVars(config map[string]string) (map[string]string, error) {
 	result := make(map[string]string)
 
-	if credentialsFile, ok := config[credentialsFileKey]; ok {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
 		result[gcpCredentialsFileEnvVar] = credentialsFile
 	}
 
 	return result, nil
 }
+
+// GetGCPCredentials gets the credential file required by a GCP bucket connection,
+// if the provided config doean't have the value, get it from system's environment variables
+func GetGCPCredentials(config map[string]string) string {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
+		return credentialsFile
+	} else {
+		return os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
+	}
+}

pkg/restic/gcp_test.go → pkg/repository/repoconfig/gcp_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"testing"
@@ -46,7 +46,7 @@ func TestGetGCPResticEnvVars(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			actual, err := getGCPResticEnvVars(tc.config)
+			actual, err := GetGCPResticEnvVars(tc.config)
 
 			require.NoError(t, err)
 

pkg/repository/repoprovider/repo_provider.go

@@ -0,0 +1,19 @@
+package repoprovider
+
+import "context"
+
+type RepositoryProvider interface {
+	InitRepo(ctx context.Context, bsl string) error
+
+	ConnectToRepo(ctx context.Context, bsl string) error
+
+	PrepareRepo(ctx context.Context, bsl string) error
+
+	PruneRepo(ctx context.Context, bsl string) error
+
+	PruneRepoQuick(ctx context.Context, bsl string) error
+
+	EnsureUnlockRepo(ctx context.Context, bsl string) error
+
+	Forget(ctx context.Context, snapshotID, bsl string) error
+}