Skip to content

Commit

Permalink
Merge pull request #5179 from Lyndon-Li/udmrepo-dev-02
Browse files Browse the repository at this point in the history 
Kopia Integration: Unified Repository Provider - Implementation
  • Loading branch information
@qiuming-best
qiuming-best authored Aug 16, 2022
2 parents 5b6d361 + 649c3a7 commit 71e4430
Show file tree
Hide file tree
Showing 13 changed files with 1,168 additions and 219 deletions.
1 change: 1 addition & 0 deletions changelogs/unreleased/5179-lyndon
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add changes for Kopia Integration: Unified Repository Provider - method implementation
3 changes: 1 addition & 2 deletions pkg/repository/provider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,9 @@ import (
)

// RepoParam includes the parameters to manipulate a backup repository
// SubDir is used to generate the path in the backup storage
type RepoParam struct {
SubDir string
BackupLocation *velerov1api.BackupStorageLocation
BackupRepo *velerov1api.BackupRepository
}

type Provider interface {
Expand Down
248 changes: 199 additions & 49 deletions pkg/repository/provider/unified_repo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config"
repokey "github.com/vmware-tanzu/velero/pkg/repository/keys"
"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
"github.com/vmware-tanzu/velero/pkg/util/ownership"
reposervice "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/service"
)

type unifiedRepoProvider struct {
Expand All @@ -49,28 +49,28 @@ var getS3BucketRegion = repoconfig.GetAWSBucketRegion
var getAzureStorageDomain = repoconfig.GetAzureStorageDomain

type localFuncTable struct {
getRepoPassword func(credentials.SecretStore, RepoParam) (string, error)
getStorageVariables func(*velerov1api.BackupStorageLocation, string) (map[string]string, error)
getStorageCredentials func(*velerov1api.BackupStorageLocation, credentials.FileStore) (map[string]string, error)
}

var funcTable = localFuncTable{
getRepoPassword: getRepoPassword,
getStorageVariables: getStorageVariables,
getStorageCredentials: getStorageCredentials,
}

const (
repoOpDescFullMaintain = "full maintenance"
repoOpDescQuickMaintain = "quick maintenance"
repoOpDescForget = "forget"
)

// NewUnifiedRepoProvider creates the service provider for Unified Repo
// workPath is the path for Unified Repo to store some local information
// workPath could be empty, if so, the default path will be used
func NewUnifiedRepoProvider(
credentialGetter credentials.CredentialGetter,
workPath string,
log logrus.FieldLogger,
) (Provider, error) {
repo := unifiedRepoProvider{
credentialGetter: credentialGetter,
workPath: workPath,
log: log,
}

Expand All @@ -89,12 +89,18 @@ func (urp *unifiedRepoProvider) InitRepo(ctx context.Context, param RepoParam) e

log.Debug("Start to init repo")

repoOption, err := urp.getRepoOption(param)
repoOption, err := udmrepo.NewRepoOptions(
udmrepo.WithPassword(urp, param),
udmrepo.WithConfigFile(urp.workPath, string(param.BackupLocation.UID)),
udmrepo.WithStoreOptions(urp, param),
udmrepo.WithDescription(repoOpDescFullMaintain),
)

if err != nil {
return errors.Wrap(err, "error to get repo options")
}

err = urp.repoService.Init(ctx, repoOption, true)
err = urp.repoService.Init(ctx, *repoOption, true)
if err != nil {
return errors.Wrap(err, "error to init backup repo")
}
Expand All @@ -105,22 +111,124 @@ func (urp *unifiedRepoProvider) InitRepo(ctx context.Context, param RepoParam) e
}

func (urp *unifiedRepoProvider) ConnectToRepo(ctx context.Context, param RepoParam) error {
///TODO
log := urp.log.WithFields(logrus.Fields{
"BSL name": param.BackupLocation.Name,
"BSL UID": param.BackupLocation.UID,
})

log.Debug("Start to connect repo")

repoOption, err := udmrepo.NewRepoOptions(
udmrepo.WithPassword(urp, param),
udmrepo.WithConfigFile(urp.workPath, string(param.BackupLocation.UID)),
udmrepo.WithStoreOptions(urp, param),
udmrepo.WithDescription(repoOpDescFullMaintain),
)

if err != nil {
return errors.Wrap(err, "error to get repo options")
}

err = urp.repoService.Init(ctx, *repoOption, false)
if err != nil {
return errors.Wrap(err, "error to connect backup repo")
}

log.Debug("Connect repo complete")

return nil
}

func (urp *unifiedRepoProvider) PrepareRepo(ctx context.Context, param RepoParam) error {
///TODO
log := urp.log.WithFields(logrus.Fields{
"BSL name": param.BackupLocation.Name,
"BSL UID": param.BackupLocation.UID,
})

log.Debug("Start to prepare repo")

repoOption, err := udmrepo.NewRepoOptions(
udmrepo.WithPassword(urp, param),
udmrepo.WithConfigFile(urp.workPath, string(param.BackupLocation.UID)),
udmrepo.WithStoreOptions(urp, param),
udmrepo.WithDescription(repoOpDescFullMaintain),
)

if err != nil {
return errors.Wrap(err, "error to get repo options")
}

err = urp.repoService.Init(ctx, *repoOption, false)
if err == nil {
log.Debug("Repo has already been initialized remotely")
return nil
}

err = urp.repoService.Init(ctx, *repoOption, true)
if err != nil {
return errors.Wrap(err, "error to init backup repo")
}

log.Debug("Prepare repo complete")

return nil
}

func (urp *unifiedRepoProvider) PruneRepo(ctx context.Context, param RepoParam) error {
///TODO
log := urp.log.WithFields(logrus.Fields{
"BSL name": param.BackupLocation.Name,
"BSL UID": param.BackupLocation.UID,
})

log.Debug("Start to prune repo")

repoOption, err := udmrepo.NewRepoOptions(
udmrepo.WithPassword(urp, param),
udmrepo.WithConfigFile(urp.workPath, string(param.BackupLocation.UID)),
udmrepo.WithGenOptions(map[string]string{udmrepo.GenOptionMaintainMode: udmrepo.GenOptionMaintainFull}),
udmrepo.WithDescription(repoOpDescFullMaintain),
)

if err != nil {
return errors.Wrap(err, "error to get repo options")
}

err = urp.repoService.Maintain(ctx, *repoOption)
if err != nil {
return errors.Wrap(err, "error to prune backup repo")
}

log.Debug("Prune repo complete")

return nil
}

func (urp *unifiedRepoProvider) PruneRepoQuick(ctx context.Context, param RepoParam) error {
///TODO
log := urp.log.WithFields(logrus.Fields{
"BSL name": param.BackupLocation.Name,
"BSL UID": param.BackupLocation.UID,
})

log.Debug("Start to prune repo quick")

repoOption, err := udmrepo.NewRepoOptions(
udmrepo.WithPassword(urp, param),
udmrepo.WithConfigFile(urp.workPath, string(param.BackupLocation.UID)),
udmrepo.WithGenOptions(map[string]string{udmrepo.GenOptionMaintainMode: udmrepo.GenOptionMaintainQuick}),
udmrepo.WithDescription(repoOpDescQuickMaintain),
)

if err != nil {
return errors.Wrap(err, "error to get repo options")
}

err = urp.repoService.Maintain(ctx, *repoOption)
if err != nil {
return errors.Wrap(err, "error to prune backup repo quick")
}

log.Debug("Prune repo quick complete")

return nil
}

Expand All @@ -129,60 +237,108 @@ func (urp *unifiedRepoProvider) EnsureUnlockRepo(ctx context.Context, param Repo
}

func (urp *unifiedRepoProvider) Forget(ctx context.Context, snapshotID string, param RepoParam) error {
///TODO
log := urp.log.WithFields(logrus.Fields{
"BSL name": param.BackupLocation.Name,
"BSL UID": param.BackupLocation.UID,
"snapshotID": snapshotID,
})

log.Debug("Start to forget snapshot")

repoOption, err := udmrepo.NewRepoOptions(
udmrepo.WithPassword(urp, param),
udmrepo.WithConfigFile(urp.workPath, string(param.BackupLocation.UID)),
udmrepo.WithDescription(repoOpDescForget),
)

if err != nil {
return errors.Wrap(err, "error to get repo options")
}

bkRepo, err := urp.repoService.Open(ctx, *repoOption)
if err != nil {
return errors.Wrap(err, "error to open backup repo")
}

defer func() {
c := bkRepo.Close(ctx)
if c != nil {
log.WithError(c).Error("Failed to close repo")
}
}()

err = bkRepo.DeleteManifest(ctx, udmrepo.ID(snapshotID))
if err != nil {
return errors.Wrap(err, "error to delete manifest")
}

log.Debug("Forget snapshot complete")

return nil
}

func getRepoPassword(secretStore credentials.SecretStore, param RepoParam) (string, error) {
if secretStore == nil {
return "", errors.New("invalid credentials interface")
func (urp *unifiedRepoProvider) GetPassword(param interface{}) (string, error) {
repoParam, ok := param.(RepoParam)
if !ok {
return "", errors.New("invalid parameter")
}

buf, err := secretStore.Get(repokey.RepoKeySelector())
repoPassword, err := getRepoPassword(urp.credentialGetter.FromSecret, repoParam)
if err != nil {
return "", errors.Wrap(err, "error to get password buffer")
return "", errors.Wrap(err, "error to get repo password")
}

return strings.TrimSpace(string(buf)), nil
return repoPassword, nil
}

func (urp *unifiedRepoProvider) getRepoOption(param RepoParam) (udmrepo.RepoOptions, error) {
repoPassword, err := funcTable.getRepoPassword(urp.credentialGetter.FromSecret, param)
if err != nil {
return udmrepo.RepoOptions{}, errors.Wrap(err, "error to get repo password")
func (urp *unifiedRepoProvider) GetStoreType(param interface{}) (string, error) {
repoParam, ok := param.(RepoParam)
if !ok {
return "", errors.New("invalid parameter")
}

storeVar, err := funcTable.getStorageVariables(param.BackupLocation, param.SubDir)
if err != nil {
return udmrepo.RepoOptions{}, errors.Wrap(err, "error to get storage variables")
return getStorageType(repoParam.BackupLocation), nil
}

func (urp *unifiedRepoProvider) GetStoreOptions(param interface{}) (map[string]string, error) {
repoParam, ok := param.(RepoParam)
if !ok {
return map[string]string{}, errors.New("invalid parameter")
}

storeCred, err := funcTable.getStorageCredentials(param.BackupLocation, urp.credentialGetter.FromFile)
storeVar, err := funcTable.getStorageVariables(repoParam.BackupLocation, repoParam.BackupRepo.Spec.VolumeNamespace)
if err != nil {
return udmrepo.RepoOptions{}, errors.Wrap(err, "error to get repo credentials")
return map[string]string{}, errors.Wrap(err, "error to get storage variables")
}

repoOption := udmrepo.RepoOptions{
StorageType: getStorageType(param.BackupLocation),
RepoPassword: repoPassword,
ConfigFilePath: getRepoConfigFile(urp.workPath, string(param.BackupLocation.UID)),
Ownership: udmrepo.OwnershipOptions{
Username: ownership.GetRepositoryOwner().Username,
DomainName: ownership.GetRepositoryOwner().DomainName,
},
StorageOptions: make(map[string]string),
GeneralOptions: make(map[string]string),
storeCred, err := funcTable.getStorageCredentials(repoParam.BackupLocation, urp.credentialGetter.FromFile)
if err != nil {
return map[string]string{}, errors.Wrap(err, "error to get repo credentials")
}

storeOptions := make(map[string]string)
for k, v := range storeVar {
repoOption.StorageOptions[k] = v
storeOptions[k] = v
}

for k, v := range storeCred {
repoOption.StorageOptions[k] = v
storeOptions[k] = v
}

return repoOption, nil
return storeOptions, nil
}

func getRepoPassword(secretStore credentials.SecretStore, param RepoParam) (string, error) {
if secretStore == nil {
return "", errors.New("invalid credentials interface")
}

rawPass, err := secretStore.Get(repokey.RepoKeySelector())
if err != nil {
return "", errors.Wrap(err, "error to get password")
}

return strings.TrimSpace(rawPass), nil
}

func getStorageType(backupLocation *velerov1api.BackupStorageLocation) string {
Expand Down Expand Up @@ -304,12 +460,6 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo
return result, nil
}

func getRepoConfigFile(workPath string, repoID string) string {
///TODO: call udmrepo to get config file
return ""
}

func createRepoService(log logrus.FieldLogger) udmrepo.BackupRepoService {
///TODO: call udmrepo create repo service
return nil
return reposervice.Create(log)
}
Loading

0 comments on commit 71e4430

Please sign in to comment.