From c39b4b2a532e3e5d90b4f4a675ed980641ac8e4b Mon Sep 17 00:00:00 2001 From: Julia Bier Date: Tue, 15 Oct 2024 13:43:46 -0400 Subject: [PATCH 1/6] implement wildcard support in backups --- pkg/resourcesets/collector.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/resourcesets/collector.go b/pkg/resourcesets/collector.go index 20f42297..00e7019d 100644 --- a/pkg/resourcesets/collector.go +++ b/pkg/resourcesets/collector.go @@ -1,6 +1,7 @@ package resourcesets import ( + "bytes" "context" "encoding/json" "fmt" @@ -15,6 +16,7 @@ import ( k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/server/options/encryptionconfig" "k8s.io/apiserver/pkg/storage/value" "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" @@ -387,7 +389,8 @@ func (h *ResourceHandler) WriteBackupObjects(backupPath string) error { } gr := schema.ParseGroupResource(gvResource.Name + "." + gv.Group) - encryptionTransformer := h.TransformerMap[gr] + var staticTransformers encryptionconfig.StaticTransformers = h.TransformerMap + encryptionTransformer := staticTransformers.TransformerForResource(gr) additionalAuthenticatedData := objName if gvResource.Namespaced { additionalAuthenticatedData = fmt.Sprintf("%s#%s", metadata["namespace"].(string), additionalAuthenticatedData) @@ -438,9 +441,11 @@ func writeToBackup(ctx context.Context, resource map[string]interface{}, backupP if err != nil { return fmt.Errorf("error converting resource to JSON: %v", err) } - resourceBytes, err = json.Marshal(encrypted) - if err != nil { - return fmt.Errorf("error converting encrypted resource to JSON: %v", err) + if !bytes.Equal(resourceBytes, encrypted) { + resourceBytes, err = json.Marshal(encrypted) + if err != nil { + return fmt.Errorf("error converting encrypted resource to JSON: %v", err) + } } } if _, err := f.Write(resourceBytes); err != nil { From 00c61843d38fa8f699c16aea985b31f020269dd3 Mon Sep 17 00:00:00 2001 From: Julia Bier Date: Tue, 15 Oct 2024 13:46:37 -0400 Subject: [PATCH 2/6] implement wildcard support in restores --- pkg/controllers/restore/download.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/restore/download.go b/pkg/controllers/restore/download.go index ca7cec10..36b840e4 100644 --- a/pkg/controllers/restore/download.go +++ b/pkg/controllers/restore/download.go @@ -15,6 +15,7 @@ import ( "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/server/options/encryptionconfig" "k8s.io/apiserver/pkg/storage/value" ) @@ -101,10 +102,12 @@ func (h *handler) loadDataFromFile(tarContent *tar.Header, readData []byte, namespace = splitPath[1] additionalAuthenticatedData = fmt.Sprintf("%s#%s", namespace, name) } + gvrStr := splitPath[0] gvr := getGVR(gvrStr) + var staticTransformers encryptionconfig.StaticTransformers = transformerMap + decryptionTransformer := staticTransformers.TransformerForResource(gvr.GroupResource()) - decryptionTransformer := transformerMap[gvr.GroupResource()] if decryptionTransformer != nil { var encryptedBytes []byte if err := json.Unmarshal(readData, &encryptedBytes); err != nil { From e49397b0d0e3f7dc0f1fbbf84a7eee856e9b0129 Mon Sep 17 00:00:00 2001 From: Julia Bier Date: Tue, 15 Oct 2024 14:34:38 -0400 Subject: [PATCH 3/6] fix spacing --- pkg/controllers/restore/download.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/restore/download.go b/pkg/controllers/restore/download.go index 36b840e4..28ac6dda 100644 --- a/pkg/controllers/restore/download.go +++ b/pkg/controllers/restore/download.go @@ -105,9 +105,9 @@ func (h *handler) loadDataFromFile(tarContent *tar.Header, readData []byte, gvrStr := splitPath[0] gvr := getGVR(gvrStr) + var staticTransformers encryptionconfig.StaticTransformers = transformerMap decryptionTransformer := staticTransformers.TransformerForResource(gvr.GroupResource()) - if decryptionTransformer != nil { var encryptedBytes []byte if err := json.Unmarshal(readData, &encryptedBytes); err != nil { From 2adc8d13cee0537e7339009f57307e1fd5d4fe5e Mon Sep 17 00:00:00 2001 From: Julia Bier Date: Tue, 15 Oct 2024 16:20:36 -0400 Subject: [PATCH 4/6] check for encrypted files --- pkg/controllers/restore/download.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/restore/download.go b/pkg/controllers/restore/download.go index 28ac6dda..9cfe9be4 100644 --- a/pkg/controllers/restore/download.go +++ b/pkg/controllers/restore/download.go @@ -8,6 +8,7 @@ import ( "io" "io/ioutil" "os" + "reflect" "strings" v1 "github.com/rancher/backup-restore-operator/pkg/apis/resources.cattle.io/v1" @@ -108,7 +109,7 @@ func (h *handler) loadDataFromFile(tarContent *tar.Header, readData []byte, var staticTransformers encryptionconfig.StaticTransformers = transformerMap decryptionTransformer := staticTransformers.TransformerForResource(gvr.GroupResource()) - if decryptionTransformer != nil { + if !reflect.ValueOf(decryptionTransformer).IsZero() { var encryptedBytes []byte if err := json.Unmarshal(readData, &encryptedBytes); err != nil { logrus.Errorf("Error unmarshaling encrypted data for resource [%v]: %v", gvr.GroupResource(), err) From f00624c47c8ad052df16bd0b94a0153c62f67370 Mon Sep 17 00:00:00 2001 From: Julia Bier Date: Tue, 15 Oct 2024 16:34:08 -0400 Subject: [PATCH 5/6] remove deprecated ioutils references --- pkg/controllers/backup/controller.go | 7 +++---- pkg/controllers/backup/upload.go | 3 +-- pkg/controllers/restore/download.go | 3 +-- pkg/crds/crd.go | 4 ++-- pkg/objectstore/s3minio.go | 3 +-- pkg/util/util.go | 3 +-- 6 files changed, 9 insertions(+), 14 deletions(-) diff --git a/pkg/controllers/backup/controller.go b/pkg/controllers/backup/controller.go index 6a8b69f5..e5d5451f 100644 --- a/pkg/controllers/backup/controller.go +++ b/pkg/controllers/backup/controller.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -145,8 +144,8 @@ func (h *handler) OnBackupChange(_ string, backup *v1.Backup) (*v1.Backup, error logrus.Infof("For backup CR %v, filename: %v", backup.Name, backupFileName) // create a temp dir to write all backup files to, delete this before returning. - // empty dir param in ioutil.TempDir defaults to os.TempDir - tmpBackupPath, err := ioutil.TempDir("", backupFileName) + // empty dir param in os.MkdirTemp. defaults to os.TempDir + tmpBackupPath, err := os.MkdirTemp("", backupFileName) if err != nil { return h.setReconcilingCondition(backup, fmt.Errorf("error creating temp dir: %v", err)) } @@ -259,7 +258,7 @@ func (h *handler) performBackup(backup *v1.Backup, tmpBackupPath, backupFileName if err != nil { return err } - err = ioutil.WriteFile(filepath.Join(filtersPath, "filters.json"), filters, os.ModePerm) + err = os.WriteFile(filepath.Join(filtersPath, "filters.json"), filters, os.ModePerm) if err != nil { return err } diff --git a/pkg/controllers/backup/upload.go b/pkg/controllers/backup/upload.go index e068680c..78a8e7c7 100644 --- a/pkg/controllers/backup/upload.go +++ b/pkg/controllers/backup/upload.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" "strings" @@ -17,7 +16,7 @@ import ( ) func (h *handler) uploadToS3(backup *v1.Backup, objectStore *v1.S3ObjectStore, tmpBackupPath, gzipFile string) error { - tmpBackupGzipFilepath, err := ioutil.TempDir("", "uploadpath") + tmpBackupGzipFilepath, err := os.MkdirTemp("", "uploadpath") if err != nil { return err } diff --git a/pkg/controllers/restore/download.go b/pkg/controllers/restore/download.go index 9cfe9be4..95b368d2 100644 --- a/pkg/controllers/restore/download.go +++ b/pkg/controllers/restore/download.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "os" "reflect" "strings" @@ -66,7 +65,7 @@ func (h *handler) LoadFromTarGzip(tarGzFilePath string, transformerMap map[schem if tarContent.Typeflag != tar.TypeReg { continue } - readData, err := ioutil.ReadAll(tarball) + readData, err := io.ReadAll(tarball) if err != nil { return err } diff --git a/pkg/crds/crd.go b/pkg/crds/crd.go index d82ac1e8..b04b1156 100644 --- a/pkg/crds/crd.go +++ b/pkg/crds/crd.go @@ -3,7 +3,7 @@ package crds import ( "encoding/json" "fmt" - "io/ioutil" + "os" "strings" resources "github.com/rancher/backup-restore-operator/pkg/apis/resources.cattle.io/v1" @@ -42,7 +42,7 @@ func WriteCRD() error { } filename := fmt.Sprintf("./charts/rancher-backup-crd/templates/%s.yaml", strings.ToLower(crd.Spec.Names.Kind)) - err = ioutil.WriteFile(filename, yamlBytes, 0644) + err = os.WriteFile(filename, yamlBytes, 0644) if err != nil { return err } diff --git a/pkg/objectstore/s3minio.go b/pkg/objectstore/s3minio.go index 243bb3c0..8691149a 100644 --- a/pkg/objectstore/s3minio.go +++ b/pkg/objectstore/s3minio.go @@ -8,7 +8,6 @@ import ( "encoding/pem" "fmt" "io" - "io/ioutil" "net/http" "os" "path" @@ -266,7 +265,7 @@ func readS3EndpointCA(endpointCA string) ([]byte, error) { if err == nil { log.Info("reading s3-endpoint-ca as a base64 string") } else { - ca, err = ioutil.ReadFile(endpointCA) + ca, err = os.ReadFile(endpointCA) log.Infof("reading s3-endpoint-ca from [%v]", endpointCA) } return ca, err diff --git a/pkg/util/util.go b/pkg/util/util.go index a337d996..eeb2ed04 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -3,7 +3,6 @@ package util import ( "context" "fmt" - "io/ioutil" "os" "reflect" @@ -36,7 +35,7 @@ func GetEncryptionTransformers(encryptionConfigSecretName string, secrets v1core if !ok { return nil, fmt.Errorf("no encryptionConfig provided") } - err = ioutil.WriteFile(encryptionProviderConfigKey, encryptionConfigBytes, os.ModePerm) + err = os.WriteFile(encryptionProviderConfigKey, encryptionConfigBytes, os.ModePerm) defer os.Remove(encryptionProviderConfigKey) if err != nil { From 3681ecdcfe360883a505f8231c4b0f5d2bc5855c Mon Sep 17 00:00:00 2001 From: Julia Bier Date: Wed, 16 Oct 2024 17:36:47 -0400 Subject: [PATCH 6/6] refactor empty transformer conditionals --- pkg/controllers/restore/download.go | 4 ++-- pkg/resourcesets/collector.go | 13 ++++++------- pkg/util/util.go | 5 +++++ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/controllers/restore/download.go b/pkg/controllers/restore/download.go index 95b368d2..c003a3a7 100644 --- a/pkg/controllers/restore/download.go +++ b/pkg/controllers/restore/download.go @@ -7,11 +7,11 @@ import ( "fmt" "io" "os" - "reflect" "strings" v1 "github.com/rancher/backup-restore-operator/pkg/apis/resources.cattle.io/v1" "github.com/rancher/backup-restore-operator/pkg/objectstore" + "github.com/rancher/backup-restore-operator/pkg/util" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -108,7 +108,7 @@ func (h *handler) loadDataFromFile(tarContent *tar.Header, readData []byte, var staticTransformers encryptionconfig.StaticTransformers = transformerMap decryptionTransformer := staticTransformers.TransformerForResource(gvr.GroupResource()) - if !reflect.ValueOf(decryptionTransformer).IsZero() { + if decryptionTransformer != nil && !util.IsDefaultEncryptionTransformer(decryptionTransformer) { var encryptedBytes []byte if err := json.Unmarshal(readData, &encryptedBytes); err != nil { logrus.Errorf("Error unmarshaling encrypted data for resource [%v]: %v", gvr.GroupResource(), err) diff --git a/pkg/resourcesets/collector.go b/pkg/resourcesets/collector.go index 00e7019d..512cb63b 100644 --- a/pkg/resourcesets/collector.go +++ b/pkg/resourcesets/collector.go @@ -1,7 +1,6 @@ package resourcesets import ( - "bytes" "context" "encoding/json" "fmt" @@ -11,6 +10,7 @@ import ( "strings" v1 "github.com/rancher/backup-restore-operator/pkg/apis/resources.cattle.io/v1" + "github.com/rancher/backup-restore-operator/pkg/util" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -436,16 +436,15 @@ func writeToBackup(ctx context.Context, resource map[string]interface{}, backupP if err != nil { return fmt.Errorf("error converting resource to JSON: %v", err) } - if transformer != nil { + if transformer != nil && !util.IsDefaultEncryptionTransformer(transformer) { encrypted, err := transformer.TransformToStorage(ctx, resourceBytes, value.DefaultContext(additionalAuthenticatedData)) if err != nil { return fmt.Errorf("error converting resource to JSON: %v", err) } - if !bytes.Equal(resourceBytes, encrypted) { - resourceBytes, err = json.Marshal(encrypted) - if err != nil { - return fmt.Errorf("error converting encrypted resource to JSON: %v", err) - } + + resourceBytes, err = json.Marshal(encrypted) + if err != nil { + return fmt.Errorf("error converting encrypted resource to JSON: %v", err) } } if _, err := f.Write(resourceBytes); err != nil { diff --git a/pkg/util/util.go b/pkg/util/util.go index eeb2ed04..608fb749 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/server/options/encryptionconfig" "k8s.io/apiserver/pkg/storage/value" + "k8s.io/apiserver/pkg/storage/value/encrypt/identity" ) const ( @@ -59,6 +60,10 @@ func GetObjectQueue(l interface{}, capacity int) chan interface{} { return c } +func IsDefaultEncryptionTransformer(transformer value.Transformer) bool { + return transformer == identity.NewEncryptCheckTransformer() +} + func ErrList(e []error) error { if len(e) > 0 { return fmt.Errorf("%v", e)