From 2e8cffc0e0add8b4180f148e8f308081ce31dc7b Mon Sep 17 00:00:00 2001 From: ayushsatyam146 Date: Fri, 3 May 2024 17:47:52 +0530 Subject: [PATCH] Add Manifestival Transformer for CRDs Adding a TruncateField transformer and using it to truncate CRD description fields from the kodata/release.yaml manifest. The CRDs for Shipwright Build objects are very large, resulting in a validation error from Kubernetes when we try to create the CRDs with Manifestival. --- controllers/shipwrightbuild_controller.go | 1 + .../testdata/test-truncate-crd-field.yaml | 27 +++++++ pkg/common/util.go | 40 ++++++++++ pkg/common/util_test.go | 73 ++++++++++++++++++- 4 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 pkg/common/testdata/test-truncate-crd-field.yaml diff --git a/controllers/shipwrightbuild_controller.go b/controllers/shipwrightbuild_controller.go index 01c3614c7..560cddc3e 100644 --- a/controllers/shipwrightbuild_controller.go +++ b/controllers/shipwrightbuild_controller.go @@ -171,6 +171,7 @@ func (r *ShipwrightBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ images := common.ToLowerCaseKeys(common.ImagesFromEnv(common.ShipwrightImagePrefix)) transformerfncs := []manifestival.Transformer{} + transformerfncs = append(transformerfncs, common.TruncateCRDFieldTransformer("description", 50)) if common.IsOpenShiftPlatform() { transformerfncs = append(transformerfncs, manifestival.InjectNamespace(targetNamespace)) transformerfncs = append(transformerfncs, common.DeploymentImages(images)) diff --git a/pkg/common/testdata/test-truncate-crd-field.yaml b/pkg/common/testdata/test-truncate-crd-field.yaml new file mode 100644 index 000000000..80a2de3c5 --- /dev/null +++ b/pkg/common/testdata/test-truncate-crd-field.yaml @@ -0,0 +1,27 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: test.crd.com +spec: + group: crd.com + versions: + - name: v1 + served: true + description: This is a long string that should be truncated + storage: true + schema: + openAPIV3Schema: + properties: + description: This is a long string that should be truncated + spec: + properties: + field1: + type: string + description: This is a long string that should be truncated + scope: Namespaced + names: + plural: tests + singular: test + kind: Test + shortNames: + - tst \ No newline at end of file diff --git a/pkg/common/util.go b/pkg/common/util.go index 8683ae204..ad3b9634c 100644 --- a/pkg/common/util.go +++ b/pkg/common/util.go @@ -87,6 +87,46 @@ func ToLowerCaseKeys(keyValues map[string]string) map[string]string { return newMap } +// truncateNestedFields truncates the named "field" from the given data object and all of its sub-objects to maxLength characters. +func truncateNestedFields(data map[string]interface{}, maxLength int, field string) { + queue := []map[string]interface{}{data} + + for len(queue) > 0 { + curr := queue[0] + queue = queue[1:] + + for key, value := range curr { + if key == field { + if str, ok := value.(string); ok && len(str) > maxLength { + curr[key] = str[:maxLength] + } + } else { + if subObj, ok := value.(map[string]interface{}); ok { + queue = append(queue, subObj) + } else if subObjs, ok := value.([]interface{}); ok { + for _, subObj := range subObjs { + if subObjMap, ok := subObj.(map[string]interface{}); ok { + queue = append(queue, subObjMap) + } + } + } + } + } + } +} + +// TruncateCRDFieldTransformer returns a manifestival.Transformer that truncates the value of the given field within a CRD spec to the provided max length. +func TruncateCRDFieldTransformer(field string, maxLength int) manifestival.Transformer { + return func(u *unstructured.Unstructured) error { + if u.GetKind() != "CustomResourceDefinition" { + return nil + } + data := u.Object + truncateNestedFields(data, maxLength, field) + return nil + } +} + // deploymentImages replaces container and env vars images. func DeploymentImages(images map[string]string) manifestival.Transformer { return func(u *unstructured.Unstructured) error { diff --git a/pkg/common/util_test.go b/pkg/common/util_test.go index e1cb515f5..8a26a7c43 100644 --- a/pkg/common/util_test.go +++ b/pkg/common/util_test.go @@ -1,13 +1,14 @@ package common import ( + "os" "path" "testing" mf "github.com/manifestival/manifestival" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - + "gopkg.in/yaml.v2" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -62,6 +63,76 @@ func TestDeploymentImages(t *testing.T) { }) } +func TestTruncateNestedFields(t *testing.T) { + RegisterFailHandler(Fail) + t.Run("test truncation of manifests", func(t *testing.T) { + testData := map[string]interface{}{ + "field1": "This is a long string that should be truncated", + "field2": map[string]interface{}{ + "field1": "This is another long string that should be truncated", + }, + } + + expected := map[string]interface{}{ + "field1": "This is a ", + "field2": map[string]interface{}{ + "field1": "This is an", + }, + } + + truncateNestedFields(testData, 10, "field1") + Expect(testData).To(Equal(expected)) + }) +} + +func CheckNestedFieldLengthWithinLimit(data map[string]interface{}, maxLength int, field string) bool { + isFieldSizeInLimit := true + queue := []map[string]interface{}{data} + + for len(queue) > 0 { + curr := queue[0] + queue = queue[1:] + + for key, value := range curr { + if key == field { + if str, ok := value.(string); ok { + isFieldSizeInLimit = isFieldSizeInLimit && (len(str) <= maxLength) + } + } else { + if subObj, ok := value.(map[string]interface{}); ok { + queue = append(queue, subObj) + } else if subObjs, ok := value.([]interface{}); ok { + for _, subObj := range subObjs { + if subObjMap, ok := subObj.(map[string]interface{}); ok { + queue = append(queue, subObjMap) + } + } + } + } + } + } + + return isFieldSizeInLimit +} + +func TestTruncateCRDFieldTransformer(t *testing.T) { + RegisterFailHandler(Fail) + t.Run("test truncate CRD field Transformer", func(t *testing.T) { + testData, err := os.ReadFile(path.Join("testdata", "test-truncate-crd-field.yaml")) + Expect(err).NotTo(HaveOccurred()) + + u := &unstructured.Unstructured{} + err = yaml.Unmarshal(testData, u) + Expect(err).NotTo(HaveOccurred()) + + transformFunc := TruncateCRDFieldTransformer("description", 10) + err = transformFunc(u) + Expect(err).NotTo(HaveOccurred(),"failed to transform CRD field") + isDescriptionTruncated := CheckNestedFieldLengthWithinLimit(u.Object, 10, "description") + Expect(isDescriptionTruncated).To(Equal(true)) + }) +} + func deploymentFor(t *testing.T, unstr unstructured.Unstructured) *appsv1.Deployment { deployment := &appsv1.Deployment{} err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstr.Object, deployment)