Replacements will generate string array and object replacements (#481)

Add config key for policyrun rentation time and featuregate

Add CreateOrGetWithRetry method to create resource

Signed-off-by: yuzhipeng <[email protected]>

config/feature_flags.go

@@ -44,6 +44,13 @@ const (
 	// TemplateRenderRetentionTimeKey represent the config key for how long templatrender will remain
 	// retentionTime seems like a better name than delayAfterCompleted so that we use rententionTime here
 	TemplateRenderRetentionTimeKey = "templateRender.retentionTime"
+
+	// PolicyRunRetentionTimeKey represent the config key for how long policyRun will remain
+	PolicyRunRetentionTimeKey = "policyRun.retentionTime"
+
+	// PolicyCheckEnabledFeatureKey indicates the configuration key of the policy check feature gate.
+	// If the value is true, the feature is enabled cluster-wide.
+	PolicyCheckEnabledFeatureKey = "policy.check.enabled"
 )
 
 const (
@@ -75,6 +82,13 @@ const (
 
 	// DefaultTemplateRenderRetentionTime represents default duration how long the templatrender will remain
 	DefaultTemplateRenderRetentionTime FeatureValue = "30m"
+
+	// DefaultPolicyRunRetentionTime represents default duration how long the policyRun will remain
+	DefaultPolicyRunRetentionTime = "30m"
+
+	// DefaultPolicyCheckEnabled indicates the default value of the policy check feature gate.
+	// If the corresponding key does not exist, the default value is returned.
+	DefaultPolicyCheckEnabled FeatureValue = "true"
 )
 
 // defaultFeatureValue defines the default value for the feature switch.
@@ -87,6 +101,8 @@ var defaultFeatureValue = map[string]FeatureValue{
 	BuildMRCheckTimeoutKey:                 DefaultMRCheckTimeout,
 	TemplateRenderCheckTimeoutKey:          DefaultTemplateRenderCheckTimeout,
 	TemplateRenderRetentionTimeKey:         DefaultTemplateRenderRetentionTime,
+	PolicyRunRetentionTimeKey:              DefaultPolicyRunRetentionTime,
+	PolicyCheckEnabledFeatureKey:           DefaultPolicyCheckEnabled,
 }
 
 // FeatureFlags holds the features configurations

controllers/retry.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2023 The Katanomi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/util/retry"
+	"knative.dev/pkg/logging"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// CreateOrGetWithRetry will retry to create source multi times if encounter error
+// however if error is alreadyExist then will get the resource and return it
+func CreateOrGetWithRetry(ctx context.Context, clt client.Client, obj client.Object) error {
+	logger := logging.FromContext(ctx)
+	if clt == nil || obj == nil {
+		return fmt.Errorf("client or obj is nil")
+	}
+
+	createObj := func() error {
+		err := clt.Create(ctx, obj)
+		if errors.IsAlreadyExists(err) {
+			logger.Warnw("obj %s already exists, try to get it", "object", fmt.Sprintf("%s/%s/%s", obj.GetObjectKind(), obj.GetNamespace(), obj.GetName()))
+			return clt.Get(ctx, client.ObjectKeyFromObject(obj), obj)
+		}
+		return err
+	}
+	retriable := func(err error) bool {
+		return err != nil
+	}
+
+	return retry.OnError(retry.DefaultRetry, retriable, createObj)
+}

controllers/retry_test.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2023 The Katanomi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/katanomi/pkg/testing"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+var _ = Describe("CreateOrGetWithRetry", func() {
+
+	var (
+		ctx        context.Context
+		client     client.Client
+		err        error
+		object     *v1.Pod
+		objectList *v1.PodList
+		length     int
+
+		scheme *runtime.Scheme
+	)
+
+	BeforeEach(func() {
+		ctx = context.TODO()
+		scheme = runtime.NewScheme()
+		object = &v1.Pod{}
+		objectList = &v1.PodList{}
+		objectList = &v1.PodList{}
+		v1.AddToScheme(scheme)
+	})
+
+	JustBeforeEach(func() {
+		err = CreateOrGetWithRetry(ctx, client, object)
+		Expect(client.List(ctx, objectList)).To(BeNil())
+		length = len(objectList.Items)
+	})
+
+	When("cluster has no object", func() {
+
+		BeforeEach(func() {
+			testing.MustLoadYaml("testdata/pod.yaml", object)
+			client = fake.NewClientBuilder().WithScheme(scheme).WithObjects().Build()
+		})
+		It("err is nil and pod is 1", func() {
+			Expect(err).To(BeNil())
+			Expect(length).To(Equal(1))
+		})
+	})
+
+	When("cluster has object exist", func() {
+
+		BeforeEach(func() {
+			existObject := &v1.Pod{}
+			testing.MustLoadYaml("testdata/pod.yaml", object)
+			testing.MustLoadYaml("testdata/pod.yaml", existObject)
+			client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(existObject).Build()
+		})
+		It("err is nil and pod is 1", func() {
+			Expect(err).To(BeNil())
+			Expect(length).To(Equal(1))
+		})
+	})
+})

controllers/testdata/pod.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test

tekton/apply.go

@@ -0,0 +1,140 @@
+/*
+Copyright 2023 The Katanomi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tekton
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/tektoncd/pipeline/pkg/apis/config"
+	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
+)
+
+var (
+	paramPatterns = []string{
+		"params.%s",
+		"params[%q]",
+		"params['%s']",
+	}
+)
+
+const (
+	// objectIndividualVariablePattern is the reference pattern for object individual keys params.<object_param_name>.<key_name>
+	objectIndividualVariablePattern = "params.%s.%s"
+)
+
+// reference from  https://github.com/katanomi/pipeline/blob/a9210847e1cb183797379917bec8dfd221450321/pkg/reconciler/pipelinerun/resources/apply.go#L106
+// and do some refactor base that
+
+// Replacements return replacements base on the params spec and provided params values
+func Replacements(ctx context.Context, paramSpecs []v1beta1.ParamSpec, params []v1beta1.Param) (stringReplacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) {
+
+	ctx = config.EnableAlphaAPIFields(ctx)
+
+	strings, arrays, objects := paramDefaultReplacements(ctx, paramSpecs)
+	// Set and overwrite params with the ones from the parameters provided
+	valueStrings, valueArrays, valueObjects := paramValueReplacements(ctx, params)
+
+	for k, v := range valueStrings {
+		strings[k] = v
+	}
+	for k, v := range valueArrays {
+		arrays[k] = v
+	}
+	for k, v := range valueObjects {
+		objects[k] = v
+	}
+
+	return strings, arrays, objects
+}
+
+func paramDefaultReplacements(ctx context.Context, paramSpecs []v1beta1.ParamSpec) (map[string]string, map[string][]string, map[string]map[string]string) {
+
+	cfg := config.FromContextOrDefaults(ctx)
+
+	stringReplacements := map[string]string{}
+	arrayReplacements := map[string][]string{}
+	objectReplacements := map[string]map[string]string{}
+
+	// Set all the default replacements
+	for _, p := range paramSpecs {
+		if p.Default == nil {
+			continue
+		}
+		switch p.Default.Type {
+		case v1beta1.ParamTypeArray:
+			for _, pattern := range paramPatterns {
+				// array indexing for param is alpha feature
+				if cfg.FeatureFlags.EnableAPIFields == config.AlphaAPIFields {
+					for i := 0; i < len(p.Default.ArrayVal); i++ {
+						stringReplacements[fmt.Sprintf(pattern+"[%d]", p.Name, i)] = p.Default.ArrayVal[i]
+					}
+				}
+				arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.ArrayVal
+			}
+		case v1beta1.ParamTypeObject:
+			for _, pattern := range paramPatterns {
+				objectReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.ObjectVal
+			}
+			for k, v := range p.Default.ObjectVal {
+				stringReplacements[fmt.Sprintf(objectIndividualVariablePattern, p.Name, k)] = v
+			}
+		default:
+			for _, pattern := range paramPatterns {
+				stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.StringVal
+			}
+		}
+	}
+	return stringReplacements, arrayReplacements, objectReplacements
+}
+
+func paramValueReplacements(ctx context.Context, params []v1beta1.Param) (map[string]string, map[string][]string, map[string]map[string]string) {
+	// stringReplacements is used for standard single-string stringReplacements,
+	// while arrayReplacements/objectReplacements contains arrays/objects that need to be further processed.
+	stringReplacements := map[string]string{}
+	arrayReplacements := map[string][]string{}
+	objectReplacements := map[string]map[string]string{}
+	cfg := config.FromContextOrDefaults(ctx)
+
+	for _, p := range params {
+		switch p.Value.Type {
+		case v1beta1.ParamTypeArray:
+			for _, pattern := range paramPatterns {
+				// array indexing for param is alpha feature
+				if cfg.FeatureFlags.EnableAPIFields == config.AlphaAPIFields {
+					for i := 0; i < len(p.Value.ArrayVal); i++ {
+						stringReplacements[fmt.Sprintf(pattern+"[%d]", p.Name, i)] = p.Value.ArrayVal[i]
+					}
+				}
+				arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.ArrayVal
+			}
+		case v1beta1.ParamTypeObject:
+			for _, pattern := range paramPatterns {
+				objectReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.ObjectVal
+			}
+			for k, v := range p.Value.ObjectVal {
+				stringReplacements[fmt.Sprintf(objectIndividualVariablePattern, p.Name, k)] = v
+			}
+		default:
+			for _, pattern := range paramPatterns {
+				stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.StringVal
+			}
+		}
+	}
+
+	return stringReplacements, arrayReplacements, objectReplacements
+}