diff --git a/config/feature_flags.go b/config/feature_flags.go index fb10710d..54138d98 100644 --- a/config/feature_flags.go +++ b/config/feature_flags.go @@ -44,6 +44,13 @@ const ( // TemplateRenderRetentionTimeKey represent the config key for how long templatrender will remain // retentionTime seems like a better name than delayAfterCompleted so that we use rententionTime here TemplateRenderRetentionTimeKey = "templateRender.retentionTime" + + // PolicyRunRetentionTimeKey represent the config key for how long policyRun will remain + PolicyRunRetentionTimeKey = "policyRun.retentionTime" + + // PolicyCheckEnabledFeatureKey indicates the configuration key of the policy check feature gate. + // If the value is true, the feature is enabled cluster-wide. + PolicyCheckEnabledFeatureKey = "policy.check.enabled" ) const ( @@ -75,6 +82,13 @@ const ( // DefaultTemplateRenderRetentionTime represents default duration how long the templatrender will remain DefaultTemplateRenderRetentionTime FeatureValue = "30m" + + // DefaultPolicyRunRetentionTime represents default duration how long the policyRun will remain + DefaultPolicyRunRetentionTime = "30m" + + // DefaultPolicyCheckEnabled indicates the default value of the policy check feature gate. + // If the corresponding key does not exist, the default value is returned. + DefaultPolicyCheckEnabled FeatureValue = "true" ) // defaultFeatureValue defines the default value for the feature switch. @@ -87,6 +101,8 @@ var defaultFeatureValue = map[string]FeatureValue{ BuildMRCheckTimeoutKey: DefaultMRCheckTimeout, TemplateRenderCheckTimeoutKey: DefaultTemplateRenderCheckTimeout, TemplateRenderRetentionTimeKey: DefaultTemplateRenderRetentionTime, + PolicyRunRetentionTimeKey: DefaultPolicyRunRetentionTime, + PolicyCheckEnabledFeatureKey: DefaultPolicyCheckEnabled, } // FeatureFlags holds the features configurations diff --git a/controllers/retry.go b/controllers/retry.go new file mode 100644 index 00000000..bd3f843a --- /dev/null +++ b/controllers/retry.go @@ -0,0 +1,50 @@ +/* +Copyright 2023 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/util/retry" + "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// CreateOrGetWithRetry will retry to create source multi times if encounter error +// however if error is alreadyExist then will get the resource and return it +func CreateOrGetWithRetry(ctx context.Context, clt client.Client, obj client.Object) error { + logger := logging.FromContext(ctx) + if clt == nil || obj == nil { + return fmt.Errorf("client or obj is nil") + } + + createObj := func() error { + err := clt.Create(ctx, obj) + if errors.IsAlreadyExists(err) { + logger.Warnw("obj %s already exists, try to get it", "object", fmt.Sprintf("%s/%s/%s", obj.GetObjectKind(), obj.GetNamespace(), obj.GetName())) + return clt.Get(ctx, client.ObjectKeyFromObject(obj), obj) + } + return err + } + retriable := func(err error) bool { + return err != nil + } + + return retry.OnError(retry.DefaultRetry, retriable, createObj) +} diff --git a/controllers/retry_test.go b/controllers/retry_test.go new file mode 100644 index 00000000..b64067c2 --- /dev/null +++ b/controllers/retry_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2023 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + + "github.com/katanomi/pkg/testing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var _ = Describe("CreateOrGetWithRetry", func() { + + var ( + ctx context.Context + client client.Client + err error + object *v1.Pod + objectList *v1.PodList + length int + + scheme *runtime.Scheme + ) + + BeforeEach(func() { + ctx = context.TODO() + scheme = runtime.NewScheme() + object = &v1.Pod{} + objectList = &v1.PodList{} + objectList = &v1.PodList{} + v1.AddToScheme(scheme) + }) + + JustBeforeEach(func() { + err = CreateOrGetWithRetry(ctx, client, object) + Expect(client.List(ctx, objectList)).To(BeNil()) + length = len(objectList.Items) + }) + + When("cluster has no object", func() { + + BeforeEach(func() { + testing.MustLoadYaml("testdata/pod.yaml", object) + client = fake.NewClientBuilder().WithScheme(scheme).WithObjects().Build() + }) + It("err is nil and pod is 1", func() { + Expect(err).To(BeNil()) + Expect(length).To(Equal(1)) + }) + }) + + When("cluster has object exist", func() { + + BeforeEach(func() { + existObject := &v1.Pod{} + testing.MustLoadYaml("testdata/pod.yaml", object) + testing.MustLoadYaml("testdata/pod.yaml", existObject) + client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(existObject).Build() + }) + It("err is nil and pod is 1", func() { + Expect(err).To(BeNil()) + Expect(length).To(Equal(1)) + }) + }) +}) diff --git a/controllers/testdata/pod.yaml b/controllers/testdata/pod.yaml new file mode 100644 index 00000000..114f6238 --- /dev/null +++ b/controllers/testdata/pod.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test diff --git a/tekton/apply.go b/tekton/apply.go new file mode 100644 index 00000000..8f00b6d9 --- /dev/null +++ b/tekton/apply.go @@ -0,0 +1,140 @@ +/* +Copyright 2023 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tekton + +import ( + "context" + "fmt" + + "github.com/tektoncd/pipeline/pkg/apis/config" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" +) + +var ( + paramPatterns = []string{ + "params.%s", + "params[%q]", + "params['%s']", + } +) + +const ( + // objectIndividualVariablePattern is the reference pattern for object individual keys params.. + objectIndividualVariablePattern = "params.%s.%s" +) + +// reference from https://github.com/katanomi/pipeline/blob/a9210847e1cb183797379917bec8dfd221450321/pkg/reconciler/pipelinerun/resources/apply.go#L106 +// and do some refactor base that + +// Replacements return replacements base on the params spec and provided params values +func Replacements(ctx context.Context, paramSpecs []v1beta1.ParamSpec, params []v1beta1.Param) (stringReplacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) { + + ctx = config.EnableAlphaAPIFields(ctx) + + strings, arrays, objects := paramDefaultReplacements(ctx, paramSpecs) + // Set and overwrite params with the ones from the parameters provided + valueStrings, valueArrays, valueObjects := paramValueReplacements(ctx, params) + + for k, v := range valueStrings { + strings[k] = v + } + for k, v := range valueArrays { + arrays[k] = v + } + for k, v := range valueObjects { + objects[k] = v + } + + return strings, arrays, objects +} + +func paramDefaultReplacements(ctx context.Context, paramSpecs []v1beta1.ParamSpec) (map[string]string, map[string][]string, map[string]map[string]string) { + + cfg := config.FromContextOrDefaults(ctx) + + stringReplacements := map[string]string{} + arrayReplacements := map[string][]string{} + objectReplacements := map[string]map[string]string{} + + // Set all the default replacements + for _, p := range paramSpecs { + if p.Default == nil { + continue + } + switch p.Default.Type { + case v1beta1.ParamTypeArray: + for _, pattern := range paramPatterns { + // array indexing for param is alpha feature + if cfg.FeatureFlags.EnableAPIFields == config.AlphaAPIFields { + for i := 0; i < len(p.Default.ArrayVal); i++ { + stringReplacements[fmt.Sprintf(pattern+"[%d]", p.Name, i)] = p.Default.ArrayVal[i] + } + } + arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.ArrayVal + } + case v1beta1.ParamTypeObject: + for _, pattern := range paramPatterns { + objectReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.ObjectVal + } + for k, v := range p.Default.ObjectVal { + stringReplacements[fmt.Sprintf(objectIndividualVariablePattern, p.Name, k)] = v + } + default: + for _, pattern := range paramPatterns { + stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.StringVal + } + } + } + return stringReplacements, arrayReplacements, objectReplacements +} + +func paramValueReplacements(ctx context.Context, params []v1beta1.Param) (map[string]string, map[string][]string, map[string]map[string]string) { + // stringReplacements is used for standard single-string stringReplacements, + // while arrayReplacements/objectReplacements contains arrays/objects that need to be further processed. + stringReplacements := map[string]string{} + arrayReplacements := map[string][]string{} + objectReplacements := map[string]map[string]string{} + cfg := config.FromContextOrDefaults(ctx) + + for _, p := range params { + switch p.Value.Type { + case v1beta1.ParamTypeArray: + for _, pattern := range paramPatterns { + // array indexing for param is alpha feature + if cfg.FeatureFlags.EnableAPIFields == config.AlphaAPIFields { + for i := 0; i < len(p.Value.ArrayVal); i++ { + stringReplacements[fmt.Sprintf(pattern+"[%d]", p.Name, i)] = p.Value.ArrayVal[i] + } + } + arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.ArrayVal + } + case v1beta1.ParamTypeObject: + for _, pattern := range paramPatterns { + objectReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.ObjectVal + } + for k, v := range p.Value.ObjectVal { + stringReplacements[fmt.Sprintf(objectIndividualVariablePattern, p.Name, k)] = v + } + default: + for _, pattern := range paramPatterns { + stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.StringVal + } + } + } + + return stringReplacements, arrayReplacements, objectReplacements +} diff --git a/tekton/apply_test.go b/tekton/apply_test.go new file mode 100644 index 00000000..fbe0a976 --- /dev/null +++ b/tekton/apply_test.go @@ -0,0 +1,165 @@ +/* +Copyright 2023 The Katanomi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tekton + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" +) + +func Test_IsClustertemplate(t *testing.T) { + RegisterTestingT(t) + + expectObjects := map[string]map[string]string{ + "params.Object": { + "object1": "{abc:def}", + }, + "params[\"Object\"]": { + "object1": "{abc:def}", + }, + "params['Object']": { + "object1": "{abc:def}", + }, + } + expectArrays := map[string][]string{ + "params[\"Array\"]": {"array1", "array2"}, + "params['Array']": {"array1", "array2"}, + "params.Array": {"array1", "array2"}, + } + expectStrings := map[string]string{ + "params.String": "string", + "params[\"String\"]": "string", + "params['String']": "string", + + "params[\"Array\"][0]": "array1", + "params.Array[0]": "array1", + "params['Array'][0]": "array1", + + "params.Array[1]": "array2", + "params['Array'][1]": "array2", + "params[\"Array\"][1]": "array2", + + "params.Object.object1": "{abc:def}", + } + + for _, c := range []struct { + description string + ctx context.Context + paramSpec []v1beta1.ParamSpec + params []v1beta1.Param + expectStrings map[string]string + expectArrays map[string][]string + expectObjects map[string]map[string]string + }{ + { + description: "empty", + ctx: context.Background(), + paramSpec: []v1beta1.ParamSpec{}, + params: []v1beta1.Param{}, + expectObjects: map[string]map[string]string{}, + expectArrays: map[string][]string{}, + expectStrings: map[string]string{}, + }, + { + description: "use default strings arrays and objects", + ctx: context.Background(), + paramSpec: []v1beta1.ParamSpec{ + { + Name: "String", + Type: v1beta1.ParamTypeString, + Default: &v1beta1.ParamValue{ + Type: v1beta1.ParamTypeString, + StringVal: "string", + }, + }, + { + Name: "Array", + Type: v1beta1.ParamTypeArray, + Default: &v1beta1.ParamValue{ + Type: v1beta1.ParamTypeArray, + ArrayVal: []string{"array1", "array2"}, + }, + }, + { + Name: "Object", + Type: v1beta1.ParamTypeArray, + Default: &v1beta1.ParamValue{ + Type: v1beta1.ParamTypeObject, + ObjectVal: map[string]string{"object1": "{abc:def}"}, + }, + }, + }, + params: []v1beta1.Param{}, + expectObjects: expectObjects, + expectArrays: expectArrays, + expectStrings: expectStrings, + }, + + { + description: "use default and value strings arrays and objects value will override default", + ctx: context.Background(), + paramSpec: []v1beta1.ParamSpec{ + { + Name: "String", + Type: v1beta1.ParamTypeString, + Default: &v1beta1.ParamValue{ + Type: v1beta1.ParamTypeString, + StringVal: "default", + }, + }, + { + Name: "Object", + Type: v1beta1.ParamTypeArray, + Default: &v1beta1.ParamValue{ + Type: v1beta1.ParamTypeObject, + ObjectVal: map[string]string{"object1": "{abc:def}"}, + }, + }, + }, + params: []v1beta1.Param{ + { + Name: "String", + Value: v1beta1.ParamValue{ + Type: v1beta1.ParamTypeString, + StringVal: "string", + }, + }, + { + Name: "Array", + Value: v1beta1.ParamValue{ + Type: v1beta1.ParamTypeArray, + ArrayVal: []string{"array1", "array2"}, + }, + }, + }, + expectObjects: expectObjects, + expectArrays: expectArrays, + expectStrings: expectStrings, + }, + } { + + t.Logf("<=== starting %s...", c.description) + strings, arrays, objects := Replacements(c.ctx, c.paramSpec, c.params) + Expect(strings).To(Equal(c.expectStrings)) + Expect(arrays).To(Equal(c.expectArrays)) + Expect(objects).To(Equal(c.expectObjects)) + t.Logf("===> passed %s...", c.description) + } +}