From cfbc2dbc7791cb0ed8ade9e14e72fc144feb303f Mon Sep 17 00:00:00 2001 From: Yongxuan Zhang Date: Sat, 21 Oct 2023 17:24:56 +0000 Subject: [PATCH] [TEP-0145] Add CEL evaluation This commit adds CEL evaluation. Users are able to use CEL in WhenExpression if the feature flag enable-cel-in-whenexpression is enabled.If the evluation is false, the PipelineTask will be skipped. Signed-off-by: Yongxuan Zhang yongxuanzhang@google.com --- config/config-feature-flags.yaml | 1 - docs/pipelines.md | 67 +++++- ...pipelinerun-with-cel-when-expressions.yaml | 154 +++++++++++++ pkg/apis/pipeline/v1/pipelinerun_types.go | 2 + pkg/apis/pipeline/v1/when_types.go | 4 +- pkg/apis/pipeline/v1beta1/when_types.go | 4 +- pkg/reconciler/pipelinerun/pipelinerun.go | 10 + .../pipelinerun/pipelinerun_test.go | 204 ++++++++++++++++++ .../resources/pipelinerunresolution.go | 62 ++++++ .../resources/pipelinerunresolution_test.go | 178 +++++++++++++++ test/custom_task_test.go | 7 +- test/e2e-tests.sh | 6 + 12 files changed, 690 insertions(+), 9 deletions(-) create mode 100644 examples/v1/pipelineruns/alpha/pipelinerun-with-cel-when-expressions.yaml diff --git a/config/config-feature-flags.yaml b/config/config-feature-flags.yaml index 50eaa68519b..436d72e833d 100644 --- a/config/config-feature-flags.yaml +++ b/config/config-feature-flags.yaml @@ -122,5 +122,4 @@ data: # allowing examination of the logs on the pods from cancelled taskruns keep-pod-on-cancel: "false" # Setting this flag to "true" will enable the CEL evaluation in WhenExpression - # This feature is in preview mode and not implemented yet. Please check #7244 for the updates. enable-cel-in-whenexpression: "false" diff --git a/docs/pipelines.md b/docs/pipelines.md index 300835e9ddf..a8eeacc8341 100644 --- a/docs/pipelines.md +++ b/docs/pipelines.md @@ -779,9 +779,70 @@ There are a lot of scenarios where `when` expressions can be really useful. Some > :seedling: **`CEL in WhenExpression` is an [alpha](install.md#alpha-features) feature.** > The `enable-cel-in-whenexpression` feature flag must be set to `"true"` to enable the use of `CEL` in `WhenExpression`. -> -> :warning: This feature is in a preview mode. -> It is still in a very early stage of development and is not yet fully functional + +CEL (Common Expression Language) is a declarative language designed for simplicity, speed, safety, and portability which can be used to express a wide variety of conditions and computations. + +You can define a CEL expression in `WhenExpression` to guard the execution of a `Task`. You can use a single line of CEL string to replace current `WhenExpressions`'s `input`+`operator`+`values`. For example: + +```yaml +# current WhenExpressions +when: + - input: "foo" + operator: "in" + values: ["foo", "bar"] + - input: "duh" + operator: "notin" + values: ["foo", "bar"] + +# with cel +when: + - cel: "'foo' in ['foo', 'bar']" + - cel: "!('duh' in ['foo', 'bar'])" +``` + +CEL can offer more conditional functions, such as numeric comparisons (e.g. `>`, `<=`, etc), logic operators (e.g. `OR`, `AND`), Regex Pattern Matching. For example: + +```yaml + when: + # test coverage result is larger than 90% + - cel: "'$(tasks.unit-test.results.test-coverage)' > 0.9" + # params is not empty, or params2 is 8.5 or 8.6 + - cel: "'$(params.param1)' != '' || '$(params.param2)' == '8.5' || '$(params.param2)' == '8.6'" + # param branch matches pattern `release/.*` + - cel: "'$(params.branch)'.matches('release/.*')" +``` + +##### Variable substitution in CEL + +`CEL` supports [string substitutions](https://github.com/tektoncd/pipeline/blob/main/docs/variables.md#variables-available-in-a-pipeline), you can reference string, array indexing or object value of a param/result. For example: + +```yaml + when: + # string result + - cel: "$(tasks.unit-test.results.test-coverage) > 0.9" + # array indexing result + - cel: "$(tasks.unit-test.results.test-coverage[0]) > 0.9" + # object result key + - cel: "'$(tasks.objectTask.results.repo.url)'.matches('github.com/tektoncd/.*')" + # string param + - cel: "'$(params.foo)' == 'foo'" + # array indexing + - cel: "'$(params.branch[0])' == 'foo'" + # object param key + - cel: "'$(params.repo.url)'.matches('github.com/tektoncd/.*')" +``` + +**Note:** the reference needs to be wrapped with single quotes. +Whole `Array` and `Object` replacements are not supported yet. The following usage is not supported: + +```yaml + when: + - cel: "'foo' in '$(params.array_params[*]']" + - cel: "'foo' in '$(params.object_params[*]']" +``` + +In addition to the custom function extension listed above, you can craft any valid CEL expression as defined by the [cel-spec language definition](https://github.com/google/cel-spec/blob/master/doc/langdef.md) + #### Guarding a `Task` and its dependent `Tasks` diff --git a/examples/v1/pipelineruns/alpha/pipelinerun-with-cel-when-expressions.yaml b/examples/v1/pipelineruns/alpha/pipelinerun-with-cel-when-expressions.yaml new file mode 100644 index 00000000000..2c91a6c3251 --- /dev/null +++ b/examples/v1/pipelineruns/alpha/pipelinerun-with-cel-when-expressions.yaml @@ -0,0 +1,154 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: guarded-pr-by-cel- +spec: + pipelineSpec: + params: + - name: path + type: string + description: The path of the file to be created + workspaces: + - name: source + description: | + This workspace is shared among all the pipeline tasks to read/write common resources + tasks: + - name: create-file # when expression using parameter, evaluates to true + when: + - cel: "'$(params.path)' == 'README.md'" + workspaces: + - name: source + workspace: source + taskSpec: + workspaces: + - name: source + description: The workspace to create the readme file in + steps: + - name: write-new-stuff + image: ubuntu + script: 'touch $(workspaces.source.path)/README.md' + - name: check-file + params: + - name: path + value: "$(params.path)" + workspaces: + - name: source + workspace: source + runAfter: + - create-file + taskSpec: + params: + - name: path + workspaces: + - name: source + description: The workspace to check for the file + results: + - name: exists + description: indicates whether the file exists or is missing + steps: + - name: check-file + image: alpine + script: | + if test -f $(workspaces.source.path)/$(params.path); then + printf yes | tee $(results.exists.path) + else + printf no | tee $(results.exists.path) + fi + - name: echo-file-exists # when expression using task result, evaluates to true + when: + - cel: "'$(tasks.check-file.results.exists)' == 'yes'" + taskSpec: + steps: + - name: echo + image: ubuntu + script: 'echo file exists' + - name: task-should-be-skipped-1 + when: + - cel: "'$(tasks.check-file.results.exists)'=='missing'" # when expression using task result, evaluates to false + taskSpec: + steps: + - name: echo + image: ubuntu + script: exit 1 + - name: task-should-be-skipped-2 # when expression using parameter, evaluates to false + when: + - cel: "'$(params.path)'!='README.md'" + taskSpec: + steps: + - name: echo + image: ubuntu + script: exit 1 + - name: task-should-be-skipped-3 # task with when expression and run after + runAfter: + - echo-file-exists + when: + - cel: "'monday'=='friday'" + taskSpec: + steps: + - name: echo + image: ubuntu + script: exit 1 + finally: + - name: finally-task-should-be-skipped-1 # when expression using execution status, evaluates to false + when: + - cel: "'$(tasks.echo-file-exists.status)'=='Failure'" + taskSpec: + steps: + - name: echo + image: ubuntu + script: exit 1 + - name: finally-task-should-be-skipped-2 # when expression using task result, evaluates to false + when: + - cel: "'$(tasks.check-file.results.exists)'=='missing'" + taskSpec: + steps: + - name: echo + image: ubuntu + script: exit 1 + - name: finally-task-should-be-skipped-3 # when expression using parameter, evaluates to false + when: + - cel: "'$(params.path)'!='README.md'" + taskSpec: + steps: + - name: echo + image: ubuntu + script: exit 1 + - name: finally-task-should-be-skipped-4 # when expression using tasks execution status, evaluates to false + when: + - cel: "'$(tasks.status)'=='Failure'" + taskSpec: + steps: + - name: echo + image: ubuntu + script: exit 1 + - name: finally-task-should-be-skipped-5 # when expression using tasks execution status, evaluates to false + when: + - cel: "'$(tasks.status)'=='Succeeded'" + taskSpec: + steps: + - name: echo + image: ubuntu + script: exit 1 + - name: finally-task-should-be-executed # when expression using execution status, tasks execution status, param, and results + when: + - cel: "'$(tasks.echo-file-exists.status)'=='Succeeded'" + - cel: "'$(tasks.status)'=='Completed'" + - cel: "'$(tasks.check-file.results.exists)'=='yes'" + - cel: "'$(params.path)'=='README.md'" + taskSpec: + steps: + - name: echo + image: ubuntu + script: 'echo finally done' + params: + - name: path + value: README.md + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 16Mi diff --git a/pkg/apis/pipeline/v1/pipelinerun_types.go b/pkg/apis/pipeline/v1/pipelinerun_types.go index fda0168c09b..88aad636b89 100644 --- a/pkg/apis/pipeline/v1/pipelinerun_types.go +++ b/pkg/apis/pipeline/v1/pipelinerun_types.go @@ -407,6 +407,8 @@ const ( PipelineRunReasonResourceVerificationFailed PipelineRunReason = "ResourceVerificationFailed" // ReasonCreateRunFailed indicates that the pipeline fails to create the taskrun or other run resources PipelineRunReasonCreateRunFailed PipelineRunReason = "CreateRunFailed" + // ReasonCELEvaluationFailed indicates the pipeline fails the CEL evaluation + PipelineRunReasonCELEvaluationFailed PipelineRunReason = "CELEvaluationFailed" ) func (t PipelineRunReason) String() string { diff --git a/pkg/apis/pipeline/v1/when_types.go b/pkg/apis/pipeline/v1/when_types.go index 45a8bdbd98c..7fd81bc6aa2 100644 --- a/pkg/apis/pipeline/v1/when_types.go +++ b/pkg/apis/pipeline/v1/when_types.go @@ -63,6 +63,7 @@ func (we *WhenExpression) isTrue() bool { func (we *WhenExpression) applyReplacements(replacements map[string]string, arrayReplacements map[string][]string) WhenExpression { replacedInput := substitution.ApplyReplacements(we.Input, replacements) + replacedCEL := substitution.ApplyReplacements(we.CEL, replacements) var replacedValues []string for _, val := range we.Values { @@ -79,13 +80,14 @@ func (we *WhenExpression) applyReplacements(replacements map[string]string, arra } } - return WhenExpression{Input: replacedInput, Operator: we.Operator, Values: replacedValues} + return WhenExpression{Input: replacedInput, Operator: we.Operator, Values: replacedValues, CEL: replacedCEL} } // GetVarSubstitutionExpressions extracts all the values between "$(" and ")" in a When Expression func (we *WhenExpression) GetVarSubstitutionExpressions() ([]string, bool) { var allExpressions []string allExpressions = append(allExpressions, validateString(we.Input)...) + allExpressions = append(allExpressions, validateString(we.CEL)...) for _, value := range we.Values { allExpressions = append(allExpressions, validateString(value)...) } diff --git a/pkg/apis/pipeline/v1beta1/when_types.go b/pkg/apis/pipeline/v1beta1/when_types.go index 76a78ea0ea1..764acf98f59 100644 --- a/pkg/apis/pipeline/v1beta1/when_types.go +++ b/pkg/apis/pipeline/v1beta1/when_types.go @@ -63,6 +63,7 @@ func (we *WhenExpression) isTrue() bool { func (we *WhenExpression) applyReplacements(replacements map[string]string, arrayReplacements map[string][]string) WhenExpression { replacedInput := substitution.ApplyReplacements(we.Input, replacements) + replacedCEL := substitution.ApplyReplacements(we.CEL, replacements) var replacedValues []string for _, val := range we.Values { @@ -79,13 +80,14 @@ func (we *WhenExpression) applyReplacements(replacements map[string]string, arra } } - return WhenExpression{Input: replacedInput, Operator: we.Operator, Values: replacedValues} + return WhenExpression{Input: replacedInput, Operator: we.Operator, Values: replacedValues, CEL: replacedCEL} } // GetVarSubstitutionExpressions extracts all the values between "$(" and ")" in a When Expression func (we *WhenExpression) GetVarSubstitutionExpressions() ([]string, bool) { var allExpressions []string allExpressions = append(allExpressions, validateString(we.Input)...) + allExpressions = append(allExpressions, validateString(we.CEL)...) for _, value := range we.Values { allExpressions = append(allExpressions, validateString(value)...) } diff --git a/pkg/reconciler/pipelinerun/pipelinerun.go b/pkg/reconciler/pipelinerun/pipelinerun.go index 90a61b73420..ba861be168d 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun.go +++ b/pkg/reconciler/pipelinerun/pipelinerun.go @@ -626,6 +626,16 @@ func (c *Reconciler) reconcile(ctx context.Context, pr *v1.PipelineRun, getPipel } } + // Evaluate the CEL of PipelineTask after the variable substitutions and validations. + for _, rpt := range pipelineRunFacts.State { + err := rpt.EvaluateCEL() + if err != nil { + logger.Errorf("Error evaluating CEL %s: %v", pr.Name, err) + pr.Status.MarkFailed(string(v1.PipelineRunReasonCELEvaluationFailed), err.Error()) + return controller.NewPermanentError(err) + } + } + // check if pipeline run is not gracefully cancelled and there are active task runs, which require cancelling if pr.IsGracefullyCancelled() && pipelineRunFacts.IsRunning() { // If the pipelinerun is cancelled, cancel tasks, but run finally diff --git a/pkg/reconciler/pipelinerun/pipelinerun_test.go b/pkg/reconciler/pipelinerun/pipelinerun_test.go index e118173020c..d727a4c8aa4 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun_test.go +++ b/pkg/reconciler/pipelinerun/pipelinerun_test.go @@ -4023,6 +4023,210 @@ status: } } +func TestReconcileWithCELWhenExpressionsWithTaskResultsAndParams(t *testing.T) { + ps := []*v1.Pipeline{parse.MustParseV1Pipeline(t, ` +metadata: + name: test-pipeline + namespace: foo +spec: + params: + - name: run + type: string + tasks: + - name: a-task + taskRef: + name: a-task + - name: b-task + taskRef: + name: b-task + when: + - cel: "'$(tasks.a-task.results.aResult)' == 'aResultValue'" + - name: c-task + taskRef: + name: c-task + when: + - cel: "'$(tasks.a-task.results.aResult)' == 'missing'" + - cel: "'$(params.run)'!='yes'" + - name: d-task + runAfter: + - c-task + taskRef: + name: d-task +`)} + prs := []*v1.PipelineRun{parse.MustParseV1PipelineRun(t, ` +metadata: + name: test-pipeline-run-different-service-accs + namespace: foo +spec: + params: + - name: run + value: "yes" + pipelineRef: + name: test-pipeline + taskRunTemplate: + serviceAccountName: test-sa-0 +`)} + ts := []*v1.Task{ + {ObjectMeta: baseObjectMeta("a-task", "foo")}, + {ObjectMeta: baseObjectMeta("b-task", "foo")}, + {ObjectMeta: baseObjectMeta("c-task", "foo")}, + {ObjectMeta: baseObjectMeta("d-task", "foo")}, + } + trs := []*v1.TaskRun{mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("test-pipeline-run-different-service-accs-a-task-xxyyy", "foo", "test-pipeline-run-different-service-accs", + "test-pipeline", "a-task", true), + ` +spec: + serviceAccountName: test-sa + taskRef: + name: hello-world + timeout: 1h0m0s +status: + conditions: + - status: "True" + type: Succeeded + results: + - name: aResult + value: aResultValue +`)} + cms := []*corev1.ConfigMap{ + { + ObjectMeta: metav1.ObjectMeta{Name: config.GetFeatureFlagsConfigName(), Namespace: system.Namespace()}, + Data: map[string]string{ + "enable-cel-in-whenexpression": "true", + }, + }, + } + d := test.Data{ + PipelineRuns: prs, + Pipelines: ps, + Tasks: ts, + TaskRuns: trs, + ConfigMaps: cms, + } + prt := newPipelineRunTest(t, d) + defer prt.Cancel() + + wantEvents := []string{ + "Normal Started", + "Normal Running Tasks Completed: 1 \\(Failed: 0, Cancelled 0\\), Incomplete: 2, Skipped: 1", + } + pipelineRun, clients := prt.reconcileRun("foo", "test-pipeline-run-different-service-accs", wantEvents, false) + + expectedTaskRunName := "test-pipeline-run-different-service-accs-b-task" + expectedTaskRun := mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta(expectedTaskRunName, "foo", "test-pipeline-run-different-service-accs", "test-pipeline", "b-task", false), + ` +spec: + serviceAccountName: test-sa-0 + taskRef: + name: b-task + kind: Task +`) + // Check that the expected TaskRun was created + actual, err := clients.Pipeline.TektonV1().TaskRuns("foo").List(prt.TestAssets.Ctx, metav1.ListOptions{ + LabelSelector: "tekton.dev/pipelineTask=b-task,tekton.dev/pipelineRun=test-pipeline-run-different-service-accs", + Limit: 1, + }) + + if err != nil { + t.Fatalf("Failure to list TaskRun's %s", err) + } + if len(actual.Items) != 1 { + t.Fatalf("Expected 1 TaskRuns got %d", len(actual.Items)) + } + actualTaskRun := actual.Items[0] + if d := cmp.Diff(expectedTaskRun, &actualTaskRun, ignoreResourceVersion, ignoreTypeMeta); d != "" { + t.Errorf("expected to see TaskRun %v created. Diff %s", expectedTaskRunName, diff.PrintWantGot(d)) + } + + expectedWhenExpressionsInTaskRun := []v1.WhenExpression{{ + CEL: "'aResultValue' == 'aResultValue'", + }} + verifyTaskRunStatusesWhenExpressions(t, pipelineRun.Status, expectedTaskRunName, expectedWhenExpressionsInTaskRun) + + actualSkippedTasks := pipelineRun.Status.SkippedTasks + expectedSkippedTasks := []v1.SkippedTask{{ + Name: "c-task", + Reason: v1.WhenExpressionsSkip, + WhenExpressions: v1.WhenExpressions{{ + CEL: "'aResultValue' == 'missing'", + }, { + CEL: "'yes'!='yes'", + }}, + }} + if d := cmp.Diff(expectedSkippedTasks, actualSkippedTasks); d != "" { + t.Errorf("expected to find Skipped Tasks %v. Diff %s", expectedSkippedTasks, diff.PrintWantGot(d)) + } + + skippedTasks := []string{"c-task"} + for _, skippedTask := range skippedTasks { + labelSelector := fmt.Sprintf("tekton.dev/pipelineTask=%s,tekton.dev/pipelineRun=test-pipeline-run-different-service-accs", skippedTask) + actualSkippedTask, err := clients.Pipeline.TektonV1().TaskRuns("foo").List(prt.TestAssets.Ctx, metav1.ListOptions{ + LabelSelector: labelSelector, + Limit: 1, + }) + if err != nil { + t.Fatalf("Failure to list TaskRun's %s", err) + } + if len(actualSkippedTask.Items) != 0 { + t.Fatalf("Expected 0 TaskRuns got %d", len(actualSkippedTask.Items)) + } + } +} + +func TestReconcile_InvalidCELWhenExpressions(t *testing.T) { + ps := []*v1.Pipeline{parse.MustParseV1Pipeline(t, ` +metadata: + name: test-pipeline + namespace: foo +spec: + params: + - name: run + type: string + tasks: + - name: a-task + taskRef: + name: a-task + when: + - cel: "$(params.run)=='yes'" +`)} + prs := []*v1.PipelineRun{parse.MustParseV1PipelineRun(t, ` +metadata: + name: test-pipeline-run-different-service-accs + namespace: foo +spec: + params: + - name: run + value: "yes" + pipelineRef: + name: test-pipeline + taskRunTemplate: + serviceAccountName: test-sa-0 +`)} + ts := []*v1.Task{ + {ObjectMeta: baseObjectMeta("a-task", "foo")}, + } + cms := []*corev1.ConfigMap{ + { + ObjectMeta: metav1.ObjectMeta{Name: config.GetFeatureFlagsConfigName(), Namespace: system.Namespace()}, + Data: map[string]string{ + "enable-cel-in-whenexpression": "true", + }, + }, + } + d := test.Data{ + PipelineRuns: prs, + Pipelines: ps, + Tasks: ts, + ConfigMaps: cms, + } + prt := newPipelineRunTest(t, d) + defer prt.Cancel() + pipelineRun, _ := prt.reconcileRun("foo", "test-pipeline-run-different-service-accs", []string{}, true) + checkPipelineRunConditionStatusAndReason(t, pipelineRun, corev1.ConditionFalse, string(v1.PipelineRunReasonCELEvaluationFailed)) +} + // TestReconcileWithAffinityAssistantStatefulSet tests that given a pipelineRun with workspaces, // an Affinity Assistant StatefulSet is created for each PVC workspace and // that the Affinity Assistant names is propagated to TaskRuns. diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go index 09ea5f8ccf6..c615a25441d 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go @@ -22,6 +22,8 @@ import ( "fmt" "sort" + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types" "github.com/tektoncd/pipeline/pkg/apis/config" "github.com/tektoncd/pipeline/pkg/apis/pipeline" v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" @@ -66,6 +68,48 @@ type ResolvedPipelineTask struct { PipelineTask *v1.PipelineTask ResolvedTask *resources.ResolvedTask ResultsCache map[string][]string + // EvaluatedCEL is used to store the results of evaluated CEL expression + EvaluatedCEL map[string]bool +} + +// EvaluateCEL evaluate the CEL expressions, and store the evaluated results in EvaluatedCEL +func (t *ResolvedPipelineTask) EvaluateCEL() error { + if t.PipelineTask != nil { + if len(t.EvaluatedCEL) == 0 { + t.EvaluatedCEL = make(map[string]bool) + } + for _, we := range t.PipelineTask.When { + if we.CEL == "" { + continue + } + _, ok := t.EvaluatedCEL[we.CEL] + if !ok { + // Create a program environment configured with the standard library of CEL functions and macros + env, _ := cel.NewEnv() + // Parse and Check the CEL to get the Abstract Syntax Tree + ast, iss := env.Compile(we.CEL) + if iss.Err() != nil { + return iss.Err() + } + // Generate an evaluable instance of the Ast within the environment + prg, err := env.Program(ast) + if err != nil { + return err + } + // Evaluate the CEL expression + out, _, err := prg.Eval(map[string]interface{}{}) + if err != nil { + return err + } + if out.ConvertToType(types.BoolType).Value() == true { + t.EvaluatedCEL[we.CEL] = true + } else { + t.EvaluatedCEL[we.CEL] = false + } + } + } + } + return nil } // isDone returns true only if the task is skipped, succeeded or failed @@ -271,6 +315,8 @@ func (t *ResolvedPipelineTask) skip(facts *PipelineRunFacts) TaskSkipStatus { skippingReason = v1.ParentTasksSkip case t.skipBecauseResultReferencesAreMissing(facts): skippingReason = v1.MissingResultsSkip + case t.skipBecauseCELExpressionsEvaluatedToFalse(facts): + skippingReason = v1.WhenExpressionsSkip case t.skipBecauseWhenExpressionsEvaluatedToFalse(facts): skippingReason = v1.WhenExpressionsSkip case t.skipBecausePipelineRunPipelineTimeoutReached(facts): @@ -305,6 +351,20 @@ func (t *ResolvedPipelineTask) Skip(facts *PipelineRunFacts) TaskSkipStatus { return facts.SkipCache[t.PipelineTask.Name] } +// skipBecauseCELExpressionsEvaluatedToFalse confirms that the CEL when expressions have completed evaluating, and +// it returns true if any of the CEL when expressions evaluate to false +func (t *ResolvedPipelineTask) skipBecauseCELExpressionsEvaluatedToFalse(facts *PipelineRunFacts) bool { + if t.checkParentsDone(facts) { + for _, we := range t.PipelineTask.When { + if we.CEL != "" && !t.EvaluatedCEL[we.CEL] { + return true + } + } + return false + } + return false +} + // skipBecauseWhenExpressionsEvaluatedToFalse confirms that the when expressions have completed evaluating, and // it returns true if any of the when expressions evaluate to false func (t *ResolvedPipelineTask) skipBecauseWhenExpressionsEvaluatedToFalse(facts *PipelineRunFacts) bool { @@ -424,6 +484,8 @@ func (t *ResolvedPipelineTask) IsFinallySkipped(facts *PipelineRunFacts) TaskSki switch { case t.skipBecauseResultReferencesAreMissing(facts): skippingReason = v1.MissingResultsSkip + case t.skipBecauseCELExpressionsEvaluatedToFalse(facts): + skippingReason = v1.WhenExpressionsSkip case t.skipBecauseWhenExpressionsEvaluatedToFalse(facts): skippingReason = v1.WhenExpressionsSkip case t.skipBecausePipelineRunPipelineTimeoutReached(facts): diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go index 70415f16c8f..d18c328013c 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go @@ -4812,3 +4812,181 @@ func TestCreateResultsCacheMatrixedTaskRuns(t *testing.T) { }) } } + +func TestEvaluateCEL_valid(t *testing.T) { + for _, tc := range []struct { + name string + rpt *ResolvedPipelineTask + want map[string]bool + }{{ + name: "empty CEL", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "", + }}, + }, + }, + want: map[string]bool{}, + }, { + name: "equal", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "'foo'=='foo'", + }}, + }, + }, + want: map[string]bool{ + "'foo'=='foo'": true, + }, + }, { + name: "not equal", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "'bar'!='foo'", + }}, + }, + }, + want: map[string]bool{ + "'bar'!='foo'": true, + }, + }, { + name: "in", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "'foo' in ['foo', 'bar']", + }}, + }, + }, + want: map[string]bool{ + "'foo' in ['foo', 'bar']": true, + }, + }, { + name: "not in", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "!('duh' in ['foo', 'bar'])", + }}, + }, + }, + want: map[string]bool{ + "!('duh' in ['foo', 'bar'])": true, + }, + }, { + name: "greater than", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "'0.95'>'0.9'", + }}, + }, + }, + want: map[string]bool{ + "'0.95'>'0.9'": true, + }, + }, { + name: "less than", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "'0.85'<'0.9'", + }}, + }, + }, + want: map[string]bool{ + "'0.85'<'0.9'": true, + }, + }, { + name: "or", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "'foo'=='foo'||false", + }}, + }, + }, + want: map[string]bool{ + "'foo'=='foo'||false": true, + }, + }, { + name: "and", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "'foo'=='foo'&&true", + }}, + }, + }, + want: map[string]bool{ + "'foo'=='foo'&&true": true, + }, + }, { + name: "regex", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "'release/v1'.matches('release/.*')", + }}, + }, + }, + want: map[string]bool{ + "'release/v1'.matches('release/.*')": true, + }, + }, { + name: "multiple CEL when expressions", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "'foo'=='foo'", + }, { + CEL: "'foo'!='foo'", + }, { + CEL: "'foo'!='bar'", + }}, + }, + }, + want: map[string]bool{ + "'foo'!='bar'": true, + "'foo'!='foo'": false, + "'foo'=='foo'": true, + }, + }} { + t.Run(tc.name, func(t *testing.T) { + err := tc.rpt.EvaluateCEL() + if err != nil { + t.Fatalf("Got unexpected err:%v", err) + } + if !cmp.Equal(tc.want, tc.rpt.EvaluatedCEL) { + t.Errorf("Did not get the expected EvaluatedCEL want %v, got: %v", tc.want, tc.rpt.EvaluatedCEL) + } + }) + } +} + +func TestEvaluateCEL_invalid(t *testing.T) { + for _, tc := range []struct { + name string + rpt *ResolvedPipelineTask + }{{ + name: "compile error - token unrecogniezd", + rpt: &ResolvedPipelineTask{ + PipelineTask: &v1.PipelineTask{ + When: v1.WhenExpressions{{ + CEL: "$(params.foo)=='foo'", + }}, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.rpt.EvaluateCEL() + if err == nil { + t.Fatalf("Expected err but got nil") + } + }) + } +} diff --git a/test/custom_task_test.go b/test/custom_task_test.go index 604f66db2b5..965fe6bb905 100644 --- a/test/custom_task_test.go +++ b/test/custom_task_test.go @@ -729,9 +729,10 @@ func resetConfigMap(ctx context.Context, t *testing.T, c *clients, namespace, co func getFeatureFlagsBaseOnAPIFlag(t *testing.T) *config.FeatureFlags { t.Helper() alphaFeatureFlags, err := config.NewFeatureFlagsFromMap(map[string]string{ - "enable-api-fields": "alpha", - "results-from": "sidecar-logs", - "enable-tekton-oci-bundles": "true", + "enable-api-fields": "alpha", + "results-from": "sidecar-logs", + "enable-tekton-oci-bundles": "true", + "enable-cel-in-whenexpression": "true", }) if err != nil { t.Fatalf("error creating alpha feature flags configmap: %v", err) diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 97f62d4d172..2216f69dbf7 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -63,6 +63,12 @@ function set_feature_gate() { jsonpatch=$(printf "{\"data\": {\"enable-api-fields\": \"%s\"}}" $1) echo "feature-flags ConfigMap patch: ${jsonpatch}" kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + if [ "$gate" == "alpha" ]; then + echo "enable feature flags in alpha phase" + jsonpatch=$(printf "{\"data\": {\"enable-cel-in-whenexpression\": \"true\"}}") + echo "feature-flags ConfigMap patch: ${jsonpatch}" + kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + fi } function set_result_extraction_method() {