[TEP-0145] Add CEL evaluation

This commit adds CEL evaluation. Users are able to use CEL in WhenExpression if the feature flag enable-cel-in-whenexpression is enabled.If the evluation is false, the PipelineTask will be skipped. Signed-off-by: Yongxuan Zhang [email protected]
tektoncd · Oct 23, 2023 · da25b08 · da25b08
1 parent dcd34c1
commit da25b08
Show file tree

Hide file tree

Showing 14 changed files with 643 additions and 11 deletions.
diff --git a/config/config-feature-flags.yaml b/config/config-feature-flags.yaml
@@ -122,5 +122,4 @@ data:
   # allowing examination of the logs on the pods from cancelled taskruns
   keep-pod-on-cancel: "false"
   # Setting this flag to "true" will enable the CEL evaluation in WhenExpression
-  # This feature is in preview mode and not implemented yet. Please check #7244 for the updates.
   enable-cel-in-whenexpression: "false"
diff --git a/docs/pipeline-api.md b/docs/pipeline-api.md
@@ -1932,7 +1932,10 @@ ParamValue
 <th>Description</th>
 </tr>
 </thead>
-<tbody><tr><td><p>&#34;Cancelled&#34;</p></td>
+<tbody><tr><td><p>&#34;CELEvaluationFailed&#34;</p></td>
+<td><p>ReasonCELEvaluationFailed indicates the pipeline fails the CEL evaluation</p>
+</td>
+</tr><tr><td><p>&#34;Cancelled&#34;</p></td>
 <td><p>PipelineRunReasonCancelled is the reason set when the PipelineRun cancelled by the user
 This reason may be found with a corev1.ConditionFalse status, if the cancellation was processed successfully
 This reason may be found with a corev1.ConditionUnknown status, if the cancellation is being processed or failed</p>

diff --git a/docs/pipelines.md b/docs/pipelines.md
@@ -779,9 +779,70 @@ There are a lot of scenarios where `when` expressions can be really useful. Some
 
 > :seedling: **`CEL in WhenExpression` is an [alpha](additional-configs.md#alpha-features) feature.**
 > The `enable-cel-in-whenexpression` feature flag must be set to `"true"` to enable the use of `CEL` in `WhenExpression`.
->
-> :warning: This feature is in a preview mode.
-> It is still in a very early stage of development and is not yet fully functional
+
+CEL (Common Expression Language) is a declarative language designed for simplicity, speed, safety, and portability which can be used to express a wide variety of conditions and computations.
+
+You can define a CEL expression in `WhenExpression` to guard the execution of a `Task`. You can use a single line of CEL string to replace current `WhenExpressions`'s `input`+`operator`+`values`. For example:
+
+```yaml
+# current WhenExpressions
+when:
+  - input: "foo"
+    operator: "in"
+    values: ["foo", "bar"]
+  - input: "duh"
+    operator: "notin"
+    values: ["foo", "bar"]
+
+# with cel
+when:
+  - cel: "'foo' in ['foo', 'bar']"
+  - cel: "!('duh' in ['foo', 'bar'])"
+```
+
+CEL can offer more conditional functions, such as numeric comparisons (e.g. `>`, `<=`, etc), logic operators (e.g. `OR`, `AND`), Regex Pattern Matching. For example:
+
+```yaml
+  when:
+    # test coverage result is larger than 90%
+    - cel: "'$(tasks.unit-test.results.test-coverage)' > 0.9"
+    # params is not empty, or params2 is 8.5 or 8.6
+    - cel: "'$(params.param1)' != '' || '$(params.param2)' == '8.5' || '$(params.param2)' == '8.6'"
+    # param branch matches pattern `release/.*`
+    - cel: "'$(params.branch)'.matches('release/.*')"
+```
+
+##### Variable substitution in CEL
+
+`CEL` supports [string substitutions](https://github.com/tektoncd/pipeline/blob/main/docs/variables.md#variables-available-in-a-pipeline), you can reference string, array indexing or object value of a param/result. For example:
+
+```yaml
+  when:
+    # string result
+    - cel: "$(tasks.unit-test.results.test-coverage) > 0.9"
+    # array indexing result
+    - cel: "$(tasks.unit-test.results.test-coverage[0]) > 0.9"
+    # object result key
+    - cel: "'$(tasks.objectTask.results.repo.url)'.matches('github.com/tektoncd/.*')"
+    # string param
+    - cel: "'$(params.foo)' == 'foo'"
+    # array indexing
+    - cel: "'$(params.branch[0])' == 'foo'"
+    # object param key
+    - cel: "'$(params.repo.url)'.matches('github.com/tektoncd/.*')"
+```
+
+**Note:** the reference needs to be wrapped with single quotes.
+Whole `Array` and `Object` replacements are not supported yet. The following usage is not supported:
+
+```yaml
+  when:
+    - cel: "'foo' in '$(params.array_params[*]']"
+    - cel: "'foo' in '$(params.object_params[*]']"
+```
+
+In addition to the custom function extension listed above, you can craft any valid CEL expression as defined by the [cel-spec language definition](https://github.com/google/cel-spec/blob/master/doc/langdef.md)
+
 
 `CEL` expression is validated at admission webhook and a validation error will be returned if the expression is invalid.
 

diff --git a/examples/v1/pipelineruns/alpha/pipelinerun-with-cel-when-expressions.yaml b/examples/v1/pipelineruns/alpha/pipelinerun-with-cel-when-expressions.yaml
@@ -0,0 +1,154 @@
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  generateName: guarded-pr-by-cel-
+spec:
+  pipelineSpec:
+    params:
+      - name: path
+        type: string
+        description: The path of the file to be created
+    workspaces:
+      - name: source
+        description: |
+          This workspace is shared among all the pipeline tasks to read/write common resources
+    tasks:
+      - name: create-file  # when expression using parameter, evaluates to true
+        when:
+          - cel: "'$(params.path)' == 'README.md'"
+        workspaces:
+          - name: source
+            workspace: source
+        taskSpec:
+          workspaces:
+            - name: source
+              description: The workspace to create the readme file in
+          steps:
+            - name: write-new-stuff
+              image: ubuntu
+              script: 'touch $(workspaces.source.path)/README.md'
+      - name: check-file
+        params:
+          - name: path
+            value: "$(params.path)"
+        workspaces:
+          - name: source
+            workspace: source
+        runAfter:
+          - create-file
+        taskSpec:
+          params:
+            - name: path
+          workspaces:
+            - name: source
+              description: The workspace to check for the file
+          results:
+            - name: exists
+              description: indicates whether the file exists or is missing
+          steps:
+            - name: check-file
+              image: alpine
+              script: |
+                if test -f $(workspaces.source.path)/$(params.path); then
+                  printf yes | tee $(results.exists.path)
+                else
+                  printf no | tee $(results.exists.path)
+                fi
+      - name: echo-file-exists  # when expression using task result, evaluates to true
+        when:
+          - cel: "'$(tasks.check-file.results.exists)' == 'yes'"
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: 'echo file exists'
+      - name: task-should-be-skipped-1
+        when:
+          - cel: "'$(tasks.check-file.results.exists)'=='missing'" # when expression using task result, evaluates to false
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: exit 1
+      - name: task-should-be-skipped-2  # when expression using parameter, evaluates to false
+        when:
+          - cel: "'$(params.path)'!='README.md'"
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: exit 1
+      - name: task-should-be-skipped-3  # task with when expression and run after
+        runAfter:
+          - echo-file-exists
+        when:
+          - cel: "'monday'=='friday'"
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: exit 1
+    finally:
+      - name: finally-task-should-be-skipped-1  # when expression using execution status, evaluates to false
+        when:
+          - cel: "'$(tasks.echo-file-exists.status)'=='Failure'"
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: exit 1
+      - name: finally-task-should-be-skipped-2  # when expression using task result, evaluates to false
+        when:
+          - cel: "'$(tasks.check-file.results.exists)'=='missing'"
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: exit 1
+      - name: finally-task-should-be-skipped-3  # when expression using parameter, evaluates to false
+        when:
+          - cel: "'$(params.path)'!='README.md'"
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: exit 1
+      - name: finally-task-should-be-skipped-4  # when expression using tasks execution status, evaluates to false
+        when:
+          - cel: "'$(tasks.status)'=='Failure'"
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: exit 1
+      - name: finally-task-should-be-skipped-5  # when expression using tasks execution status, evaluates to false
+        when:
+          - cel: "'$(tasks.status)'=='Succeeded'"
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: exit 1
+      - name: finally-task-should-be-executed  # when expression using execution status, tasks execution status, param, and results
+        when:
+          - cel: "'$(tasks.echo-file-exists.status)'=='Succeeded'"
+          - cel: "'$(tasks.status)'=='Completed'"
+          - cel: "'$(tasks.check-file.results.exists)'=='yes'"
+          - cel: "'$(params.path)'=='README.md'"
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              script: 'echo finally done'
+  params:
+    - name: path
+      value: README.md
+  workspaces:
+    - name: source
+      volumeClaimTemplate:
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 16Mi
diff --git a/go.mod b/go.mod
@@ -190,7 +190,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/hashicorp/go-version v1.6.0
+	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

diff --git a/pkg/apis/pipeline/v1/pipelinerun_types.go b/pkg/apis/pipeline/v1/pipelinerun_types.go
@@ -407,6 +407,8 @@ const (
 	PipelineRunReasonResourceVerificationFailed PipelineRunReason = "ResourceVerificationFailed"
 	// ReasonCreateRunFailed indicates that the pipeline fails to create the taskrun or other run resources
 	PipelineRunReasonCreateRunFailed PipelineRunReason = "CreateRunFailed"
+	// ReasonCELEvaluationFailed indicates the pipeline fails the CEL evaluation
+	PipelineRunReasonCELEvaluationFailed PipelineRunReason = "CELEvaluationFailed"
 )
 
 func (t PipelineRunReason) String() string {

diff --git a/pkg/apis/pipeline/v1/when_types.go b/pkg/apis/pipeline/v1/when_types.go
@@ -63,6 +63,7 @@ func (we *WhenExpression) isTrue() bool {
 
 func (we *WhenExpression) applyReplacements(replacements map[string]string, arrayReplacements map[string][]string) WhenExpression {
 	replacedInput := substitution.ApplyReplacements(we.Input, replacements)
+	replacedCEL := substitution.ApplyReplacements(we.CEL, replacements)
 
 	var replacedValues []string
 	for _, val := range we.Values {
@@ -79,13 +80,14 @@ func (we *WhenExpression) applyReplacements(replacements map[string]string, arra
 		}
 	}
 
-	return WhenExpression{Input: replacedInput, Operator: we.Operator, Values: replacedValues}
+	return WhenExpression{Input: replacedInput, Operator: we.Operator, Values: replacedValues, CEL: replacedCEL}
 }
 
 // GetVarSubstitutionExpressions extracts all the values between "$(" and ")" in a When Expression
 func (we *WhenExpression) GetVarSubstitutionExpressions() ([]string, bool) {
 	var allExpressions []string
 	allExpressions = append(allExpressions, validateString(we.Input)...)
+	allExpressions = append(allExpressions, validateString(we.CEL)...)
 	for _, value := range we.Values {
 		allExpressions = append(allExpressions, validateString(value)...)
 	}

diff --git a/pkg/apis/pipeline/v1beta1/when_types.go b/pkg/apis/pipeline/v1beta1/when_types.go
@@ -63,6 +63,7 @@ func (we *WhenExpression) isTrue() bool {
 
 func (we *WhenExpression) applyReplacements(replacements map[string]string, arrayReplacements map[string][]string) WhenExpression {
 	replacedInput := substitution.ApplyReplacements(we.Input, replacements)
+	replacedCEL := substitution.ApplyReplacements(we.CEL, replacements)
 
 	var replacedValues []string
 	for _, val := range we.Values {
@@ -79,13 +80,14 @@ func (we *WhenExpression) applyReplacements(replacements map[string]string, arra
 		}
 	}
 
-	return WhenExpression{Input: replacedInput, Operator: we.Operator, Values: replacedValues}
+	return WhenExpression{Input: replacedInput, Operator: we.Operator, Values: replacedValues, CEL: replacedCEL}
 }
 
 // GetVarSubstitutionExpressions extracts all the values between "$(" and ")" in a When Expression
 func (we *WhenExpression) GetVarSubstitutionExpressions() ([]string, bool) {
 	var allExpressions []string
 	allExpressions = append(allExpressions, validateString(we.Input)...)
+	allExpressions = append(allExpressions, validateString(we.CEL)...)
 	for _, value := range we.Values {
 		allExpressions = append(allExpressions, validateString(value)...)
 	}

diff --git a/pkg/reconciler/pipelinerun/pipelinerun.go b/pkg/reconciler/pipelinerun/pipelinerun.go
@@ -626,6 +626,16 @@ func (c *Reconciler) reconcile(ctx context.Context, pr *v1.PipelineRun, getPipel
 		}
 	}
 
+	// Evaluate the CEL of PipelineTask after the variable substitutions and validations.
+	for _, rpt := range pipelineRunFacts.State {
+		err := rpt.EvaluateCEL()
+		if err != nil {
+			logger.Errorf("Error evaluating CEL %s: %v", pr.Name, err)
+			pr.Status.MarkFailed(string(v1.PipelineRunReasonCELEvaluationFailed), err.Error())
+			return controller.NewPermanentError(err)
+		}
+	}
+
 	// check if pipeline run is not gracefully cancelled and there are active task runs, which require cancelling
 	if pr.IsGracefullyCancelled() && pipelineRunFacts.IsRunning() {
 		// If the pipelinerun is cancelled, cancel tasks, but run finally