test: add chainsaw tests for reports generated by policy using an ass…

…ertion tree (kyverno#10779)

Signed-off-by: Charles-Edouard Brétéché <[email protected]>
Co-authored-by: shuting <[email protected]>

test/conformance/chainsaw/reports/admission/exception/exception.yaml

@@ -7,6 +7,7 @@ spec:
     - policyName: require-labels
       ruleNames:
         - require-team
+        - require-team-assert
   match:
     any:
       - resources:

test/conformance/chainsaw/reports/admission/exception/policy.yaml

@@ -18,3 +18,16 @@ spec:
           metadata:
             labels:
               team: '?*'
+    - name: require-team-assert
+      match:
+        any:
+          - resources:
+              kinds:
+                - ConfigMap
+      validate:
+        message: 'The label `team` is required.'
+        assert:
+          object:
+            metadata:
+              labels:
+                team: {}

test/conformance/chainsaw/reports/admission/exception/report-assert.yaml

@@ -17,9 +17,16 @@ results:
   source: kyverno
   properties:
     exceptions: mynewpolex
+- policy: require-labels
+  result: skip
+  rule: require-team-assert
+  scored: true
+  source: kyverno
+  properties:
+    exceptions: mynewpolex
 summary:
   error: 0
   fail: 0
   pass: 0
-  skip: 1
+  skip: 2
   warn: 0

test/conformance/chainsaw/reports/admission/namespaceselector-assert/README.md

@@ -0,0 +1,8 @@
+## Description
+
+This test validate the reporting ability for a audit policy with the `namespaceSelector` defined.
+
+## Expected Behavior
+
+A policy report should be created for the pod `test-audit-reports-namespacesselector/audit-pod`, but not for `test-non-audit-reports-namespacesselector/non-audit-pod` as the namespace selector doesn't match.
+

test/conformance/chainsaw/reports/admission/namespaceselector-assert/chainsaw-test.yaml

@@ -0,0 +1,25 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: namespaceselector
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-assert.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: pods.yaml
+    - assert:
+        file: pods.yaml
+  - name: step-03
+    try:
+    - assert:
+        file: report-expected.yaml
+    - error:
+        file: report-unexpected.yaml

test/conformance/chainsaw/reports/admission/namespaceselector-assert/pods.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: audit-pod
+  namespace: test-audit-reports-namespacesselector
+spec:
+  containers:
+  - image: nginx:latest
+    name: audit-pod
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: non-audit-pod
+  namespace: test-non-audit-reports-namespacesselector
+spec:
+  containers:
+  - image: nginx:latest
+    name: non-audit-pod

test/conformance/chainsaw/reports/admission/namespaceselector-assert/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: test-audit-reports-namespacesselector
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/reports/admission/namespaceselector-assert/policy.yaml

@@ -0,0 +1,39 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: test-audit-reports-namespacesselector
+  annotations:  
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  background: false
+  mutateExistingOnPolicyUpdate: false
+  validationFailureAction: Audit
+  rules:
+  - name: test-audit-reports-namespacesselector
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+          namespaceSelector:
+            matchExpressions:
+              - key: org
+                operator: Exists
+    validate:
+      assert:
+        assert:
+          metadata:
+            annotations:
+              validate: namespaceselector
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    org: kyverno-test
+  name: test-audit-reports-namespacesselector
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-non-audit-reports-namespacesselector

test/conformance/chainsaw/reports/admission/namespaceselector-assert/report-expected.yaml

@@ -0,0 +1,17 @@
+apiVersion: wgpolicyk8s.io/v1alpha2
+kind: PolicyReport
+metadata:
+  namespace: test-audit-reports-namespacesselector
+  ownerReferences:
+  - apiVersion: v1
+    kind: Pod
+    name: audit-pod
+scope:
+  apiVersion: v1
+  kind: Pod
+  name: audit-pod
+  namespace: test-audit-reports-namespacesselector
+results:
+- policy: test-audit-reports-namespacesselector
+  result: fail
+  rule: test-audit-reports-namespacesselector

test/conformance/chainsaw/reports/admission/namespaceselector-assert/report-unexpected.yaml

@@ -0,0 +1,8 @@
+apiVersion: wgpolicyk8s.io/v1alpha2
+kind: PolicyReport
+metadata:
+  namespace: test-non-audit-reports-namespacesselector
+  ownerReferences:
+  - apiVersion: v1
+    kind: Pod
+    name: non-audit-pod

test/conformance/chainsaw/reports/background/exception-assert/README.md

@@ -0,0 +1,16 @@
+## Description
+
+This test creates a policy, a policy exception and a configmap.
+It makes sure the generated background scan report contains a skipped result instead of a failed one.
+
+## Steps
+
+1.  - Create a confimap named `emergency`
+2.  - Create a cluster policy
+    - Assert the policy becomes ready
+3.  - Create a policy exception for the cluster policy created above, configured to apply to configmap named `emergency`
+4.  - Assert that a policy report exists with a skipped result
+
+## Reference Issue(s)
+
+https://github.com/kyverno/kyverno/issues/7287

test/conformance/chainsaw/reports/background/exception-assert/chainsaw-test.yaml

@@ -0,0 +1,27 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: exception
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: configmap.yaml
+    - assert:
+        file: configmap.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-assert.yaml
+  - name: step-03
+    try:
+    - apply:
+        file: exception.yaml
+  - name: step-04
+    try:
+    - assert:
+        file: report-assert.yaml

test/conformance/chainsaw/reports/background/exception-assert/configmap.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: emergency

test/conformance/chainsaw/reports/background/exception-assert/exception.yaml

@@ -0,0 +1,17 @@
+apiVersion: kyverno.io/v2
+kind: PolicyException
+metadata:
+  name: mynewpolex
+  namespace: kyverno
+spec:
+  exceptions:
+    - policyName: require-labels
+      ruleNames:
+        - require-team
+  match:
+    any:
+      - resources:
+          kinds:
+            - ConfigMap
+          names:
+            - emergency

test/conformance/chainsaw/reports/background/exception-assert/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/reports/background/exception-assert/policy.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterPolicy
+metadata:
+  name: require-labels
+spec:
+  validationFailureAction: Enforce
+  admission: false
+  background: true
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - ConfigMap
+      validate:
+        message: 'The label `team` is required.'
+        assert:
+          object:
+            metadata:
+              labels:
+                team: {}

test/conformance/chainsaw/reports/background/exception-assert/report-assert.yaml

@@ -0,0 +1,25 @@
+apiVersion: wgpolicyk8s.io/v1alpha2
+kind: PolicyReport
+metadata:
+  ownerReferences:
+  - apiVersion: v1
+    kind: ConfigMap
+    name: emergency
+scope:
+  apiVersion: v1
+  kind: ConfigMap
+  name: emergency
+results:
+- policy: require-labels
+  result: skip
+  rule: require-team
+  scored: true
+  source: kyverno
+  properties:
+    exceptions: mynewpolex
+summary:
+  error: 0
+  fail: 0
+  pass: 0
+  skip: 1
+  warn: 0