[chore] Add a smoke test that is run as part of the scorecard tests V2

.github/workflows/scorecard.yaml

@@ -26,16 +26,13 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
 
-      - name: "install kuttl and kind"
+      - name: "install kuttl"
         run: ./hack/install-kuttl.sh
 
-      - name: "start kind"
+      - name: "prepare cluster"
         env:
           KUBE_VERSION: ${{ matrix.kube-version }}
-        run: kind create cluster --config kind-$KUBE_VERSION.yaml
+        run: make prepare-e2e KUBE_VERSION=$KUBE_VERSION VERSION=e2e
 
-      - name: "wait until cluster is ready"
-        run:  kubectl wait --timeout=5m --for=condition=available deployment/coredns -n kube-system
-
-      - name: "run scorecard test"
+      - name: "run scorecard tests"
         run: make scorecard-tests

Makefile

@@ -64,6 +64,11 @@ OPERATOR_SDK_VERSION ?= 1.27.0
 
 CERTMANAGER_VERSION ?= 1.10.0
 
+SCORECARD_KUTTL_TESTS := \
+	smoke-simplest \
+	smoke-sidecar \
+	instrumentation-java
+
 ifndef ignore-not-found
   ignore-not-found = false
 endif
@@ -190,8 +195,9 @@ prepare-e2e: kuttl set-image-controller container container-target-allocator con
 	TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh
 
 .PHONY: scorecard-tests
-scorecard-tests: operator-sdk
-	$(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1)
+scorecard-tests: operator-sdk bundle-scorecard-kuttl-tests
+	kubectl apply -f tests/scorecard/rbac.yaml
+	$(OPERATOR_SDK) scorecard -w=15m bundle || (echo "scorecard test failed" && exit 1)
 
 
 # Build the container image, used only for local dev purposes
@@ -377,6 +383,11 @@ bundle: kustomize operator-sdk manifests set-image-controller
 	$(OPERATOR_SDK) bundle validate ./bundle
 	./hack/ignore-createdAt-bundle.sh
 
+# Copy selected e2e tests to the kuttl scorecard tests of the bundle
+.PHONY: bundle-scorecard-kuttl-tests
+bundle-scorecard-kuttl-tests:
+	for t in $(SCORECARD_KUTTL_TESTS); do cp -r tests/e2e/$$t bundle/tests/scorecard/kuttl; done
+
 # Build the bundle image, used only for local dev purposes
 .PHONY: bundle-build
 bundle-build:

bundle/tests/scorecard/config.yaml

@@ -2,6 +2,7 @@ apiVersion: scorecard.operatorframework.io/v1alpha3
 kind: Configuration
 metadata:
   name: config
+serviceaccount: scorecard-admin
 stages:
 - parallel: true
   tests:
@@ -45,6 +46,12 @@ stages:
     storage:
       spec:
         mountPath: {}
+  - image: docker.io/andreasgerstmayr/scorecard-test-kuttl:1681909083
+    labels:
+      suite: kuttl
+    storage:
+      spec:
+        mountPath: {}
 storage:
   spec:
     mountPath: {}

bundle/tests/scorecard/kuttl/kuttl-test.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+parallel: 1
+startControlPlane: false
+timeout: 120
+suppress:
+  - events

config/scorecard/bases/config.yaml

@@ -2,6 +2,7 @@ apiVersion: scorecard.operatorframework.io/v1alpha3
 kind: Configuration
 metadata:
   name: config
+serviceaccount: scorecard-admin
 stages:
 - parallel: true
   tests: []

config/scorecard/kustomization.yaml

@@ -13,4 +13,10 @@ patchesJson6902:
     version: v1alpha3
     kind: Configuration
     name: config
+- path: patches/kuttl.config.yaml
+  target:
+    group: scorecard.operatorframework.io
+    version: v1alpha3
+    kind: Configuration
+    name: config
 #+kubebuilder:scaffold:patchesJson6902

config/scorecard/patches/kuttl.config.yaml

@@ -0,0 +1,9 @@
+- op: add
+  path: /stages/0/tests/-
+  value:
+    # TODO: Switch to quay.io/operator-framework/scorecard-test-kuttl image
+    # once https://github.com/operator-framework/operator-sdk/pull/6401 is merged.
+    # https://github.com/open-telemetry/opentelemetry-operator/issues/1721
+    image: docker.io/andreasgerstmayr/scorecard-test-kuttl:1681909083
+    labels:
+      suite: kuttl

tests/scorecard/rbac.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: scorecard-admin
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: scorecard-admin-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: scorecard-admin
+  namespace: default