Add a smoke test that is run as part of the scorecard tests

Signed-off-by: Kevin Earls <[email protected]>

Setup service account and e2e env for kuttl scorecard tests

Signed-off-by: Andreas Gerstmayr <[email protected]>

.github/workflows/scorecard.yaml

@@ -26,16 +26,13 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
 
-      - name: "install kuttl and kind"
+      - name: "install kuttl"
         run: ./hack/install-kuttl.sh
 
-      - name: "start kind"
+      - name: "prepare cluster"
         env:
           KUBE_VERSION: ${{ matrix.kube-version }}
-        run: kind create cluster --config kind-$KUBE_VERSION.yaml
+        run: make KUBE_VERSION=$KUBE_VERSION prepare-scorecard-tests
 
-      - name: "wait until cluster is ready"
-        run:  kubectl wait --timeout=5m --for=condition=available deployment/coredns -n kube-system
-
-      - name: "run scorecard test"
+      - name: "run scorecard tests"
         run: make scorecard-tests

Makefile

@@ -176,9 +176,14 @@ prepare-e2e: kuttl set-test-image-vars set-image-controller container container-
 	$(KUSTOMIZE) build config/default -o tests/_build/manifests/01-opentelemetry-operator.yaml
 	$(KUSTOMIZE) build config/crd -o tests/_build/crds/
 
+.PHONY: prepare-scorecard-tests
+prepare-scorecard-tests: prepare-e2e cert-manager
+	kubectl apply -f tests/_build/manifests/01-opentelemetry-operator.yaml
+	kubectl apply -f tests/scorecard/rbac.yaml
+
 .PHONY: scorecard-tests
 scorecard-tests: operator-sdk
-	$(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1)
+	$(OPERATOR_SDK) scorecard --wait-time=5m --service-account=scorecard-admin bundle || (echo "scorecard test failed" && exit 1)
 
 .PHONY: set-test-image-vars
 set-test-image-vars:

bundle/tests/scorecard/config.yaml

@@ -45,6 +45,13 @@ stages:
     storage:
       spec:
         mountPath: {}
+  - image: quay.io/operator-framework/scorecard-test-kuttl:v2.0.0
+    labels:
+      suite: kuttlsuite
+      test: kuttltest1
+    storage:
+      spec:
+        mountPath: {}
 storage:
   spec:
     mountPath: {}

bundle/tests/scorecard/kuttl/kuttl-test.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+timeout: 150
+startControlPlane: false

bundle/tests/scorecard/kuttl/smoketest/00-assert.yaml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: simplest-collector
+status:
+  readyReplicas: 1
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: simplest-collector-headless
+spec:
+  ports:
+  - appProtocol: grpc
+    name: jaeger-grpc
+    port: 14250
+    protocol: TCP
+    targetPort: 14250
+  - appProtocol: grpc
+    name: otlp-grpc
+    port: 4317
+    protocol: TCP
+    targetPort: 4317
+  - appProtocol: http
+    name: otlp-http
+    port: 4318
+    protocol: TCP
+    targetPort: 4318
+  - appProtocol: http
+    name: otlp-http-legacy
+    port: 55681
+    protocol: TCP
+    targetPort: 4318
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: simplest-collector
+spec:
+  ports:
+  - appProtocol: grpc
+    name: jaeger-grpc
+    port: 14250
+    protocol: TCP
+    targetPort: 14250
+  - appProtocol: grpc
+    name: otlp-grpc
+    port: 4317
+    protocol: TCP
+    targetPort: 4317
+  - appProtocol: http
+    name: otlp-http
+    port: 4318
+    protocol: TCP
+    targetPort: 4318
+  - appProtocol: http
+    name: otlp-http-legacy
+    port: 55681
+    protocol: TCP
+    targetPort: 4318

bundle/tests/scorecard/kuttl/smoketest/00-install.yaml

@@ -0,0 +1,25 @@
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: simplest
+spec:
+  config: |
+    receivers:
+      jaeger:
+        protocols:
+          grpc:
+      otlp:
+        protocols:
+          grpc:
+          http:
+    processors:
+
+    exporters:
+      logging:
+
+    service:
+      pipelines:
+        traces:
+          receivers: [jaeger,otlp]
+          processors: []
+          exporters: [logging]

config/scorecard/bases/config.yaml

@@ -3,5 +3,5 @@ kind: Configuration
 metadata:
   name: config
 stages:
-- parallel: true
-  tests: []
+  - parallel: true
+    tests: []

config/scorecard/kustomization.yaml

@@ -13,4 +13,10 @@ patchesJson6902:
     version: v1alpha3
     kind: Configuration
     name: config
+- path: patches/smoke.config.yaml
+  target:
+    group: scorecard.operatorframework.io
+    version: v1alpha3
+    kind: Configuration
+    name: config
 #+kubebuilder:scaffold:patchesJson6902

config/scorecard/patches/smoke.config.yaml

@@ -0,0 +1,7 @@
+- op: add
+  path: /stages/0/tests/-
+  value:
+    image: quay.io/operator-framework/scorecard-test-kuttl:v2.0.0
+    labels:
+      suite: kuttlsuite
+      test: kuttltest1

tests/scorecard/rbac.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: scorecard-admin
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: scorecard-admin-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: scorecard-admin
+  namespace: default