From 0a544af21a9706e4605bf64623de744317245c85 Mon Sep 17 00:00:00 2001 From: Andreas Gerstmayr Date: Tue, 20 Dec 2022 14:58:53 +0100 Subject: [PATCH 1/5] Add a smoke test that is run as part of the scorecard tests Co-authored-by: Kevin Earls Setup service account and e2e env for kuttl scorecard tests Signed-off-by: Andreas Gerstmayr --- .github/workflows/scorecard.yaml | 11 ++-- Makefile | 6 +- bundle/tests/scorecard/config.yaml | 7 ++ bundle/tests/scorecard/kuttl/kuttl-test.yaml | 4 ++ .../scorecard/kuttl/smoketest/00-assert.yaml | 64 +++++++++++++++++++ .../scorecard/kuttl/smoketest/00-install.yaml | 25 ++++++++ config/scorecard/bases/config.yaml | 4 +- config/scorecard/kustomization.yaml | 6 ++ config/scorecard/patches/smoke.config.yaml | 7 ++ tests/scorecard/rbac.yaml | 17 +++++ 10 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 bundle/tests/scorecard/kuttl/kuttl-test.yaml create mode 100644 bundle/tests/scorecard/kuttl/smoketest/00-assert.yaml create mode 100644 bundle/tests/scorecard/kuttl/smoketest/00-install.yaml create mode 100644 config/scorecard/patches/smoke.config.yaml create mode 100644 tests/scorecard/rbac.yaml diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index 8149825859..467beb7454 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -26,16 +26,13 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: "install kuttl and kind" + - name: "install kuttl" run: ./hack/install-kuttl.sh - - name: "start kind" + - name: "prepare cluster" env: KUBE_VERSION: ${{ matrix.kube-version }} - run: kind create cluster --config kind-$KUBE_VERSION.yaml + run: make prepare-scorecard-tests KUBE_VERSION=$KUBE_VERSION VERSION=e2e - - name: "wait until cluster is ready" - run: kubectl wait --timeout=5m --for=condition=available deployment/coredns -n kube-system - - - name: "run scorecard test" + - name: "run scorecard tests" run: make scorecard-tests diff --git a/Makefile b/Makefile index 6554052029..6329ac2688 100644 --- a/Makefile +++ b/Makefile @@ -189,9 +189,13 @@ e2e-log-operator: prepare-e2e: kuttl set-image-controller container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server install-openshift-routes load-image-all deploy TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh +.PHONY: prepare-scorecard-tests +prepare-scorecard-tests: prepare-e2e + kubectl apply -f tests/scorecard/rbac.yaml + .PHONY: scorecard-tests scorecard-tests: operator-sdk - $(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1) + $(OPERATOR_SDK) scorecard --wait-time=5m --service-account=scorecard-admin bundle || (echo "scorecard test failed" && exit 1) # Build the container image, used only for local dev purposes diff --git a/bundle/tests/scorecard/config.yaml b/bundle/tests/scorecard/config.yaml index efe0cbadca..00b5a9f23d 100644 --- a/bundle/tests/scorecard/config.yaml +++ b/bundle/tests/scorecard/config.yaml @@ -45,6 +45,13 @@ stages: storage: spec: mountPath: {} + - image: quay.io/operator-framework/scorecard-test-kuttl:v2.0.0 + labels: + suite: kuttlsuite + test: kuttltest1 + storage: + spec: + mountPath: {} storage: spec: mountPath: {} diff --git a/bundle/tests/scorecard/kuttl/kuttl-test.yaml b/bundle/tests/scorecard/kuttl/kuttl-test.yaml new file mode 100644 index 0000000000..20800f8eef --- /dev/null +++ b/bundle/tests/scorecard/kuttl/kuttl-test.yaml @@ -0,0 +1,4 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +timeout: 150 +startControlPlane: false diff --git a/bundle/tests/scorecard/kuttl/smoketest/00-assert.yaml b/bundle/tests/scorecard/kuttl/smoketest/00-assert.yaml new file mode 100644 index 0000000000..f19dd651af --- /dev/null +++ b/bundle/tests/scorecard/kuttl/smoketest/00-assert.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simplest-collector +status: + readyReplicas: 1 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: simplest-collector-headless +spec: + ports: + - appProtocol: grpc + name: jaeger-grpc + port: 14250 + protocol: TCP + targetPort: 14250 + - appProtocol: grpc + name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + - appProtocol: http + name: otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 + - appProtocol: http + name: otlp-http-legacy + port: 55681 + protocol: TCP + targetPort: 4318 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: simplest-collector +spec: + ports: + - appProtocol: grpc + name: jaeger-grpc + port: 14250 + protocol: TCP + targetPort: 14250 + - appProtocol: grpc + name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + - appProtocol: http + name: otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 + - appProtocol: http + name: otlp-http-legacy + port: 55681 + protocol: TCP + targetPort: 4318 diff --git a/bundle/tests/scorecard/kuttl/smoketest/00-install.yaml b/bundle/tests/scorecard/kuttl/smoketest/00-install.yaml new file mode 100644 index 0000000000..eb4cc2b39e --- /dev/null +++ b/bundle/tests/scorecard/kuttl/smoketest/00-install.yaml @@ -0,0 +1,25 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: simplest +spec: + config: | + receivers: + jaeger: + protocols: + grpc: + otlp: + protocols: + grpc: + http: + processors: + + exporters: + logging: + + service: + pipelines: + traces: + receivers: [jaeger,otlp] + processors: [] + exporters: [logging] \ No newline at end of file diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml index c77047841e..707a5c25fd 100644 --- a/config/scorecard/bases/config.yaml +++ b/config/scorecard/bases/config.yaml @@ -3,5 +3,5 @@ kind: Configuration metadata: name: config stages: -- parallel: true - tests: [] + - parallel: true + tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml index 50cd2d084e..d36175b6a5 100644 --- a/config/scorecard/kustomization.yaml +++ b/config/scorecard/kustomization.yaml @@ -13,4 +13,10 @@ patchesJson6902: version: v1alpha3 kind: Configuration name: config +- path: patches/smoke.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config #+kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/smoke.config.yaml b/config/scorecard/patches/smoke.config.yaml new file mode 100644 index 0000000000..40a7f2f6de --- /dev/null +++ b/config/scorecard/patches/smoke.config.yaml @@ -0,0 +1,7 @@ +- op: add + path: /stages/0/tests/- + value: + image: quay.io/operator-framework/scorecard-test-kuttl:v2.0.0 + labels: + suite: kuttlsuite + test: kuttltest1 diff --git a/tests/scorecard/rbac.yaml b/tests/scorecard/rbac.yaml new file mode 100644 index 0000000000..13f0964e8e --- /dev/null +++ b/tests/scorecard/rbac.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: scorecard-admin +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: scorecard-admin-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: scorecard-admin + namespace: default From ee1f87769821d5fb2c45456e43cad95d1ab47b11 Mon Sep 17 00:00:00 2001 From: Andreas Gerstmayr Date: Mon, 6 Feb 2023 17:26:34 +0100 Subject: [PATCH 2/5] Specify scorecard tests serviceaccount in config.yaml instead of the Makefile Signed-off-by: Andreas Gerstmayr --- Makefile | 2 +- bundle/tests/scorecard/config.yaml | 1 + config/scorecard/bases/config.yaml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6329ac2688..f55a4d98ac 100644 --- a/Makefile +++ b/Makefile @@ -195,7 +195,7 @@ prepare-scorecard-tests: prepare-e2e .PHONY: scorecard-tests scorecard-tests: operator-sdk - $(OPERATOR_SDK) scorecard --wait-time=5m --service-account=scorecard-admin bundle || (echo "scorecard test failed" && exit 1) + $(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1) # Build the container image, used only for local dev purposes diff --git a/bundle/tests/scorecard/config.yaml b/bundle/tests/scorecard/config.yaml index 00b5a9f23d..6cb201cea4 100644 --- a/bundle/tests/scorecard/config.yaml +++ b/bundle/tests/scorecard/config.yaml @@ -2,6 +2,7 @@ apiVersion: scorecard.operatorframework.io/v1alpha3 kind: Configuration metadata: name: config +serviceaccount: scorecard-admin stages: - parallel: true tests: diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml index 707a5c25fd..3311fee65f 100644 --- a/config/scorecard/bases/config.yaml +++ b/config/scorecard/bases/config.yaml @@ -2,6 +2,7 @@ apiVersion: scorecard.operatorframework.io/v1alpha3 kind: Configuration metadata: name: config +serviceaccount: scorecard-admin stages: - parallel: true tests: [] From f0661f223c6e8360e668f68ec1d7d31c08a2fe4f Mon Sep 17 00:00:00 2001 From: Andreas Gerstmayr Date: Tue, 18 Apr 2023 18:25:23 +0200 Subject: [PATCH 3/5] Move selected e2e tests to scorecard tests Signed-off-by: Andreas Gerstmayr --- .github/workflows/scorecard.yaml | 2 +- Makefile | 31 +++++++-- bundle/tests/scorecard/config.yaml | 5 +- bundle/tests/scorecard/kuttl/kuttl-test.yaml | 5 +- .../scorecard/kuttl/smoketest/00-assert.yaml | 64 ------------------- .../scorecard/kuttl/smoketest/00-install.yaml | 25 -------- config/scorecard/bases/config.yaml | 4 +- config/scorecard/kustomization.yaml | 2 +- config/scorecard/patches/kuttl.config.yaml | 6 ++ config/scorecard/patches/smoke.config.yaml | 7 -- 10 files changed, 41 insertions(+), 110 deletions(-) delete mode 100644 bundle/tests/scorecard/kuttl/smoketest/00-assert.yaml delete mode 100644 bundle/tests/scorecard/kuttl/smoketest/00-install.yaml create mode 100644 config/scorecard/patches/kuttl.config.yaml delete mode 100644 config/scorecard/patches/smoke.config.yaml diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index 467beb7454..637ce2e9ed 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -32,7 +32,7 @@ jobs: - name: "prepare cluster" env: KUBE_VERSION: ${{ matrix.kube-version }} - run: make prepare-scorecard-tests KUBE_VERSION=$KUBE_VERSION VERSION=e2e + run: make prepare-e2e KUBE_VERSION=$KUBE_VERSION VERSION=e2e - name: "run scorecard tests" run: make scorecard-tests diff --git a/Makefile b/Makefile index f55a4d98ac..d3174b9925 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,23 @@ OPERATOR_SDK_VERSION ?= 1.27.0 CERTMANAGER_VERSION ?= 1.10.0 +SCORECARD_KUTTL_TESTS := \ + ingress \ + instrumentation-dotnet \ + instrumentation-dotnet-multicontainer \ + instrumentation-java \ + instrumentation-java-multicontainer \ + instrumentation-nodejs \ + instrumentation-nodejs-multicontainer \ + instrumentation-python \ + instrumentation-python-multicontainer \ + smoke-pod-annotations \ + smoke-restarting-deployment \ + smoke-sidecar \ + smoke-simplest \ + smoke-statefulset \ + statefulset-features \ + ifndef ignore-not-found ignore-not-found = false endif @@ -189,13 +206,10 @@ e2e-log-operator: prepare-e2e: kuttl set-image-controller container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server install-openshift-routes load-image-all deploy TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh -.PHONY: prepare-scorecard-tests -prepare-scorecard-tests: prepare-e2e - kubectl apply -f tests/scorecard/rbac.yaml - .PHONY: scorecard-tests -scorecard-tests: operator-sdk - $(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1) +scorecard-tests: operator-sdk bundle-scorecard-kuttl-tests + kubectl apply -f tests/scorecard/rbac.yaml + $(OPERATOR_SDK) scorecard -w=15m bundle || (echo "scorecard test failed" && exit 1) # Build the container image, used only for local dev purposes @@ -381,6 +395,11 @@ bundle: kustomize operator-sdk manifests set-image-controller $(OPERATOR_SDK) bundle validate ./bundle ./hack/ignore-createdAt-bundle.sh +# Copy selected e2e tests to the kuttl scorecard tests of the bundle +.PHONY: bundle-scorecard-kuttl-tests +bundle-scorecard-kuttl-tests: + for t in $(SCORECARD_KUTTL_TESTS); do cp -r tests/e2e/$$t bundle/tests/scorecard/kuttl; done + # Build the bundle image, used only for local dev purposes .PHONY: bundle-build bundle-build: diff --git a/bundle/tests/scorecard/config.yaml b/bundle/tests/scorecard/config.yaml index 6cb201cea4..95cb08e14c 100644 --- a/bundle/tests/scorecard/config.yaml +++ b/bundle/tests/scorecard/config.yaml @@ -46,10 +46,9 @@ stages: storage: spec: mountPath: {} - - image: quay.io/operator-framework/scorecard-test-kuttl:v2.0.0 + - image: docker.io/andreasgerstmayr/scorecard-test-kuttl:1681909083 labels: - suite: kuttlsuite - test: kuttltest1 + suite: kuttl storage: spec: mountPath: {} diff --git a/bundle/tests/scorecard/kuttl/kuttl-test.yaml b/bundle/tests/scorecard/kuttl/kuttl-test.yaml index 20800f8eef..85baa9f354 100644 --- a/bundle/tests/scorecard/kuttl/kuttl-test.yaml +++ b/bundle/tests/scorecard/kuttl/kuttl-test.yaml @@ -1,4 +1,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestSuite -timeout: 150 +parallel: 1 startControlPlane: false +timeout: 120 +suppress: + - events diff --git a/bundle/tests/scorecard/kuttl/smoketest/00-assert.yaml b/bundle/tests/scorecard/kuttl/smoketest/00-assert.yaml deleted file mode 100644 index f19dd651af..0000000000 --- a/bundle/tests/scorecard/kuttl/smoketest/00-assert.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: simplest-collector -status: - readyReplicas: 1 - ---- - -apiVersion: v1 -kind: Service -metadata: - name: simplest-collector-headless -spec: - ports: - - appProtocol: grpc - name: jaeger-grpc - port: 14250 - protocol: TCP - targetPort: 14250 - - appProtocol: grpc - name: otlp-grpc - port: 4317 - protocol: TCP - targetPort: 4317 - - appProtocol: http - name: otlp-http - port: 4318 - protocol: TCP - targetPort: 4318 - - appProtocol: http - name: otlp-http-legacy - port: 55681 - protocol: TCP - targetPort: 4318 - ---- - -apiVersion: v1 -kind: Service -metadata: - name: simplest-collector -spec: - ports: - - appProtocol: grpc - name: jaeger-grpc - port: 14250 - protocol: TCP - targetPort: 14250 - - appProtocol: grpc - name: otlp-grpc - port: 4317 - protocol: TCP - targetPort: 4317 - - appProtocol: http - name: otlp-http - port: 4318 - protocol: TCP - targetPort: 4318 - - appProtocol: http - name: otlp-http-legacy - port: 55681 - protocol: TCP - targetPort: 4318 diff --git a/bundle/tests/scorecard/kuttl/smoketest/00-install.yaml b/bundle/tests/scorecard/kuttl/smoketest/00-install.yaml deleted file mode 100644 index eb4cc2b39e..0000000000 --- a/bundle/tests/scorecard/kuttl/smoketest/00-install.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: opentelemetry.io/v1alpha1 -kind: OpenTelemetryCollector -metadata: - name: simplest -spec: - config: | - receivers: - jaeger: - protocols: - grpc: - otlp: - protocols: - grpc: - http: - processors: - - exporters: - logging: - - service: - pipelines: - traces: - receivers: [jaeger,otlp] - processors: [] - exporters: [logging] \ No newline at end of file diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml index 3311fee65f..6f3be72131 100644 --- a/config/scorecard/bases/config.yaml +++ b/config/scorecard/bases/config.yaml @@ -4,5 +4,5 @@ metadata: name: config serviceaccount: scorecard-admin stages: - - parallel: true - tests: [] +- parallel: true + tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml index d36175b6a5..b223cee49f 100644 --- a/config/scorecard/kustomization.yaml +++ b/config/scorecard/kustomization.yaml @@ -13,7 +13,7 @@ patchesJson6902: version: v1alpha3 kind: Configuration name: config -- path: patches/smoke.config.yaml +- path: patches/kuttl.config.yaml target: group: scorecard.operatorframework.io version: v1alpha3 diff --git a/config/scorecard/patches/kuttl.config.yaml b/config/scorecard/patches/kuttl.config.yaml new file mode 100644 index 0000000000..5bcfbda107 --- /dev/null +++ b/config/scorecard/patches/kuttl.config.yaml @@ -0,0 +1,6 @@ +- op: add + path: /stages/0/tests/- + value: + image: docker.io/andreasgerstmayr/scorecard-test-kuttl:1681909083 + labels: + suite: kuttl diff --git a/config/scorecard/patches/smoke.config.yaml b/config/scorecard/patches/smoke.config.yaml deleted file mode 100644 index 40a7f2f6de..0000000000 --- a/config/scorecard/patches/smoke.config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - image: quay.io/operator-framework/scorecard-test-kuttl:v2.0.0 - labels: - suite: kuttlsuite - test: kuttltest1 From 6d4821ccebfa3ab92155ccc33e36a8491825191f Mon Sep 17 00:00:00 2001 From: Andreas Gerstmayr Date: Tue, 25 Apr 2023 15:30:50 +0200 Subject: [PATCH 4/5] Update list of scorecard tests Signed-off-by: Andreas Gerstmayr --- Makefile | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index d3174b9925..e60c8afd51 100644 --- a/Makefile +++ b/Makefile @@ -65,21 +65,9 @@ OPERATOR_SDK_VERSION ?= 1.27.0 CERTMANAGER_VERSION ?= 1.10.0 SCORECARD_KUTTL_TESTS := \ - ingress \ - instrumentation-dotnet \ - instrumentation-dotnet-multicontainer \ - instrumentation-java \ - instrumentation-java-multicontainer \ - instrumentation-nodejs \ - instrumentation-nodejs-multicontainer \ - instrumentation-python \ - instrumentation-python-multicontainer \ - smoke-pod-annotations \ - smoke-restarting-deployment \ - smoke-sidecar \ smoke-simplest \ - smoke-statefulset \ - statefulset-features \ + smoke-sidecar \ + instrumentation-java ifndef ignore-not-found ignore-not-found = false From 32c8d66ac0c92cb59b7c07a446f653d374727083 Mon Sep 17 00:00:00 2001 From: Andreas Gerstmayr Date: Mon, 8 May 2023 15:17:15 +0200 Subject: [PATCH 5/5] add comment to switch to upstream again once a PR is merged Signed-off-by: Andreas Gerstmayr --- config/scorecard/patches/kuttl.config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/scorecard/patches/kuttl.config.yaml b/config/scorecard/patches/kuttl.config.yaml index 5bcfbda107..f7f5269ca4 100644 --- a/config/scorecard/patches/kuttl.config.yaml +++ b/config/scorecard/patches/kuttl.config.yaml @@ -1,6 +1,9 @@ - op: add path: /stages/0/tests/- value: + # TODO: Switch to quay.io/operator-framework/scorecard-test-kuttl image + # once https://github.com/operator-framework/operator-sdk/pull/6401 is merged. + # https://github.com/open-telemetry/opentelemetry-operator/issues/1721 image: docker.io/andreasgerstmayr/scorecard-test-kuttl:1681909083 labels: suite: kuttl