From 669c45157322368c2176a89b70a5285a70471223 Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk Date: Mon, 3 Jul 2023 11:48:51 +0100 Subject: [PATCH] Adds E2E coverage reporting Signed-off-by: Mikalai Radchuk --- .github/workflows/e2e.yaml | 15 +++++++- .github/workflows/unit-test.yaml | 1 + .gitignore | 5 +-- Makefile | 19 +++++++--- config/e2e/kustomization.yaml | 9 +++++ config/e2e/manager_e2e_coverage_copy_pod.yaml | 36 +++++++++++++++++++ config/e2e/manager_e2e_coverage_patch.yaml | 21 +++++++++++ config/e2e/manager_e2e_coverage_pvc.yaml | 10 ++++++ hack/e2e-coverage.sh | 33 +++++++++++++++++ 9 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 config/e2e/kustomization.yaml create mode 100644 config/e2e/manager_e2e_coverage_copy_pod.yaml create mode 100644 config/e2e/manager_e2e_coverage_patch.yaml create mode 100644 config/e2e/manager_e2e_coverage_pvc.yaml create mode 100755 hack/e2e-coverage.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d562d21fd..e92cdaf9c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -22,4 +22,17 @@ jobs: - name: Run e2e tests run: | - make e2e + # By default make stops building on first non-zero exit code which + # in case of E2E tests will mean that code coverage will only be + # collected on successful runs. We want to collect coverage even + # after failing tests. + # With -k flag make will continue the build, but will return non-zero + # exit code in case of any errors. + make -k e2e + + - uses: codecov/codecov-action@v3 + with: + files: e2e-cover.out + flags: e2e + fail_ci_if_error: true + functionalities: fixes diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index a169f2960..bba7222ab 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -25,5 +25,6 @@ jobs: - uses: codecov/codecov-action@v3 with: files: cover.out + flags: unit fail_ci_if_error: true functionalities: fixes diff --git a/.gitignore b/.gitignore index d6b142bfe..10817bcdf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,9 @@ Dockerfile.cross # Test binary, build with `go test -c` *.test -# Output of the go coverage tool, specifically when used with LiteIDE +# Output of the go coverage tools *.out +coverage # Release output dist/** @@ -35,4 +36,4 @@ install.sh .\#* # documentation website asset folder -docs/_site \ No newline at end of file +docs/_site diff --git a/Makefile b/Makefile index ed615c694..364a91439 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,8 @@ KIND_CLUSTER_NAME ?= operator-controller CONTAINER_RUNTIME ?= docker +KUSTOMIZE_BUILD_DIR ?= config/default + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -103,7 +105,13 @@ test-unit: $(SETUP_ENVTEST) ## Run the unit tests eval $$($(SETUP_ENVTEST) use -p env $(ENVTEST_VERSION)) && go test -tags $(GO_BUILD_TAGS) -count=1 -short $(UNIT_TEST_DIRS) -coverprofile cover.out e2e: KIND_CLUSTER_NAME=operator-controller-e2e -e2e: run kind-load-test-artifacts test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster +e2e: KUSTOMIZE_BUILD_DIR=config/e2e +e2e: GO_BUILD_FLAGS=-cover +e2e: run kind-load-test-artifacts test-e2e e2e-coverage kind-cluster-cleanup ## Run e2e test suite on local kind cluster + +.PHONY: e2e-coverage +e2e-coverage: + COVERAGE_OUTPUT=./e2e-cover.out ./hack/e2e-coverage.sh kind-load: $(KIND) ## Loads the currently constructed image onto the cluster $(KIND) load docker-image $(IMG) --name $(KIND_CLUSTER_NAME) @@ -115,6 +123,7 @@ kind-cluster: $(KIND) kind-cluster-cleanup ## Standup a kind cluster kind-cluster-cleanup: $(KIND) ## Delete the kind cluster $(KIND) delete cluster --name ${KIND_CLUSTER_NAME} +.PHONY: kind-load-test-artifacts kind-load-test-artifacts: $(KIND) ## Load the e2e testdata container images into a kind cluster $(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.47.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0 $(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/plain-v0/plain.v0.1.0 -t localhost/testdata/bundles/plain-v0/plain:v0.1.0 @@ -174,7 +183,7 @@ release: $(GORELEASER) ## Runs goreleaser for the operator-controller. By defaul quickstart: export MANIFEST="https://github.com/operator-framework/operator-controller/releases/download/$(VERSION)/operator-controller.yaml" quickstart: $(KUSTOMIZE) generate ## Generate the installation release manifests and scripts - $(KUSTOMIZE) build config/default | sed "s/:devel/:$(VERSION)/g" > operator-controller.yaml + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) | sed "s/:devel/:$(VERSION)/g" > operator-controller.yaml envsubst '$$CATALOGD_VERSION,$$CERT_MGR_VERSION,$$RUKPAK_VERSION,$$MANIFEST' < scripts/install.tpl.sh > install.sh ##@ Deployment @@ -186,7 +195,7 @@ endif .PHONY: install install: export MANIFEST="./operator-controller.yaml" install: manifests $(KUSTOMIZE) generate ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/default > operator-controller.yaml + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) > operator-controller.yaml envsubst '$$CATALOGD_VERSION,$$CERT_MGR_VERSION,$$RUKPAK_VERSION,$$MANIFEST' < scripts/install.tpl.sh | bash -s .PHONY: uninstall @@ -196,8 +205,8 @@ uninstall: manifests $(KUSTOMIZE) ## Uninstall CRDs from the K8s cluster specifi .PHONY: deploy deploy: manifests $(KUSTOMIZE) ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | kubectl apply -f - + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) | kubectl apply -f - .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) | kubectl delete --ignore-not-found=$(ignore-not-found) -f - diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml new file mode 100644 index 000000000..7d0ba86c5 --- /dev/null +++ b/config/e2e/kustomization.yaml @@ -0,0 +1,9 @@ +namespace: operator-controller-system + +resources: +- ../default +- manager_e2e_coverage_pvc.yaml +- manager_e2e_coverage_copy_pod.yaml + +patches: +- path: manager_e2e_coverage_patch.yaml diff --git a/config/e2e/manager_e2e_coverage_copy_pod.yaml b/config/e2e/manager_e2e_coverage_copy_pod.yaml new file mode 100644 index 000000000..45139d062 --- /dev/null +++ b/config/e2e/manager_e2e_coverage_copy_pod.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Pod +metadata: + name: e2e-coverage-copy-pod + labels: + app.kubernetes.io/name: e2e-coverage-copy-pod + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/component: e2e-coverage + app.kubernetes.io/created-by: operator-controller + app.kubernetes.io/part-of: operator-controller + app.kubernetes.io/managed-by: kustomize +spec: + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: tar + image: busybox:1.36 + command: ["sleep", "infinity"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + volumeMounts: + - name: e2e-coverage-volume + mountPath: /e2e-coverage + readOnly: true + volumes: + - name: e2e-coverage-volume + persistentVolumeClaim: + claimName: e2e-coverage + readOnly: true diff --git a/config/e2e/manager_e2e_coverage_patch.yaml b/config/e2e/manager_e2e_coverage_patch.yaml new file mode 100644 index 000000000..bda011daf --- /dev/null +++ b/config/e2e/manager_e2e_coverage_patch.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + - name: manager + env: + - name: GOCOVERDIR + value: /e2e-coverage + volumeMounts: + - name: e2e-coverage-volume + mountPath: /e2e-coverage + volumes: + - name: e2e-coverage-volume + persistentVolumeClaim: + claimName: e2e-coverage diff --git a/config/e2e/manager_e2e_coverage_pvc.yaml b/config/e2e/manager_e2e_coverage_pvc.yaml new file mode 100644 index 000000000..126d4d4e6 --- /dev/null +++ b/config/e2e/manager_e2e_coverage_pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: e2e-coverage +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 64Mi diff --git a/hack/e2e-coverage.sh b/hack/e2e-coverage.sh new file mode 100755 index 000000000..4b24bfe33 --- /dev/null +++ b/hack/e2e-coverage.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -euo pipefail + +COVERAGE_OUTPUT="${COVERAGE_OUTPUT:-e2e-cover.out}" + +OPERATOR_CONTROLLER_NAMESPACE="operator-controller-system" +OPERATOR_CONTROLLER_MANAGER_DEPLOYMENT_NAME="operator-controller-controller-manager" +COPY_POD_NAME="e2e-coverage-copy-pod" + +# Create a temporary directory for coverage +COVERAGE_DIR=$(mktemp -d) + +# Trap to cleanup temporary directory on script exit +function cleanup { + rm -rf "$COVERAGE_DIR" +} +trap cleanup EXIT + +# Coverage-instrumented binary produces coverage on termination, +# so we scale down the manager before gathering the coverage +kubectl -n "$OPERATOR_CONTROLLER_NAMESPACE" scale deployment/"$OPERATOR_CONTROLLER_MANAGER_DEPLOYMENT_NAME" --replicas=0 + +# Wait for the copy pod to be ready +kubectl -n "$OPERATOR_CONTROLLER_NAMESPACE" wait --for=condition=ready pod "$COPY_POD_NAME" + +# Copy the coverage data from the temporary pod +kubectl -n "$OPERATOR_CONTROLLER_NAMESPACE" cp "$COPY_POD_NAME":/e2e-coverage/ "$COVERAGE_DIR" + +# Convert binary coverage data files into the textual format +go tool covdata textfmt -i "$COVERAGE_DIR" -o "$COVERAGE_OUTPUT" + +echo "Coverage report generated successfully at: $COVERAGE_OUTPUT"