Skip to content

Commit

Permalink
Adds E2E coverage reporting
Browse files Browse the repository at this point in the history
Signed-off-by: Mikalai Radchuk <[email protected]>
  • Loading branch information
Mikalai Radchuk authored and ncdc committed Jul 28, 2023
1 parent a90c035 commit 669c451
Show file tree
Hide file tree
Showing 9 changed files with 141 additions and 8 deletions.
15 changes: 14 additions & 1 deletion .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,17 @@ jobs:

- name: Run e2e tests
run: |
make e2e
# By default make stops building on first non-zero exit code which
# in case of E2E tests will mean that code coverage will only be
# collected on successful runs. We want to collect coverage even
# after failing tests.
# With -k flag make will continue the build, but will return non-zero
# exit code in case of any errors.
make -k e2e
- uses: codecov/codecov-action@v3
with:
files: e2e-cover.out
flags: e2e
fail_ci_if_error: true
functionalities: fixes
1 change: 1 addition & 0 deletions .github/workflows/unit-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,6 @@ jobs:
- uses: codecov/codecov-action@v3
with:
files: cover.out
flags: unit
fail_ci_if_error: true
functionalities: fixes
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@ Dockerfile.cross
# Test binary, build with `go test -c`
*.test

# Output of the go coverage tool, specifically when used with LiteIDE
# Output of the go coverage tools
*.out
coverage

# Release output
dist/**
Expand All @@ -35,4 +36,4 @@ install.sh
.\#*

# documentation website asset folder
docs/_site
docs/_site
19 changes: 14 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ KIND_CLUSTER_NAME ?= operator-controller

CONTAINER_RUNTIME ?= docker

KUSTOMIZE_BUILD_DIR ?= config/default

# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
GOBIN=$(shell go env GOPATH)/bin
Expand Down Expand Up @@ -103,7 +105,13 @@ test-unit: $(SETUP_ENVTEST) ## Run the unit tests
eval $$($(SETUP_ENVTEST) use -p env $(ENVTEST_VERSION)) && go test -tags $(GO_BUILD_TAGS) -count=1 -short $(UNIT_TEST_DIRS) -coverprofile cover.out

e2e: KIND_CLUSTER_NAME=operator-controller-e2e
e2e: run kind-load-test-artifacts test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
e2e: KUSTOMIZE_BUILD_DIR=config/e2e
e2e: GO_BUILD_FLAGS=-cover
e2e: run kind-load-test-artifacts test-e2e e2e-coverage kind-cluster-cleanup ## Run e2e test suite on local kind cluster

.PHONY: e2e-coverage
e2e-coverage:
COVERAGE_OUTPUT=./e2e-cover.out ./hack/e2e-coverage.sh

kind-load: $(KIND) ## Loads the currently constructed image onto the cluster
$(KIND) load docker-image $(IMG) --name $(KIND_CLUSTER_NAME)
Expand All @@ -115,6 +123,7 @@ kind-cluster: $(KIND) kind-cluster-cleanup ## Standup a kind cluster
kind-cluster-cleanup: $(KIND) ## Delete the kind cluster
$(KIND) delete cluster --name ${KIND_CLUSTER_NAME}

.PHONY: kind-load-test-artifacts
kind-load-test-artifacts: $(KIND) ## Load the e2e testdata container images into a kind cluster
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.47.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/plain-v0/plain.v0.1.0 -t localhost/testdata/bundles/plain-v0/plain:v0.1.0
Expand Down Expand Up @@ -174,7 +183,7 @@ release: $(GORELEASER) ## Runs goreleaser for the operator-controller. By defaul

quickstart: export MANIFEST="https://github.com/operator-framework/operator-controller/releases/download/$(VERSION)/operator-controller.yaml"
quickstart: $(KUSTOMIZE) generate ## Generate the installation release manifests and scripts
$(KUSTOMIZE) build config/default | sed "s/:devel/:$(VERSION)/g" > operator-controller.yaml
$(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) | sed "s/:devel/:$(VERSION)/g" > operator-controller.yaml
envsubst '$$CATALOGD_VERSION,$$CERT_MGR_VERSION,$$RUKPAK_VERSION,$$MANIFEST' < scripts/install.tpl.sh > install.sh

##@ Deployment
Expand All @@ -186,7 +195,7 @@ endif
.PHONY: install
install: export MANIFEST="./operator-controller.yaml"
install: manifests $(KUSTOMIZE) generate ## Install CRDs into the K8s cluster specified in ~/.kube/config.
$(KUSTOMIZE) build config/default > operator-controller.yaml
$(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) > operator-controller.yaml
envsubst '$$CATALOGD_VERSION,$$CERT_MGR_VERSION,$$RUKPAK_VERSION,$$MANIFEST' < scripts/install.tpl.sh | bash -s

.PHONY: uninstall
Expand All @@ -196,8 +205,8 @@ uninstall: manifests $(KUSTOMIZE) ## Uninstall CRDs from the K8s cluster specifi
.PHONY: deploy
deploy: manifests $(KUSTOMIZE) ## Deploy controller to the K8s cluster specified in ~/.kube/config.
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
$(KUSTOMIZE) build config/default | kubectl apply -f -
$(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) | kubectl apply -f -

.PHONY: undeploy
undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
$(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
9 changes: 9 additions & 0 deletions config/e2e/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
namespace: operator-controller-system

resources:
- ../default
- manager_e2e_coverage_pvc.yaml
- manager_e2e_coverage_copy_pod.yaml

patches:
- path: manager_e2e_coverage_patch.yaml
36 changes: 36 additions & 0 deletions config/e2e/manager_e2e_coverage_copy_pod.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
apiVersion: v1
kind: Pod
metadata:
name: e2e-coverage-copy-pod
labels:
app.kubernetes.io/name: e2e-coverage-copy-pod
app.kubernetes.io/instance: controller-manager
app.kubernetes.io/component: e2e-coverage
app.kubernetes.io/created-by: operator-controller
app.kubernetes.io/part-of: operator-controller
app.kubernetes.io/managed-by: kustomize
spec:
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 65532
seccompProfile:
type: RuntimeDefault
containers:
- name: tar
image: busybox:1.36
command: ["sleep", "infinity"]
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- "ALL"
volumeMounts:
- name: e2e-coverage-volume
mountPath: /e2e-coverage
readOnly: true
volumes:
- name: e2e-coverage-volume
persistentVolumeClaim:
claimName: e2e-coverage
readOnly: true
21 changes: 21 additions & 0 deletions config/e2e/manager_e2e_coverage_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
spec:
template:
spec:
containers:
- name: kube-rbac-proxy
- name: manager
env:
- name: GOCOVERDIR
value: /e2e-coverage
volumeMounts:
- name: e2e-coverage-volume
mountPath: /e2e-coverage
volumes:
- name: e2e-coverage-volume
persistentVolumeClaim:
claimName: e2e-coverage
10 changes: 10 additions & 0 deletions config/e2e/manager_e2e_coverage_pvc.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: e2e-coverage
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 64Mi
33 changes: 33 additions & 0 deletions hack/e2e-coverage.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/bash

set -euo pipefail

COVERAGE_OUTPUT="${COVERAGE_OUTPUT:-e2e-cover.out}"

OPERATOR_CONTROLLER_NAMESPACE="operator-controller-system"
OPERATOR_CONTROLLER_MANAGER_DEPLOYMENT_NAME="operator-controller-controller-manager"
COPY_POD_NAME="e2e-coverage-copy-pod"

# Create a temporary directory for coverage
COVERAGE_DIR=$(mktemp -d)

# Trap to cleanup temporary directory on script exit
function cleanup {
rm -rf "$COVERAGE_DIR"
}
trap cleanup EXIT

# Coverage-instrumented binary produces coverage on termination,
# so we scale down the manager before gathering the coverage
kubectl -n "$OPERATOR_CONTROLLER_NAMESPACE" scale deployment/"$OPERATOR_CONTROLLER_MANAGER_DEPLOYMENT_NAME" --replicas=0

# Wait for the copy pod to be ready
kubectl -n "$OPERATOR_CONTROLLER_NAMESPACE" wait --for=condition=ready pod "$COPY_POD_NAME"

# Copy the coverage data from the temporary pod
kubectl -n "$OPERATOR_CONTROLLER_NAMESPACE" cp "$COPY_POD_NAME":/e2e-coverage/ "$COVERAGE_DIR"

# Convert binary coverage data files into the textual format
go tool covdata textfmt -i "$COVERAGE_DIR" -o "$COVERAGE_OUTPUT"

echo "Coverage report generated successfully at: $COVERAGE_OUTPUT"

0 comments on commit 669c451

Please sign in to comment.