From cde2829f5f51c03da57434ed08653994e44bdba5 Mon Sep 17 00:00:00 2001 From: Szilard Parrag Date: Wed, 24 Jan 2024 16:08:44 +0100 Subject: [PATCH] feat(ci): enable e2e-test in ci Signed-off-by: Szilard Parrag --- .github/workflows/ci.yaml | 11 ++ .github/workflows/e2e.yaml | 161 ++++++++++++++++++++++++++++++ Dockerfile | 2 +- Makefile | 34 ++++++- cmd/main.go | 2 +- config/manager/kustomization.yaml | 6 ++ config/manager/manager.yaml | 1 + e2e/e2e_test.sh | 20 +++- 8 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/e2e.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f516a01..28e2315 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,3 +77,14 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: make license-check + + artifacts: + name: Artifacts + uses: ./.github/workflows/artifacts.yaml + with: + publish: ${{ github.event_name == 'push' }} + permissions: + contents: read + packages: write + id-token: write + security-events: write \ No newline at end of file diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 0000000..ea3d9a3 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,161 @@ +name: E2E tests + +on: + push: + branches: + - master + - "release-[0-9]+.[0-9]+*" + pull_request: + +env: + GO_VERSION: '1.21' + KUBECTL_VERSION: 'v1.24.1' + +jobs: + build: + name: Image build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and export + uses: docker/build-push-action@v4 + with: + context: . + tags: controller:latest + cache-from: type=gha + cache-to: type=gha,mode=max + outputs: type=docker,dest=/tmp/controller.tar + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: controller + path: /tmp/controller.tar + + go: + name: Go end2end tests + runs-on: ubuntu-latest + needs: build + strategy: + fail-fast: false +# matrix: +# SHARD: [0] +# SHARDS: [1] + + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: controller + path: /tmp + + - name: Load image + run: | + docker load --input /tmp/controller.tar + docker image ls -a + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + with: + version: ${{ env.KUBECTL_VERSION }} + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Run e2e tests + run: make test-e2e-ci +# env: +# SHARD: ${{ matrix.SHARD }} +# SHARDS: ${{ matrix.SHARDS }} + + - name: Archive Test Results + if: always() + uses: actions/upload-artifact@v3 + with: + name: go-e2e-test-cluster-logs + path: build/_test + retention-days: 5 + + e2e-test: + name: Shell script tests with different k8s versions + runs-on: ubuntu-latest + needs: build + strategy: + fail-fast: false + matrix: + kube: ["1.26", "1.27", "1.28", "1.29"] + + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: controller + path: /tmp + + - name: Load image + run: | + docker load --input /tmp/controller.tar + docker image ls -a + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + with: + version: ${{ env.KUBECTL_VERSION }} + + - name: Checkout code + uses: actions/checkout@v3 + + # See https://github.com/kubernetes-sigs/kind/releases/tag/v0.20.0 + - name: Determine KinD node image version + id: node_image + run: | + case ${{ matrix.kube }} in + 1.26) + NODE_IMAGE=kindest/node:v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb ;; + 1.27) + NODE_IMAGE=kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72 ;; + 1.28) + NODE_IMAGE=kindest/node:v1.28.0@sha256:b7a4cad12c197af3ba43202d3efe03246b3f0793f162afb40a33c923952d5b31 ;; + 1.29) + NODE_IMAGE=kindest/node:v1.29.0@sha256:eaa1450915475849a73a9227b8f201df25e55e268e5d619312131292e324d570 ;; + esac + + echo "image=$NODE_IMAGE" >> $GITHUB_OUTPUT + + - name: Make setup + run: make kind-cluster stern + env: + KIND_IMAGE: ${{ steps.node_image.outputs.image }} + + - name: Test script for E2E + run: make e2e-test-ci + + - name: Print last 10k kubernetes logs from default, collector and example-tenant-ns namespaces + if: always() + run: | + mkdir -p build/_test + bin/stern -n default,collector,example-tenant-ns ".*" --tail 100000 --no-follow > build/_test/cluster.logs + + - name: Archive Test Results + if: always() + uses: actions/upload-artifact@v3 + with: + name: script-e2e-test-cluster-logs + path: build/_test + retention-days: 5 diff --git a/Dockerfile b/Dockerfile index c389c09..aa74435 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.20 as builder +FROM golang:1.21 as builder ARG TARGETOS ARG TARGETARCH diff --git a/Makefile b/Makefile index 93cad3c..207a873 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,13 @@ export PATH := $(BIN):$(PATH) GOVERSION := $(shell go env GOVERSION) +KIND := ${BIN}/kind +KIND_VERSION ?= v0.20.0 +KIND_IMAGE ?= kindest/node:v1.29.0@sha256:eaa1450915475849a73a9227b8f201df25e55e268e5d619312131292e324d570 +KIND_CLUSTER ?= kind + +CI_MODE_ENABLED := "" + IMG ?= controller:latest # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.28.0 @@ -130,7 +137,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform ##@ Deployment ifndef ignore-not-found - ignore-not-found = false +ignore-not-found = false endif .PHONY: install @@ -197,8 +204,13 @@ tidy: ## Tidy Go modules find . -iname "go.mod" -not -path "./.devcontainer/*" | xargs -L1 sh -c 'cd $$(dirname $$0); go mod tidy' .PHONY: e2e-test -e2e-test: ## Run e2e tests, make sure subscription operator is running somewhere - cd e2e && timeout --foreground 15m ./e2e_test.sh || (echo "E2E test failed"; exit 1) +e2e-test: ## Run e2e tests + cd e2e && export CI_MODE=$(CI_MODE_ENABLED) && timeout --foreground 15m ./e2e_test.sh || (echo "E2E test failed"; exit 1) + +.PHONY: e2e-test-ci +e2e-test-ci: CI_MODE_ENABLED=1 +e2e-test-ci: IMG="controller:latest" ## Run e2e tests, telemetry collector runs inside k8s +e2e-test-ci: docker-build e2e-test .PHONY: check-diff check-diff: generate @@ -209,8 +221,22 @@ license-check: ${LICENSEI} .licensei.cache ## Run license check ${LICENSEI} check ${LICENSEI} header +stern: | ${BIN} + GOBIN=${BIN} go install github.com/stern/stern@latest + +.PHONY: kind-cluster +kind-cluster: ${KIND} + kind create cluster --name $(KIND_CLUSTER) --image $(KIND_IMAGE) + ## target: ci-run -## target: licensei + +${KIND}: ${KIND}_${KIND_VERSION}_${GOVERSION} | ${BIN} + ln -sf $(notdir $<) $@ + +${KIND}_${KIND_VERSION}_${GOVERSION}: IMPORT_PATH := sigs.k8s.io/kind +${KIND}_${KIND_VERSION}_${GOVERSION}: VERSION := ${KIND_VERSION} +${KIND}_${KIND_VERSION}_${GOVERSION}: | ${BIN} + ${go_install_binary} ${LICENSEI}: ${LICENSEI}_${LICENSEI_VERSION}_${GOVERSION} | ${BIN} ln -sf $(notdir $<) $@ diff --git a/cmd/main.go b/cmd/main.go index 6c48ce6..3294704 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -112,7 +112,7 @@ func main() { os.Exit(1) } - setupLog.Info("starting manager") + setupLog.Info("starting telemetry controller manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b8..ad13e96 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: controller + newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ecd3b66..068ec9a 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -71,6 +71,7 @@ spec: args: - --leader-elect image: controller:latest + imagePullPolicy: IfNotPresent name: manager securityContext: allowPrivilegeEscalation: false diff --git a/e2e/e2e_test.sh b/e2e/e2e_test.sh index 7535aab..45f3ffa 100755 --- a/e2e/e2e_test.sh +++ b/e2e/e2e_test.sh @@ -9,7 +9,8 @@ create_if_does_not_exist() { } KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME_E2E:-so-e2e} -# Backup current kubernetes context +NO_KIND_CLEANUP=${NO_KIND_CLEANUP:-} + # Backup current kubernetes context CURRENT_K8S_CTX=$(kubectl config view | grep "current" | cut -f 2 -d : | xargs) # Prepare env @@ -38,8 +39,21 @@ kubectl wait --namespace opentelemetry-operator-system --for=condition=available # Use example kubectl apply -f ../docs/examples/simple-demo - -(cd .. && timeout 5m make run &) +if [[ -z "${CI_MODE}" ]]; then + $(cd .. && timeout 5m make run &) + #cd - +else + kind load docker-image controller:latest --name "${KIND_CLUSTER_NAME}" + cd .. && make deploy && cd - + # helm upgrade --install \ + # --debug \ + # --wait \ + # --create-namespace \ + # --namespace example-tenant-ns \ + # -f values.yaml \ + # telemetry-controller \ + # "../charts/telemetry-controller" +fi # Create log-generator helm install --wait --create-namespace --namespace example-tenant-ns --generate-name oci://ghcr.io/kube-logging/helm-charts/log-generator