feat(ci): enable e2e-test in ci

Signed-off-by: Szilard Parrag <[email protected]>

.github/workflows/ci.yaml

@@ -77,3 +77,14 @@ jobs:
           env:
             GITHUB_TOKEN: ${{ github.token }}
           run: make license-check
+
+  artifacts:
+    name: Artifacts
+    uses: ./.github/workflows/artifacts.yaml
+    with:
+      publish: ${{ github.event_name == 'push' }}
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      security-events: write

.github/workflows/e2e.yaml

@@ -0,0 +1,161 @@
+name: E2E tests
+
+on:
+  push:
+    branches:
+      - master
+      - "release-[0-9]+.[0-9]+*"
+  pull_request:
+
+env:
+  GO_VERSION: '1.21'
+  KUBECTL_VERSION: 'v1.24.1'
+
+jobs:
+  build:
+    name: Image build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and export
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          tags: controller:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          outputs: type=docker,dest=/tmp/controller.tar
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: controller
+          path: /tmp/controller.tar
+
+  go:
+    name: Go end2end tests
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      fail-fast: false
+#      matrix:
+#        SHARD: [0]
+#        SHARDS: [1]
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: controller
+          path: /tmp
+
+      - name: Load image
+        run: |
+          docker load --input /tmp/controller.tar
+          docker image ls -a
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Set up kubectl
+        uses: azure/setup-kubectl@v3
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Run e2e tests
+        run: make test-e2e-ci
+#        env:
+#          SHARD: ${{ matrix.SHARD }}
+#          SHARDS: ${{ matrix.SHARDS }}
+
+      - name: Archive Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-e2e-test-cluster-logs
+          path: build/_test
+          retention-days: 5
+
+  e2e-test:
+    name: Shell script tests with different k8s versions
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        kube: ["1.26", "1.27", "1.28", "1.29"]
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: controller
+          path: /tmp
+
+      - name: Load image
+        run: |
+          docker load --input /tmp/controller.tar
+          docker image ls -a
+
+      - name: Set up kubectl
+        uses: azure/setup-kubectl@v3
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      # See https://github.com/kubernetes-sigs/kind/releases/tag/v0.20.0
+      - name: Determine KinD node image version
+        id: node_image
+        run: |
+          case ${{ matrix.kube }} in
+            1.26)
+              NODE_IMAGE=kindest/node:v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb ;;
+            1.27)
+              NODE_IMAGE=kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72 ;;
+            1.28)
+              NODE_IMAGE=kindest/node:v1.28.0@sha256:b7a4cad12c197af3ba43202d3efe03246b3f0793f162afb40a33c923952d5b31 ;;
+            1.29)
+              NODE_IMAGE=kindest/node:v1.29.0@sha256:eaa1450915475849a73a9227b8f201df25e55e268e5d619312131292e324d570 ;;
+            esac
+
+            echo "image=$NODE_IMAGE" >> $GITHUB_OUTPUT
+
+      - name: Make setup
+        run: make kind-cluster stern
+        env:
+          KIND_IMAGE: ${{ steps.node_image.outputs.image }}
+
+      - name: Test script for E2E
+        run: make e2e-test-ci
+
+      - name: Print last 10k kubernetes logs from default, collector and example-tenant-ns namespaces
+        if: always()
+        run: |
+          mkdir -p build/_test
+          bin/stern -n default,collector,example-tenant-ns ".*" --tail 100000 --no-follow > build/_test/cluster.logs
+
+      - name: Archive Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: script-e2e-test-cluster-logs
+          path: build/_test
+          retention-days: 5

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.20 as builder
+FROM golang:1.21 as builder
 ARG TARGETOS
 ARG TARGETARCH
 

Makefile

@@ -6,6 +6,13 @@ export PATH := $(BIN):$(PATH)
 
 GOVERSION := $(shell go env GOVERSION)
 
+KIND := ${BIN}/kind
+KIND_VERSION ?= v0.20.0
+KIND_IMAGE ?= kindest/node:v1.29.0@sha256:eaa1450915475849a73a9227b8f201df25e55e268e5d619312131292e324d570 
+KIND_CLUSTER ?= kind
+
+CI_MODE_ENABLED := ""
+
 IMG ?= controller:latest
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.28.0
@@ -130,7 +137,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform
 ##@ Deployment
 
 ifndef ignore-not-found
-  ignore-not-found = false
+ignore-not-found = false
 endif
 
 .PHONY: install
@@ -197,8 +204,13 @@ tidy: ## Tidy Go modules
 	find . -iname "go.mod" -not -path "./.devcontainer/*" | xargs -L1 sh -c 'cd $$(dirname $$0); go mod tidy'
 
 .PHONY: e2e-test
-e2e-test: ## Run e2e tests, make sure subscription operator is running somewhere
-	cd e2e && timeout --foreground 15m ./e2e_test.sh || (echo "E2E test failed"; exit 1)
+e2e-test: ## Run e2e tests
+	cd e2e && export CI_MODE=$(CI_MODE_ENABLED) && timeout --foreground 15m ./e2e_test.sh || (echo "E2E test failed"; exit 1)
+
+.PHONY: e2e-test-ci
+e2e-test-ci: CI_MODE_ENABLED=1 
+e2e-test-ci: IMG="controller:latest" ## Run e2e tests, telemetry collector runs inside k8s
+e2e-test-ci: docker-build e2e-test
 
 .PHONY: check-diff
 check-diff: generate
@@ -209,8 +221,22 @@ license-check: ${LICENSEI} .licensei.cache ## Run license check
 	${LICENSEI} check
 	${LICENSEI} header
 
+stern: | ${BIN}
+	GOBIN=${BIN} go install github.com/stern/stern@latest
+
+.PHONY: kind-cluster
+kind-cluster: ${KIND}
+	kind create cluster --name $(KIND_CLUSTER) --image $(KIND_IMAGE)
+
 ## target: ci-run
-## target: licensei
+
+${KIND}: ${KIND}_${KIND_VERSION}_${GOVERSION} | ${BIN}
+	ln -sf $(notdir $<) $@
+
+${KIND}_${KIND_VERSION}_${GOVERSION}: IMPORT_PATH := sigs.k8s.io/kind
+${KIND}_${KIND_VERSION}_${GOVERSION}: VERSION := ${KIND_VERSION}
+${KIND}_${KIND_VERSION}_${GOVERSION}: | ${BIN}
+	${go_install_binary}
 
 ${LICENSEI}: ${LICENSEI}_${LICENSEI_VERSION}_${GOVERSION} | ${BIN}
 	ln -sf $(notdir $<) $@

cmd/main.go

@@ -112,7 +112,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	setupLog.Info("starting manager")
+	setupLog.Info("starting telemetry controller manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)

config/manager/kustomization.yaml

@@ -1,2 +1,8 @@
 resources:
 - manager.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: controller
+  newName: controller
+  newTag: latest

config/manager/manager.yaml

@@ -71,6 +71,7 @@ spec:
         args:
         - --leader-elect
         image: controller:latest
+        imagePullPolicy: IfNotPresent
         name: manager
         securityContext:
           allowPrivilegeEscalation: false

e2e/e2e_test.sh

@@ -9,7 +9,8 @@ create_if_does_not_exist() {
 }
 
 KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME_E2E:-so-e2e}
-# Backup current kubernetes context
+NO_KIND_CLEANUP=${NO_KIND_CLEANUP:-}
+  # Backup current kubernetes context
 CURRENT_K8S_CTX=$(kubectl config view | grep "current" | cut -f 2 -d : | xargs)
 
 # Prepare env
@@ -38,8 +39,21 @@ kubectl wait --namespace opentelemetry-operator-system --for=condition=available
 # Use example
 kubectl apply -f ../docs/examples/simple-demo
 
-
-(cd .. && timeout 5m make run &)
+if [[ -z "${CI_MODE}" ]]; then
+  $(cd .. && timeout 5m make run &)
+  #cd -
+else
+  kind load docker-image controller:latest --name "${KIND_CLUSTER_NAME}"
+  cd .. && make deploy && cd -
+  # helm upgrade --install \
+  #   --debug \
+  #   --wait \
+  #   --create-namespace \
+  #   --namespace example-tenant-ns \
+  #   -f values.yaml \
+  #   telemetry-controller \
+  #   "../charts/telemetry-controller" 
+fi
 
 # Create log-generator
 helm install --wait --create-namespace --namespace example-tenant-ns --generate-name oci://ghcr.io/kube-logging/helm-charts/log-generator