Skip to content

Commit

Permalink
[chore] Change the E2E automation to run the tests in OpenShift (open…
Browse files Browse the repository at this point in the history
…-telemetry#1254)

* Run the E2E tests in OpenShift

Signed-off-by: Israel Blancas <[email protected]>

* Set version for GitHub Actions workflow

Signed-off-by: Israel Blancas <[email protected]>

* Fix CI

Signed-off-by: Israel Blancas <[email protected]>

* Apply some changes to simplify the upgrade E2E test

Signed-off-by: Israel Blancas <[email protected]>

* Fix CI

Signed-off-by: Israel Blancas <[email protected]>

* Fix CI

Signed-off-by: Israel Blancas <[email protected]>

* Install OpenShift routes only if needed

Signed-off-by: Israel Blancas <[email protected]>

* Trigger Build

Signed-off-by: Israel Blancas <[email protected]>

* Remove cert-manager installation from deployment step

Signed-off-by: Israel Blancas <[email protected]>

* Trigger Build

Signed-off-by: Israel Blancas <[email protected]>

* Ensure the webhook don't fail when kubectl apply

Signed-off-by: Israel Blancas <[email protected]>

* Fix coding standard issues

Signed-off-by: Israel Blancas <[email protected]>

* Fix coding standard issues

Signed-off-by: Israel Blancas <[email protected]>

* Increase timeout for golangci

Signed-off-by: Israel Blancas <[email protected]>

* Apply changes requested in CR

Signed-off-by: Israel Blancas <[email protected]>

* Change timeout to 500ms

Signed-off-by: Israel Blancas <[email protected]>

* Change program file name

Signed-off-by: Israel Blancas <[email protected]>

* Do the check without depending on a manifest file

Signed-off-by: Israel Blancas <[email protected]>

* Change program file name

Signed-off-by: Israel Blancas <[email protected]>

Signed-off-by: Israel Blancas <[email protected]>
  • Loading branch information
iblancasa authored Dec 20, 2022
1 parent 776a143 commit 97c0349
Show file tree
Hide file tree
Showing 10 changed files with 185 additions and 44 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/continuous-integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
- name: Lint
uses: golangci/golangci-lint-action@v3
with:
args: -v
args: -v --timeout 5m
version: v1.48
working-directory: ${{ matrix.workdir }}

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ jobs:
- name: "run tests"
env:
KUBE_VERSION: ${{ matrix.kube-version }}
run: make prepare-e2e e2e e2e-upgrade KUBE_VERSION=$KUBE_VERSION
run: make prepare-e2e e2e e2e-upgrade KUBE_VERSION=$KUBE_VERSION VERSION=e2e

- name: "log operator if failed"
if: ${{ failure() }}
Expand Down
43 changes: 28 additions & 15 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ else
GOTEST_OPTS=-race -v
endif

START_KIND_CLUSTER ?= true

KUBE_VERSION ?= 1.24
KIND_CONFIG ?= kind-$(KUBE_VERSION).yaml

Expand Down Expand Up @@ -117,6 +119,7 @@ set-image-controller: manifests kustomize
.PHONY: deploy
deploy: set-image-controller
$(KUSTOMIZE) build config/default | kubectl apply -f -
go run hack/check-operator-ready.go 300

# Undeploy controller in the current Kubernetes context, configured in ~/.kube/config
.PHONY: undeploy
Expand Down Expand Up @@ -162,7 +165,7 @@ e2e:

# end-to-end-test for testing upgrading
.PHONY: e2e-upgrade
e2e-upgrade:
e2e-upgrade: undeploy
$(KUTTL) test --config kuttl-test-upgrade.yaml

.PHONY: e2e-log-operator
Expand All @@ -171,38 +174,38 @@ e2e-log-operator:
kubectl get deploy -A

.PHONY: prepare-e2e
prepare-e2e: kuttl set-test-image-vars set-image-controller container container-target-allocator start-kind install-metrics-server install-openshift-routes load-image-all
mkdir -p tests/_build/crds tests/_build/manifests
$(KUSTOMIZE) build config/default -o tests/_build/manifests/01-opentelemetry-operator.yaml
$(KUSTOMIZE) build config/crd -o tests/_build/crds/
prepare-e2e: kuttl set-image-controller container container-target-allocator start-kind cert-manager install-metrics-server install-openshift-routes load-image-all deploy
TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) ./hack/modify-test-images.sh

.PHONY: scorecard-tests
scorecard-tests: operator-sdk
$(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1)

.PHONY: set-test-image-vars
set-test-image-vars:
$(eval IMG=local/opentelemetry-operator:e2e)
$(eval TARGETALLOCATOR_IMG=local/opentelemetry-operator-targetallocator:e2e)

# Build the container image, used only for local dev purposes
# buildx is used to ensure same results for arm based systems (m1/2 chips)
.PHONY: container
container:
docker buildx build --platform linux/${ARCH} -t ${IMG} --build-arg VERSION_PKG=${VERSION_PKG} --build-arg VERSION=${VERSION} --build-arg VERSION_DATE=${VERSION_DATE} --build-arg OTELCOL_VERSION=${OTELCOL_VERSION} --build-arg TARGETALLOCATOR_VERSION=${TARGETALLOCATOR_VERSION} --build-arg AUTO_INSTRUMENTATION_JAVA_VERSION=${AUTO_INSTRUMENTATION_JAVA_VERSION} --build-arg AUTO_INSTRUMENTATION_NODEJS_VERSION=${AUTO_INSTRUMENTATION_NODEJS_VERSION} --build-arg AUTO_INSTRUMENTATION_PYTHON_VERSION=${AUTO_INSTRUMENTATION_PYTHON_VERSION} --build-arg AUTO_INSTRUMENTATION_DOTNET_VERSION=${AUTO_INSTRUMENTATION_DOTNET_VERSION} .
docker buildx build --load --platform linux/${ARCH} -t ${IMG} --build-arg VERSION_PKG=${VERSION_PKG} --build-arg VERSION=${VERSION} --build-arg VERSION_DATE=${VERSION_DATE} --build-arg OTELCOL_VERSION=${OTELCOL_VERSION} --build-arg TARGETALLOCATOR_VERSION=${TARGETALLOCATOR_VERSION} --build-arg AUTO_INSTRUMENTATION_JAVA_VERSION=${AUTO_INSTRUMENTATION_JAVA_VERSION} --build-arg AUTO_INSTRUMENTATION_NODEJS_VERSION=${AUTO_INSTRUMENTATION_NODEJS_VERSION} --build-arg AUTO_INSTRUMENTATION_PYTHON_VERSION=${AUTO_INSTRUMENTATION_PYTHON_VERSION} --build-arg AUTO_INSTRUMENTATION_DOTNET_VERSION=${AUTO_INSTRUMENTATION_DOTNET_VERSION} .

# Push the container image, used only for local dev purposes
.PHONY: container-push
container-push:
docker push ${IMG}

.PHONY: container-target-allocator-push
container-target-allocator-push:
docker push ${TARGETALLOCATOR_IMG}

.PHONY: container-target-allocator
container-target-allocator:
docker buildx build --platform linux/${ARCH} -t ${TARGETALLOCATOR_IMG} cmd/otel-allocator
docker buildx build --load --platform linux/${ARCH} -t ${TARGETALLOCATOR_IMG} cmd/otel-allocator

.PHONY: start-kind
start-kind:
ifeq (true,$(START_KIND_CLUSTER))
kind create cluster --config $(KIND_CONFIG)
endif

.PHONY: install-metrics-server
install-metrics-server:
Expand All @@ -216,12 +219,22 @@ install-openshift-routes:
load-image-all: load-image-operator load-image-target-allocator

.PHONY: load-image-operator
load-image-operator:
kind load docker-image local/opentelemetry-operator:e2e
load-image-operator: container
ifeq (true,$(START_KIND_CLUSTER))
kind load docker-image $(IMG)
else
$(MAKE) container-push
endif


.PHONY: load-image-target-allocator
load-image-target-allocator:
kind load docker-image ${TARGETALLOCATOR_IMG}
load-image-target-allocator: container-target-allocator
ifeq (true,$(START_KIND_CLUSTER))
kind load docker-image $(TARGETALLOCATOR_IMG)
else
$(MAKE) container-target-allocator-push
endif


.PHONY: cert-manager
cert-manager: cmctl
Expand Down
128 changes: 128 additions & 0 deletions hack/check-operator-ready.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"context"
"fmt"
"os"
"path/filepath"
"time"

appsv1 "k8s.io/api/apps/v1"
k8sruntime "k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"

"github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"sigs.k8s.io/controller-runtime/pkg/client"

otelv1alpha1 "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1"
)

var scheme *k8sruntime.Scheme

func init() {
scheme = k8sruntime.NewScheme()
utilruntime.Must(otelv1alpha1.AddToScheme(scheme))
utilruntime.Must(appsv1.AddToScheme(scheme))
}

func main() {
var timeout int
var kubeconfigPath string

defaultKubeconfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config")

pflag.IntVar(&timeout, "timeout", 300, "The timeout for the check.")
pflag.StringVar(&kubeconfigPath, "kubeconfig-path", defaultKubeconfigPath, "Absolute path to the KubeconfigPath file")
pflag.Parse()

pollInterval := 500 * time.Millisecond
timeoutPoll := time.Duration(timeout) * time.Second

config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
if err != nil {
println("Error reading the kubeconfig:", err.Error())
os.Exit(1)
}

clusterClient, err := client.New(config, client.Options{Scheme: scheme})
if err != nil {
println("Creating the Kubernetes client", err)
os.Exit(1)
}

fmt.Println("Waiting until the OTEL Collector Operator is deployed")
operatorDeployment := &appsv1.Deployment{}

err = wait.Poll(pollInterval, timeoutPoll, func() (done bool, err error) {
err = clusterClient.Get(
context.Background(),
client.ObjectKey{
Name: "opentelemetry-operator-controller-manager",
Namespace: "opentelemetry-operator-system",
},
operatorDeployment,
)
if err != nil {
fmt.Println(err)
return false, nil
}
return true, nil
})

if err != nil {
fmt.Println(err)
}
fmt.Println("OTEL Collector Operator is deployed properly!")

// Sometimes, the deployment of the OTEL Operator is ready but, when
// creating new instances of the OTEL Collector, the webhook is not reachable
// and kubectl apply fails. This code deployes an OTEL Collector instance
// until success (or timeout)
collectorInstance := otelv1alpha1.OpenTelemetryCollector{
ObjectMeta: metav1.ObjectMeta{
Name: "operator-check",
Namespace: "default",
},
}

// Ensure the collector is not there before the check
_ = clusterClient.Delete(context.Background(), &collectorInstance)

fmt.Println("Ensure the creation of OTEL Collectors is available")
err = wait.Poll(pollInterval, timeoutPoll, func() (done bool, err error) {
err = clusterClient.Create(
context.Background(),
&collectorInstance,
)
if err != nil {
fmt.Println(err)
return false, nil
}
return true, nil
})

if err != nil {
fmt.Println(err)
os.Exit(1)
}

_ = clusterClient.Delete(context.Background(), &collectorInstance)
}
17 changes: 13 additions & 4 deletions hack/install-metrics-server.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,16 @@
#!/bin/bash

# Install metrics-server on kind clusters for autoscale tests. Note: This is not needed for minikube,
# Install metrics-server on kind clusters for autoscale tests.
# Note: This is not needed for minikube,
# you can just add --addons "metrics-server" to the start command.
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
kubectl patch deployment -n kube-system metrics-server --type "json" -p '[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": --kubelet-insecure-tls}]'
kubectl wait --for=condition=available deployment/metrics-server -n kube-system --timeout=5m


if [[ "$(kubectl api-resources)" =~ "openshift" ]]; then
echo "Connected to an OpenShift cluster. metrics-server installation is not needed"
elif [[ "$(kubectl get deployment metrics-server -n kube-system 2>&1 )" =~ "NotFound" ]]; then
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
kubectl patch deployment -n kube-system metrics-server --type "json" -p '[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": --kubelet-insecure-tls}]'
kubectl wait --for=condition=available deployment/metrics-server -n kube-system --timeout=5m
else
echo "metrics-server is installed. Skipping installation"
fi
12 changes: 8 additions & 4 deletions hack/install-openshift-routes.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
#!/bin/bash

kubectl apply -f https://raw.githubusercontent.com/openshift/router/release-4.12/deploy/router_rbac.yaml
kubectl apply -f https://raw.githubusercontent.com/openshift/router/release-4.12/deploy/route_crd.yaml
kubectl apply -f https://raw.githubusercontent.com/openshift/router/release-4.12/deploy/router.yaml
kubectl wait --for=condition=available deployment/ingress-router -n openshift-ingress --timeout=5m
if [[ "$(kubectl api-resources)" =~ "openshift" ]]; then
echo "Connected to an OpenShift cluster. OpenShift routes installation is not needed"
else
kubectl apply -f https://raw.githubusercontent.com/openshift/router/release-4.12/deploy/router_rbac.yaml
kubectl apply -f https://raw.githubusercontent.com/openshift/router/release-4.12/deploy/route_crd.yaml
kubectl apply -f https://raw.githubusercontent.com/openshift/router/release-4.12/deploy/router.yaml
kubectl wait --for=condition=available deployment/ingress-router -n openshift-ingress --timeout=5m
fi
4 changes: 4 additions & 0 deletions hack/modify-test-images.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/bin/bash

sed -i "s#local/opentelemetry-operator-targetallocator:e2e#${TARGETALLOCATOR_IMG}#g" tests/e2e/smoke-targetallocator/00-install.yaml
sed -i "s#local/opentelemetry-operator-targetallocator:e2e#${TARGETALLOCATOR_IMG}#g" tests/e2e/targetallocator-features/00-install.yaml
9 changes: 1 addition & 8 deletions kuttl-test-upgrade.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,9 @@
# create a new one when the selector changed.
apiVersion: kuttl.dev/v1beta1
kind: TestSuite
crdDir: ./tests/_build/crds/
artifactsDir: ./tests/_build/artifacts/
kindContainers:
- local/opentelemetry-operator:e2e
- ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator:v0.49.0
commands:
- command: make cert-manager
- command: kubectl apply -f ./tests/e2e-upgrade/upgrade-test/opentelemetry-operator-v0.49.0.yaml
- command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
- command: sleep 60s
- command: go run hack/check-operator-ready.go
testDirs:
- ./tests/e2e-upgrade/
timeout: 300
8 changes: 0 additions & 8 deletions kuttl-test.yaml
Original file line number Diff line number Diff line change
@@ -1,14 +1,6 @@
apiVersion: kuttl.dev/v1beta1
kind: TestSuite
crdDir: ./tests/_build/crds/
artifactsDir: ./tests/_build/artifacts/
kindContainers:
- local/opentelemetry-operator:e2e
commands:
- command: make cert-manager
- command: kubectl apply -f ./tests/_build/manifests/01-opentelemetry-operator.yaml
- command: kubectl wait --timeout=5m --for=condition=available deployment opentelemetry-operator-controller-manager -n opentelemetry-operator-system
- command: sleep 5s
testDirs:
- ./tests/e2e/
timeout: 150
4 changes: 1 addition & 3 deletions tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
apiVersion: kuttl.dev/v1beta1
kind: TestStep
commands:
- command: kubectl apply -f ../../_build/manifests/01-opentelemetry-operator.yaml
- command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
- command: sleep 60s
- script: cd ../../../ && make deploy VERSION=e2e

0 comments on commit 97c0349

Please sign in to comment.