From 312e4a5142b5604b28a5d504bf099855d89651d7 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Mon, 12 Aug 2024 04:50:40 +0000 Subject: [PATCH] chart(test): test scenario on existing KEDA Signed-off-by: Viet Nguyen Duc --- .circleci/config.yml | 13 ++----------- .github/workflows/helm-chart-test.yml | 13 +++---------- Makefile | 12 ++++++------ tests/charts/make/chart_cluster_setup.sh | 11 ----------- tests/charts/make/chart_test.sh | 11 +++++++++++ 5 files changed, 22 insertions(+), 38 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c88ea673a..cda8c93a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,6 @@ workflows: test-strategy: disabled cluster: 'minikube' helm-version: 'v3.10.3' - test-existing-keda: false test-upgrade: true - kubernetes-test: name: "K8s test - Autoscaling Jobs" @@ -21,7 +20,6 @@ workflows: test-strategy: job cluster: 'minikube' helm-version: 'v3.11.3' - test-existing-keda: true test-upgrade: true - kubernetes-test: name: "K8s test - Autoscaling Deployments" @@ -31,7 +29,6 @@ workflows: test-strategy: deployment cluster: 'minikube' helm-version: 'v3.12.3' - test-existing-keda: true test-upgrade: true - kubernetes-test: name: "K8s test - Autoscaling Jobs - HTTPS" @@ -41,7 +38,6 @@ workflows: test-strategy: job_https cluster: 'minikube' helm-version: 'v3.13.3' - test-existing-keda: true test-upgrade: true - kubernetes-test: name: "K8s test - Autoscaling Jobs - Ingress hostname" @@ -51,7 +47,6 @@ workflows: test-strategy: job_hostname cluster: 'minikube' helm-version: 'v3.14.4' - test-existing-keda: false test-upgrade: true - kubernetes-test: name: "K8s test - Autoscaling Deployments - HTTPS" @@ -61,7 +56,6 @@ workflows: test-strategy: deployment_https cluster: 'minikube' helm-version: 'v3.15.3' - test-existing-keda: false test-upgrade: true - docker-test: name: "Docker test - Use random user (true)" @@ -194,8 +188,6 @@ jobs: type: string helm-version: type: string - test-existing-keda: - type: boolean test-upgrade: type: boolean executor: << parameters.machine-type >> @@ -207,7 +199,6 @@ jobs: CLUSTER: << parameters.cluster >> KUBERNETES_VERSION: << parameters.k8s-version >> HELM_VERSION: << parameters.helm-version >> - TEST_EXISTING_KEDA: << parameters.test-existing-keda >> TEST_UPGRADE_CHART: << parameters.test-upgrade >> steps: - run: @@ -228,7 +219,7 @@ jobs: make chart_setup_env make set_containerd_image_store CLUSTER=${CLUSTER} KUBERNETES_VERSION=${KUBERNETES_VERSION} NAME=${NAMESPACE} VERSION=${BRANCH} \ - BUILD_DATE=${BUILD_DATE} TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} make chart_cluster_setup + BUILD_DATE=${BUILD_DATE} make chart_cluster_setup - run: name: "Build Docker images" no_output_timeout: 30m @@ -260,7 +251,7 @@ jobs: N=3 while [ $N -gt 0 ]; do PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} \ - TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY} \ + TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY} \ && make test_video_integrity || true if [ $? -eq 0 ]; then echo "Tests passed" diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 10c6982e2..995abba7c 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -38,44 +38,37 @@ jobs: test-strategy: disabled cluster: 'minikube' helm-version: 'v3.10.3' - test-existing-keda: false test-upgrade: true - k8s-version: 'v1.26.15' test-strategy: job cluster: 'minikube' helm-version: 'v3.11.3' - test-existing-keda: true test-upgrade: true - k8s-version: 'v1.27.16' test-strategy: deployment cluster: 'minikube' helm-version: 'v3.12.3' - test-existing-keda: true test-upgrade: true - k8s-version: 'v1.28.12' test-strategy: job_https cluster: 'minikube' helm-version: 'v3.13.3' - test-existing-keda: true test-upgrade: true - k8s-version: 'v1.29.7' test-strategy: job_hostname cluster: 'minikube' helm-version: 'v3.14.4' - test-existing-keda: false test-upgrade: true - k8s-version: 'v1.30.3' test-strategy: deployment_https cluster: 'minikube' helm-version: 'v3.15.3' - test-existing-keda: false test-upgrade: true env: CLUSTER: ${{ matrix.cluster }} KUBERNETES_VERSION: ${{ matrix.k8s-version }} ARTIFACT_NAME: "${{ matrix.k8s-version }}-${{ matrix.test-strategy }}" HELM_VERSION: ${{ matrix.helm-version }} - TEST_EXISTING_KEDA: ${{ matrix.test-existing-keda }} TEST_UPGRADE_CHART: ${{ matrix.test-upgrade }} steps: - name: Free Disk Space (Ubuntu) @@ -141,7 +134,7 @@ jobs: with: timeout_minutes: 10 max_attempts: 3 - command: CLUSTER=${CLUSTER} KUBERNETES_VERSION=${KUBERNETES_VERSION} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} make chart_cluster_setup + command: CLUSTER=${CLUSTER} KUBERNETES_VERSION=${KUBERNETES_VERSION} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup - name: Test chart template run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test_template - name: Test set custom CA certificate @@ -162,12 +155,12 @@ jobs: timeout_minutes: 30 max_attempts: 3 command: | - NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} \ + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} \ && make test_video_integrity - name: Test chart upgrade if: (matrix.test-upgrade == true) run: | - NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} SET_MAX_REPLICAS=10 TEST_UPGRADE_CHART=${TEST_UPGRADE_CHART} make chart_test_autoscaling_${{ matrix.test-strategy }} + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SET_MAX_REPLICAS=10 TEST_UPGRADE_CHART=${TEST_UPGRADE_CHART} make chart_test_autoscaling_${{ matrix.test-strategy }} - name: Cleanup Kubernetes cluster if: always() run: CLUSTER=${CLUSTER} make chart_cluster_cleanup diff --git a/Makefile b/Makefile index b5ce033dd..9ce7233d6 100644 --- a/Makefile +++ b/Makefile @@ -796,18 +796,18 @@ chart_test_autoscaling_deployment_https: ./tests/charts/make/chart_test.sh DeploymentAutoscaling chart_test_autoscaling_deployment: - PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium CHART_ENABLE_TRACING=true \ + PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true \ SECURE_CONNECTION_SERVER=true SECURE_USE_EXTERNAL_CERT=true SERVICE_TYPE_NODEPORT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=31444 \ SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \ - TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_enableTracing_secureServer_externalCerts_nodePort_autoScaling_scaledObject_subPath.yaml" \ + TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_enableTracing_secureServer_externalCerts_nodePort_autoScaling_scaledObject_existingKEDA_subPath.yaml" \ ./tests/charts/make/chart_test.sh DeploymentAutoscaling chart_test_autoscaling_job_https: - PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true \ + PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true \ SECURE_CONNECTION_SERVER=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 SUB_PATH=/ \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \ - TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_basicAuth_secureServer_autoScaling_scaledJob.yaml" \ + TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_basicAuth_secureServer_autoScaling_scaledJob_existingKEDA.yaml" \ ./tests/charts/make/chart_test.sh JobAutoscaling chart_test_autoscaling_job_hostname: @@ -818,10 +818,10 @@ chart_test_autoscaling_job_hostname: ./tests/charts/make/chart_test.sh JobAutoscaling chart_test_autoscaling_job: - PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true \ + PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true TEST_CHROMIUM=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true \ SECURE_INGRESS_ONLY_CONFIG_INLINE=true SECURE_USE_EXTERNAL_CERT=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=selenium-grid.prod SUB_PATH=/ SELENIUM_GRID_PORT=443 \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \ - TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_nodeChromium_enableTracing_fullDistributed_secureIngress_externalCerts_ingressHostName_ingressTLSInline_autoScaling_scaledJob_.yaml" \ + TEMPLATE_OUTPUT_FILENAME="k8s_fullDistributed_secureIngress_externalCerts_ingressHostName_ingressTLSInline_autoScaling_scaledJob_existingKEDA_prefixSelenium_nodeChromium_enableTracing.yaml" \ ./tests/charts/make/chart_test.sh JobAutoscaling chart_test_language_bindings: diff --git a/tests/charts/make/chart_cluster_setup.sh b/tests/charts/make/chart_cluster_setup.sh index 1a55f60f5..ded500a19 100755 --- a/tests/charts/make/chart_cluster_setup.sh +++ b/tests/charts/make/chart_cluster_setup.sh @@ -50,12 +50,6 @@ elif [ "${CLUSTER}" = "minikube" ]; then sudo chown -R $USER $HOME/.kube $HOME/.minikube fi -if [ "${TEST_EXISTING_KEDA}" = "true" ]; then - helm repo add kedacore https://kedacore.github.io/charts - echo "Install KEDA core on kind kubernetes cluster" - helm upgrade -i ${KEDA_NAMESPACE} -n ${KEDA_NAMESPACE} --create-namespace --set webhooks.enabled=false kedacore/keda -fi - if [ "${CLUSTER}" = "kind" ]; then echo "Load built local Docker Images into Kind Cluster" image_list=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep ${NAMESPACE} | grep ${BUILD_DATE:-$VERSION}) @@ -63,8 +57,3 @@ if [ "${CLUSTER}" = "kind" ]; then kind load docker-image --name ${CLUSTER_NAME} "$image" done fi - -if [ "${TEST_EXISTING_KEDA}" = "true" ]; then - echo "Wait for KEDA core to be ready" - kubectl -n ${KEDA_NAMESPACE} wait --for=condition=ready pod -l app.kubernetes.io/instance=${KEDA_NAMESPACE} --timeout 180s -fi diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 1cc162646..c74746af0 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -315,6 +315,17 @@ if [ "${RENDER_HELM_TEMPLATE_ONLY}" = "true" ]; then exit 0 fi +if [ "${TEST_EXISTING_KEDA}" = "true" ] && [ "${TEST_UPGRADE_CHART}" != "true" ]; then + helm repo add kedacore https://kedacore.github.io/charts + echo "Install KEDA core on kind kubernetes cluster" + helm upgrade -i ${KEDA_NAMESPACE} -n ${KEDA_NAMESPACE} --create-namespace --set webhooks.enabled=false kedacore/keda +fi + +if [ "${TEST_EXISTING_KEDA}" = "true" ] && [ "${TEST_UPGRADE_CHART}" != "true" ]; then + echo "Wait for KEDA core to be ready" + kubectl -n ${KEDA_NAMESPACE} wait --for=condition=ready pod -l app.kubernetes.io/instance=${KEDA_NAMESPACE} --timeout 180s +fi + echo "Deploy Selenium Grid Chart" helm upgrade --install ${HELM_COMMAND_ARGS}