diff --git a/Makefile b/Makefile index 6853a676d..b2ac24195 100644 --- a/Makefile +++ b/Makefile @@ -364,30 +364,34 @@ test_video: video hub chrome firefox edge docker run -v $$(pwd):$$(pwd) -w $$(pwd) $(FFMPEG_BASED_NAME)/ffmpeg:$(FFMPEG_BASED_TAG) -v error -i ./tests/videos/edge_video.mp4 -f null - 2>error.log chart_setup_env: - ./tests/K8s/chart_setup_env.sh + ./tests/charts/make/chart_setup_env.sh chart_test: chart_lint \ + chart_test_template \ chart_install_chrome \ chart_install_firefox \ chart_install_edge +chart_test_template: + ./tests/charts/bootstrap.sh + chart_cluster_setup: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_cluster_setup.sh + VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_cluster_setup.sh chart_lint: - ./tests/K8s/chart_lint.sh + ./tests/charts/make/chart_lint.sh chart_install_chrome: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_install.sh NodeChrome + VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_install.sh NodeChrome chart_install_firefox: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_install.sh NodeFirefox + VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_install.sh NodeFirefox chart_install_edge: - VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_install.sh NodeEdge + VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_install.sh NodeEdge chart_cluster_cleanup: - ./tests/K8s/chart_cluster_cleanup.sh + ./tests/charts/make/chart_cluster_cleanup.sh .PHONY: \ all \ diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index ea0bb31dd..a1c6eff5b 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -36,13 +36,13 @@ helm install selenium-grid --set ingress.hostname=selenium-grid.k8s.local docker Selenium Grid has the ability to autoscaling browser nodes up/down based on the pending requests in the session queue. -To do this [KEDA](https://keda.sh/docs/2.12/scalers/selenium-grid-scaler/) is used. When enabling +To do this [KEDA](https://keda.sh/docs/latest/scalers/selenium-grid-scaler/) is used. When enabling autoscaling using `autoscaling.enabling` KEDA is installed automatically. To instead use an existing installation of KEDA you can enable autoscaling with `autoscaling.enableWithExistingKEDA` instead. KEDA can scale either with -[deployments](https://keda.sh/docs/2.12/concepts/scaling-deployments/#scaling-of-deployments-and-statefulsets) -or [jobs](https://keda.sh/docs/2.12/concepts/scaling-jobs/) and the charts support both types. This +[deployments](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaling-of-deployments-and-statefulsets) +or [jobs](https://keda.sh/docs/latest/concepts/scaling-jobs/) and the charts support both types. This chart support both modes. It is controlled with `autoscaling.scalingType` that can be set to either job (default) or deployment. diff --git a/charts/selenium-grid/TESTING.md b/charts/selenium-grid/TESTING.md index f9fa772d0..90c57c115 100644 --- a/charts/selenium-grid/TESTING.md +++ b/charts/selenium-grid/TESTING.md @@ -4,28 +4,46 @@ All related testing to this helm chart will be documented in this file. ## Test Traceability Matrix -| Features | TC Description | Coverage | -|------------------------|----------------------------------------------------------------------|----------| -| Basic Auth | Basic Auth is disabled | ✓ | -| | Basic Auth is enabled | ✗ | -| Auto scaling | Auto scaling with `enableWithExistingKEDA` is `true` | ✓ | -| | Auto scaling with `scalingType` is `job` | ✓ | -| | Auto scaling with `scalingType` is `deployment` | ✗ | -| | Auto scaling with `autoscaling.scaledOptions.minReplicaCount` is `0` | ✓ | -| Ingress | Ingress is enabled without `hostname` | ✓ | -| | Ingress is enabled with `hostname` is set | ✗ | -| | Hub `sub-path` is set with Ingress `ImplementationSpecific` paths | ✓ | -| Distributed components | `isolateComponents` is enabled | ✓ | -| | `isolateComponents` is disabled | ✗ | -| Browser Nodes | Node `nameOverride` is set | ✓ | -| | Sanity tests in node | ✓ | -| | Video recorder is enabled in node | ✗ | -| | Node `extraEnvironmentVariables` is set value | ✓ | -| General | Set new image registry via `global.seleniumGrid.imageRegistry` | ✓ | -| Tracing | Enable tracing via `SE_ENABLE_TRACING` | ✓ | -| | Disable tracing via `SE_ENABLE_TRACING` | ✗ | - -## Build & test Docker images with Helm charts +| Features | TC Description | Coverage | Test via | +|------------------------|----------------------------------------------------------------------|----------|----------| +| Basic Auth | Basic Auth is disabled | ✓ | Cluster | +| | Basic Auth is enabled | ✗ | | +| Auto scaling | Auto scaling with `enableWithExistingKEDA` is `true` | ✓ | Cluster | +| | Auto scaling with `scalingType` is `job` | ✓ | Cluster | +| | Auto scaling with `scalingType` is `deployment` | ✗ | | +| | Auto scaling with `autoscaling.scaledOptions.minReplicaCount` is `0` | ✓ | Cluster | +| Ingress | Ingress is enabled without `hostname` | ✓ | Cluster | +| | Ingress is enabled with `hostname` is set | ✗ | | +| | Hub `sub-path` is set with Ingress `ImplementationSpecific` paths | ✓ | Cluster | +| Distributed components | `isolateComponents` is enabled | ✓ | Cluster | +| | `isolateComponents` is disabled | ✗ | | +| Browser Nodes | Node `nameOverride` is set | ✓ | Cluster | +| | Sanity tests in node | ✓ | Cluster | +| | Video recorder is enabled in node | ✗ | | +| | Node `extraEnvironmentVariables` is set value | ✓ | Cluster | +| General | Set new image registry via `global.seleniumGrid.imageRegistry` | ✓ | Cluster | +| | Components are able to set `.affinity` | ✓ | Template | +| Tracing | Enable tracing via `SE_ENABLE_TRACING` | ✓ | Cluster | +| | Disable tracing via `SE_ENABLE_TRACING` | ✗ | | + +## Test Chart Template +- By using `helm template` command, the chart template is tested without installing it to Kubernetes cluster. +- Templates are rendered and the output as a YAML manifest file. The manifest file is then asserted with [pyyaml](https://pyyaml.org/wiki/PyYAMLDocumentation). +- Set of values are used to render the templates located in [tests/charts/templates/render](../../tests/charts/templates/render). + +```bash +# Back to root directory +cd ../.. + +# Build chart dependencies and lint +make chart_lint + +# Test chart template +make chart_test_template +``` +- Build chart dependencies and lint requires [Chart Testing `ct`](https://github.com/helm/chart-testing). There is a config file [ct.yaml](../../tests/charts/config/ct.yaml) to configure the chart testing. + +## Build & test Docker images with deploy to Kubernetes cluster Noted: These `make` commands are composed and tested on Linux x86_64. Run entire commands to build and test Docker images with Helm charts in local environment. @@ -48,3 +66,5 @@ make chart_test # Cleanup Kubernetes cluster make chart_cluster_cleanup ``` +- Setup Kubernetes environment requires [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/) and [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). +- Set of values are used to deploy the chart to Kubernetes cluster located in [tests/charts/ci](../../tests/charts/ci). diff --git a/tests/charts/bootstrap.sh b/tests/charts/bootstrap.sh new file mode 100755 index 000000000..d961b63ae --- /dev/null +++ b/tests/charts/bootstrap.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +mkdir -p tests/tests +cd tests || true + +if [ "${CI:-false}" = "false" ]; then + pip3 install virtualenv | grep -v 'Requirement already satisfied' + virtualenv docker-selenium-tests + source docker-selenium-tests/bin/activate +fi + +python -m pip install pyyaml==6.0.1 \ + | grep -v 'Requirement already satisfied' + +cd .. +helm template dummy --values tests/charts/templates/render/dummy.yaml \ + charts/selenium-grid > ./tests/tests/output_deployment.yaml + +python tests/charts/templates/test.py "./tests/tests/output_deployment.yaml" +ret_code=$? + +if [ "${CI:-false}" = "false" ]; then + deactivate +fi + +exit $ret_code diff --git a/charts/selenium-grid/ci/NodeChrome-values.yaml b/tests/charts/ci/NodeChrome-values.yaml similarity index 77% rename from charts/selenium-grid/ci/NodeChrome-values.yaml rename to tests/charts/ci/NodeChrome-values.yaml index 131434058..09396f42d 100644 --- a/charts/selenium-grid/ci/NodeChrome-values.yaml +++ b/tests/charts/ci/NodeChrome-values.yaml @@ -1,4 +1,4 @@ -# This is used in Helm chart testing. This disables the basic auth on selenium grid +# This is used in Helm chart testing # Configuration for chrome nodes chromeNode: nameOverride: my-chrome-name diff --git a/charts/selenium-grid/ci/NodeEdge-values.yaml b/tests/charts/ci/NodeEdge-values.yaml similarity index 77% rename from charts/selenium-grid/ci/NodeEdge-values.yaml rename to tests/charts/ci/NodeEdge-values.yaml index aec663509..27220e275 100644 --- a/charts/selenium-grid/ci/NodeEdge-values.yaml +++ b/tests/charts/ci/NodeEdge-values.yaml @@ -1,4 +1,4 @@ -# This is used in Helm chart testing. This disables the basic auth on selenium grid +# This is used in Helm chart testing # Configuration for chrome nodes chromeNode: enabled: false diff --git a/charts/selenium-grid/ci/NodeFirefox-values.yaml b/tests/charts/ci/NodeFirefox-values.yaml similarity index 77% rename from charts/selenium-grid/ci/NodeFirefox-values.yaml rename to tests/charts/ci/NodeFirefox-values.yaml index 8ccc9bd41..21bb6a285 100644 --- a/charts/selenium-grid/ci/NodeFirefox-values.yaml +++ b/tests/charts/ci/NodeFirefox-values.yaml @@ -1,4 +1,4 @@ -# This is used in Helm chart testing. This disables the basic auth on selenium grid +# This is used in Helm chart testing # Configuration for chrome nodes chromeNode: enabled: false diff --git a/charts/selenium-grid/ci/auth-ingress-values.yaml b/tests/charts/ci/auth-ingress-values.yaml similarity index 100% rename from charts/selenium-grid/ci/auth-ingress-values.yaml rename to tests/charts/ci/auth-ingress-values.yaml diff --git a/charts/selenium-grid/ci/autoscaling-values.yaml b/tests/charts/ci/autoscaling-values.yaml similarity index 100% rename from charts/selenium-grid/ci/autoscaling-values.yaml rename to tests/charts/ci/autoscaling-values.yaml diff --git a/charts/selenium-grid/ci/tracing-values.yaml b/tests/charts/ci/tracing-values.yaml similarity index 100% rename from charts/selenium-grid/ci/tracing-values.yaml rename to tests/charts/ci/tracing-values.yaml diff --git a/tests/K8s/chart-testing.yaml b/tests/charts/config/ct.yaml similarity index 100% rename from tests/K8s/chart-testing.yaml rename to tests/charts/config/ct.yaml diff --git a/tests/K8s/kind-cluster-config.yaml b/tests/charts/config/kind-cluster.yaml similarity index 100% rename from tests/K8s/kind-cluster-config.yaml rename to tests/charts/config/kind-cluster.yaml diff --git a/tests/K8s/chart_cluster_cleanup.sh b/tests/charts/make/chart_cluster_cleanup.sh similarity index 100% rename from tests/K8s/chart_cluster_cleanup.sh rename to tests/charts/make/chart_cluster_cleanup.sh diff --git a/tests/K8s/chart_cluster_setup.sh b/tests/charts/make/chart_cluster_setup.sh similarity index 92% rename from tests/K8s/chart_cluster_setup.sh rename to tests/charts/make/chart_cluster_setup.sh index b46231449..76b27872d 100755 --- a/tests/K8s/chart_cluster_setup.sh +++ b/tests/charts/make/chart_cluster_setup.sh @@ -8,7 +8,7 @@ KEDA_NAMESPACE=${KEDA_NAMESPACE:-"keda"} INGRESS_NAMESPACE=${INGRESS_NAMESPACE:-"ingress-nginx"} SUB_PATH=${SUB_PATH:-"/selenium"} CHART_PATH=${CHART_PATH:-"charts/selenium-grid"} -TEST_VALUES_PATH=${TEST_VALUES_PATH:-"charts/selenium-grid/ci"} +TEST_VALUES_PATH=${TEST_VALUES_PATH:-"tests/charts/ci"} SELENIUM_GRID_HOST=${SELENIUM_GRID_HOST:-"localhost"} SELENIUM_GRID_PORT=${SELENIUM_GRID_PORT:-"80"} WAIT_TIMEOUT=${WAIT_TIMEOUT:-"90s"} @@ -17,7 +17,7 @@ SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the clust # Function to clean up for retry step on workflow cleanup() { if [ "${SKIP_CLEANUP}" = "false" ]; then - ./tests/K8s/chart_cluster_cleanup.sh + ./tests/charts/make/chart_cluster_cleanup.sh fi } @@ -33,7 +33,7 @@ on_failure() { trap 'on_failure' ERR echo "Create Kind cluster" -kind create cluster --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests/K8s/kind-cluster-config.yaml +kind create cluster --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests/charts/config/kind-cluster.yaml echo "Install KEDA core on kind kubernetes cluster" kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.12.1/keda-2.12.1-core.yaml diff --git a/tests/K8s/chart_install.sh b/tests/charts/make/chart_install.sh similarity index 97% rename from tests/K8s/chart_install.sh rename to tests/charts/make/chart_install.sh index 9beffae4a..d89f96487 100755 --- a/tests/K8s/chart_install.sh +++ b/tests/charts/make/chart_install.sh @@ -8,7 +8,7 @@ KEDA_NAMESPACE=${KEDA_NAMESPACE:-"keda"} INGRESS_NAMESPACE=${INGRESS_NAMESPACE:-"ingress-nginx"} SUB_PATH=${SUB_PATH:-"/selenium"} CHART_PATH=${CHART_PATH:-"charts/selenium-grid"} -TEST_VALUES_PATH=${TEST_VALUES_PATH:-"charts/selenium-grid/ci"} +TEST_VALUES_PATH=${TEST_VALUES_PATH:-"tests/charts/ci"} SELENIUM_GRID_HOST=${SELENIUM_GRID_HOST:-"localhost"} SELENIUM_GRID_PORT=${SELENIUM_GRID_PORT:-"80"} MATRIX_BROWSER=${1:-"NodeChrome"} diff --git a/tests/K8s/chart_lint.sh b/tests/charts/make/chart_lint.sh similarity index 89% rename from tests/K8s/chart_lint.sh rename to tests/charts/make/chart_lint.sh index 09c46edc1..db92cf0e0 100755 --- a/tests/K8s/chart_lint.sh +++ b/tests/charts/make/chart_lint.sh @@ -22,7 +22,8 @@ python -m pip install yamale==4.0.4 \ | grep -v 'Requirement already satisfied' cd .. -ct lint --all --config tests/K8s/chart-testing.yaml +rm -rf ./charts/**/Chart.lock +ct lint --all --config tests/charts/config/ct.yaml if [ "${CI:-false}" = "false" ]; then deactivate diff --git a/tests/K8s/chart_setup_env.sh b/tests/charts/make/chart_setup_env.sh similarity index 100% rename from tests/K8s/chart_setup_env.sh rename to tests/charts/make/chart_setup_env.sh diff --git a/tests/charts/templates/render/dummy.yaml b/tests/charts/templates/render/dummy.yaml new file mode 100644 index 000000000..da60ed3a6 --- /dev/null +++ b/tests/charts/templates/render/dummy.yaml @@ -0,0 +1,24 @@ +# This is dummy values file for chart template testing +global: + seleniumGrid: + affinity: &affinity + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - selenium + topologyKey: "kubernetes.io/hostname" + +isolateComponents: true + +chromeNode: + affinity: *affinity + +firefoxNode: + affinity: *affinity + +edgeNode: + affinity: *affinity diff --git a/tests/charts/templates/test.py b/tests/charts/templates/test.py new file mode 100644 index 000000000..b004c2a8b --- /dev/null +++ b/tests/charts/templates/test.py @@ -0,0 +1,42 @@ +import yaml +import unittest +import sys +import logging + +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) + +def load_template(yaml_file): + try: + with open(yaml_file, 'r') as file: + documents = yaml.safe_load_all(file) + list_of_documents = [doc for doc in documents] + return list_of_documents + except yaml.YAMLError as error: + print("Error in configuration file: ", error) + +class ChartTemplateTests(unittest.TestCase): + def test_set_affinity(self): + resources_name = ['selenium-chrome-node', 'selenium-distributor', 'selenium-edge-node', 'selenium-firefox-node', + 'selenium-event-bus', 'selenium-router', 'selenium-session-map', 'selenium-session-queue'] + count = 0 + for doc in LIST_OF_DOCUMENTS: + if doc['metadata']['name'] in resources_name and doc['kind'] == 'Deployment': + self.assertTrue(doc['spec']['template']['spec']['affinity']['podAffinity']['requiredDuringSchedulingIgnoredDuringExecution'][0]['labelSelector']['matchExpressions'] is not None) + count += 1 + self.assertEqual(count, len(resources_name), "Not all resources have affinity set") + +if __name__ == '__main__': + failed = False + try: + FILE_NAME = sys.argv[1] + LIST_OF_DOCUMENTS = load_template(FILE_NAME) + suite = unittest.TestLoader().loadTestsFromTestCase(ChartTemplateTests) + test_runner = unittest.TextTestRunner(verbosity=3) + failed = not test_runner.run(suite).wasSuccessful() + except Exception as e: + logger.fatal(e) + failed = True + + if failed: + exit(1)