From 3d5da12bb65b0aa21128c45b3f536be3d5ca7882 Mon Sep 17 00:00:00 2001 From: Niladri Halder Date: Mon, 4 Nov 2024 19:38:08 +0000 Subject: [PATCH] test: add basic sanity test for upgrade Signed-off-by: Niladri Halder --- .github/workflows/k8s-ci.yml | 11 +- .gitignore | 3 +- pytest.ini | 7 + scripts/helm/install.sh | 4 - scripts/python/generate-test-tag.sh | 1 + scripts/python/pytest_fmt.sh | 0 shell.nix | 3 + tests/bdd/common/environment.py | 17 ++ tests/bdd/common/helm.py | 305 +++++++++++++++++++++++++++ tests/bdd/common/kubectl_mayastor.py | 41 ++++ tests/bdd/common/repo.py | 35 +++ tests/bdd/features/test_upgrade.py | 87 ++++++++ tests/bdd/features/upgrade.feature | 8 + tests/bdd/requirements.txt | 6 +- 14 files changed, 518 insertions(+), 10 deletions(-) create mode 100644 pytest.ini create mode 100644 scripts/python/pytest_fmt.sh create mode 100644 tests/bdd/common/environment.py create mode 100644 tests/bdd/common/helm.py create mode 100644 tests/bdd/common/kubectl_mayastor.py create mode 100644 tests/bdd/common/repo.py create mode 100644 tests/bdd/features/test_upgrade.py create mode 100644 tests/bdd/features/upgrade.feature diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index cb678561e..4aa1d0c4c 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -23,18 +23,21 @@ jobs: id: build run: | TAG=$(nix-shell ./shell.nix --run './scripts/python/generate-test-tag.sh') - BIN=$(mktemp -p . -d -t test-bin-XXXXXX) + TEST_DIR=$(realpath $(mktemp -d ./test-dir-XXXXXX)) nix-shell ./shell.nix --run "./scripts/python/tag-chart.sh $TAG" - RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-bins --build-binary-out $BIN --no-static-linking --skip-publish --debug + RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-binary-out $TEST_DIR --no-static-linking --skip-publish --debug echo "tag=$TAG" >> $GITHUB_OUTPUT - echo "bin=$BIN" >> $GITHUB_OUTPUT + echo "bin=$TEST_DIR" >> $GITHUB_OUTPUT - name: BootStrap k8s cluster run: | nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label" - name: Load images to Kind cluster run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/load-images-to-kind.sh --tag ${{ steps.build.outputs.tag }} --trim-debug-suffix" - name: Run Pytests - run: nix-shell ./shell.nix --run './scripts/python/test.sh' + run: | + export UPGRADE_TARGET_VERSION=${{ steps.build.outputs.tag }} + export TEST_DIR=${{ steps.build.outputs.bin }} + nix-shell ./shell.nix --run "./scripts/python/test.sh" - name: The job has failed if: ${{ failure() }} run: | diff --git a/.gitignore b/.gitignore index 026122377..6398c18f6 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,6 @@ __pycache__ /kubectl-plugin # Pytest assets -/test-bin-* +/test-dir-* tests/bdd/venv +pytest.log \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..f9405c8c8 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +log_cli = true +log_cli_level = INFO +log_cli_format = %(asctime)s [%(levelname)s] %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S +log_file = pytest.log +log_file_level = DEBUG diff --git a/scripts/helm/install.sh b/scripts/helm/install.sh index 66d6e5d6f..88228d2d9 100755 --- a/scripts/helm/install.sh +++ b/scripts/helm/install.sh @@ -40,10 +40,6 @@ die() { exit "${_return}" } -nvme_ana_check() { - cat /sys/module/nvme_core/parameters/multipath -} - while [ "$#" -gt 0 ]; do case $1 in -h|--help) diff --git a/scripts/python/generate-test-tag.sh b/scripts/python/generate-test-tag.sh index e5b3652c5..d943a35a7 100755 --- a/scripts/python/generate-test-tag.sh +++ b/scripts/python/generate-test-tag.sh @@ -3,6 +3,7 @@ SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" ROOT_DIR="$SCRIPT_DIR/../.." +# Imports source "$ROOT_DIR"/scripts/utils/log.sh source "$ROOT_DIR"/scripts/utils/repo.sh diff --git a/scripts/python/pytest_fmt.sh b/scripts/python/pytest_fmt.sh new file mode 100644 index 000000000..e69de29bb diff --git a/shell.nix b/shell.nix index cab3ca6e1..d4aa69795 100644 --- a/shell.nix +++ b/shell.nix @@ -18,6 +18,8 @@ in mkShell { name = "extensions-shell"; buildInputs = [ + autoflake + black cacert cargo-expand cargo-udeps @@ -27,6 +29,7 @@ mkShell { cowsay git helm-docs + isort kubectl kubernetes-helm-wrapped llvmPackages.libclang diff --git a/tests/bdd/common/environment.py b/tests/bdd/common/environment.py new file mode 100644 index 000000000..f0d0b1257 --- /dev/null +++ b/tests/bdd/common/environment.py @@ -0,0 +1,17 @@ +import logging +import os + +logger = logging.getLogger(__name__) + + +def get_env(variable: str): + try: + value = os.getenv(variable) + if len(value) == 0: + raise ValueError("Env {variable} is empty") + logger.info(f"Found env {variable}={value}") + return value + + except Exception as e: + logger.error(f"Failed to get env {variable}: {e}") + return None diff --git a/tests/bdd/common/helm.py b/tests/bdd/common/helm.py new file mode 100644 index 000000000..b3766f497 --- /dev/null +++ b/tests/bdd/common/helm.py @@ -0,0 +1,305 @@ +import json +import logging +import os +import subprocess +from enum import Enum +from shutil import which + +from common import repo +from common.environment import get_env + +logger = logging.getLogger(__name__) + +helm_bin = which("helm") + + +def repo_ls(): + try: + result = subprocess.run( + [helm_bin, "repo", "ls", "-o", "json"], + capture_output=True, + check=True, + text=True, + ) + return json.loads(result.stdout.strip()) + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command 'helm repo ls -o json' failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + +def repo_add_mayastor(): + repos = repo_ls() + if repos is not None: + for r in repos: + if r["url"] == "https://openebs.github.io/mayastor-extensions": + return r["name"] + + try: + repo_name = "mayastor" + subprocess.run( + [ + helm_bin, + "repo", + "add", + repo_name, + "https://openebs.github.io/mayastor-extensions", + ], + capture_output=True, + check=True, + text=True, + ) + + subprocess.run( + [ + helm_bin, + "repo", + "update", + ], + capture_output=True, + check=True, + text=True, + ) + return repo_name + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command 'helm repo add mayastor https://openebs.github.io/mayastor-extensions' failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + +def latest_chart_so_far(version=None): + if version is None: + v = get_env("UPGRADE_TARGET_VERSION") + if v is None: + version = generate_test_tag() + else: + version = v + + repo_name = repo_add_mayastor() + assert repo_name is not None + + helm_search_command = [ + helm_bin, + "search", + "repo", + repo_name + "/mayastor", + "--version", + "<" + version, + "-o", + "json", + ] + try: + result = subprocess.run( + helm_search_command, + capture_output=True, + check=True, + text=True, + ) + result_chart_info = json.loads(result.stdout.strip()) + return result_chart_info[0]["version"] + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command {helm_search_command} failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + +class ChartSource(Enum): + HOSTED = "mayastor" + LOCAL = [ + "/bin/bash", + "-c", + os.path.join(repo.root_dir(), "scripts/helm/install.sh") + + " --dep-update --wait", + ] + + +class HelmReleaseClient: + """ + A client for interacting with Helm releases in a specified Kubernetes namespace. + + Attributes: + namespace (str): The Kubernetes namespace where the Helm releases are managed. + """ + + def __init__(self): + """ + Initializes the HelmReleaseClient. + """ + self.namespace = "mayastor" + + def get_metadata_mayastor(self): + command = [ + helm_bin, + "get", + "metadata", + "mayastor", + "-n", + self.namespace, + "-o", + "json", + ] + try: + result = subprocess.run( + command, + capture_output=True, + check=True, + text=True, + ) + return json.loads(result.stdout.strip()) + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command '{command}' failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + def list(self): + """ + Lists the deployed Helm releases in the specified namespace. + + Executes the 'helm ls' command to retrieve a list of deployed releases. + + Returns: + str: A newline-separated string of deployed release names, or None if an error occurs. + """ + try: + result = subprocess.run( + [ + helm_bin, + "ls", + "-n", + self.namespace, + "--deployed", + "--short", + ], + capture_output=True, + check=True, + text=True, + ) + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command 'helm ls -n {self.namespace} --deployed --short' failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + def release_is_deployed(self, release_name: str): + releases = self.list() + if releases is not None: + for release in releases: + if release == release_name: + return True + return False + + def install_mayastor(self, source: ChartSource, version=None): + if self.release_is_deployed("mayastor"): + logger.error( + f"WARN: Helm release 'mayastor' already exists in the 'mayastor' namespace." + ) + return + + install_command = [] + if source == ChartSource.HOSTED: + repo_name = repo_add_mayastor() + assert repo_name is not None + + install_command += [ + helm_bin, + "install", + "mayastor", + repo_name + "/" + source.value, + "--namespace=" + self.namespace, + "--create-namespace", + "--set", + "obs.callhome.sendReport=false,localpv-provisioner.analytics.enabled=false,etcd.livenessProbe.initialDelaySeconds=5,etcd.readinessProbe.initialDelaySeconds=5,etcd.replicaCount=1,eventing.enabled=false", + ] + if version is not None: + install_command += ["--version=" + version] + install_command += ["--wait"] + logger.info( + f"Installing mayastor helm chart from hosted registry, version='{version}'" + ) + + if source == ChartSource.LOCAL: + install_command = source.value + logger.info("Installing mayastor helm chart from local directory") + + try: + result = subprocess.run( + install_command, + capture_output=True, + check=True, + text=True, + ) + logger.info("Installation succeeded") + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command {install_command} failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + +def generate_test_tag(): + generate_test_tag_script = [ + "/bin/bash", + "-c", + os.path.join(repo.root_dir(), "scripts/python/generate-test-tag.sh"), + ] + try: + result = subprocess.run( + generate_test_tag_script, + capture_output=True, + check=True, + text=True, + ) + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command {generate_test_tag_script} failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None diff --git a/tests/bdd/common/kubectl_mayastor.py b/tests/bdd/common/kubectl_mayastor.py new file mode 100644 index 000000000..8ca4d2d8b --- /dev/null +++ b/tests/bdd/common/kubectl_mayastor.py @@ -0,0 +1,41 @@ +import logging +import os +import subprocess +from shutil import which + +from common.environment import get_env + +logger = logging.getLogger(__name__) + + +def get_bin_path(): + bins = get_env("TEST_DIR") + if bins: + return os.path.join(bins, "kubectl-plugin/bin/kubectl-mayastor") + logging.warning(f"Environmental variable 'BIN' is not set") + return which("kubectl-mayastor") + + +def kubectl_mayastor(args: list[str]): + command = [get_bin_path()] + command.extend(args) + logger.info(f"Running kubectl-mayastor command: {command}") + + try: + result = subprocess.run( + command, + capture_output=True, + check=True, + text=True, + ) + logger.info(f"kubectl-mayastor command succeeded") + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error(f"Error: command '{command}' failed with exit code {e.returncode}") + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None diff --git a/tests/bdd/common/repo.py b/tests/bdd/common/repo.py new file mode 100644 index 000000000..fa28c65f2 --- /dev/null +++ b/tests/bdd/common/repo.py @@ -0,0 +1,35 @@ +import logging +import os +import subprocess + +logger = logging.getLogger(__name__) + + +def root_dir(): + file_path = os.path.abspath(__file__) + return file_path.split("tests/bdd")[0] + + +def run_script(path: str, args: str = ""): + script = os.path.join(root_dir(), path) + " " + args + logger.info(f"Running script '{script}'") + command = ["/bin/bash", "-c", script] + try: + result = subprocess.run( + command, + capture_output=True, + check=True, + shell=True, + text=True, + ) + logger.info(f"Script succeeded") + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error(f"Error: command {command} failed with exit code {e.returncode}") + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None diff --git a/tests/bdd/features/test_upgrade.py b/tests/bdd/features/test_upgrade.py new file mode 100644 index 000000000..775859e9f --- /dev/null +++ b/tests/bdd/features/test_upgrade.py @@ -0,0 +1,87 @@ +"""Upgrade feature tests.""" + +import logging + +from common import helm, repo +from common.environment import get_env +from common.helm import ChartSource +from common.kubectl_mayastor import kubectl_mayastor +from kubernetes import client, config +from pytest_bdd import given, scenario, then, when +from retrying import retry + +logger = logging.getLogger(__name__) + +helm_client = helm.HelmReleaseClient() + + +@scenario("upgrade.feature", "upgrade command is issued") +def test_upgrade_command_is_issued(): + """upgrade command is issued.""" + + +@given("an installed mayastor helm chart") +def an_installed_mayastor_helm_chart(): + """an installed mayastor helm chart.""" + assert ( + helm_client.install_mayastor(ChartSource.HOSTED, helm.latest_chart_so_far()) + is not None + ) + + +@when("a kubectl mayastor upgrade command is issued") +def a_kubectl_mayastor_upgrade_command_is_issued(): + """a kubectl mayastor upgrade command is issued.""" + assert kubectl_mayastor(["upgrade"]) is not None + + +@then("the installed chart should be upgraded to the kubectl mayastor plugin's version") +def the_installed_chart_should_be_upgraded_to_the_kubectl_mayastor_plugins_version(): + """the installed chart should be upgraded to the kubectl mayastor plugin's version.""" + + upgrade_target_version = get_env("UPGRADE_TARGET_VERSION") + if upgrade_target_version is None: + upgrade_target_version = repo.run_script("scripts/python/generate-test-tag.sh") + upgrade_target_version = upgrade_target_version.lstrip("v") + logger.info(f"Value of upgrade_target_version={upgrade_target_version}") + + @retry( + stop_max_attempt_number=450, + wait_fixed=2000, + ) + def helm_upgrade_succeeded(): + logger.info("Checking if helm upgrade succeeded...") + metadata = helm_client.get_metadata_mayastor() + logger.debug(f"helm get metadata output={metadata}") + logger.debug(f"upgrade_target_version={upgrade_target_version}") + if metadata: + assert metadata["version"] == upgrade_target_version + return + raise ValueError("helm get metadata returned a None") + + @retry( + stop_max_attempt_number=600, + wait_fixed=2000, + ) + def data_plane_upgrade_succeeded(): + logger.info("Checking if data-plane upgrade succeeded...") + config.load_kube_config() + v1 = client.CoreV1Api() + label_selector = "app=io-engine" + pods = v1.list_namespaced_pod( + namespace="mayastor", label_selector=label_selector + ) + switch = True + for pod in pods.items: + for i, container in enumerate(pod.spec.containers): + if container.name == "io-engine": + logger.info( + f"pod.metadata.name={pod.metadata.name}, pod.spec.containers[{i}].image={container.image}" + ) + switch = switch and container.image.endswith(":develop") + logger.info(f"Value of 'switch' after the AND={switch}") + break + assert switch + + helm_upgrade_succeeded() + data_plane_upgrade_succeeded() diff --git a/tests/bdd/features/upgrade.feature b/tests/bdd/features/upgrade.feature new file mode 100644 index 000000000..8d4d9b61f --- /dev/null +++ b/tests/bdd/features/upgrade.feature @@ -0,0 +1,8 @@ +Feature: Upgrade + + Background: + Given an installed mayastor helm chart + + Scenario: upgrade command is issued + When a kubectl mayastor upgrade command is issued + Then the installed chart should be upgraded to the kubectl mayastor plugin's version diff --git a/tests/bdd/requirements.txt b/tests/bdd/requirements.txt index ffc56cb63..f054f2bb6 100644 --- a/tests/bdd/requirements.txt +++ b/tests/bdd/requirements.txt @@ -1,2 +1,6 @@ -pytest-bdd==7.3.0 kubernetes==31.0.0 +pytest-bdd==7.3.0 +pytest==8.3.3 +requests==2.26.0 +retrying==1.3.4 +semver==3.0.2