From c1429957ad3dec3b3dd8d09595b22e5b5be95b44 Mon Sep 17 00:00:00 2001 From: Niladri Halder Date: Sat, 12 Oct 2024 08:57:42 +0000 Subject: [PATCH] ci: add ci to run pytests Signed-off-by: Niladri Halder --- .github/workflows/k8s-ci.yml | 18 +++++-- .gitignore | 6 +-- default.nix | 13 ++++- dependencies/control-plane | 2 +- nix/overlay.nix | 12 ++++- nix/pkgs/extensions/cargo-project.nix | 11 ++++- nix/pkgs/extensions/default.nix | 22 ++++++--- scripts/helm/publish-chart-yaml.sh | 60 +++-------------------- scripts/k8s/load-images-to-kind.sh | 69 +++++++++++++++++++++++++++ scripts/k8s/setup-io-prereq.sh | 2 +- scripts/python/generate-test-tag.sh | 69 +++++++++++++++++++++++++++ scripts/python/tag-chart.sh | 35 ++++++++++++++ scripts/python/test.sh | 58 ++++++++++++++++++++++ scripts/utils/log.sh | 4 ++ scripts/utils/repo.sh | 38 +++++++++++++++ scripts/utils/yaml.sh | 15 ++++++ shell.nix | 2 + 17 files changed, 360 insertions(+), 76 deletions(-) create mode 100755 scripts/k8s/load-images-to-kind.sh create mode 100755 scripts/python/generate-test-tag.sh create mode 100755 scripts/python/tag-chart.sh create mode 100755 scripts/python/test.sh create mode 100755 scripts/utils/repo.sh create mode 100644 scripts/utils/yaml.sh diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index b8c857c69..cb678561e 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -8,6 +8,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: recursive - uses: DeterminateSystems/nix-installer-action@v11 with: kvm: true @@ -17,12 +19,22 @@ jobs: export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r) echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV nix-shell ./scripts/k8s/shell.nix --run "echo" + - name: Build binaries and images + id: build + run: | + TAG=$(nix-shell ./shell.nix --run './scripts/python/generate-test-tag.sh') + BIN=$(mktemp -p . -d -t test-bin-XXXXXX) + nix-shell ./shell.nix --run "./scripts/python/tag-chart.sh $TAG" + RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-bins --build-binary-out $BIN --no-static-linking --skip-publish --debug + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "bin=$BIN" >> $GITHUB_OUTPUT - name: BootStrap k8s cluster run: | nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label" - - name: Install Helm Chart - run: | - nix-shell ./scripts/k8s/shell.nix --run "./scripts/helm/install.sh --dep-update --wait" + - name: Load images to Kind cluster + run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/load-images-to-kind.sh --tag ${{ steps.build.outputs.tag }} --trim-debug-suffix" + - name: Run Pytests + run: nix-shell ./shell.nix --run './scripts/python/test.sh' - name: The job has failed if: ${{ failure() }} run: | diff --git a/.gitignore b/.gitignore index f6ab4c0e6..026122377 100644 --- a/.gitignore +++ b/.gitignore @@ -22,9 +22,5 @@ __pycache__ /kubectl-plugin # Pytest assets +/test-bin-* tests/bdd/venv - -# Minikube assets -tests/bdd/minikube/.cleanup_config.yaml -tests/bdd/minikube/.cleanup_config.yaml.lock -tests/bdd/minikube/bin diff --git a/default.nix b/default.nix index f2f2486dd..1f4b579eb 100644 --- a/default.nix +++ b/default.nix @@ -1,9 +1,18 @@ -{ system ? null, allInOne ? true, incremental ? false, static ? false, img_tag ? "", tag ? "", img_org ? "", product_prefix ? "" }: +{ system ? null +, allInOne ? true +, incremental ? false +, static ? false +, img_tag ? "" +, tag ? "" +, img_org ? "" +, product_prefix ? "" +, rustFlags ? "" +}: let sources = import ./nix/sources.nix; hostSystem = (import sources.nixpkgs { }).hostPlatform.system; pkgs = import sources.nixpkgs { - overlays = [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { inherit allInOne incremental static img_tag tag img_org product_prefix; }) (import sources.rust-overlay) ]; + overlays = [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { inherit allInOne incremental static img_tag tag img_org product_prefix rustFlags; }) (import sources.rust-overlay) ]; system = if system != null then system else hostSystem; }; in diff --git a/dependencies/control-plane b/dependencies/control-plane index 934be6330..7158a9c64 160000 --- a/dependencies/control-plane +++ b/dependencies/control-plane @@ -1 +1 @@ -Subproject commit 934be6330b40f9e3c5b8b3c75cab3382ef08dcfe +Subproject commit 7158a9c64ef27726491c60a3aa3b22600c76ce0f diff --git a/nix/overlay.nix b/nix/overlay.nix index bebafb429..099e74fd9 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,4 +1,12 @@ -{ allInOne ? true, incremental ? false, static ? false, img_tag ? "", tag ? "", img_org ? "", product_prefix ? "" }: +{ allInOne ? true +, incremental ? false +, static ? false +, img_tag ? "" +, tag ? "" +, img_org ? "" +, product_prefix ? "" +, rustFlags ? "" +}: let config = import ./config.nix; img_prefix = if product_prefix == "" then config.product_prefix else product_prefix; @@ -6,7 +14,7 @@ in self: super: { sourcer = super.callPackage ./lib/sourcer.nix { }; images = super.callPackage ./pkgs/images { inherit img_tag img_org img_prefix; }; - extensions = super.callPackage ./pkgs/extensions { inherit allInOne incremental static tag; }; + extensions = super.callPackage ./pkgs/extensions { inherit allInOne incremental static tag rustFlags; }; openapi-generator = super.callPackage ./../dependencies/control-plane/nix/pkgs/openapi-generator { }; utils = super.callPackage ./pkgs/utils { inherit incremental; }; channel = import ./lib/rust.nix { pkgs = super.pkgs; }; diff --git a/nix/pkgs/extensions/cargo-project.nix b/nix/pkgs/extensions/cargo-project.nix index 3d0e57a03..60eb643e1 100644 --- a/nix/pkgs/extensions/cargo-project.nix +++ b/nix/pkgs/extensions/cargo-project.nix @@ -27,6 +27,7 @@ # for development and not for CI , incremental ? false , static ? false +, rustFlags }: let stable_channel = { @@ -105,6 +106,12 @@ let doCheck = false; }; release_build = { "release" = true; "debug" = false; }; + flags = + if builtins.stringLength rustFlags > 0 + then builtins.split " " rustFlags + else if static + then [ "-C" "target-feature=+crt-static" ] + else [ ]; in let build_with_naersk = { buildType, cargoBuildFlags }: @@ -133,7 +140,7 @@ let export OPENSSL_LIB_DIR=${static_ssl.out}/lib export OPENSSL_INCLUDE_DIR=${static_ssl.dev}/include ''; - ${if static then "RUSTFLAGS" else null} = [ "-C" "target-feature=+crt-static" ]; + ${if flags == [ ] then null else "RUSTFLAGS"} = flags; cargoLock = { lockFile = ../../../Cargo.lock; }; @@ -151,7 +158,7 @@ in build = { buildType, cargoBuildFlags ? [ ] }: if buildAllInOne then - builder { inherit buildType; cargoBuildFlags = [ "-p rpc" "-p metrics-exporter" "-p call-home" "-p upgrade" ]; } + builder { inherit buildType; cargoBuildFlags = [ "-p rpc" "-p metrics-exporter" "-p call-home" "-p upgrade" "-p kubectl-plugin" ]; } else builder { inherit buildType cargoBuildFlags; }; } diff --git a/nix/pkgs/extensions/default.nix b/nix/pkgs/extensions/default.nix index 808de4753..b0601dcca 100644 --- a/nix/pkgs/extensions/default.nix +++ b/nix/pkgs/extensions/default.nix @@ -1,4 +1,4 @@ -{ stdenv, git, lib, pkgs, allInOne, incremental, static, sourcer, tag ? "" }: +{ stdenv, git, lib, pkgs, allInOne, incremental, static, sourcer, tag ? "", rustFlags }: let versionDrv = import ../../lib/version.nix { inherit sourcer lib stdenv git tag; }; version = builtins.readFile "${versionDrv}"; @@ -8,7 +8,9 @@ let "tag_or_long" = builtins.readFile "${versionDrv.tag_or_long}"; }; project-builder = - pkgs.callPackage ../extensions/cargo-project.nix { inherit sourcer gitVersions allInOne incremental static; }; + pkgs.callPackage ../extensions/cargo-project.nix { + inherit sourcer gitVersions allInOne incremental static rustFlags; + }; installer = { pname, src, suffix ? "" }: stdenv.mkDerivation rec { inherit pname src; @@ -70,12 +72,18 @@ let pname = "obs-callhome-stats"; }; }; - kubectl-plugin = installer { - src = builder.build { - inherit buildType; - cargoBuildFlags = [ "--bin kubectl-mayastor" ]; + kubectl-plugin = rec { + recurseForDerivations = true; + plugin_builder = { buildType, builder, cargoBuildFlags ? [ "-p kubectl-plugin" ] }: builder.build { inherit buildType cargoBuildFlags; }; + plugin_installer = { pname, src }: installer { inherit pname src; }; + plugin = plugin_installer { + src = + if allInOne then + plugin_builder { inherit buildType builder; cargoBuildFlags = [ "-p kubectl-plugin" ]; } + else + plugin_builder { inherit buildType builder; cargoBuildFlags = [ "--bin kubectl-mayastor" ]; }; + pname = "kubectl-mayastor"; }; - pname = "kubectl-mayastor"; }; }; in diff --git a/scripts/helm/publish-chart-yaml.sh b/scripts/helm/publish-chart-yaml.sh index dcda2d2f9..0ca59b71f 100755 --- a/scripts/helm/publish-chart-yaml.sh +++ b/scripts/helm/publish-chart-yaml.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +SCRIPTDIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOTDIR="$SCRIPTDIR/../.." + +source "$ROOTDIR/scripts/utils/yaml.sh" +source "$ROOTDIR/scripts/utils/repo.sh" + # On a new appTag, update the Chart.yaml which is used to publish the chart to the appropriate # version and appVersion. # For this first iteration version and appVersion in the Chart.yaml *MUST* point to the stable @@ -24,46 +30,12 @@ die() set -euo pipefail -# This uses the existing remote refs for the openebs/mayastor-extensions repo to find the latest 'release/x.y' branch. -# Requires a 'git fetch origin' (origin being the remote entry for openebs/mayastor-extensions) or equivalent, if not -# done already. -latest_release_branch() { - if [ -n "$LATEST_RELEASE_BRANCH" ]; then - echo "$LATEST_RELEASE_BRANCH" - return 0 - fi - - cd "$ROOTDIR" - - # The latest release branch name is required for generating the helm chart version/appVersion - # for the 'main' branch only. - # The 'git branch' command in the below lines checks remote refs for release/x.y branch entries. - # Because the 'main' branch is not a significant branch for a user/contributor, this approach towards - # finding the latest release branch assumes that this script is used when the 'openebs/mayastor-extensions' - # repo is present amongst git remote refs. This happens automatically when the 'openebs/mayastor-extensions' - # repo is cloned, and not a user/contributor's fork. - local latest_release_branch=$(git branch \ - --all \ - --list "origin/release/*.*" \ - --format '%(refname:short)' \ - --sort 'refname' \ - | tail -n 1) - - if [ "$latest_release_branch" == "" ]; then - latest_release_branch="origin/release/0.0" - fi - - cd - >/dev/null - - echo "${latest_release_branch#*origin/}" -} - helm_testing_branch_version() { local release_branch=$1 local helm_kind="" if [[ "$check_branch" == "helm-testing/develop" ]]; then - release_branch=$(latest_release_branch) + release_branch=$(latest_release_branch "origin") helm_kind="main" elif [[ "$check_branch" =~ ^helm-testing\/release\/[0-9.]+$ ]]; then release_branch="${check_branch#helm-testing/}" @@ -203,22 +175,6 @@ index_yaml() fi } -# yq-go eats up blank lines -# this function gets around that using diff with --ignore-blank-lines -yq_ibl() -{ - set +e - diff_out=$(diff -B <(yq '.' "$2") <(yq "$1" "$2")) - error=$? - if [ "$error" != "0" ] && [ "$error" != "1" ]; then - exit "$error" - fi - if [ -n "$diff_out" ]; then - echo "$diff_out" | patch --quiet --no-backup-if-mismatch "$2" - - fi - set -euo pipefail -} - output_yaml() { newChartVersion=$1 @@ -272,8 +228,6 @@ Examples: EOF } -SCRIPTDIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" -ROOTDIR="$SCRIPTDIR/../.." CHART_FILE=${CHART_FILE:-"$ROOTDIR/chart/Chart.yaml"} CHART_VALUES=${CHART_VALUES:-"$ROOTDIR/chart/values.yaml"} CHART_DOC=${CHART_DOC:-"$ROOTDIR/chart/doc.yaml"} diff --git a/scripts/k8s/load-images-to-kind.sh b/scripts/k8s/load-images-to-kind.sh new file mode 100755 index 000000000..19cff5305 --- /dev/null +++ b/scripts/k8s/load-images-to-kind.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# Print usage options for this script. +print_help() { + cat < + +Options: + -h, --help Display this text. + --tag Input the container image tag. + --trim-debug-suffix Remove the '-debug' suffix from image names. + +Examples: + $(basename "${0}") --tag "ribbit" +EOF +} + +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/../.." +# Imports +source "$ROOT_DIR/scripts/utils/log.sh" + +set -e + +TAG= +TRIM_DEBUG_SUFFIX=0 + +while test $# -gt 0; do + arg="$1" + case "$arg" in + --tag) + test $# -lt 2 && log_fatal "missing value for the argument '$arg'" + TAG=$2 + shift + ;; + --tag=*) + TAG=${arg#*=} + ;; + --trim-debug-suffix) + TRIM_DEBUG_SUFFIX=1 + ;; + -h* | --help*) + print_help + exit 0 + ;; + *) + print_help + log_fatal "unexpected argument '$arg'" 1 + ;; + esac + shift +done + +if [ -z "$TAG" ]; then + log_fatal "requires an image tag" +fi + +IMAGE_TAG="v${TAG#v}" +# This list is static and is bound to fall out of date. +# TODO: generate the list of container images at run time from build assets. +images=("upgrade-job" "obs-callhome" "obs-callhome-stats" "metrics-exporter-io-engine") +load_cmd="kind load docker-image" +for image in "${images[@]}"; do + if [ "$TRIM_DEBUG_SUFFIX" = 1 ]; then + docker tag "openebs/mayastor-"$image"-debug":$IMAGE_TAG "openebs/mayastor-"$image:$IMAGE_TAG + fi + load_cmd+=" openebs/mayastor-"$image:$IMAGE_TAG +done +eval $load_cmd diff --git a/scripts/k8s/setup-io-prereq.sh b/scripts/k8s/setup-io-prereq.sh index b94654a92..6bba221d8 100755 --- a/scripts/k8s/setup-io-prereq.sh +++ b/scripts/k8s/setup-io-prereq.sh @@ -57,7 +57,7 @@ install_kernel_modules() { DISTRO="$(distro)" case "$DISTRO" in Ubuntu) - $SUDO apt-get install linux-modules-extra-$(uname -r) + $SUDO apt-get update && $SUDO apt-get install -y linux-modules-extra-$(uname -r) ;; NixOS | *) install_kernel_modules_nsup "$DISTRO" diff --git a/scripts/python/generate-test-tag.sh b/scripts/python/generate-test-tag.sh new file mode 100755 index 000000000..6b010afc2 --- /dev/null +++ b/scripts/python/generate-test-tag.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/../.." + +source "$ROOT_DIR"/scripts/utils/log.sh +source "$ROOT_DIR"/scripts/utils/repo.sh + +set -e + +CHART_VERSION=$(yq '.version' "$ROOT_DIR/chart/Chart.yaml") +GIT_REMOTE="origin" +TAG= + +# Print usage options for this script. +print_help() { + cat < Set the name of the git remote target. (default: "origin") + +Examples: + $(basename "${0}") -r upstream +EOF +} + +# Parse args. +while test $# -gt 0; do + arg="$1" + case "$arg" in + -r | --remote) + test $# -lt 2 && log_fatal "Missing value for the optional argument '$arg'." + GIT_REMOTE="$2" + shift + ;; + -r=* | --remote=*) + GIT_REMOTE="${arg#*=}" + ;; + -h* | --help*) + print_help + exit 0 + ;; + *) + print_help + log_fatal "unexpected argument '$arg'" 1 + ;; + esac + shift +done + +case "$CHART_VERSION" in +0.0.0) + latest_branch=$(git fetch -q && latest_release_branch $GIT_REMOTE $ROOT_DIR) + latest=${latest_branch#release/} + if [[ "$latest" =~ ^([0-9]+\.[0-9]+)$ ]]; then + latest="$latest.0" + fi + test "$(semver validate $latest)" = "valid" + TAG=$(semver bump minor $latest) + ;; +*) + test "$(semver validate $CHART_VERSION)" = "valid" + TAG=$(semver bump patch $CHART_VERSION) + ;; +esac + +echo "v${TAG#v}" diff --git a/scripts/python/tag-chart.sh b/scripts/python/tag-chart.sh new file mode 100755 index 000000000..c2e5ca108 --- /dev/null +++ b/scripts/python/tag-chart.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/../.." +# Imports +source "$ROOT_DIR/scripts/utils/log.sh" +source "$ROOT_DIR/scripts/utils/yaml.sh" + +set -e + +# No tag specified. +if [ -z "$1" ]; then + log_fatal "no tag specified" +fi + +CHART_VERSION=${1#v} +IMAGE_TAG="v$CHART_VERSION" +CHART_DIR="$ROOT_DIR/chart" +# TODO: tests should copy the chart and work with its own copy of the chart. Shouldn't modify the chart. +# chart/Chart.yaml +yq_ibl " + .version = \"$CHART_VERSION\" | + .appVersion = \"$CHART_VERSION\" | .appVersion style=\"double\" | + (.dependencies[] | select(.name == \"crds\").version) = \"$CHART_VERSION\" +" "$CHART_DIR/Chart.yaml" +# chart/charts/crds/Chart.yaml +yq_ibl ".version = \"$CHART_VERSION\"" "$CHART_DIR/charts/crds/Chart.yaml" +# chart/doc.yaml +yq_ibl ".chart.version = \"$CHART_VERSION\"" "$CHART_DIR/doc.yaml" +# chart/values.yaml +yq_ibl ".image.repoTags.extensions = \"$IMAGE_TAG\" | .image.repoTags.extensions style=\"double\" | + .image.repoTags.controlPlane = .image.tag | .image.repoTags.controlPlane style=\"double\" | + .image.repoTags.dataPlane = .image.tag | .image.repoTags.dataPlane style=\"double\" | + .image.pullPolicy = \"IfNotPresent\" +" "$CHART_DIR/values.yaml" diff --git a/scripts/python/test.sh b/scripts/python/test.sh new file mode 100755 index 000000000..dcdc88e64 --- /dev/null +++ b/scripts/python/test.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/../.." +# Imports +source "$ROOT_DIR/scripts/utils/log.sh" + +set -e + +_pytest() { + pytest "$1" || pytest_return=$? + # Exit code 5 denotes no tests were run, which is something we're ok with. + if [ "$pytest_return" = 5 ]; then + exit 0 + fi + exit $pytest_return +} + +# Print usage options for this script. +print_help() { + cat < /dev/null + + # The latest release branch name is required for generating the helm chart version/appVersion + # for the 'main' branch only. + # The 'git branch' command in the below lines checks remote refs for release/x.y branch entries. + # Because the 'main' branch is not a significant branch for a user/contributor, this approach towards + # finding the latest release branch assumes that this script is used when the 'openebs/mayastor-extensions' + # repo is present amongst git remote refs. This happens automatically when the 'openebs/mayastor-extensions' + # repo is cloned, and not a user/contributor's fork. + local latest_release_branch=$(git branch \ + --all \ + --list "$remote/release/*.*" \ + --format '%(refname:short)' \ + --sort 'refname' \ + | tail -n 1) + + if [ "$latest_release_branch" = "" ]; then + latest_release_branch="$remote/release/0.0" + fi + + popd > /dev/null + + echo "${latest_release_branch#*$remote/}" +} diff --git a/scripts/utils/yaml.sh b/scripts/utils/yaml.sh new file mode 100644 index 000000000..5a72ebe40 --- /dev/null +++ b/scripts/utils/yaml.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# yq-go eats up blank lines +# this function gets around that using diff with --ignore-blank-lines +yq_ibl() +{ + error=0 + diff_out=$(diff -B <(yq '.' "$2") <(yq "$1" "$2")) || error=$? + if [ "$error" != "0" ] && [ "$error" != "1" ]; then + exit "$error" + fi + if [ -n "$diff_out" ]; then + echo "$diff_out" | patch --quiet --no-backup-if-mismatch "$2" - + fi +} diff --git a/shell.nix b/shell.nix index 1b8d4f507..0a75423c7 100644 --- a/shell.nix +++ b/shell.nix @@ -27,6 +27,7 @@ mkShell { cowsay git helm-docs + kubectl kubernetes-helm-wrapped llvmPackages.libclang niv @@ -38,6 +39,7 @@ mkShell { python3 semver-tool utillinux + virtualenv which yq-go kind