From 662cbf898e74ad016f4b1d76cf5e9654131b70aa Mon Sep 17 00:00:00 2001 From: Eddie Torres Date: Tue, 22 Nov 2022 14:31:01 +0000 Subject: [PATCH] Add test-helm-chart target to Makefile to lint/test Helm chart upgrades Signed-off-by: Eddie Torres --- Makefile | 9 + .../templates/tests/helm-tester.yaml | 201 ++++++++++++++++++ charts/aws-ebs-csi-driver/values.yaml | 5 +- charts/chart_schema.yaml | 37 ++++ charts/config.yaml | 19 ++ charts/lintconf.yaml | 42 ++++ hack/e2e/chart-testing.sh | 17 ++ hack/e2e/run.sh | 197 +++++++++-------- 8 files changed, 438 insertions(+), 89 deletions(-) create mode 100644 charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml create mode 100644 charts/chart_schema.yaml create mode 100644 charts/config.yaml create mode 100644 charts/lintconf.yaml create mode 100644 hack/e2e/chart-testing.sh diff --git a/Makefile b/Makefile index 4d4a70efad..993e082ba0 100644 --- a/Makefile +++ b/Makefile @@ -204,6 +204,15 @@ test-e2e-external-eks: GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \ ./hack/e2e/run.sh +.PHONY: test-helm-chart +test-helm-chart: + AWS_REGION=us-west-2 \ + AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ + EBS_INSTALL_SNAPSHOT="true" \ + HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ + HELM_TEST="true" \ + ./hack/e2e/run.sh + .PHONY: verify-vendor test: verify-vendor verify: verify-vendor diff --git a/charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml b/charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml new file mode 100644 index 0000000000..9006526fc2 --- /dev/null +++ b/charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml @@ -0,0 +1,201 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: helm-sa +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: + - apiGroups: [ "" ] + resources: + - events + - nodes + - pods + - replicationcontrollers + - serviceaccounts + - configmaps + - persistentvolumes + - persistentvolumeclaims + verbs: [ "list" ] + - apiGroups: [ "" ] + resources: + - services + - nodes + - nodes/proxy + - persistentvolumes + - persistentvolumeclaims + - pods + - pods/log + verbs: [ "get" ] + - apiGroups: [ "" ] + resources: + - namespaces + - persistentvolumes + - persistentvolumeclaims + - pods + - pods/exec + verbs: [ "create" ] + - apiGroups: [ "" ] + resources: + - namespaces + - persistentvolumes + - persistentvolumeclaims + - pods + verbs: [ "delete" ] + - apiGroups: [ "" ] + resources: + - persistentvolumeclaims + verbs: [ "update" ] + - apiGroups: [ "" ] + resources: + - pods/ephemeralcontainers + verbs: [ "patch" ] + - apiGroups: [ "" ] + resources: + - serviceaccounts + - configmaps + verbs: [ "watch" ] + - apiGroups: [ "apps" ] + resources: + - replicasets + - daemonsets + verbs: [ "list" ] + - apiGroups: [ "storage.k8s.io" ] + resources: + - storageclasses + verbs: [ "create" ] + - apiGroups: [ "storage.k8s.io" ] + resources: + - storageclasses + - csinodes + verbs: [ "get" ] + - apiGroups: [ "storage.k8s.io" ] + resources: + - storageclasses + verbs: [ "delete" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: + - volumesnapshots + - volumesnapshotclasses + - volumesnapshotcontents + verbs: [ "create" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: + - volumesnapshots + - volumesnapshotclasses + - volumesnapshotcontents + verbs: [ "get" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: + - volumesnapshotcontents + verbs: [ "update" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: + - volumesnapshots + - volumesnapshotclasses + - volumesnapshotcontents + verbs: [ "delete" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - clusterroles + verbs: [ "list" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - subjectaccessreviews + verbs: [ "create" ] + - apiGroups: [ "rbac.authorization.k8s.io" ] + resources: + - clusterroles + verbs: [ "list" ] + - apiGroups: [ "rbac.authorization.k8s.io" ] + resources: + - clusterrolebindings + verbs: [ "create" ] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role-binding +subjects: + - kind: ServiceAccount + name: helm-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: test-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ConfigMap +data: + manifests.yaml: | + ShortName: ebs + StorageClass: + FromFile: storageclass.yaml + SnapshotClass: + FromName: true + DriverInfo: + Name: ebs.csi.aws.com + SupportedSizeRange: + Min: 1Gi + Max: 16Ti + SupportedFsType: + xfs: {} + ext4: {} + SupportedMountOption: + dirsync: {} + TopologyKeys: ["topology.ebs.csi.aws.com/zone"] + Capabilities: + persistence: true + fsGroup: true + block: true + exec: true + snapshotDataSource: true + pvcDataSource: false + multipods: true + controllerExpansion: true + nodeExpansion: true + volumeLimits: true + topology: true + storageclass.yaml: | + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: ebs.csi.aws.com + provisioner: ebs.csi.aws.com + volumeBindingMode: WaitForFirstConsumer +metadata: + name: manifest-config +--- +apiVersion: v1 +kind: Pod +metadata: + name: helm-test + annotations: + "helm.sh/hook": test +spec: + containers: + - name: helm-test + image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20220624-1a63fdd9f2-master + command: [ "/bin/sh", "-c" ] + args: + - | + cp /etc/config/storageclass.yaml /workspace/storageclass.yaml + go install sigs.k8s.io/kubetest2/...@latest + kubectl config set-cluster cluster --server=https://kubernetes.default --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kubectl config set-context kubetest2 --cluster=cluster + kubectl config set-credentials sa --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + kubectl config set-context kubetest2 --user=sa + kubectl config use-context kubetest2 + kubetest2 noop --run-id='e2e-kubernetes' --test=ginkgo -- --test-package-version=$(curl https://storage.googleapis.com/kubernetes-release/release/stable-1.25.txt) --skip-regex='\[Disruptive\]|\[Serial\]' --focus-regex='External.Storage' --parallel=25 --test-args='-storage.testdriver=/etc/config/manifests.yaml' + volumeMounts: + - name: config-vol + mountPath: /etc/config + serviceAccountName: helm-sa + volumes: + - name: config-vol + configMap: + name: manifest-config + restartPolicy: Never diff --git a/charts/aws-ebs-csi-driver/values.yaml b/charts/aws-ebs-csi-driver/values.yaml index d32eecbb1f..985b9465a5 100644 --- a/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/aws-ebs-csi-driver/values.yaml @@ -183,8 +183,9 @@ controller: # cpu: 100m # memory: 128Mi serviceAccount: - create: true # A service account will be created for you if set to true. Set to false if you want to use your own. - name: ebs-csi-controller-sa # Name of the service-account to be used/created. + # A service account will be created for you if set to true. Set to false if you want to use your own. + create: true + name: ebs-csi-controller-sa annotations: {} tolerations: - key: CriticalAddonsOnly diff --git a/charts/chart_schema.yaml b/charts/chart_schema.yaml new file mode 100644 index 0000000000..2a26d9ba35 --- /dev/null +++ b/charts/chart_schema.yaml @@ -0,0 +1,37 @@ +name: str() +home: str(required=False) +version: str() +apiVersion: str() +appVersion: any(str(), num(), required=False) +description: str(required=False) +keywords: list(str(), required=False) +sources: list(str(), required=False) +maintainers: list(include('maintainer'), required=False) +dependencies: list(include('dependency'), required=False) +icon: str(required=False) +engine: str(required=False) +condition: str(required=False) +tags: str(required=False) +deprecated: bool(required=False) +kubeVersion: str(required=False) +annotations: map(str(), str(), required=False) +type: str(required=False) +--- +maintainer: + name: str() + email: str(required=False) + url: str(required=False) +--- +dependency: + name: str() + version: str() + repository: str(required=False) + condition: str(required=False) + tags: list(str(), required=False) + enabled: bool(required=False) + import-values: any(list(str()), list(include('import-value')), required=False) + alias: str(required=False) +--- +import-value: + child: str() + parent: str() diff --git a/charts/config.yaml b/charts/config.yaml new file mode 100644 index 0000000000..c498d4db91 --- /dev/null +++ b/charts/config.yaml @@ -0,0 +1,19 @@ +remote: origin +since: HEAD~1 +github-instance: https://github.com +lint-conf: charts/lintconf.yaml +chart-yaml-schema: charts/chart_schema.yaml +validate-chart-schema: true +validate-yaml: true +check-version-increment: false +charts: + - charts/aws-ebs-csi-driver +helm-extra-args: "--timeout 1000s" +upgrade: true +skip-missing-values: true +namespace: kube-system +release-label: release +kubectl-timeout: 1000s +skip-clean-up: false +print-logs: true +debug: true diff --git a/charts/lintconf.yaml b/charts/lintconf.yaml new file mode 100644 index 0000000000..90f48c889b --- /dev/null +++ b/charts/lintconf.yaml @@ -0,0 +1,42 @@ +--- +rules: + braces: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + brackets: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + colons: + max-spaces-before: 0 + max-spaces-after: 1 + commas: + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: + require-starting-space: true + min-spaces-from-content: 2 + document-end: disable + document-start: disable # No --- to start a file + empty-lines: + max: 2 + max-start: 0 + max-end: 0 + hyphens: + max-spaces-after: 1 + indentation: + spaces: consistent + indent-sequences: whatever # - list indentation will handle both indentation and without + check-multi-line-strings: false + key-duplicates: enable + line-length: disable # Lines can be any length + new-line-at-end-of-file: enable + new-lines: + type: unix + trailing-spaces: enable + truthy: + level: warning diff --git a/hack/e2e/chart-testing.sh b/hack/e2e/chart-testing.sh new file mode 100644 index 0000000000..e570785979 --- /dev/null +++ b/hack/e2e/chart-testing.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -uo pipefail + +function ct_install() { + INSTALL_PATH=${1} + CHART_TESTING_VERSION=${2} + if [[ ! -e ${INSTALL_PATH}/chart-testing ]]; then + CHART_TESTING_DOWNLOAD_URL="https://github.com/helm/chart-testing/releases/download/v${CHART_TESTING_VERSION}/chart-testing_${CHART_TESTING_VERSION}_linux_amd64.tar.gz" + curl --silent --location "${CHART_TESTING_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}" + chmod +x "${INSTALL_PATH}"/ct + fi + + export PATH=/root/.local/bin:$PATH + python3 -m pip install --upgrade pip + python3 -m pip install --user yamllint yamale +} diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index 96ba8bde77..6d55c14dcf 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -22,6 +22,7 @@ source "${BASE_DIR}"/eksctl.sh source "${BASE_DIR}"/helm.sh source "${BASE_DIR}"/kops.sh source "${BASE_DIR}"/util.sh +source "${BASE_DIR}"/chart-testing.sh DRIVER_NAME=${DRIVER_NAME:-aws-ebs-csi-driver} CONTAINER_NAME=${CONTAINER_NAME:-ebs-plugin} @@ -76,6 +77,8 @@ TEST_EXTRA_FLAGS=${TEST_EXTRA_FLAGS:-} EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"false"} EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.1.0"} +HELM_TEST=${HELM_TEST:-"false"} +CHART_TESTING_VERSION=${CHART_TESTING_VERSION:-3.7.1} CLEAN=${CLEAN:-"true"} loudecho "Testing in region ${REGION} and zones ${ZONES}" @@ -99,28 +102,33 @@ loudecho "Installing helm to ${BIN_DIR}" helm_install "${BIN_DIR}" HELM_BIN=${BIN_DIR}/helm -loudecho "Installing ginkgo to ${BIN_DIR}" -GINKGO_BIN=${BIN_DIR}/ginkgo -if [[ ! -e ${GINKGO_BIN} ]]; then - pushd /tmp - GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install github.com/onsi/ginkgo/v2/ginkgo@v2.4.0 - popd - ginkgo version -fi +if [[ "${HELM_TEST}" == true ]]; then + loudecho "Installing chart-testing ${CHART_TESTING_VERSION} to ${BIN_DIR}" + ct_install "${BIN_DIR}" "${CHART_TESTING_VERSION}" + CHART_TESTING_BIN=${BIN_DIR}/ct +else + loudecho "Installing ginkgo to ${BIN_DIR}" + GINKGO_BIN=${BIN_DIR}/ginkgo + if [[ ! -e ${GINKGO_BIN} ]]; then + pushd /tmp + GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install github.com/onsi/ginkgo/v2/ginkgo@v2.2.0 + popd + ginkgo version + fi + loudecho "Installing kubetest2 to ${BIN_DIR}" + KUBETEST2_BIN=${BIN_DIR}/kubetest2 + if [[ ! -e ${KUBETEST2_BIN} ]]; then + pushd /tmp + GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install sigs.k8s.io/kubetest2/...@latest + popd + fi -loudecho "Installing kubetest2 to ${BIN_DIR}" -KUBETEST2_BIN=${BIN_DIR}/kubetest2 -if [[ ! -e ${KUBETEST2_BIN} ]]; then - pushd /tmp - GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install sigs.k8s.io/kubetest2/...@latest - popd + ecr_build_and_push "${REGION}" \ + "${AWS_ACCOUNT_ID}" \ + "${IMAGE_NAME}" \ + "${IMAGE_TAG}" fi -ecr_build_and_push "${REGION}" \ - "${AWS_ACCOUNT_ID}" \ - "${IMAGE_NAME}" \ - "${IMAGE_TAG}" - if [[ "${CLUSTER_TYPE}" == "kops" ]]; then kops_create_cluster \ "$SSH_KEY_PATH" \ @@ -165,89 +173,102 @@ if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml fi -loudecho "Deploying driver" -startSec=$(date +'%s') - -HELM_ARGS=(upgrade --install "${DRIVER_NAME}" - --namespace kube-system - --set image.repository="${IMAGE_NAME}" - --set image.tag="${IMAGE_TAG}" - --wait - --kubeconfig "${KUBECONFIG}" - ./charts/"${DRIVER_NAME}") -if [[ -f "$HELM_VALUES_FILE" ]]; then - HELM_ARGS+=(-f "${HELM_VALUES_FILE}") -fi -eval "EXPANDED_HELM_EXTRA_FLAGS=$HELM_EXTRA_FLAGS" -if [[ -n "$EXPANDED_HELM_EXTRA_FLAGS" ]]; then - HELM_ARGS+=("${EXPANDED_HELM_EXTRA_FLAGS}") -fi -set -x -"${HELM_BIN}" "${HELM_ARGS[@]}" -set +x - -endSec=$(date +'%s') -secondUsed=$(((endSec - startSec) / 1)) -# Set timeout threshold as 20 seconds for now, usually it takes less than 10s to startup -if [ $secondUsed -gt $DRIVER_START_TIME_THRESHOLD_SECONDS ]; then - loudecho "Driver start timeout, took $secondUsed but the threshold is $DRIVER_START_TIME_THRESHOLD_SECONDS. Fail the test." - exit 1 -fi -loudecho "Driver deployment complete, time used: $secondUsed seconds" - -loudecho "Testing focus ${GINKGO_FOCUS}" - -if [[ $TEST_PATH == "./tests/e2e-kubernetes/..." ]]; then - pushd ${PWD}/tests/e2e-kubernetes - packageVersion=$(echo $(cut -d '.' -f 1,2 <<< $K8S_VERSION)) - +if [[ "${HELM_TEST}" == true ]]; then + loudecho "Test and lint Helm chart with chart-testing" + pushd ${PWD}/charts/aws-ebs-csi-driver set -x set +e - kubetest2 noop \ - --run-id="e2e-kubernetes" \ - --test=ginkgo \ - -- \ - --skip-regex="${GINKGO_SKIP}" \ - --focus-regex="${GINKGO_FOCUS}" \ - --test-package-version=$(curl https://storage.googleapis.com/kubernetes-release/release/stable-$packageVersion.txt) \ - --parallel=25 \ - --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=$KUBECONFIG" - + export KUBECONFIG="${KUBECONFIG}" + ${CHART_TESTING_BIN} lint-and-install --config ${PWD}/charts/config.yaml TEST_PASSED=$? set -e set +x popd -fi - -if [[ $TEST_PATH == "./tests/e2e/..." ]]; then - eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS" +else + loudecho "Deploying driver" + startSec=$(date +'%s') + + HELM_ARGS=(upgrade --install "${DRIVER_NAME}" + --namespace kube-system + --set image.repository="${IMAGE_NAME}" + --set image.tag="${IMAGE_TAG}" + --wait + --kubeconfig "${KUBECONFIG}" + ./charts/"${DRIVER_NAME}") + if [[ -f "$HELM_VALUES_FILE" ]]; then + HELM_ARGS+=(-f "${HELM_VALUES_FILE}") + fi + eval "EXPANDED_HELM_EXTRA_FLAGS=$HELM_EXTRA_FLAGS" + if [[ -n "$EXPANDED_HELM_EXTRA_FLAGS" ]]; then + HELM_ARGS+=("${EXPANDED_HELM_EXTRA_FLAGS}") + fi set -x - set +e - ${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "${TEST_PATH}" -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${FIRST_ZONE}" "${EXPANDED_TEST_EXTRA_FLAGS}" - TEST_PASSED=$? - set -e + "${HELM_BIN}" "${HELM_ARGS[@]}" set +x -fi -OVERALL_TEST_PASSED="${TEST_PASSED}" + endSec=$(date +'%s') + secondUsed=$(((endSec - startSec) / 1)) + # Set timeout threshold as 20 seconds for now, usually it takes less than 10s to startup + if [ $secondUsed -gt $DRIVER_START_TIME_THRESHOLD_SECONDS ]; then + loudecho "Driver start timeout, took $secondUsed but the threshold is $DRIVER_START_TIME_THRESHOLD_SECONDS. Fail the test." + exit 1 + fi + loudecho "Driver deployment complete, time used: $secondUsed seconds" + + loudecho "Testing focus ${GINKGO_FOCUS}" + + if [[ $TEST_PATH == "./tests/e2e-kubernetes/..." ]]; then + pushd ${PWD}/tests/e2e-kubernetes + packageVersion=$(echo $(cut -d '.' -f 1,2 <<< $K8S_VERSION)) + + set -x + set +e + kubetest2 noop \ + --run-id="e2e-kubernetes" \ + --test=ginkgo \ + -- \ + --skip-regex="${GINKGO_SKIP}" \ + --focus-regex="${GINKGO_FOCUS}" \ + --test-package-version=$(curl https://storage.googleapis.com/kubernetes-release/release/stable-$packageVersion.txt) \ + --parallel=25 \ + --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=$KUBECONFIG" + + TEST_PASSED=$? + set -e + set +x + popd + fi -PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name) + if [[ $TEST_PATH == "./tests/e2e/..." ]]; then + eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS" + set -x + set +e + ${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "${TEST_PATH}" -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${FIRST_ZONE}" "${EXPANDED_TEST_EXTRA_FLAGS}" + TEST_PASSED=$? + set -e + set +x + fi -while IFS= read -r POD; do - loudecho "Printing pod ${POD} ${CONTAINER_NAME} container logs" - set +e - kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" \ - --kubeconfig "${KUBECONFIG}" - set -e -done <<< "${PODS}" + PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name) + + while IFS= read -r POD; do + loudecho "Printing pod ${POD} ${CONTAINER_NAME} container logs" + set +e + kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" \ + --kubeconfig "${KUBECONFIG}" + set -e + done <<< "${PODS}" +fi if [[ "${CLEAN}" == true ]]; then loudecho "Cleaning" - loudecho "Removing driver" - ${HELM_BIN} del "${DRIVER_NAME}" \ - --namespace kube-system \ - --kubeconfig "${KUBECONFIG}" + if [[ "${HELM_TEST}" != true ]]; then + loudecho "Removing driver" + ${HELM_BIN} del "${DRIVER_NAME}" \ + --namespace kube-system \ + --kubeconfig "${KUBECONFIG}" + fi if [[ "${CLUSTER_TYPE}" == "kops" ]]; then kops_delete_cluster \ @@ -263,6 +284,8 @@ else loudecho "Not cleaning" fi +OVERALL_TEST_PASSED="${TEST_PASSED}" + loudecho "OVERALL_TEST_PASSED: ${OVERALL_TEST_PASSED}" if [[ $OVERALL_TEST_PASSED -ne 0 ]]; then loudecho "FAIL!"