From 74c52c2949d3b0523ac056e9df0d297509b0c041 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 10 May 2024 10:23:09 +0300 Subject: [PATCH] Deploy/K8S pipeline post-migration fixes (#39295) (#39489) * removed make.sh, updated check step * updated steps with env setup & removed setup from hook * updated setenv.sh * moved module to env * updated PR based on comments * updated common.sh * updated k8s env setup * removed tmp folder * added section for tests in BK (cherry picked from commit 8664cae202d1bfe02d30f1c734c568377d16e08b) Co-authored-by: Olga Naydyonock --- .../deploy/kubernetes/deploy-k8s-pipeline.yml | 59 +++++++++++++------ .../deploy/kubernetes/scripts/install-kind.sh | 40 ------------- .../kubernetes/scripts/install-kubectl.sh | 42 ------------- .buildkite/deploy/kubernetes/scripts/make.sh | 7 --- .../kubernetes/scripts/setup-k8s-env.sh | 10 +++- .buildkite/env-scripts/env.sh | 2 - .buildkite/env-scripts/util.sh | 0 .buildkite/hooks/pre-command | 6 -- .buildkite/hooks/scripts/util.sh | 4 +- .buildkite/scripts/setenv.sh | 7 ++- 10 files changed, 54 insertions(+), 123 deletions(-) delete mode 100755 .buildkite/deploy/kubernetes/scripts/install-kind.sh delete mode 100755 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh delete mode 100755 .buildkite/deploy/kubernetes/scripts/make.sh mode change 100644 => 100755 .buildkite/env-scripts/util.sh diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml index bd1a493b885..c0005309457 100644 --- a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml +++ b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml @@ -3,6 +3,10 @@ env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + MODULE: "kubernetes" + + # Other deps + ASDF_KIND_VERSION: "0.20.0" steps: - group: "Deploy/K8S" @@ -10,22 +14,29 @@ steps: steps: - label: "Checks" - command: ".buildkite/deploy/kubernetes/scripts/make.sh" + command: | + set -euo pipefail + make -C deploy/kubernetes all + make check-no-changes agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-checks" + context: "deploy/k8s checks" - label: "K8S Test/K8S version: v1.29.0" key: "k8s-test-129" env: K8S_VERSION: "v1.29.0" - commands: - - "MODULE=kubernetes make -C metricbeat integration-tests" - - "make -C deploy/kubernetes test" + MODULE: "${MODULE}" + commands: | + set -euo pipefail + source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh + echo "--- Executing Tests" + make -C metricbeat integration-tests + make -C deploy/kubernetes test retry: automatic: - limit: 3 @@ -35,15 +46,19 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-test v1.29.0" + context: "deploy/k8s test v1.29.0" - label: "K8S Test/K8S version: v1.28.0" key: "k8s-test-128" env: K8S_VERSION: "v1.28.0" - commands: - - "MODULE=kubernetes make -C metricbeat integration-tests" - - "make -C deploy/kubernetes test" + MODULE: "${MODULE}" + commands: | + set -euo pipefail + source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh + echo "--- Executing Tests" + make -C metricbeat integration-tests + make -C deploy/kubernetes test retry: automatic: - limit: 3 @@ -53,15 +68,19 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-test v1.28.0" + context: "deploy/k8s test v1.28.0" - label: "K8S Test/K8S version: v1.27.3" key: "k8s-test-1273" env: K8S_VERSION: "v1.27.3" - commands: - - "MODULE=kubernetes make -C metricbeat integration-tests" - - "make -C deploy/kubernetes test" + MODULE: "${MODULE}" + commands: | + set -euo pipefail + source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh + echo "--- Executing Tests" + make -C metricbeat integration-tests + make -C deploy/kubernetes test retry: automatic: - limit: 3 @@ -71,15 +90,19 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-test v1.27.3" + context: "deploy/k8s test v1.27.3" - label: "K8S Test/K8S version: v1.26.6" key: "k8s-test-1266" env: K8S_VERSION: "v1.26.6" - commands: - - "MODULE=kubernetes make -C metricbeat integration-tests" - - "make -C deploy/kubernetes test" + MODULE: "${MODULE}" + commands: | + set -euo pipefail + source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh + echo "--- Executing Tests" + make -C metricbeat integration-tests + make -C deploy/kubernetes test retry: automatic: - limit: 3 @@ -89,4 +112,4 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "Deploy/k8s-test v1.26.6" + context: "deploy/k8s test v1.26.6" diff --git a/.buildkite/deploy/kubernetes/scripts/install-kind.sh b/.buildkite/deploy/kubernetes/scripts/install-kind.sh deleted file mode 100755 index 8c399d2de37..00000000000 --- a/.buildkite/deploy/kubernetes/scripts/install-kind.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -MSG="environment variable missing." -KIND_VERSION=${KIND_VERSION:?$MSG} -KIND_BINARY="${BIN}/kind" - -if command -v kind -then - set +e - echo "Found Kind. Checking version.." - FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}') - if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ] - then - echo "--- Versions match. No need to install Kind. Exiting." - exit 0 - fi - set -e -fi - -echo "UNMET DEP: Installing Kind" - -OS=$(uname -s| tr '[:upper:]' '[:lower:]') -ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') -if [ "${ARCH}" == "aarch64" ] ; then - ARCH_SUFFIX=arm64 -else - ARCH_SUFFIX=amd64 -fi - -if curl -sSLo "${KIND_BINARY}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then - chmod +x "${KIND_BINARY}" - echo "Kind installed: ${KIND_VERSION}" -else - echo "Something bad with the download, let's delete the corrupted binary" - if [ -e "${KIND_BINARY}" ] ; then - rm "${KIND_BINARY}" - fi - exit 1 -fi diff --git a/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh b/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh deleted file mode 100755 index 7f6c75bf3b9..00000000000 --- a/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -MSG="parameter missing." -K8S_VERSION=${K8S_VERSION:?$MSG} -KUBECTL_BINARY="${BIN}/kubectl" - -if command -v kubectl -then - set +e - echo "Found kubectl. Checking version.." - FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}') - if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ] - then - echo "Kubectl Versions match. No need to install kubectl. Exiting." - exit 0 - fi - set -e -fi - -echo "UNMET DEP: Installing kubectl" - -OS=$(uname -s| tr '[:upper:]' '[:lower:]') -ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') -if [ "${ARCH}" == "aarch64" ] ; then - ARCH_SUFFIX=arm64 -else - ARCH_SUFFIX=amd64 -fi - -if curl -sSLo "${KUBECTL_BINARY}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS}/${ARCH_SUFFIX}/kubectl" ; then - chmod +x "${KUBECTL_BINARY}" - echo "Current K8S Version: ${K8S_VERSION}" - echo "Kubectl installed: ${KUBECTL_BINARY}" -else - echo "--- Something bad with the download, let's delete the corrupted binary" - if [ -e "${KUBECTL_BINARY}" ] ; then - rm "${KUBECTL_BINARY}" - fi - exit 1 -fi - diff --git a/.buildkite/deploy/kubernetes/scripts/make.sh b/.buildkite/deploy/kubernetes/scripts/make.sh deleted file mode 100755 index 4c9120a2d90..00000000000 --- a/.buildkite/deploy/kubernetes/scripts/make.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -echo "--- Checking K8S" -make -C deploy/kubernetes all -make check-no-changes diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh index 3656318bd64..3a4418a18f9 100755 --- a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh +++ b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh @@ -4,11 +4,15 @@ set -euo pipefail source .buildkite/env-scripts/util.sh +export KUBECONFIG="${WORKSPACE}/kubecfg" +export BIN="${WORKSPACE}/bin" +add_bin_path + echo "--- Installing kind & kubectl" -retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kind.sh -retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh +asdf plugin add kind +asdf install kind $ASDF_KIND_VERSION -echo "--- Setting up kind" +echo "~~~ Setting up kind" max_retries=3 timeout=5 retries=0 diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh index b30b26c3e8a..58624e300e6 100644 --- a/.buildkite/env-scripts/env.sh +++ b/.buildkite/env-scripts/env.sh @@ -12,7 +12,6 @@ WORKSPACE="$(pwd)" BIN="${WORKSPACE}/bin" HW_TYPE="$(uname -m)" PLATFORM_TYPE="$(uname)" -TMP_FOLDER="tmp.${REPO}" SNAPSHOT="true" PYTEST_ADDOPTS="" OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" @@ -38,7 +37,6 @@ export WORKSPACE export BIN export HW_TYPE export PLATFORM_TYPE -export TMP_FOLDER export SNAPSHOT export PYTEST_ADDOPTS export OSS_MODULE_PATTERN diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh old mode 100644 new mode 100755 diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 870ece78925..1631b7dc0a1 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -91,9 +91,3 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then - source .buildkite/env-scripts/env.sh - if [[ "$BUILDKITE_STEP_KEY" == k8s-test* ]]; then - .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh - fi -fi diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh index 8ef932725c6..b441991e6fe 100755 --- a/.buildkite/hooks/scripts/util.sh +++ b/.buildkite/hooks/scripts/util.sh @@ -27,8 +27,8 @@ google_cloud_logout_active_account() { cleanup() { if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then echo "Deleting temporary files..." - if [[ -n "${BIN:-}" ]] && [[ -e "${BIN}/${TMP_FOLDER}" ]]; then - rm -rf "${BIN}/${TMP_FOLDER}.*" + if [[ -n "${BIN:-}" ]]; then + rm -rf "${BIN}" fi echo "Done." fi diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 383ff6ff56b..6351af6f5a6 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -2,20 +2,21 @@ set -euo pipefail +WORKSPACE=${WORKSPACE:-"$(pwd)"} +GO_VERSION=$(cat .go-version) + export REPO="beats" export DOCKER_REGISTRY="docker.elastic.co" export SETUP_GVM_VERSION="v0.5.1" export DOCKER_COMPOSE_VERSION="1.21.0" export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" + export ASDF_NODEJS_VERSION="18.17.1" export AWS_REGION="eu-central-1" -WORKSPACE=${WORKSPACE:-"$(pwd)"} export WORKSPACE -GO_VERSION=$(cat .go-version) export GO_VERSION - exportVars() { local platform_type="$(uname)" local arch_type="$(uname -m)"