From 729cb9174e1aac392c867c7c89d51b17972b589a Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Thu, 2 May 2024 17:01:58 +0300 Subject: [PATCH] Update auditbeat pipeline to match Jenkins steps (#39245) This commit updates the `auditbeat` pipeline to match same steps & commands as in Jenkins. Relates: https://github.com/elastic/ingest-dev/issues/3254 --------- Co-authored-by: Dimitrios Liappis (cherry picked from commit c1663a3f83974db66029da01d79aade6cefb8477) --- .buildkite/auditbeat/auditbeat-pipeline.yml | 103 +++++++++++++++----- .buildkite/hooks/pre-command | 2 +- 2 files changed, 82 insertions(+), 23 deletions(-) diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index ed19c7d9164..13e3ef2f256 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -3,7 +3,6 @@ name: "beats-auditbeat" env: AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" - AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" @@ -30,6 +29,7 @@ steps: steps: - label: ":ubuntu: Auditbeat Unit Tests" command: | + set -euo pipefail cd auditbeat mage build unitTest retry: @@ -48,6 +48,7 @@ steps: - label: ":rhel: Auditbeat Unit Tests" command: | + set -euo pipefail cd auditbeat mage build unitTest retry: @@ -118,18 +119,62 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "auditbeat: Cross compile" + context: "auditbeat: Crosscompile" + + - group: "Auditbeat Extended Tests" + key: "auditbeat-extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|integrations).*/ - - group: "Auditbeat Linux arm64 Tests" - key: "auditbeat-extended-tests-linux-arm64" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ steps: - - label: ":linux: Auditbeat Linux arm64 Unit Tests" + - label: ":linux: Auditbeat Ubuntu Integration Tests" + key: "auditbeat-extended-integ-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + command: | + set -euo pipefail + cd auditbeat + mage build integTest + retry: + automatic: + - limit: 3 + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + notify: + - github_commit_status: + context: "auditbeat: Linux Integration Tests" + + - label: ":linux: Auditbeat Ubuntu arm64 Integration Tests" + key: "auditbeat-extended-arm64-integ-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + command: | + set -euo pipefail + cd auditbeat + mage build integTest + retry: + automatic: + - limit: 3 + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + notify: + - github_commit_status: + context: "auditbeat: Linux arm64 Integration Tests" + + - label: ":linux: Auditbeat Ubuntu arm64 Unit Tests" key: "auditbeat-extended-arm64-unit-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ command: | set -euo pipefail cd auditbeat - mage unitTest + mage build unitTest retry: automatic: - limit: 3 @@ -137,51 +182,57 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "auditbeat/build/*.*" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Linux arm64 tests" + context: "auditbeat: Linux arm64 Unit Tests" - - group: "Auditbeat MacOS Extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - steps: - label: ":mac: Auditbeat macOS x86_64 Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh cd auditbeat - mage unitTest + mage build unitTest retry: automatic: - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "auditbeat/build/*.*" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Extended MacOS x86_64 Unit Tests" + context: "auditbeat: macOS x86_64 Unit Tests" - - label: ":mac: Auditbeat macOS arm64 Unit Tests" + - label: ":mac: Auditbeat macOS arm64 ARM Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh cd auditbeat - mage unitTest + mage build unitTest retry: automatic: - limit: 3 agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: "auditbeat/build/*.*" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Extended MacOS arm64 Unit Tests" + context: "auditbeat: macOS arm64 Unit Tests" - group: "Auditbeat Windows Extended Testing" key: "auditbeat-extended-tests-win" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: - label: ":windows: Auditbeat Win-2019 Unit Tests" key: "auditbeat-extended-win-2019-unit-tests" @@ -197,7 +248,9 @@ steps: machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "auditbeat/build/*.*" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" notify: - github_commit_status: context: "auditbeat: Extended Windows 2019 Unit Tests" @@ -216,7 +269,9 @@ steps: machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "auditbeat/build/*.*" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" notify: - github_commit_status: context: "auditbeat: Extended Windows 10 Unit Tests" @@ -235,7 +290,9 @@ steps: machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "auditbeat/build/*.*" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" notify: - github_commit_status: context: "auditbeat: Extended Windows 11 Unit Tests" @@ -256,6 +313,7 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true command: | + set -euo pipefail cd auditbeat mage package agents: @@ -273,6 +331,7 @@ steps: PACKAGES: "docker" SNAPSHOT: true command: | + set -euo pipefail cd auditbeat mage package agents: diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 2abf56684a3..870ece78925 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then source .buildkite/env-scripts/env.sh if [[ -z "${GO_VERSION-""}" ]]; then export GO_VERSION=$(cat "${WORKSPACE}/.go-version")