From 80dab50f0ce3dafe3125a0587db9465dac5d4fa2 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz <87968844+sharbuz@users.noreply.github.com> Date: Fri, 5 Apr 2024 11:08:01 +0300 Subject: [PATCH] replace default images (#38583) * replace default images -------------------------- Signed-off-by: Alexandros Sapranidis Co-authored-by: Alexandros Sapranidis --- .buildkite/auditbeat/auditbeat-pipeline.yml | 27 +- .../auditbeat/generate_auditbeat_pipeline.sh | 216 -------------- .buildkite/filebeat/filebeat-pipeline.yml | 25 +- .buildkite/heartbeat/heartbeat-pipeline.yml | 50 ++-- .buildkite/hooks/pre-command | 4 +- .buildkite/libbeat/pipeline.libbeat.yml | 18 +- .buildkite/metricbeat/pipeline.yml | 26 +- .buildkite/packetbeat/pipeline.packetbeat.yml | 31 +- .buildkite/scripts/cloud_tests.sh | 52 +++- .buildkite/scripts/common.sh | 30 +- .../scripts/generate_auditbeat_pipeline.sh | 265 +++++++++++++++++ .../scripts/generate_filebeat_pipeline.sh | 266 ++++++++++++++++++ .../scripts/generate_libbeat_pipeline.sh | 58 +++- .../scripts/generate_metricbeat_pipeline.sh | 121 +++++--- .../scripts/generate_packetbeat_pipeline.sh | 116 +++++--- .../scripts/generate_winlogbeat_pipeline.sh | 64 +++-- .../generate_xpack_auditbeat_pipeline.sh | 67 ++++- .../generate_xpack_dockerlogbeat_pipeline.sh | 25 +- .../generate_xpack_filebeat_pipeline.sh | 80 +++++- .../generate_xpack_heartbeat_pipeline.sh | 56 +++- .../generate_xpack_libbeat_pipeline.sh | 98 ++++--- .../generate_xpack_metricbeat_pipeline.sh | 133 +++++---- .../generate_xpack_osquerybeat_pipeline.sh | 50 +++- .../generate_xpack_packetbeat_pipeline.sh | 84 ++++-- .../generate_xpack_winlogbeat_pipeline.sh | 40 ++- .buildkite/scripts/go_int_tests.sh | 12 - .buildkite/scripts/packaging/package-step.sh | 4 +- .buildkite/scripts/py_int_tests.sh | 13 - .buildkite/scripts/setenv.sh | 40 +-- .buildkite/scripts/win_unit_tests.ps1 | 188 ------------- .buildkite/winlogbeat/pipeline.winlogbeat.yml | 23 +- .../x-pack/pipeline.xpack.auditbeat.yml | 33 ++- .../x-pack/pipeline.xpack.dockerlogbeat.yml | 27 +- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 32 ++- .../x-pack/pipeline.xpack.heartbeat.yml | 32 ++- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 30 +- .../x-pack/pipeline.xpack.metricbeat.yml | 32 ++- .../x-pack/pipeline.xpack.osquerybeat.yml | 29 +- .../x-pack/pipeline.xpack.packetbeat.yml | 32 ++- .../x-pack/pipeline.xpack.winlogbeat.yml | 19 +- 40 files changed, 1616 insertions(+), 932 deletions(-) delete mode 100644 .buildkite/auditbeat/generate_auditbeat_pipeline.sh create mode 100644 .buildkite/scripts/generate_auditbeat_pipeline.sh create mode 100644 .buildkite/scripts/generate_filebeat_pipeline.sh delete mode 100755 .buildkite/scripts/go_int_tests.sh delete mode 100755 .buildkite/scripts/py_int_tests.sh delete mode 100644 .buildkite/scripts/win_unit_tests.ps1 diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index a8def81f1fa..edbb600e385 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -2,21 +2,28 @@ name: "beats-auditbeat" env: - BEATS_PROJECT_NAME: "auditbeat" - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + BEATS_PROJECT_NAME: "auditbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: - input: "Input Parameters" @@ -62,7 +69,9 @@ steps: - label: ":linux: Load dynamic auditbeat pipeline" key: "auditbeat-pipeline" - command: ".buildkite/auditbeat/generate_auditbeat_pipeline.sh" + command: ".buildkite/scripts/generate_auditbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/auditbeat/generate_auditbeat_pipeline.sh b/.buildkite/auditbeat/generate_auditbeat_pipeline.sh deleted file mode 100644 index 56befb51d25..00000000000 --- a/.buildkite/auditbeat/generate_auditbeat_pipeline.sh +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.auditbeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - group: "Auditbeat Mandatory Testing" - key: "mandatory-tests" - - steps: - - label: ":ubuntu: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":rhel: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: rhel/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2016 Unit Tests" - key: "windows-2016" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: windows-2016/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2022 Unit Tests" - key: "windows-2022" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: windows-2022/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":linux: Crosscompile" - command: - - "make -C auditbeat crosscompile" - env: - GOX_FLAGS: "-arch amd64" - notify: - - github_commit_status: - context: "Auditbeat: Crosscompile" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "macos-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":mac: MacOS ARM Unit Tests" - key: "macos-arm64-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: MacOS ARM Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" - key: "extended-arm64-unit-test" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - group: "Windows Extended Testing" - key: "extended-tests-win" - - steps: - - label: ":windows:-2019 Unit Tests" - key: "windows-2019-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-11 Unit Tests" - key: "windows-11-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-11 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-10 Unit Tests" - key: "windows-10-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-10 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - group: "Packaging" - key: "packaging" - depends_on: - - "mandatory-tests" - - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" - notify: - - github_commit_status: - context: "Auditbeat: Packaging" - -YAML -fi - -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 9daeef287d9..a22e96c974a 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -2,20 +2,27 @@ name: "beats-filebeat" env: + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" BEATS_PROJECT_NAME: "filebeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: - input: "Input Parameters" @@ -61,7 +68,9 @@ steps: - label: ":linux: Load dynamic filebeat pipeline" key: "filebeat-pipeline" - command: ".buildkite/filebeat/generate_filebeat_pipeline.sh" + command: ".buildkite/scripts/generate_filebeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index e63fb1c60e2..9b1ea1984a7 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -4,12 +4,12 @@ env: BEATS_PROJECT_NAME: "heartbeat" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_MACOS_ARM: "generic-13-ventura-arm" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" @@ -17,6 +17,10 @@ env: GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + #Packaging + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PACKAGING_ARM_PLATFORMS: "linux/arm64" + steps: - group: "Heartbeat Mandatory Testing" key: "mandatory-tests" @@ -24,7 +28,7 @@ steps: steps: - label: ":ubuntu: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: ubuntu/Unit Tests" @@ -37,7 +41,7 @@ steps: - "heartbeat/build/*.json" - label: ":rhel:-9 Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: rhel-9/Unit Tests" @@ -51,10 +55,12 @@ steps: - label: ":windows:-2016 Unit Test" key: "windows-2016" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: windows-2016/Unit Tests" + context: "Heartbeat: Windows 2016/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -66,10 +72,12 @@ steps: - label: ":windows:-2022 Unit Test" key: "windows-2022" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: windows-2022/Unit Tests" + context: "Heartbeat: Windows 2022/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -80,7 +88,7 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Go Integration Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" notify: - github_commit_status: context: "Heartbeat: Go Integration Tests" @@ -93,7 +101,7 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Python Integration Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage pythonIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" notify: - github_commit_status: context: "Heartbeat: Python Integration Tests" @@ -113,7 +121,7 @@ steps: - label: ":linux: ARM64 Unit Tests" key: "arm-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: Unit Tests ARM" @@ -126,7 +134,7 @@ steps: - label: ":mac: MacOS Unit Tests" key: "macos-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: MacOS Unit Tests" @@ -138,7 +146,7 @@ steps: - label: ":mac: MacOS ARM Unit Tests" key: "macos-extended-arm" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: MacOS ARM Unit Tests" @@ -154,10 +162,12 @@ steps: steps: - label: ":windows:-2019 Unit Tests" key: "windows-extended-2019" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: Win-2019 Unit Tests" + context: "Heartbeat: Windows 2019/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -169,10 +179,12 @@ steps: - label: ":windows:-11 Unit Tests" key: "windows-extended-11" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: Win-11 Unit Tests" + context: "Heartbeat: Windows 11/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -184,10 +196,12 @@ steps: - label: ":windows:-10 Unit Tests" key: "windows-extended-10" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: Win-10 Unit Tests" + context: "Heartbeat: Windows 10/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_10}" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 5f625b35c59..52bee3fb674 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -9,11 +9,13 @@ source .buildkite/env-scripts/util.sh AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" -#PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then source .buildkite/env-scripts/env.sh + if [[ -z "${GO_VERSION-""}" ]]; then + export GO_VERSION=$(cat "${WORKSPACE}/.go-version") + fi # Images with prefix "platform-ingest-beats-*" has Go pre-setup. # Image itself takes care of Go version download/installation based on .go-version file diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 83fda5662c7..46ddeca92a4 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -2,13 +2,19 @@ name: "beats-libbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" BEATS_PROJECT_NAME: "libbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -40,6 +46,8 @@ steps: - label: ":linux: Load dynamic Libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_libbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 04f3b44575e..c7dcc01e625 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -2,20 +2,26 @@ name: "beats-metricbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "metricbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -47,6 +53,8 @@ steps: - label: ":linux: Load dynamic metricbeat pipeline" key: "metricbeat-pipeline" command: ".buildkite/scripts/generate_metricbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 8e606e16622..9247b144667 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -2,20 +2,27 @@ name: "beats-packetbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_RHEL9_X86_64: "family/core-rhel-9" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "packetbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "packetbeat" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -55,6 +62,8 @@ steps: - label: ":linux: Load dynamic packetbeat pipeline" key: "packetbeat-pipeline" command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index d96baf670a9..3ca52305188 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -euo pipefail # What Terraform Module will run if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then @@ -7,19 +8,52 @@ elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then export MODULE_DIR="x-pack/filebeat/input/awss3/_meta/terraform" fi -source .buildkite/scripts/install_tools.sh +teardown() { + popd + # Teardown resources after using them + echo "~~~ Terraform Cleanup" + tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK -set -euo pipefail + echo "~~~ Docker Compose Cleanup" + docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +} + +tf_cleanup() { + DIRECTORY=${1:-.} -trap 'teardown || true; unset_secrets' EXIT + for tfstate in $(find $DIRECTORY -name terraform.tfstate); do + cd $(dirname $tfstate) + terraform init + if ! terraform destroy -auto-approve; then + echo "+++ Failed to Terraform destroy the resources" + fi + cd - + done +} + +trap 'teardown' EXIT # Prepare the cloud resources using Terraform -startCloudTestEnv "${MODULE_DIR}" +#startCloudTestEnv "${MODULE_DIR}" +echo "~~~ Loading creds" +set +o xtrace +export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY +export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY +export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" +set -o xtrace + +echo "~~~ Run docker-compose services for emulated cloud env" +docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +echo "~~~ Initialize TF cloud resources" +pushd "$MODULE_DIR" +export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') +export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" +export TF_VAR_CREATED_DATE=$(date +%s) +export TF_VAR_ENVIRONMENT="ci" +export TF_VAR_REPO="${REPO}" +terraform init && terraform apply -auto-approve # Run tests -echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" -pushd "${BEATS_PROJECT_NAME}" > /dev/null - +echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" mage build test - -popd > /dev/null diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 3c968af2088..3875fc5955d 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -23,6 +23,7 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" [ -z "${run_xpack_winlogbeat+x}" ] && run_xpack_winlogbeat="$(buildkite-agent meta-data get run_xpack_winlogbeat --default "false")" [ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")" +[ -z "${run_xpack_dockerlogbeat+x}" ] && run_xpack_dockerlogbeat="$(buildkite-agent meta-data get run_xpack_dockerlogbeat --default "false")" [ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")" [ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")" [ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")" @@ -63,7 +64,7 @@ auditbeat_changeset=( filebeat_changeset=( "^filebeat/.*" - ) + ) metricbeat_changeset=( "^metricbeat/.*" @@ -195,7 +196,7 @@ case "${BUILDKITE_PIPELINE_SLUG}" in BEAT_CHANGESET_REFERENCE=${xpack_winlogbeat_changeset[@]} ;; *) - echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." + echo "~~~ The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." ;; esac @@ -210,7 +211,7 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_win_tests" - echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" + echo "--- Beats project name is $BEATS_XPACK_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" "${xpack_changeset[@]}" @@ -223,7 +224,7 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_PROJECT_NAME}_win_tests" - echo "Beats project name is $BEATS_PROJECT_NAME" + echo "--- Beats project name is $BEATS_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" "${oss_changeset[@]}" @@ -251,19 +252,6 @@ with_docker_compose() { docker-compose version } -with_Terraform() { - echo "Setting up the Terraform environment..." - local path_to_file="${WORKSPACE}/terraform.zip" - create_workspace - check_platform_architeture - retry 5 curl -sSL -o ${path_to_file} "https://releases.hashicorp.com/terraform/${ASDF_TERRAFORM_VERSION}/terraform_${ASDF_TERRAFORM_VERSION}_${platform_type_lowercase}_${go_arch_type}.zip" - unzip -q ${path_to_file} -d ${BIN}/ - rm ${path_to_file} - chmod +x ${BIN}/terraform - export PATH="${BIN}:${PATH}" - terraform version -} - create_workspace() { if [[ ! -d "${BIN}" ]]; then mkdir -p "${BIN}" @@ -466,7 +454,7 @@ are_conditions_met_macos_tests() { } are_conditions_met_win_tests() { - if are_conditions_met_mandatory_tests; then + if are_conditions_met_mandatory_tests; then if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_WIN_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_WIN_LABEL} || "${!TRIGGER_SPECIFIC_WIN_TESTS}" == "true" ]]; then return 0 @@ -529,7 +517,11 @@ defineModuleFromTheChangeSet() { fi done if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - export MODULE="aws" + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + export MODULE="aws" + else + export MODULE="kubernetes" + fi else export MODULE="${changed_modules}" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved # export MODULE="${changed_modules}" # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved diff --git a/.buildkite/scripts/generate_auditbeat_pipeline.sh b/.buildkite/scripts/generate_auditbeat_pipeline.sh new file mode 100644 index 00000000000..d514c5d79d8 --- /dev/null +++ b/.buildkite/scripts/generate_auditbeat_pipeline.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.auditbeat-dynamic.yml" + +# TODO: steps: must be always included +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + - group: "Mandatory Testing" + key: "mandatory-tests" + + steps: + - label: ":ubuntu: Ubuntu Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":rhel: RHEL Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":linux: Crosscompile" + command: "make -C $BEATS_PROJECT_NAME crosscompile" + env: + GOX_FLAGS: "-arch amd64" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Crosscompile" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "macos-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":mac: MacOS ARM Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS ARM Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: Ubuntu ARM Unit Tests" + key: "extended-arm64-unit-test" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + +YAML +fi + +if are_conditions_met_win_tests; then + cat >> $pipelineName <<- YAML + - group: "Windows Extended Testing" + key: "extended-tests-win" + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then +cat >> $pipelineName <<- YAML + - group: "Packaging" + key: "packaging" + depends_on: + - "mandatory-tests" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" + + +YAML +fi + +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName \ No newline at end of file diff --git a/.buildkite/scripts/generate_filebeat_pipeline.sh b/.buildkite/scripts/generate_filebeat_pipeline.sh new file mode 100644 index 00000000000..a3fe1108dc3 --- /dev/null +++ b/.buildkite/scripts/generate_filebeat_pipeline.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.filebeat-dynamic.yml" + +# TODO: steps: must be always included +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + + +steps: + - group: "Mandatory Testing" + key: "mandatory-tests" + + steps: + - label: ":ubuntu: Ubuntu Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Ubuntu Go Integration Tests" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Ubuntu Python Integration Tests" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":windows: Windows 2016 Unit Tests" + key: "windows-2016-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 2022 Unit Tests" + key: "windows-2022-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "macos-unit-tests-extended" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":mac: MacOS ARM Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS ARM Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + + - label: ":linux: Ubuntu ARM Unit Tests" + key: "extended-arm64-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" +YAML +fi + +if are_conditions_met_win_tests; then + cat >> $pipelineName <<- YAML + + - group: "Windows Extended Testing" + key: "extended-tests-win" + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "windows-extended-2019" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 11 Unit Tests" + key: "windows-extended-11" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 10 Unit Tests" + key: "windows-extended-10" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then +cat >> $pipelineName <<- YAML + + - group: "Packaging" + key: "packaging" + depends_on: + - "mandatory-tests" + + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" + +YAML +fi + +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 0674d0b186b..37132e74abf 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -17,48 +17,71 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh" + command: "make -C $BEATS_PROJECT_NAME crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cross compile" - label: ":testengine: Stress Tests" key: "mandatory-stress-test" - command: ".buildkite/scripts/stress_tests.sh" + command: "cd $BEATS_PROJECT_NAME && make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS='| go-junit-report > libbeat-stress-test.xml' stress-tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Stress Tests" YAML else @@ -73,20 +96,25 @@ if are_conditions_met_arm_tests; then - group: "Extended Tests" key: "extended-tests" steps: - - label: ":linux: Arm64 Unit Tests" + - label: ":linux: Ubuntu ARM64 Unit Tests" key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM64 Unit Tests" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 477f8fb25a3..a0386aeb1b7 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -17,42 +17,68 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" - label: ":go: Go Intergration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh" + command: "make -C $BEATS_PROJECT_NAME crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cross compile" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -65,7 +91,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline @@ -73,38 +104,30 @@ steps: - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - label: ":windows: Windows 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else echo "The conditions don't match to requirements for generating pipeline steps." @@ -124,7 +147,13 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + YAML fi @@ -143,17 +172,22 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" @@ -161,12 +195,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 97bdb531a6b..e4870437ac1 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -17,25 +17,37 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":rhel: RHEL-9 Unit Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + + - label: ":rhel: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -48,43 +60,39 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" - group: "Extended Windowds Tests" key: "extended-win-tests" steps: - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_11}" + image: "{{matrix.image}}" machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -111,21 +119,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -144,30 +162,38 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh index ce812016e79..ede70538c2c 100755 --- a/.buildkite/scripts/generate_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh @@ -18,15 +18,22 @@ steps: - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh" + command: "make -C $BEATS_PROJECT_NAME crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cross compile" - label: ":windows: Windows 2016/2019/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -40,7 +47,12 @@ steps: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2019}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline @@ -48,28 +60,29 @@ steps: - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows 10/11 Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else echo "The conditions don't match to requirements for generating pipeline steps." @@ -90,20 +103,23 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh index f29e6152e60..eb9c8269f3d 100755 --- a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh @@ -25,20 +25,32 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":rhel: RHEL-9 Unit Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit (MODULE) Tests" + + - label: ":rhel: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -51,7 +63,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -60,7 +77,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -74,8 +93,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -102,21 +125,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -144,6 +177,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -155,12 +191,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh index 46e92f8ddf9..f9ca4e7f767 100755 --- a/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh @@ -22,7 +22,13 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -33,7 +39,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" YAML fi @@ -61,6 +72,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -72,12 +86,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh b/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh index 80d28770bf0..78a351a078d 100755 --- a/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh @@ -22,7 +22,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -31,7 +36,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" @@ -42,10 +52,17 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -58,7 +75,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -67,7 +89,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -81,7 +105,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -108,21 +137,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -132,15 +171,22 @@ if are_conditions_met_aws_tests; then - label: ":linux: Cloud Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" + skip: "Temporary disable, additional conditions required, we hit AWS limits" env: MODULE: $MODULE + ASDF_TERRAFORM_VERSION: 1.0.2 agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cloud Tests" YAML fi @@ -168,6 +214,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -179,12 +228,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh index b51ca0ab3b9..bec8e9bf96d 100755 --- a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh @@ -17,28 +17,40 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" # ## TODO: there are windows test failures already reported # ## https://github.com/elastic/beats/issues/23957 and https://github.com/elastic/beats/issues/23958 # ## waiting for being fixed. # - label: ":windows: Windows Unit Tests - {{matrix.image}}" -# command: ".buildkite/scripts/win_unit_tests.ps1" +# command: +# - "Set-Location -Path $BEATS_PROJECT_NAME" +# - "New-Item -ItemType Directory -Force -Path 'build'" +# - "mage unitTest" # key: "mandatory-win-unit-tests" # agents: # provider: "gcp" @@ -51,7 +63,9 @@ steps: # image: # - "${IMAGE_WIN_2016}" # - "${IMAGE_WIN_2022}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" # ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -60,7 +74,10 @@ steps: # steps: # - label: ":windows: Windows Unit Tests - {{matrix.image}}" -# command: ".buildkite/scripts/win_unit_tests.ps1" +# command: +# - "Set-Location -Path $BEATS_PROJECT_NAME" +# - "New-Item -ItemType Directory -Force -Path 'build'" +# - "mage unitTest" # key: "extended-win-unit-tests" # agents: # provider: "gcp" @@ -74,7 +91,9 @@ steps: # - "${IMAGE_WIN_10}" # - "${IMAGE_WIN_11}" # - "${IMAGE_WIN_2019}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" YAML else @@ -95,7 +114,12 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi @@ -123,6 +147,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -134,12 +161,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh index 66f0750ab6f..7e3d5086288 100755 --- a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh @@ -17,33 +17,50 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage -w reader/etw build goUnitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -56,44 +73,40 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage -w reader\etw build goUnitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_11}" + image: "{{matrix.image}}" machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -108,20 +121,25 @@ if are_conditions_met_arm_tests; then - group: "Extended Tests" key: "extended-tests" steps: - - label: ":linux: Arm64 Unit Tests" + - label: ":linux: Ubuntu ARM64 Unit Tests" key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM64 Unit Tests" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index ddc3ce2c8f2..d38c3123656 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -17,37 +17,60 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: | + cd $BEATS_PROJECT_NAME + mage build unitTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":go: Go Integration Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + + - label: ":go: Go (MODULE) Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: | + cd $BEATS_PROJECT_NAME + mage goIntegTest env: MODULE: $MODULE agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":python: Python Integration Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go (MODULE) Integration Tests" + + - label: ":python: Python (MODULE) Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: | + cd $BEATS_PROJECT_NAME + mage pythonIntegTest env: MODULE: $MODULE agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python (MODULE) Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -60,44 +83,40 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2019}" + image: "{{matrix.image}}" machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -125,23 +144,35 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: Cloud Tests" + - label: ":linux: Cloud (MODULE) Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" + skip: "Does not belong to a stage, exists but not run" env: MODULE: $MODULE + ASDF_TERRAFORM_VERSION: 1.0.2 agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cloud (MODULE) Tests" YAML fi @@ -169,6 +200,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -180,12 +214,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P -echo "--- Loading dynamic steps" +echo "~~~ Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh b/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh index d6887c4c382..5372d7f3faa 100755 --- a/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh @@ -22,7 +22,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -31,10 +36,17 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -47,7 +59,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -56,7 +73,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -70,7 +89,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -91,7 +115,12 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi @@ -119,12 +148,15 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index 4eb2a1c3e04..c29b6c5d9bc 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -22,7 +22,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":linux: Ubuntu System Tests" key: "mandatory-linux-system-test" @@ -31,20 +36,33 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":rhel: RHEL-9 Unit Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu System Tests" + + - label: ":rhel: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: + - "Set-Location -Path $BEATS_PROJECT_NAME" + - "New-Item -ItemType Directory -Force -Path 'build'" + - "mage unitTest" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -57,7 +75,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved # - label: ":windows: Windows 2022 System Tests" @@ -69,7 +92,9 @@ steps: # machineType: "${GCP_WIN_MACHINE_TYPE}" # disk_size: 100 # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -78,7 +103,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -92,7 +119,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved # - label: ":windows: Windows 10 System Tests" @@ -104,7 +136,9 @@ steps: # machineType: "${GCP_WIN_MACHINE_TYPE}" # disk_size: 100 # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" YAML else @@ -131,21 +165,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -173,6 +217,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -184,12 +231,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh index 108a70c1562..ad5ebd44583 100755 --- a/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh @@ -18,7 +18,9 @@ steps: - label: ":windows: Windows 2019 Unit (MODULE) Tests" key: "mandatory-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest env: MODULE: $MODULE agents: @@ -27,10 +29,17 @@ steps: machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows (MODULE) {{matrix.image}} Unit Tests" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -43,7 +52,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline @@ -53,7 +67,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -67,7 +83,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -99,12 +120,15 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh deleted file mode 100755 index b4c519f4512..00000000000 --- a/.buildkite/scripts/go_int_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/install_tools.sh - -set -euo pipefail - -echo "--- Run Go Intergration Tests for $BEATS_PROJECT_NAME" -pushd "${BEATS_PROJECT_NAME}" > /dev/null - -mage goIntegTest - -popd > /dev/null diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh index 9eddfafcfba..5f04ed3849b 100755 --- a/.buildkite/scripts/packaging/package-step.sh +++ b/.buildkite/scripts/packaging/package-step.sh @@ -18,7 +18,7 @@ if are_files_changed "$changeset"; then - label: ":ubuntu: ${BEATS_PROJECT_NAME}/Packaging Linux X86" key: "package-linux-x86" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: $PACKAGING_PLATFORMS SNAPSHOT: true command: ".buildkite/scripts/packaging/package.sh" notify: @@ -32,7 +32,7 @@ if are_files_changed "$changeset"; then - label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM" key: "package-linux-arm" env: - PLATFORMS: "linux/arm64" + PLATFORMS: $PACKAGING_ARM_PLATFORMS PACKAGES: "docker" SNAPSHOT: true command: ".buildkite/scripts/packaging/package.sh" diff --git a/.buildkite/scripts/py_int_tests.sh b/.buildkite/scripts/py_int_tests.sh deleted file mode 100755 index 19fa8796c3e..00000000000 --- a/.buildkite/scripts/py_int_tests.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/install_tools.sh - -set -euo pipefail - -echo "--- Run Python Intergration Tests for $BEATS_PROJECT_NAME" - -pushd "${BEATS_PROJECT_NAME}" > /dev/null - -mage pythonIntegTest - -popd > /dev/null diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index f0116c6b308..f6b03f53f9b 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -2,37 +2,19 @@ set -euo pipefail -REPO="beats" -TMP_FOLDER="tmp.${REPO}" -DOCKER_REGISTRY="docker.elastic.co" -SETUP_GVM_VERSION="v0.5.1" -DOCKER_COMPOSE_VERSION="1.21.0" -DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" -SETUP_WIN_PYTHON_VERSION="3.11.0" -NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) -GO_VERSION=$(cat .go-version) -ASDF_MAGE_VERSION="1.15.0" -PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" -PACKAGING_ARM_PLATFORMS="linux/arm64" -ASDF_TERRAFORM_VERSION="1.0.2" -AWS_REGION="eu-central-1" -NODEJS_VERSION="18.17.1" +export REPO="beats" +export DOCKER_REGISTRY="docker.elastic.co" +export SETUP_GVM_VERSION="v0.5.1" +export DOCKER_COMPOSE_VERSION="1.21.0" +export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" +export ASDF_NODEJS_VERSION="18.17.1" +export AWS_REGION="eu-central-1" -export SETUP_GVM_VERSION -export DOCKER_COMPOSE_VERSION -export DOCKER_COMPOSE_VERSION_AARCH64 -export SETUP_WIN_PYTHON_VERSION -export NMAP_WIN_VERSION +WORKSPACE=${WORKSPACE:-"$(pwd)"} +export WORKSPACE +GO_VERSION=$(cat .go-version) export GO_VERSION -export ASDF_MAGE_VERSION -export PACKAGING_PLATFORMS -export PACKAGING_ARM_PLATFORMS -export REPO -export TMP_FOLDER -export DOCKER_REGISTRY -export ASDF_TERRAFORM_VERSION -export AWS_REGION -export NODEJS_VERSION + exportVars() { local platform_type="$(uname)" diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 deleted file mode 100644 index 6c3af6e9321..00000000000 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ /dev/null @@ -1,188 +0,0 @@ -param( - [string]$testType = "unittest" -) - -$ErrorActionPreference = "Stop" # set -e -$WorkFolder = $env:BEATS_PROJECT_NAME -$WORKSPACE = Get-Location -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF { - Write-Host "-- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} - -function retry { - param( - [int]$retries, - [ScriptBlock]$scriptBlock - ) - $count = 0 - while ($count -lt $retries) { - $count++ - try { - & $scriptBlock - return - } catch { - $exitCode = $_.Exception.ErrorCode - Write-Host "Retry $count/$retries exited $exitCode, retrying..." - Start-Sleep -Seconds ([Math]::Pow(2, $count)) - } - } - Write-Host "Retry $count/$retries exited, no more retries left." -} - -function verifyFileChecksum { - param ( - [string]$filePath, - [string]$checksumFilePath - ) - $actualHash = (Get-FileHash -Algorithm SHA256 -Path $filePath).Hash - $checksumData = Get-Content -Path $checksumFilePath - $expectedHash = ($checksumData -split "\s+")[0] - if ($actualHash -eq $expectedHash) { - Write-Host "CheckSum is checked. File is correct. Original checkSum is: $expectedHash " - return $true - } else { - Write-Host "CheckSum is wrong. File can be corrupted or modified. Current checksum is: $actualHash, the original checksum is: $expectedHash" - return $false - } -} - -function withGolang($version) { - Write-Host "-- Installing Go $version --" - $goDownloadPath = Join-Path $env:TEMP "go_installer.msi" - $goInstallerUrl = "https://golang.org/dl/go$version.windows-amd64.msi" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -Uri $goInstallerUrl -OutFile $goDownloadPath - } - Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $goDownloadPath /quiet" -Wait - $env:GOPATH = "${env:ProgramFiles}\Go" - $env:GOBIN = "${env:GOPATH}\bin" - $env:Path += ";$env:GOPATH;$env:GOBIN" - go version - installGoDependencies -} - -function withPython($version) { - Write-Host "-- Installing Python $version --" - [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" - $pyDownloadPath = Join-Path $env:TEMP "python-$version-amd64.exe" - $pyInstallerUrl = "https://www.python.org/ftp/python/$version/python-$version-amd64.exe" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -UseBasicParsing -Uri $pyInstallerUrl -OutFile $pyDownloadPath - } - Start-Process -FilePath $pyDownloadPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0" -Wait - $pyBinPath = "${env:ProgramFiles}\Python311" - $env:Path += ";$pyBinPath" - python --version -} - -function withMinGW { - Write-Host "-- Installing MinGW --" - [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" - $gwInstallerUrl = "https://github.com/brechtsanders/winlibs_mingw/releases/download/12.1.0-14.0.6-10.0.0-ucrt-r3/winlibs-x86_64-posix-seh-gcc-12.1.0-llvm-14.0.6-mingw-w64ucrt-10.0.0-r3.zip" - $gwInstallerCheckSumUrl = "$gwInstallerUrl.sha256" - $gwDownloadPath = "$env:TEMP\winlibs-x86_64.zip" - $gwDownloadCheckSumPath = "$env:TEMP\winlibs-x86_64.zip.sha256" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -Uri $gwInstallerUrl -OutFile $gwDownloadPath - Invoke-WebRequest -Uri $gwInstallerCheckSumUrl -OutFile $gwDownloadCheckSumPath - } - $comparingResult = verifyFileChecksum -filePath $gwDownloadPath -checksumFilePath $gwDownloadCheckSumPath - if ($comparingResult) { - Expand-Archive -Path $gwDownloadPath -DestinationPath "$env:TEMP" - $gwBinPath = "$env:TEMP\mingw64\bin" - $env:Path += ";$gwBinPath" - } else { - exit 1 - } - -} -function installGoDependencies { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report/v2" - "gotest.tools/gotestsum" - ) - foreach ($pkg in $installPackages) { - go install "$pkg@latest" - } -} - -function withNmap($version) { - Write-Host "-- Installing Nmap $version --" - [Net.ServicePointManager]::SecurityProtocol = "tls, tls11, tls12, ssl3" - $nmapInstallerUrl = "https://nmap.org/dist/nmap-$version-setup.exe" - $nmapDownloadPath = "$env:TEMP\nmap-$version-setup.exe" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -UseBasicParsing -Uri $nmapInstallerUrl -OutFile $nmapDownloadPath - } - Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait -} -function google_cloud_auth { - $tempFileName = "google-cloud-credentials.json" - $secretFileLocation = Join-Path $env:TEMP $tempFileName - $null = New-Item -ItemType File -Path $secretFileLocation - Set-Content -Path $secretFileLocation -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET - gcloud auth activate-service-account --key-file $secretFileLocation > $null 2>&1 - $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation -} - -function google_cloud_auth_cleanup { - if (Test-Path $env:GOOGLE_APPLICATION_CREDENTIALS) { - Remove-Item $env:GOOGLE_APPLICATION_CREDENTIALS -Force - Remove-Item Env:\GOOGLE_APPLICATION_CREDENTIALS - } else { - Write-Host "No GCP credentials were added" - } -} - -fixCRLF - -withGolang $env:GO_VERSION - -withPython $env:SETUP_WIN_PYTHON_VERSION - -withMinGW - -if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-packetbeat" -or $env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-filebeat") { - withNmap $env:NMAP_WIN_VERSION -} - -$ErrorActionPreference = "Continue" # set +e - -Set-Location -Path $WorkFolder - -$magefile = "$WORKSPACE\$WorkFolder\.magefile" -$env:MAGEFILE_CACHE = $magefile - -New-Item -ItemType Directory -Force -Path "build" - -if ($testType -eq "unittest") { - if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { - mage -w reader/etw build goUnitTest - } else { - mage build unitTest - } -} -elseif ($testType -eq "systemtest") { - try { - google_cloud_auth - mage systemTest - } finally { - google_cloud_auth_cleanup - } -} -else { - Write-Host "Unknown test type. Please specify 'unittest' or 'systemtest'." -} - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index 7c5ee4d1609..254370db030 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -2,16 +2,23 @@ name: "beats-winlogbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + BEATS_PROJECT_NAME: "winlogbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - BEATS_PROJECT_NAME: "winlogbeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -35,6 +42,8 @@ steps: - label: ":linux: Load dynamic winlogbeat pipeline" key: "winlogbeat-pipeline" command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index d88bf2a4ff0..c0c92cc3ae1 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -2,21 +2,28 @@ name: "beats-xpack-auditbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/auditbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/auditbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -57,6 +64,8 @@ steps: - label: ":linux: Load dynamic x-pack auditbeat pipeline" key: "xpack-auditbeat-pipeline" command: ".buildkite/scripts/generate_xpack_auditbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index bcc2610e175..f55b812cd17 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -2,13 +2,28 @@ name: "beats-xpack-dockerlogbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "x-pack/dockerlogbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -33,6 +48,8 @@ steps: - label: ":linux: Load dynamic x-pack dockerlogbeat pipeline" key: "xpack-dockerlogbeat-pipeline" command: ".buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index a324353b65f..02b1cf0ddea 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -2,20 +2,28 @@ name: "beats-xpack-filebeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/filebeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/filebeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -64,6 +72,8 @@ steps: - label: ":linux: Load dynamic x-pack filebeat pipeline" key: "xpack-filebeat-pipeline" command: ".buildkite/scripts/generate_xpack_filebeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 2804e98996a..3952b43c45e 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -3,20 +3,28 @@ name: "beats-xpack-heartbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/heartbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/heartbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -49,6 +57,8 @@ steps: - label: ":linux: Load dynamic x-pack heartbeat pipeline" key: "xpack-heartbeat-pipeline" command: ".buildkite/scripts/generate_xpack_heartbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 0c745b1a09d..c029951f010 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -2,18 +2,28 @@ name: "beats-xpack-libbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/libbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/libbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -45,6 +55,8 @@ steps: - label: ":linux: Load dynamic x-pack libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 216f3134344..855c59b8bac 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -2,20 +2,28 @@ name: "beats-xpack-metricbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/metricbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/metricbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -56,6 +64,8 @@ steps: - label: ":linux: Load dynamic x-pack metricbeat pipeline" key: "xpack-metricbeat-pipeline" command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index 57726c03828..7cd699b5331 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -2,17 +2,28 @@ name: "beats-xpack-osquerybeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/osquerybeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - BEATS_PROJECT_NAME: "x-pack/osquerybeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -45,6 +56,8 @@ steps: - label: ":linux: Load dynamic x-pack osquerybeat pipeline" key: "xpack-osquerybeat-pipeline" command: ".buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 750b59e716d..5e3ce87e2bf 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -2,20 +2,28 @@ name: "beats-xpack-packetbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/packetbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/packetbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -56,6 +64,8 @@ steps: - label: ":linux: Load dynamic x-pack packetbeat pipeline" key: "packetbeat-pipeline" command: ".buildkite/scripts/generate_xpack_packetbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index 5c8acefd698..80b70e287d9 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -3,15 +3,22 @@ name: "beats-xpack-winlogbeat" env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" BEATS_PROJECT_NAME: "x-pack/winlogbeat" + #Packaging + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PACKAGING_ARM_PLATFORMS: "linux/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 + steps: - input: "Input Parameters" @@ -34,6 +41,8 @@ steps: - label: ":linux: Load dynamic x-pack winlogbeat pipeline" key: "xpack-winlogbeat-pipeline" command: ".buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"