From 80dab50f0ce3dafe3125a0587db9465dac5d4fa2 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz <87968844+sharbuz@users.noreply.github.com> Date: Fri, 5 Apr 2024 11:08:01 +0300 Subject: [PATCH 1/4] replace default images (#38583) * replace default images -------------------------- Signed-off-by: Alexandros Sapranidis Co-authored-by: Alexandros Sapranidis --- .buildkite/auditbeat/auditbeat-pipeline.yml | 27 +- .../auditbeat/generate_auditbeat_pipeline.sh | 216 -------------- .buildkite/filebeat/filebeat-pipeline.yml | 25 +- .buildkite/heartbeat/heartbeat-pipeline.yml | 50 ++-- .buildkite/hooks/pre-command | 4 +- .buildkite/libbeat/pipeline.libbeat.yml | 18 +- .buildkite/metricbeat/pipeline.yml | 26 +- .buildkite/packetbeat/pipeline.packetbeat.yml | 31 +- .buildkite/scripts/cloud_tests.sh | 52 +++- .buildkite/scripts/common.sh | 30 +- .../scripts/generate_auditbeat_pipeline.sh | 265 +++++++++++++++++ .../scripts/generate_filebeat_pipeline.sh | 266 ++++++++++++++++++ .../scripts/generate_libbeat_pipeline.sh | 58 +++- .../scripts/generate_metricbeat_pipeline.sh | 121 +++++--- .../scripts/generate_packetbeat_pipeline.sh | 116 +++++--- .../scripts/generate_winlogbeat_pipeline.sh | 64 +++-- .../generate_xpack_auditbeat_pipeline.sh | 67 ++++- .../generate_xpack_dockerlogbeat_pipeline.sh | 25 +- .../generate_xpack_filebeat_pipeline.sh | 80 +++++- .../generate_xpack_heartbeat_pipeline.sh | 56 +++- .../generate_xpack_libbeat_pipeline.sh | 98 ++++--- .../generate_xpack_metricbeat_pipeline.sh | 133 +++++---- .../generate_xpack_osquerybeat_pipeline.sh | 50 +++- .../generate_xpack_packetbeat_pipeline.sh | 84 ++++-- .../generate_xpack_winlogbeat_pipeline.sh | 40 ++- .buildkite/scripts/go_int_tests.sh | 12 - .buildkite/scripts/packaging/package-step.sh | 4 +- .buildkite/scripts/py_int_tests.sh | 13 - .buildkite/scripts/setenv.sh | 40 +-- .buildkite/scripts/win_unit_tests.ps1 | 188 ------------- .buildkite/winlogbeat/pipeline.winlogbeat.yml | 23 +- .../x-pack/pipeline.xpack.auditbeat.yml | 33 ++- .../x-pack/pipeline.xpack.dockerlogbeat.yml | 27 +- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 32 ++- .../x-pack/pipeline.xpack.heartbeat.yml | 32 ++- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 30 +- .../x-pack/pipeline.xpack.metricbeat.yml | 32 ++- .../x-pack/pipeline.xpack.osquerybeat.yml | 29 +- .../x-pack/pipeline.xpack.packetbeat.yml | 32 ++- .../x-pack/pipeline.xpack.winlogbeat.yml | 19 +- 40 files changed, 1616 insertions(+), 932 deletions(-) delete mode 100644 .buildkite/auditbeat/generate_auditbeat_pipeline.sh create mode 100644 .buildkite/scripts/generate_auditbeat_pipeline.sh create mode 100644 .buildkite/scripts/generate_filebeat_pipeline.sh delete mode 100755 .buildkite/scripts/go_int_tests.sh delete mode 100755 .buildkite/scripts/py_int_tests.sh delete mode 100644 .buildkite/scripts/win_unit_tests.ps1 diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index a8def81f1fa0..edbb600e3852 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -2,21 +2,28 @@ name: "beats-auditbeat" env: - BEATS_PROJECT_NAME: "auditbeat" - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + BEATS_PROJECT_NAME: "auditbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: - input: "Input Parameters" @@ -62,7 +69,9 @@ steps: - label: ":linux: Load dynamic auditbeat pipeline" key: "auditbeat-pipeline" - command: ".buildkite/auditbeat/generate_auditbeat_pipeline.sh" + command: ".buildkite/scripts/generate_auditbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/auditbeat/generate_auditbeat_pipeline.sh b/.buildkite/auditbeat/generate_auditbeat_pipeline.sh deleted file mode 100644 index 56befb51d254..000000000000 --- a/.buildkite/auditbeat/generate_auditbeat_pipeline.sh +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.auditbeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - group: "Auditbeat Mandatory Testing" - key: "mandatory-tests" - - steps: - - label: ":ubuntu: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":rhel: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: rhel/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2016 Unit Tests" - key: "windows-2016" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: windows-2016/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2022 Unit Tests" - key: "windows-2022" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: windows-2022/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":linux: Crosscompile" - command: - - "make -C auditbeat crosscompile" - env: - GOX_FLAGS: "-arch amd64" - notify: - - github_commit_status: - context: "Auditbeat: Crosscompile" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "macos-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":mac: MacOS ARM Unit Tests" - key: "macos-arm64-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: MacOS ARM Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" - key: "extended-arm64-unit-test" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - group: "Windows Extended Testing" - key: "extended-tests-win" - - steps: - - label: ":windows:-2019 Unit Tests" - key: "windows-2019-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-11 Unit Tests" - key: "windows-11-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-11 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-10 Unit Tests" - key: "windows-10-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-10 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - group: "Packaging" - key: "packaging" - depends_on: - - "mandatory-tests" - - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" - notify: - - github_commit_status: - context: "Auditbeat: Packaging" - -YAML -fi - -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 9daeef287d91..a22e96c974aa 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -2,20 +2,27 @@ name: "beats-filebeat" env: + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" BEATS_PROJECT_NAME: "filebeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: - input: "Input Parameters" @@ -61,7 +68,9 @@ steps: - label: ":linux: Load dynamic filebeat pipeline" key: "filebeat-pipeline" - command: ".buildkite/filebeat/generate_filebeat_pipeline.sh" + command: ".buildkite/scripts/generate_filebeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index e63fb1c60e2b..9b1ea1984a74 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -4,12 +4,12 @@ env: BEATS_PROJECT_NAME: "heartbeat" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_MACOS_ARM: "generic-13-ventura-arm" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" @@ -17,6 +17,10 @@ env: GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + #Packaging + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PACKAGING_ARM_PLATFORMS: "linux/arm64" + steps: - group: "Heartbeat Mandatory Testing" key: "mandatory-tests" @@ -24,7 +28,7 @@ steps: steps: - label: ":ubuntu: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: ubuntu/Unit Tests" @@ -37,7 +41,7 @@ steps: - "heartbeat/build/*.json" - label: ":rhel:-9 Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: rhel-9/Unit Tests" @@ -51,10 +55,12 @@ steps: - label: ":windows:-2016 Unit Test" key: "windows-2016" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: windows-2016/Unit Tests" + context: "Heartbeat: Windows 2016/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -66,10 +72,12 @@ steps: - label: ":windows:-2022 Unit Test" key: "windows-2022" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: windows-2022/Unit Tests" + context: "Heartbeat: Windows 2022/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -80,7 +88,7 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Go Integration Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" notify: - github_commit_status: context: "Heartbeat: Go Integration Tests" @@ -93,7 +101,7 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Python Integration Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage pythonIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" notify: - github_commit_status: context: "Heartbeat: Python Integration Tests" @@ -113,7 +121,7 @@ steps: - label: ":linux: ARM64 Unit Tests" key: "arm-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: Unit Tests ARM" @@ -126,7 +134,7 @@ steps: - label: ":mac: MacOS Unit Tests" key: "macos-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: MacOS Unit Tests" @@ -138,7 +146,7 @@ steps: - label: ":mac: MacOS ARM Unit Tests" key: "macos-extended-arm" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: MacOS ARM Unit Tests" @@ -154,10 +162,12 @@ steps: steps: - label: ":windows:-2019 Unit Tests" key: "windows-extended-2019" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: Win-2019 Unit Tests" + context: "Heartbeat: Windows 2019/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -169,10 +179,12 @@ steps: - label: ":windows:-11 Unit Tests" key: "windows-extended-11" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: Win-11 Unit Tests" + context: "Heartbeat: Windows 11/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -184,10 +196,12 @@ steps: - label: ":windows:-10 Unit Tests" key: "windows-extended-10" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: Win-10 Unit Tests" + context: "Heartbeat: Windows 10/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_10}" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 5f625b35c591..52bee3fb674c 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -9,11 +9,13 @@ source .buildkite/env-scripts/util.sh AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" -#PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then source .buildkite/env-scripts/env.sh + if [[ -z "${GO_VERSION-""}" ]]; then + export GO_VERSION=$(cat "${WORKSPACE}/.go-version") + fi # Images with prefix "platform-ingest-beats-*" has Go pre-setup. # Image itself takes care of Go version download/installation based on .go-version file diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 83fda5662c74..46ddeca92a4e 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -2,13 +2,19 @@ name: "beats-libbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" BEATS_PROJECT_NAME: "libbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -40,6 +46,8 @@ steps: - label: ":linux: Load dynamic Libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_libbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 04f3b44575e4..c7dcc01e625e 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -2,20 +2,26 @@ name: "beats-metricbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "metricbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -47,6 +53,8 @@ steps: - label: ":linux: Load dynamic metricbeat pipeline" key: "metricbeat-pipeline" command: ".buildkite/scripts/generate_metricbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 8e606e16622e..9247b1446676 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -2,20 +2,27 @@ name: "beats-packetbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_RHEL9_X86_64: "family/core-rhel-9" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "packetbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "packetbeat" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -55,6 +62,8 @@ steps: - label: ":linux: Load dynamic packetbeat pipeline" key: "packetbeat-pipeline" command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index d96baf670a9a..3ca52305188b 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -euo pipefail # What Terraform Module will run if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then @@ -7,19 +8,52 @@ elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then export MODULE_DIR="x-pack/filebeat/input/awss3/_meta/terraform" fi -source .buildkite/scripts/install_tools.sh +teardown() { + popd + # Teardown resources after using them + echo "~~~ Terraform Cleanup" + tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK -set -euo pipefail + echo "~~~ Docker Compose Cleanup" + docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +} + +tf_cleanup() { + DIRECTORY=${1:-.} -trap 'teardown || true; unset_secrets' EXIT + for tfstate in $(find $DIRECTORY -name terraform.tfstate); do + cd $(dirname $tfstate) + terraform init + if ! terraform destroy -auto-approve; then + echo "+++ Failed to Terraform destroy the resources" + fi + cd - + done +} + +trap 'teardown' EXIT # Prepare the cloud resources using Terraform -startCloudTestEnv "${MODULE_DIR}" +#startCloudTestEnv "${MODULE_DIR}" +echo "~~~ Loading creds" +set +o xtrace +export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY +export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY +export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" +set -o xtrace + +echo "~~~ Run docker-compose services for emulated cloud env" +docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +echo "~~~ Initialize TF cloud resources" +pushd "$MODULE_DIR" +export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') +export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" +export TF_VAR_CREATED_DATE=$(date +%s) +export TF_VAR_ENVIRONMENT="ci" +export TF_VAR_REPO="${REPO}" +terraform init && terraform apply -auto-approve # Run tests -echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" -pushd "${BEATS_PROJECT_NAME}" > /dev/null - +echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" mage build test - -popd > /dev/null diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 3c968af20883..3875fc5955d1 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -23,6 +23,7 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" [ -z "${run_xpack_winlogbeat+x}" ] && run_xpack_winlogbeat="$(buildkite-agent meta-data get run_xpack_winlogbeat --default "false")" [ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")" +[ -z "${run_xpack_dockerlogbeat+x}" ] && run_xpack_dockerlogbeat="$(buildkite-agent meta-data get run_xpack_dockerlogbeat --default "false")" [ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")" [ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")" [ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")" @@ -63,7 +64,7 @@ auditbeat_changeset=( filebeat_changeset=( "^filebeat/.*" - ) + ) metricbeat_changeset=( "^metricbeat/.*" @@ -195,7 +196,7 @@ case "${BUILDKITE_PIPELINE_SLUG}" in BEAT_CHANGESET_REFERENCE=${xpack_winlogbeat_changeset[@]} ;; *) - echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." + echo "~~~ The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." ;; esac @@ -210,7 +211,7 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_win_tests" - echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" + echo "--- Beats project name is $BEATS_XPACK_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" "${xpack_changeset[@]}" @@ -223,7 +224,7 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_PROJECT_NAME}_win_tests" - echo "Beats project name is $BEATS_PROJECT_NAME" + echo "--- Beats project name is $BEATS_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" "${oss_changeset[@]}" @@ -251,19 +252,6 @@ with_docker_compose() { docker-compose version } -with_Terraform() { - echo "Setting up the Terraform environment..." - local path_to_file="${WORKSPACE}/terraform.zip" - create_workspace - check_platform_architeture - retry 5 curl -sSL -o ${path_to_file} "https://releases.hashicorp.com/terraform/${ASDF_TERRAFORM_VERSION}/terraform_${ASDF_TERRAFORM_VERSION}_${platform_type_lowercase}_${go_arch_type}.zip" - unzip -q ${path_to_file} -d ${BIN}/ - rm ${path_to_file} - chmod +x ${BIN}/terraform - export PATH="${BIN}:${PATH}" - terraform version -} - create_workspace() { if [[ ! -d "${BIN}" ]]; then mkdir -p "${BIN}" @@ -466,7 +454,7 @@ are_conditions_met_macos_tests() { } are_conditions_met_win_tests() { - if are_conditions_met_mandatory_tests; then + if are_conditions_met_mandatory_tests; then if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_WIN_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_WIN_LABEL} || "${!TRIGGER_SPECIFIC_WIN_TESTS}" == "true" ]]; then return 0 @@ -529,7 +517,11 @@ defineModuleFromTheChangeSet() { fi done if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - export MODULE="aws" + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + export MODULE="aws" + else + export MODULE="kubernetes" + fi else export MODULE="${changed_modules}" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved # export MODULE="${changed_modules}" # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved diff --git a/.buildkite/scripts/generate_auditbeat_pipeline.sh b/.buildkite/scripts/generate_auditbeat_pipeline.sh new file mode 100644 index 000000000000..d514c5d79d85 --- /dev/null +++ b/.buildkite/scripts/generate_auditbeat_pipeline.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.auditbeat-dynamic.yml" + +# TODO: steps: must be always included +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + - group: "Mandatory Testing" + key: "mandatory-tests" + + steps: + - label: ":ubuntu: Ubuntu Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":rhel: RHEL Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":linux: Crosscompile" + command: "make -C $BEATS_PROJECT_NAME crosscompile" + env: + GOX_FLAGS: "-arch amd64" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Crosscompile" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "macos-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":mac: MacOS ARM Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS ARM Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: Ubuntu ARM Unit Tests" + key: "extended-arm64-unit-test" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + +YAML +fi + +if are_conditions_met_win_tests; then + cat >> $pipelineName <<- YAML + - group: "Windows Extended Testing" + key: "extended-tests-win" + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then +cat >> $pipelineName <<- YAML + - group: "Packaging" + key: "packaging" + depends_on: + - "mandatory-tests" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" + + +YAML +fi + +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName \ No newline at end of file diff --git a/.buildkite/scripts/generate_filebeat_pipeline.sh b/.buildkite/scripts/generate_filebeat_pipeline.sh new file mode 100644 index 000000000000..a3fe1108dc3b --- /dev/null +++ b/.buildkite/scripts/generate_filebeat_pipeline.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.filebeat-dynamic.yml" + +# TODO: steps: must be always included +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + + +steps: + - group: "Mandatory Testing" + key: "mandatory-tests" + + steps: + - label: ":ubuntu: Ubuntu Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Ubuntu Go Integration Tests" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Ubuntu Python Integration Tests" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":windows: Windows 2016 Unit Tests" + key: "windows-2016-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 2022 Unit Tests" + key: "windows-2022-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "macos-unit-tests-extended" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":mac: MacOS ARM Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS ARM Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + + - label: ":linux: Ubuntu ARM Unit Tests" + key: "extended-arm64-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" +YAML +fi + +if are_conditions_met_win_tests; then + cat >> $pipelineName <<- YAML + + - group: "Windows Extended Testing" + key: "extended-tests-win" + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "windows-extended-2019" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 11 Unit Tests" + key: "windows-extended-11" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 10 Unit Tests" + key: "windows-extended-10" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then +cat >> $pipelineName <<- YAML + + - group: "Packaging" + key: "packaging" + depends_on: + - "mandatory-tests" + + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" + +YAML +fi + +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 0674d0b186b5..37132e74abf5 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -17,48 +17,71 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh" + command: "make -C $BEATS_PROJECT_NAME crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cross compile" - label: ":testengine: Stress Tests" key: "mandatory-stress-test" - command: ".buildkite/scripts/stress_tests.sh" + command: "cd $BEATS_PROJECT_NAME && make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS='| go-junit-report > libbeat-stress-test.xml' stress-tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Stress Tests" YAML else @@ -73,20 +96,25 @@ if are_conditions_met_arm_tests; then - group: "Extended Tests" key: "extended-tests" steps: - - label: ":linux: Arm64 Unit Tests" + - label: ":linux: Ubuntu ARM64 Unit Tests" key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM64 Unit Tests" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 477f8fb25a34..a0386aeb1b75 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -17,42 +17,68 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" - label: ":go: Go Intergration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh" + command: "make -C $BEATS_PROJECT_NAME crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cross compile" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -65,7 +91,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline @@ -73,38 +104,30 @@ steps: - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - label: ":windows: Windows 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else echo "The conditions don't match to requirements for generating pipeline steps." @@ -124,7 +147,13 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + YAML fi @@ -143,17 +172,22 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" @@ -161,12 +195,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 97bdb531a6b0..e4870437ac16 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -17,25 +17,37 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":rhel: RHEL-9 Unit Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + + - label: ":rhel: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -48,43 +60,39 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" - group: "Extended Windowds Tests" key: "extended-win-tests" steps: - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_11}" + image: "{{matrix.image}}" machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -111,21 +119,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -144,30 +162,38 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh index ce812016e795..ede70538c2c5 100755 --- a/.buildkite/scripts/generate_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh @@ -18,15 +18,22 @@ steps: - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh" + command: "make -C $BEATS_PROJECT_NAME crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cross compile" - label: ":windows: Windows 2016/2019/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -40,7 +47,12 @@ steps: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2019}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline @@ -48,28 +60,29 @@ steps: - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows 10/11 Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else echo "The conditions don't match to requirements for generating pipeline steps." @@ -90,20 +103,23 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh index f29e6152e602..eb9c8269f3db 100755 --- a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh @@ -25,20 +25,32 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":rhel: RHEL-9 Unit Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit (MODULE) Tests" + + - label: ":rhel: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -51,7 +63,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -60,7 +77,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -74,8 +93,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -102,21 +125,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -144,6 +177,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -155,12 +191,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh index 46e92f8ddf97..f9ca4e7f7677 100755 --- a/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh @@ -22,7 +22,13 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -33,7 +39,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" YAML fi @@ -61,6 +72,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -72,12 +86,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh b/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh index 80d28770bf05..78a351a078d9 100755 --- a/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh @@ -22,7 +22,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -31,7 +36,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" @@ -42,10 +52,17 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -58,7 +75,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -67,7 +89,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -81,7 +105,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -108,21 +137,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -132,15 +171,22 @@ if are_conditions_met_aws_tests; then - label: ":linux: Cloud Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" + skip: "Temporary disable, additional conditions required, we hit AWS limits" env: MODULE: $MODULE + ASDF_TERRAFORM_VERSION: 1.0.2 agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cloud Tests" YAML fi @@ -168,6 +214,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -179,12 +228,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh index b51ca0ab3b9e..bec8e9bf96dd 100755 --- a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh @@ -17,28 +17,40 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" # ## TODO: there are windows test failures already reported # ## https://github.com/elastic/beats/issues/23957 and https://github.com/elastic/beats/issues/23958 # ## waiting for being fixed. # - label: ":windows: Windows Unit Tests - {{matrix.image}}" -# command: ".buildkite/scripts/win_unit_tests.ps1" +# command: +# - "Set-Location -Path $BEATS_PROJECT_NAME" +# - "New-Item -ItemType Directory -Force -Path 'build'" +# - "mage unitTest" # key: "mandatory-win-unit-tests" # agents: # provider: "gcp" @@ -51,7 +63,9 @@ steps: # image: # - "${IMAGE_WIN_2016}" # - "${IMAGE_WIN_2022}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" # ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -60,7 +74,10 @@ steps: # steps: # - label: ":windows: Windows Unit Tests - {{matrix.image}}" -# command: ".buildkite/scripts/win_unit_tests.ps1" +# command: +# - "Set-Location -Path $BEATS_PROJECT_NAME" +# - "New-Item -ItemType Directory -Force -Path 'build'" +# - "mage unitTest" # key: "extended-win-unit-tests" # agents: # provider: "gcp" @@ -74,7 +91,9 @@ steps: # - "${IMAGE_WIN_10}" # - "${IMAGE_WIN_11}" # - "${IMAGE_WIN_2019}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" YAML else @@ -95,7 +114,12 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi @@ -123,6 +147,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -134,12 +161,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh index 66f0750ab6fa..7e3d5086288f 100755 --- a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh @@ -17,33 +17,50 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage -w reader/etw build goUnitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -56,44 +73,40 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage -w reader\etw build goUnitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_11}" + image: "{{matrix.image}}" machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -108,20 +121,25 @@ if are_conditions_met_arm_tests; then - group: "Extended Tests" key: "extended-tests" steps: - - label: ":linux: Arm64 Unit Tests" + - label: ":linux: Ubuntu ARM64 Unit Tests" key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM64 Unit Tests" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index ddc3ce2c8f20..d38c31236562 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -17,37 +17,60 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: | + cd $BEATS_PROJECT_NAME + mage build unitTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":go: Go Integration Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + + - label: ":go: Go (MODULE) Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: | + cd $BEATS_PROJECT_NAME + mage goIntegTest env: MODULE: $MODULE agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":python: Python Integration Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go (MODULE) Integration Tests" + + - label: ":python: Python (MODULE) Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: | + cd $BEATS_PROJECT_NAME + mage pythonIntegTest env: MODULE: $MODULE agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python (MODULE) Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -60,44 +83,40 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2019}" + image: "{{matrix.image}}" machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -125,23 +144,35 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: Cloud Tests" + - label: ":linux: Cloud (MODULE) Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" + skip: "Does not belong to a stage, exists but not run" env: MODULE: $MODULE + ASDF_TERRAFORM_VERSION: 1.0.2 agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cloud (MODULE) Tests" YAML fi @@ -169,6 +200,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -180,12 +214,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P -echo "--- Loading dynamic steps" +echo "~~~ Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh b/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh index d6887c4c3826..5372d7f3faa7 100755 --- a/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh @@ -22,7 +22,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -31,10 +36,17 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -47,7 +59,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -56,7 +73,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -70,7 +89,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -91,7 +115,12 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi @@ -119,12 +148,15 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index 4eb2a1c3e049..c29b6c5d9bc1 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -22,7 +22,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":linux: Ubuntu System Tests" key: "mandatory-linux-system-test" @@ -31,20 +36,33 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":rhel: RHEL-9 Unit Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu System Tests" + + - label: ":rhel: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: + - "Set-Location -Path $BEATS_PROJECT_NAME" + - "New-Item -ItemType Directory -Force -Path 'build'" + - "mage unitTest" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -57,7 +75,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved # - label: ":windows: Windows 2022 System Tests" @@ -69,7 +92,9 @@ steps: # machineType: "${GCP_WIN_MACHINE_TYPE}" # disk_size: 100 # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -78,7 +103,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -92,7 +119,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved # - label: ":windows: Windows 10 System Tests" @@ -104,7 +136,9 @@ steps: # machineType: "${GCP_WIN_MACHINE_TYPE}" # disk_size: 100 # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" YAML else @@ -131,21 +165,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -173,6 +217,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -184,12 +231,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh index 108a70c15622..ad5ebd445837 100755 --- a/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh @@ -18,7 +18,9 @@ steps: - label: ":windows: Windows 2019 Unit (MODULE) Tests" key: "mandatory-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest env: MODULE: $MODULE agents: @@ -27,10 +29,17 @@ steps: machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows (MODULE) {{matrix.image}} Unit Tests" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -43,7 +52,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline @@ -53,7 +67,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -67,7 +83,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -99,12 +120,15 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh deleted file mode 100755 index b4c519f45126..000000000000 --- a/.buildkite/scripts/go_int_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/install_tools.sh - -set -euo pipefail - -echo "--- Run Go Intergration Tests for $BEATS_PROJECT_NAME" -pushd "${BEATS_PROJECT_NAME}" > /dev/null - -mage goIntegTest - -popd > /dev/null diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh index 9eddfafcfba6..5f04ed3849bc 100755 --- a/.buildkite/scripts/packaging/package-step.sh +++ b/.buildkite/scripts/packaging/package-step.sh @@ -18,7 +18,7 @@ if are_files_changed "$changeset"; then - label: ":ubuntu: ${BEATS_PROJECT_NAME}/Packaging Linux X86" key: "package-linux-x86" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: $PACKAGING_PLATFORMS SNAPSHOT: true command: ".buildkite/scripts/packaging/package.sh" notify: @@ -32,7 +32,7 @@ if are_files_changed "$changeset"; then - label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM" key: "package-linux-arm" env: - PLATFORMS: "linux/arm64" + PLATFORMS: $PACKAGING_ARM_PLATFORMS PACKAGES: "docker" SNAPSHOT: true command: ".buildkite/scripts/packaging/package.sh" diff --git a/.buildkite/scripts/py_int_tests.sh b/.buildkite/scripts/py_int_tests.sh deleted file mode 100755 index 19fa8796c3e7..000000000000 --- a/.buildkite/scripts/py_int_tests.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/install_tools.sh - -set -euo pipefail - -echo "--- Run Python Intergration Tests for $BEATS_PROJECT_NAME" - -pushd "${BEATS_PROJECT_NAME}" > /dev/null - -mage pythonIntegTest - -popd > /dev/null diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index f0116c6b3086..f6b03f53f9bd 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -2,37 +2,19 @@ set -euo pipefail -REPO="beats" -TMP_FOLDER="tmp.${REPO}" -DOCKER_REGISTRY="docker.elastic.co" -SETUP_GVM_VERSION="v0.5.1" -DOCKER_COMPOSE_VERSION="1.21.0" -DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" -SETUP_WIN_PYTHON_VERSION="3.11.0" -NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) -GO_VERSION=$(cat .go-version) -ASDF_MAGE_VERSION="1.15.0" -PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" -PACKAGING_ARM_PLATFORMS="linux/arm64" -ASDF_TERRAFORM_VERSION="1.0.2" -AWS_REGION="eu-central-1" -NODEJS_VERSION="18.17.1" +export REPO="beats" +export DOCKER_REGISTRY="docker.elastic.co" +export SETUP_GVM_VERSION="v0.5.1" +export DOCKER_COMPOSE_VERSION="1.21.0" +export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" +export ASDF_NODEJS_VERSION="18.17.1" +export AWS_REGION="eu-central-1" -export SETUP_GVM_VERSION -export DOCKER_COMPOSE_VERSION -export DOCKER_COMPOSE_VERSION_AARCH64 -export SETUP_WIN_PYTHON_VERSION -export NMAP_WIN_VERSION +WORKSPACE=${WORKSPACE:-"$(pwd)"} +export WORKSPACE +GO_VERSION=$(cat .go-version) export GO_VERSION -export ASDF_MAGE_VERSION -export PACKAGING_PLATFORMS -export PACKAGING_ARM_PLATFORMS -export REPO -export TMP_FOLDER -export DOCKER_REGISTRY -export ASDF_TERRAFORM_VERSION -export AWS_REGION -export NODEJS_VERSION + exportVars() { local platform_type="$(uname)" diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 deleted file mode 100644 index 6c3af6e9321c..000000000000 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ /dev/null @@ -1,188 +0,0 @@ -param( - [string]$testType = "unittest" -) - -$ErrorActionPreference = "Stop" # set -e -$WorkFolder = $env:BEATS_PROJECT_NAME -$WORKSPACE = Get-Location -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF { - Write-Host "-- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} - -function retry { - param( - [int]$retries, - [ScriptBlock]$scriptBlock - ) - $count = 0 - while ($count -lt $retries) { - $count++ - try { - & $scriptBlock - return - } catch { - $exitCode = $_.Exception.ErrorCode - Write-Host "Retry $count/$retries exited $exitCode, retrying..." - Start-Sleep -Seconds ([Math]::Pow(2, $count)) - } - } - Write-Host "Retry $count/$retries exited, no more retries left." -} - -function verifyFileChecksum { - param ( - [string]$filePath, - [string]$checksumFilePath - ) - $actualHash = (Get-FileHash -Algorithm SHA256 -Path $filePath).Hash - $checksumData = Get-Content -Path $checksumFilePath - $expectedHash = ($checksumData -split "\s+")[0] - if ($actualHash -eq $expectedHash) { - Write-Host "CheckSum is checked. File is correct. Original checkSum is: $expectedHash " - return $true - } else { - Write-Host "CheckSum is wrong. File can be corrupted or modified. Current checksum is: $actualHash, the original checksum is: $expectedHash" - return $false - } -} - -function withGolang($version) { - Write-Host "-- Installing Go $version --" - $goDownloadPath = Join-Path $env:TEMP "go_installer.msi" - $goInstallerUrl = "https://golang.org/dl/go$version.windows-amd64.msi" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -Uri $goInstallerUrl -OutFile $goDownloadPath - } - Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $goDownloadPath /quiet" -Wait - $env:GOPATH = "${env:ProgramFiles}\Go" - $env:GOBIN = "${env:GOPATH}\bin" - $env:Path += ";$env:GOPATH;$env:GOBIN" - go version - installGoDependencies -} - -function withPython($version) { - Write-Host "-- Installing Python $version --" - [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" - $pyDownloadPath = Join-Path $env:TEMP "python-$version-amd64.exe" - $pyInstallerUrl = "https://www.python.org/ftp/python/$version/python-$version-amd64.exe" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -UseBasicParsing -Uri $pyInstallerUrl -OutFile $pyDownloadPath - } - Start-Process -FilePath $pyDownloadPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0" -Wait - $pyBinPath = "${env:ProgramFiles}\Python311" - $env:Path += ";$pyBinPath" - python --version -} - -function withMinGW { - Write-Host "-- Installing MinGW --" - [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" - $gwInstallerUrl = "https://github.com/brechtsanders/winlibs_mingw/releases/download/12.1.0-14.0.6-10.0.0-ucrt-r3/winlibs-x86_64-posix-seh-gcc-12.1.0-llvm-14.0.6-mingw-w64ucrt-10.0.0-r3.zip" - $gwInstallerCheckSumUrl = "$gwInstallerUrl.sha256" - $gwDownloadPath = "$env:TEMP\winlibs-x86_64.zip" - $gwDownloadCheckSumPath = "$env:TEMP\winlibs-x86_64.zip.sha256" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -Uri $gwInstallerUrl -OutFile $gwDownloadPath - Invoke-WebRequest -Uri $gwInstallerCheckSumUrl -OutFile $gwDownloadCheckSumPath - } - $comparingResult = verifyFileChecksum -filePath $gwDownloadPath -checksumFilePath $gwDownloadCheckSumPath - if ($comparingResult) { - Expand-Archive -Path $gwDownloadPath -DestinationPath "$env:TEMP" - $gwBinPath = "$env:TEMP\mingw64\bin" - $env:Path += ";$gwBinPath" - } else { - exit 1 - } - -} -function installGoDependencies { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report/v2" - "gotest.tools/gotestsum" - ) - foreach ($pkg in $installPackages) { - go install "$pkg@latest" - } -} - -function withNmap($version) { - Write-Host "-- Installing Nmap $version --" - [Net.ServicePointManager]::SecurityProtocol = "tls, tls11, tls12, ssl3" - $nmapInstallerUrl = "https://nmap.org/dist/nmap-$version-setup.exe" - $nmapDownloadPath = "$env:TEMP\nmap-$version-setup.exe" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -UseBasicParsing -Uri $nmapInstallerUrl -OutFile $nmapDownloadPath - } - Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait -} -function google_cloud_auth { - $tempFileName = "google-cloud-credentials.json" - $secretFileLocation = Join-Path $env:TEMP $tempFileName - $null = New-Item -ItemType File -Path $secretFileLocation - Set-Content -Path $secretFileLocation -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET - gcloud auth activate-service-account --key-file $secretFileLocation > $null 2>&1 - $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation -} - -function google_cloud_auth_cleanup { - if (Test-Path $env:GOOGLE_APPLICATION_CREDENTIALS) { - Remove-Item $env:GOOGLE_APPLICATION_CREDENTIALS -Force - Remove-Item Env:\GOOGLE_APPLICATION_CREDENTIALS - } else { - Write-Host "No GCP credentials were added" - } -} - -fixCRLF - -withGolang $env:GO_VERSION - -withPython $env:SETUP_WIN_PYTHON_VERSION - -withMinGW - -if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-packetbeat" -or $env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-filebeat") { - withNmap $env:NMAP_WIN_VERSION -} - -$ErrorActionPreference = "Continue" # set +e - -Set-Location -Path $WorkFolder - -$magefile = "$WORKSPACE\$WorkFolder\.magefile" -$env:MAGEFILE_CACHE = $magefile - -New-Item -ItemType Directory -Force -Path "build" - -if ($testType -eq "unittest") { - if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { - mage -w reader/etw build goUnitTest - } else { - mage build unitTest - } -} -elseif ($testType -eq "systemtest") { - try { - google_cloud_auth - mage systemTest - } finally { - google_cloud_auth_cleanup - } -} -else { - Write-Host "Unknown test type. Please specify 'unittest' or 'systemtest'." -} - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index 7c5ee4d16090..254370db030b 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -2,16 +2,23 @@ name: "beats-winlogbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + BEATS_PROJECT_NAME: "winlogbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - BEATS_PROJECT_NAME: "winlogbeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -35,6 +42,8 @@ steps: - label: ":linux: Load dynamic winlogbeat pipeline" key: "winlogbeat-pipeline" command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index d88bf2a4ff05..c0c92cc3ae13 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -2,21 +2,28 @@ name: "beats-xpack-auditbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/auditbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/auditbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -57,6 +64,8 @@ steps: - label: ":linux: Load dynamic x-pack auditbeat pipeline" key: "xpack-auditbeat-pipeline" command: ".buildkite/scripts/generate_xpack_auditbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index bcc2610e175b..f55b812cd173 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -2,13 +2,28 @@ name: "beats-xpack-dockerlogbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "x-pack/dockerlogbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -33,6 +48,8 @@ steps: - label: ":linux: Load dynamic x-pack dockerlogbeat pipeline" key: "xpack-dockerlogbeat-pipeline" command: ".buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index a324353b65f9..02b1cf0ddead 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -2,20 +2,28 @@ name: "beats-xpack-filebeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/filebeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/filebeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -64,6 +72,8 @@ steps: - label: ":linux: Load dynamic x-pack filebeat pipeline" key: "xpack-filebeat-pipeline" command: ".buildkite/scripts/generate_xpack_filebeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 2804e98996a1..3952b43c45e6 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -3,20 +3,28 @@ name: "beats-xpack-heartbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/heartbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/heartbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -49,6 +57,8 @@ steps: - label: ":linux: Load dynamic x-pack heartbeat pipeline" key: "xpack-heartbeat-pipeline" command: ".buildkite/scripts/generate_xpack_heartbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 0c745b1a09d6..c029951f0107 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -2,18 +2,28 @@ name: "beats-xpack-libbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/libbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/libbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -45,6 +55,8 @@ steps: - label: ":linux: Load dynamic x-pack libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 216f31343446..855c59b8bac2 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -2,20 +2,28 @@ name: "beats-xpack-metricbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/metricbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/metricbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -56,6 +64,8 @@ steps: - label: ":linux: Load dynamic x-pack metricbeat pipeline" key: "xpack-metricbeat-pipeline" command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index 57726c038280..7cd699b53319 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -2,17 +2,28 @@ name: "beats-xpack-osquerybeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/osquerybeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - BEATS_PROJECT_NAME: "x-pack/osquerybeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -45,6 +56,8 @@ steps: - label: ":linux: Load dynamic x-pack osquerybeat pipeline" key: "xpack-osquerybeat-pipeline" command: ".buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 750b59e716d8..5e3ce87e2bf4 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -2,20 +2,28 @@ name: "beats-xpack-packetbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/packetbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/packetbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -56,6 +64,8 @@ steps: - label: ":linux: Load dynamic x-pack packetbeat pipeline" key: "packetbeat-pipeline" command: ".buildkite/scripts/generate_xpack_packetbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index 5c8acefd6989..80b70e287d96 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -3,15 +3,22 @@ name: "beats-xpack-winlogbeat" env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" BEATS_PROJECT_NAME: "x-pack/winlogbeat" + #Packaging + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PACKAGING_ARM_PLATFORMS: "linux/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 + steps: - input: "Input Parameters" @@ -34,6 +41,8 @@ steps: - label: ":linux: Load dynamic x-pack winlogbeat pipeline" key: "xpack-winlogbeat-pipeline" command: ".buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" From dbdaac363e0d64d066c7c7a7f952957c7778fcf3 Mon Sep 17 00:00:00 2001 From: Mattia Meleleo Date: Fri, 5 Apr 2024 12:19:16 +0200 Subject: [PATCH 2/4] [Auditbeat] fim(ebpf): enrich file events with process data (#38199) * fim(ebpf): enrich file events with process data * apply review suggestions * apply review suggestions * fix(fim/ebpf): move process fields to event root and insert them so keys do not contain dots * fix(fim/ebpf): refactor HostID to utilise sync.OnceValue and expose boot time * fix(fim/ebpf): refactor TicksPerSecond to utilise sync.OnceValue * fix(fim/ebpf): remove empty slice allocation * chore: go mod tidy * fix: explicitly set go 1.21.8 in go.mod * fix(fim/ebpf): nil slice of errors in TestNewEventFromEbpfEvent * fix(fim/ebpf): remove re-declaration of already ecs included fields * fix(fim/ebpf): utilise OnceValues to declutter the code * fix(fim/ebpf): remove x-pack import from OSS package * fix(fim/ebpf): propagate process fields changes to integration tests * chore: go mod tidy * ci: temporary solution to outdated docker compose python library * ci: transition to a fixed tag for docker image instead of a rolling one --------- Co-authored-by: Panos Koutsovasilis Co-authored-by: Pierre HILBERT --- CHANGELOG.next.asciidoc | 3 + NOTICE.txt | 146 ++++++++++-------- auditbeat/module/file_integrity/event.go | 46 ++++++ .../module/file_integrity/event_linux.go | 95 +++++++++++- .../module/file_integrity/event_linux_test.go | 38 +++-- auditbeat/tests/system/test_file_integrity.py | 18 +++ go.mod | 8 +- go.sum | 18 ++- libbeat/ebpf/sys/sys.go | 68 ++++++++ libbeat/ebpf/sys/time.go | 74 +++++++++ libbeat/tests/system/requirements.txt | 8 +- libbeat/tests/system/requirements_aix.txt | 8 +- metricbeat/Dockerfile | 8 +- 13 files changed, 431 insertions(+), 107 deletions(-) create mode 100644 libbeat/ebpf/sys/sys.go create mode 100644 libbeat/ebpf/sys/time.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 78801d4a00ea..f0ad6e6a5c11 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,6 +138,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Auditbeat* +- Add linux capabilities to processes in the system/process. {pull}37453[37453] +- Add opt-in eBPF backend for file_integrity module. {pull}37223[37223] +- Add process data to file events (Linux only, eBPF backend). {pull}38199[38199] *Filebeat* diff --git a/NOTICE.txt b/NOTICE.txt index c4ae3f617b57..eb8588c77835 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12287,11 +12287,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/ebpfevents -Version: v0.4.0 +Version: v0.5.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.4.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.5.0/LICENSE.txt: The https://github.com/elastic/ebpfevents repository contains source code under various licenses: @@ -22921,6 +22921,45 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/tklauser/go-sysconf +Version: v0.3.10 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/tklauser/go-sysconf@v0.3.10/LICENSE: + +BSD 3-Clause License + +Copyright (c) 2018-2021, Tobias Klauser +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/tsg/go-daemon Version: v0.0.0-20200207173439-e704b93fd89b @@ -36661,11 +36700,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/cilium/ebpf -Version: v0.12.3 +Version: v0.13.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/cilium/ebpf@v0.12.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/cilium/ebpf@v0.13.2/LICENSE: MIT License @@ -38697,11 +38736,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/frankban/quicktest -Version: v1.14.5 +Version: v1.14.3 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.3/LICENSE: MIT License @@ -39304,6 +39343,37 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/go-quicktest/qt +Version: v1.101.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-quicktest/qt@v1.101.0/LICENSE: + +MIT License + +Copyright (c) 2017 Canonical Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/go-sourcemap/sourcemap Version: v2.1.2+incompatible @@ -49663,27 +49733,6 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/pkg/diff -Version: v0.0.0-20210226163009-20ebb0f2a09e -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/pkg/diff@v0.0.0-20210226163009-20ebb0f2a09e/LICENSE: - -Copyright 2018 Joshua Bleecher Snyder - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/pmezard/go-difflib Version: v1.0.0 @@ -49967,11 +50016,11 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/client_golan -------------------------------------------------------------------------------- Dependency : github.com/rogpeppe/go-internal -Version: v1.9.0 +Version: v1.11.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/go-internal@v1.9.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/go-internal@v1.11.0/LICENSE: Copyright (c) 2018 The Go Authors. All rights reserved. @@ -50873,45 +50922,6 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/tklauser/go-sysconf -Version: v0.3.10 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/tklauser/go-sysconf@v0.3.10/LICENSE: - -BSD 3-Clause License - -Copyright (c) 2018-2021, Tobias Klauser -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/tklauser/numcpus Version: v0.4.0 diff --git a/auditbeat/module/file_integrity/event.go b/auditbeat/module/file_integrity/event.go index 22813a47f22e..b282aaaf3d29 100644 --- a/auditbeat/module/file_integrity/event.go +++ b/auditbeat/module/file_integrity/event.go @@ -134,6 +134,7 @@ type Event struct { Action Action `json:"action"` // Action (like created, updated). Hashes map[HashType]Digest `json:"hash,omitempty"` // File hashes. ParserResults mapstr.M `json:"file,omitempty"` // Results from running file parsers. + Process *Process `json:"process,omitempty"` // Process data. Available only on Linux when using the eBPF backend. // Metadata rtt time.Duration // Time taken to collect the info. @@ -141,6 +142,33 @@ type Event struct { hashFailed bool // Set when hashing the file failed. } +// Process contain information about a process. +// These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. +type Process struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + // Process name. Sometimes called program name or similar. + Name string `json:"name,omitempty"` + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + // Process id. + PID uint32 `json:"pid,omitempty"` +} + // Metadata contains file metadata. type Metadata struct { Inode uint64 `json:"inode"` @@ -354,6 +382,24 @@ func buildMetricbeatEvent(e *Event, existedBefore bool) mb.Event { } } + if e.Process != nil { + process := mapstr.M{ + "pid": e.Process.PID, + "name": e.Process.Name, + "entity_id": e.Process.EntityID, + "user": mapstr.M{ + "id": e.Process.User.ID, + "name": e.Process.User.Name, + }, + "group": mapstr.M{ + "id": e.Process.Group.ID, + "name": e.Process.Group.Name, + }, + } + + out.MetricSetFields.Put("process", process) + } + if len(e.Hashes) > 0 { hashes := make(mapstr.M, len(e.Hashes)) for hashType, digest := range e.Hashes { diff --git a/auditbeat/module/file_integrity/event_linux.go b/auditbeat/module/file_integrity/event_linux.go index 7643d03a6b42..c0eb2d57b159 100644 --- a/auditbeat/module/file_integrity/event_linux.go +++ b/auditbeat/module/file_integrity/event_linux.go @@ -26,6 +26,7 @@ import ( "strconv" "time" + "github.com/elastic/beats/v7/libbeat/ebpf/sys" "github.com/elastic/ebpfevents" ) @@ -41,7 +42,9 @@ func NewEventFromEbpfEvent( path, target string action Action metadata Metadata + process Process err error + errors []error ) switch ee.Type { case ebpfevents.EventTypeFileCreate: @@ -54,7 +57,16 @@ func NewEventFromEbpfEvent( return event, false } target = fileCreateEvent.SymlinkTargetPath + metadata, err = metadataFromFileCreate(fileCreateEvent) + if err != nil { + errors = append(errors, err) + } + + process, err = processFromFileCreate(fileCreateEvent) + if err != nil { + errors = append(errors, err) + } case ebpfevents.EventTypeFileRename: action = Moved @@ -65,7 +77,16 @@ func NewEventFromEbpfEvent( return event, false } target = fileRenameEvent.SymlinkTargetPath + metadata, err = metadataFromFileRename(fileRenameEvent) + if err != nil { + errors = append(errors, err) + } + + process, err = processFromFileRename(fileRenameEvent) + if err != nil { + errors = append(errors, err) + } case ebpfevents.EventTypeFileDelete: action = Deleted @@ -76,6 +97,11 @@ func NewEventFromEbpfEvent( return event, false } target = fileDeleteEvent.SymlinkTargetPath + + process, err = processFromFileDelete(fileDeleteEvent) + if err != nil { + errors = append(errors, err) + } case ebpfevents.EventTypeFileModify: fileModifyEvent := ee.Body.(*ebpfevents.FileModify) @@ -92,7 +118,16 @@ func NewEventFromEbpfEvent( return event, false } target = fileModifyEvent.SymlinkTargetPath + metadata, err = metadataFromFileModify(fileModifyEvent) + if err != nil { + errors = append(errors, err) + } + + process, err = processFromFileModify(fileModifyEvent) + if err != nil { + errors = append(errors, err) + } } event := Event{ @@ -102,10 +137,8 @@ func NewEventFromEbpfEvent( Info: &metadata, Source: SourceEBPF, Action: action, - errors: make([]error, 0), - } - if err != nil { - event.errors = append(event.errors, err) + Process: &process, + errors: errors, } if event.Action == Deleted { @@ -115,7 +148,6 @@ func NewEventFromEbpfEvent( case FileType: fillHashes(&event, path, maxFileSize, hashTypes, fileParsers) case SymlinkType: - var err error event.TargetPath, err = filepath.EvalSymlinks(event.Path) if err != nil { event.errors = append(event.errors, err) @@ -147,6 +179,59 @@ func metadataFromFileModify(evt *ebpfevents.FileModify) (Metadata, error) { return md, err } +func newProcess(pid uint32, start uint64, comm string, euid, egid uint32) (Process, error) { + var ( + p Process + err error + ) + + t, err := sys.TimeFromNsSinceBoot(start) + if err != nil { + return p, err + } + + p.EntityID, err = sys.EntityID(pid, t) + if err != nil { + return p, err + } + p.Name = comm + p.PID = pid + + p.User.ID = strconv.FormatUint(uint64(euid), 10) + u, err := user.LookupId(p.User.ID) + if err == nil { + p.User.Name = u.Username + } else { + p.User.Name = "n/a" + } + + p.Group.ID = strconv.FormatUint(uint64(egid), 10) + g, err := user.LookupGroupId(p.Group.ID) + if err == nil { + p.Group.Name = g.Name + } else { + p.Group.Name = "n/a" + } + + return p, nil +} + +func processFromFileCreate(evt *ebpfevents.FileCreate) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + +func processFromFileRename(evt *ebpfevents.FileRename) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + +func processFromFileModify(evt *ebpfevents.FileModify) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + +func processFromFileDelete(evt *ebpfevents.FileDelete) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + func fillFileInfo(md *Metadata, finfo ebpfevents.FileInfo) error { md.Inode = finfo.Inode md.UID = finfo.Uid diff --git a/auditbeat/module/file_integrity/event_linux_test.go b/auditbeat/module/file_integrity/event_linux_test.go index 1a440afb8f17..beac98789099 100644 --- a/auditbeat/module/file_integrity/event_linux_test.go +++ b/auditbeat/module/file_integrity/event_linux_test.go @@ -21,7 +21,6 @@ package file_integrity import ( "os" - "os/user" "testing" "github.com/stretchr/testify/assert" @@ -40,13 +39,25 @@ func TestNewEventFromEbpfEvent(t *testing.T) { Inode: 1234, Mode: os.FileMode(0o644), Size: 2345, - Uid: 3456, - Gid: 4567, + Uid: uint32(os.Geteuid()), + Gid: uint32(os.Getegid()), }, Path: "/foo", SymlinkTargetPath: "/bar", + Creds: ebpfevents.CredInfo{ + Ruid: 1, + Rgid: 2, + Euid: uint32(os.Geteuid()), + Egid: uint32(os.Getegid()), + Suid: 5, + Sgid: 6, + }, }, } + event, ok := NewEventFromEbpfEvent( + ebpfEvent, 0, []HashType{}, []FileParser{}, func(path string) bool { return false }) + assert.True(t, ok) + expectedEvent := Event{ Action: Created, Path: "/foo", @@ -54,21 +65,22 @@ func TestNewEventFromEbpfEvent(t *testing.T) { Info: &Metadata{ Type: FileType, Inode: 1234, - UID: 3456, - GID: 4567, + UID: uint32(os.Geteuid()), + GID: uint32(os.Getegid()), Size: 2345, - Owner: "n/a", - Group: "n/a", + Owner: event.Info.Owner, + Group: event.Info.Group, Mode: os.FileMode(0o644), }, - Source: SourceEBPF, - errors: []error{user.UnknownUserIdError(3456)}, + Process: event.Process, // 1:1 copy this as it changes on every machine + Source: SourceEBPF, + errors: nil, } - - event, ok := NewEventFromEbpfEvent( - ebpfEvent, 0, []HashType{}, []FileParser{}, func(path string) bool { return false }) - assert.True(t, ok) event.Timestamp = expectedEvent.Timestamp assert.Equal(t, expectedEvent, event) + assert.NotEqual(t, "", event.Process.EntityID) + assert.NotEqual(t, 0, event.Process.PID) + assert.NotEqual(t, 0, event.Process.User.ID) + assert.NotEqual(t, "", event.Process.User.Name) } diff --git a/auditbeat/tests/system/test_file_integrity.py b/auditbeat/tests/system/test_file_integrity.py index e6b03306c3a2..64062dd0a772 100644 --- a/auditbeat/tests/system/test_file_integrity.py +++ b/auditbeat/tests/system/test_file_integrity.py @@ -5,6 +5,10 @@ from auditbeat import * +if platform.platform().split('-')[0] == 'Linux': + import pwd + + def is_root(): if 'geteuid' not in dir(os): return False @@ -100,6 +104,16 @@ def wait_startup(self, backend, dir): # may differ self.wait_log_contains(escape_path(dir), max_timeout=30, ignore_case=True) + def _assert_process_data(self, event, backend): + if backend != "ebpf": + return + assert event["process.entity_id"] != "" + assert event["process.executable"] == "pytest" + assert event["process.pid"] == os.getpid() + assert int(event["process.user.id"]) == os.geteuid() + assert event["process.user.name"] == pwd.getpwuid(os.geteuid()).pw_name + assert int(event["process.group.id"]) == os.getegid() + def _test_non_recursive(self, backend): """ file_integrity monitors watched directories (non recursive). @@ -172,6 +186,8 @@ def _test_non_recursive(self, backend): # assert file inside subdir is not reported assert self.log_contains(file3) is False + self._assert_process_data(objs[0], backend) + @unittest.skipIf(os.getenv("CI") is not None and platform.system() == 'Darwin', 'Flaky test: https://github.com/elastic/beats/issues/24678') def test_non_recursive__fsnotify(self): @@ -252,6 +268,8 @@ def _test_recursive(self, backend): file_events(objs, file1, ['created']) file_events(objs, file2, ['created']) + self._assert_process_data(objs[0], backend) + def test_recursive__fsnotify(self): self._test_recursive("fsnotify") diff --git a/go.mod b/go.mod index 745d47673b61..9e8c91fc742e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.21 +go 1.21.8 require ( cloud.google.com/go/bigquery v1.55.0 @@ -201,7 +201,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/ebpfevents v0.4.0 + github.com/elastic/ebpfevents v0.5.0 github.com/elastic/elastic-agent-autodiscover v0.6.8 github.com/elastic/elastic-agent-libs v0.7.5 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 @@ -224,6 +224,7 @@ require ( github.com/pkg/xattr v0.4.9 github.com/sergi/go-diff v1.3.1 github.com/shirou/gopsutil/v3 v3.22.10 + github.com/tklauser/go-sysconf v0.3.10 go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.8 go.elastic.co/apm/module/apmhttp/v2 v2.4.8 go.elastic.co/apm/v2 v2.4.8 @@ -271,7 +272,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cilium/ebpf v0.12.3 // indirect + github.com/cilium/ebpf v0.13.2 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -360,7 +361,6 @@ require ( github.com/sirupsen/logrus v1.9.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect diff --git a/go.sum b/go.sum index ff0abc92363a..d839df6a98e9 100644 --- a/go.sum +++ b/go.sum @@ -442,8 +442,8 @@ github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLI github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= -github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= +github.com/cilium/ebpf v0.13.2 h1:uhLimLX+jF9BTPPvoCUYh/mBeoONkjgaJ9w9fn0mRj4= +github.com/cilium/ebpf v0.13.2/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P1Hso= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -671,8 +671,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/ebpfevents v0.4.0 h1:M80eAeJnzvGQgU9cjJqkjFca9pjM3aq/TuZxJeom4bI= -github.com/elastic/ebpfevents v0.4.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= +github.com/elastic/ebpfevents v0.5.0 h1:QkyMAYWo3fXFbYtXAXU8sZu2SQ4LXVYC6gLXIWXy02E= +github.com/elastic/ebpfevents v0.5.0/go.mod h1:ESG9gw7N+n5yCCMgdg1IIJENKWSmX7+X0Fi9GUs9nvU= github.com/elastic/elastic-agent-autodiscover v0.6.8 h1:BSXz+QwjZAEt08G+T3GDGl14Bh9a6zD8luNCvZut/b8= github.com/elastic/elastic-agent-autodiscover v0.6.8/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= github.com/elastic/elastic-agent-client/v7 v7.8.1 h1:J9wZc/0mUvSEok0X5iR5+n60Jgb+AWooKddb3XgPWqM= @@ -772,8 +772,8 @@ github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -919,6 +919,8 @@ github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= @@ -1639,7 +1641,6 @@ github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1732,8 +1733,9 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rootless-containers/rootlesskit v1.1.0 h1:cRaRIYxY8oce4eE/zeAUZhgKu/4tU1p9YHN4+suwV7M= github.com/rootless-containers/rootlesskit v1.1.0/go.mod h1:H+o9ndNe7tS91WqU0/+vpvc+VaCd7TCIWaJjnV0ujUo= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= diff --git a/libbeat/ebpf/sys/sys.go b/libbeat/ebpf/sys/sys.go new file mode 100644 index 000000000000..4156321a3214 --- /dev/null +++ b/libbeat/ebpf/sys/sys.go @@ -0,0 +1,68 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package sys + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "sync" + "time" + + "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" +) + +var ( + hostInfoOnce = sync.OnceValues(func() (types.HostInfo, error) { + host, err := sysinfo.Host() + + if host == nil { + return types.HostInfo{}, err + } + + return host.Info(), err + }) +) + +// EntityID creates an ID that uniquely identifies this process across machines. +func EntityID(pid uint32, start time.Time) (string, error) { + info, err := hostInfoOnce() + if err != nil { + return "", err + } + + h := sha256.New() + if _, err := h.Write([]byte(info.UniqueID)); err != nil { + return "", err + } + if err := binary.Write(h, binary.LittleEndian, int64(pid)); err != nil { + return "", err + } + if err := binary.Write(h, binary.LittleEndian, int64(start.Nanosecond())); err != nil { + return "", err + } + + sum := h.Sum(nil) + if len(sum) > 12 { + sum = sum[:12] + } + return base64.RawStdEncoding.EncodeToString(sum), nil +} diff --git a/libbeat/ebpf/sys/time.go b/libbeat/ebpf/sys/time.go new file mode 100644 index 000000000000..7dca6454c328 --- /dev/null +++ b/libbeat/ebpf/sys/time.go @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package sys + +import ( + "sync" + "time" + + "github.com/tklauser/go-sysconf" +) + +var ( + ticksPerSecondOnce = sync.OnceValues(func() (uint64, error) { + ticks, err := sysconf.Sysconf(sysconf.SC_CLK_TCK) + if err != nil { + return 0, err + } + + return uint64(ticks), err + }) +) + +func TicksToNs(ticks uint64) (uint64, error) { + tps, err := ticksPerSecondOnce() + if err != nil { + return 0, err + } + + return ticks * uint64(time.Second.Nanoseconds()) / tps, nil +} + +func TimeFromNsSinceBoot(ns uint64) (time.Time, error) { + info, err := hostInfoOnce() + if err != nil { + return time.Time{}, err + } + + reduced, err := reduceTimestampPrecision(ns) + if err != nil { + return time.Time{}, err + } + + return info.BootTime.Add(time.Duration(reduced)), nil +} + +// When generating an `entity_id` in ECS we need to reduce the precision of a +// process's start time to that of procfs. Process start times can come from either +// eBPF (high precision) or other sources. We must reduce them all to the +// lowest common denominator such that entity ID's generated are always consistent. +func reduceTimestampPrecision(ns uint64) (uint64, error) { + tps, err := ticksPerSecondOnce() + if err != nil { + return 0, err + } + + return ns - (ns % (uint64(time.Second.Nanoseconds()) / tps)), nil +} diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index fc4227738c32..87133cda11dc 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -1,3 +1,7 @@ +requests==2.31.0 +urllib3==1.26.18 +docker==6.1.3 +docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix async-timeout==4.0.3 attrs==19.3.0 autopep8==1.5.4 @@ -13,8 +17,6 @@ cryptography==42.0.4 deepdiff==4.2.0 Deprecated==1.2.14 distro==1.9.0 -docker==6.0.1 -docker-compose==1.29.2 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -58,7 +60,6 @@ pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 redis==4.4.4 -requests==2.31.0 semver==2.8.1 six==1.15.0 stomp.py==4.1.22 @@ -67,7 +68,6 @@ texttable==0.9.1 toml==0.10.1 tomli==2.0.1 typing_extensions==4.9.0 -urllib3==1.26.18 wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/libbeat/tests/system/requirements_aix.txt b/libbeat/tests/system/requirements_aix.txt index fc4227738c32..87133cda11dc 100644 --- a/libbeat/tests/system/requirements_aix.txt +++ b/libbeat/tests/system/requirements_aix.txt @@ -1,3 +1,7 @@ +requests==2.31.0 +urllib3==1.26.18 +docker==6.1.3 +docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix async-timeout==4.0.3 attrs==19.3.0 autopep8==1.5.4 @@ -13,8 +17,6 @@ cryptography==42.0.4 deepdiff==4.2.0 Deprecated==1.2.14 distro==1.9.0 -docker==6.0.1 -docker-compose==1.29.2 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -58,7 +60,6 @@ pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 redis==4.4.4 -requests==2.31.0 semver==2.8.1 six==1.15.0 stomp.py==4.1.22 @@ -67,7 +68,6 @@ texttable==0.9.1 toml==0.10.1 tomli==2.0.1 typing_extensions==4.9.0 -urllib3==1.26.18 wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index b6da04173167..a16b942032d8 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,5 @@ FROM golang:1.21.8 +COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ RUN \ apt update \ @@ -10,6 +11,8 @@ RUN \ python3-venv \ libaio-dev \ unzip \ + libssl-dev \ + libffi-dev \ && rm -rf /var/lib/apt/lists/* # Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts @@ -19,9 +22,12 @@ RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip3 install --upgrade pip==20.1.1 -RUN pip3 install --upgrade docker-compose==1.23.2 RUN pip3 install --upgrade setuptools==47.3.2 RUN pip3 install --upgrade PyYAML==5.3.1 +RUN pip3 install requests==2.31.0 +RUN pip3 install urllib3==1.26.18 +RUN pip3 install docker==6.1.3 +RUN pip3 install git+https://github.com/pkoutsovasilis/compose@v1_fix # Oracle instant client RUN cd /usr/lib \ From a455d0508128ffc3bd2b37d6bdfe721d46b9caa6 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Fri, 5 Apr 2024 14:16:35 +0300 Subject: [PATCH 3/4] Fix the dynamic step for pipeline upload on the x-pack Heartbeat (#38736) Fix unbound BEATS_PROJECT_NAME Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/common.sh | 3 ++- .buildkite/x-pack/pipeline.xpack.heartbeat.yml | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 3875fc5955d1..64b36131caf6 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -201,7 +201,8 @@ case "${BUILDKITE_PIPELINE_SLUG}" in esac check_and_set_beat_vars() { - if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then + local BEATS_PROJECT_NAME=${BEATS_PROJECT_NAME:=""} + if [[ "${BEATS_PROJECT_NAME:=""}" == *"x-pack/"* ]]; then BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_} #replace / to _ BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-} #replace / to - for labels diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 3952b43c45e6..1b04fcf99225 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -1,7 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json name: "beats-xpack-heartbeat" - env: AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "x-pack/heartbeat" From 65ca2de7efaf9d06126246eda5b5754fcae0f18c Mon Sep 17 00:00:00 2001 From: Luca Belluccini Date: Fri, 5 Apr 2024 16:00:18 +0200 Subject: [PATCH 4/4] Include plugin.id in the plain text log format (#36727) Co-authored-by: Nathan L Smith --- filebeat/module/logstash/log/ingest/pipeline-plaintext.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/filebeat/module/logstash/log/ingest/pipeline-plaintext.yml b/filebeat/module/logstash/log/ingest/pipeline-plaintext.yml index 4c75de47c99d..e506b0bc97bc 100644 --- a/filebeat/module/logstash/log/ingest/pipeline-plaintext.yml +++ b/filebeat/module/logstash/log/ingest/pipeline-plaintext.yml @@ -13,6 +13,8 @@ processors: (.| )* patterns: + - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{NOTSPACE:logstash.log.pipeline_id}\]\[%{NOTSPACE:logstash.log.plugin_id}\] + %{GREEDYMULTILINE:message} - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{NOTSPACE:logstash.log.pipeline_id}\] %{GREEDYMULTILINE:message} - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]