From d0a3b457ec2423007bf55dbfa0234fad2bb5c20a Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 9 Apr 2024 21:43:41 +0200 Subject: [PATCH] Single pipeline entrypoint (#38783) * Single pipeline entrypoint * Fix indentations * Notifications --- .buildkite/auditbeat/auditbeat-pipeline.yml | 227 +++++++++++---- .buildkite/heartbeat/heartbeat-pipeline.yml | 123 ++++---- .buildkite/hooks/pre-command | 1 + .buildkite/pipeline.yml | 49 +++- .buildkite/pull-requests.json | 32 --- .buildkite/scripts/common.sh | 18 +- .../scripts/generate_auditbeat_pipeline.sh | 265 ------------------ 7 files changed, 275 insertions(+), 440 deletions(-) delete mode 100644 .buildkite/scripts/generate_auditbeat_pipeline.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index edbb600e3852..c88d64519532 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -2,12 +2,17 @@ name: "beats-auditbeat" env: + BEATS_PROJECT_NAME: "auditbeat" + + ASDF_MAGE_VERSION: 1.15.0 + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - BEATS_PROJECT_NAME: "auditbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" @@ -22,56 +27,174 @@ env: PACKAGING_ARM_PLATFORMS: "linux/arm64" PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - #Deps - ASDF_MAGE_VERSION: 1.15.0 +notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME" steps: - - input: "Input Parameters" - key: "run_auditbeat" - fields: - - select: "auditbeat - run_auditbeat" - key: "run_auditbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "auditbeat - run_auditbeat_macos_tests" - key: "run_auditbeat_macos_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "auditbeat - run_auditbeat_arm_tests" - key: "run_auditbeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "auditbeat - run_auditbeat_win_tests" - key: "run_auditbeat_win_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - if: "build.source == 'ui'" - - - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic auditbeat pipeline" - key: "auditbeat-pipeline" - command: ".buildkite/scripts/generate_auditbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + - group: "Auditbeat Mandatory Testing" + key: "auditbeat-mandatory-tests" + + steps: + - label: ":ubuntu: Auditbeat Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":ubuntu: Auditbeat Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":rhel: Auditbeat Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows: Auditbeat Win-2016 Auditbeat Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows: Auditbeat Win-2022 Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":linux: Auditbeat Crosscompile" + command: "make -C $BEATS_PROJECT_NAME crosscompile" + env: + GOX_FLAGS: "-arch amd64" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - group: "Auditbeat ARM Tests" + key: "auditbeat-extended-tests-arm" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + steps: + - label: ":arm: ARM64 Unit Tests" + key: "auditbeat-extended-arm64-unit-tests" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - group: "Heartbeat MacOS Extended" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: + - label: ":mac: MacOS Unit Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":mac: MacOS ARM Unit Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - group: "Auditbeat Windows Extended Testing" + key: "auditbeat-extended-tests-win" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + steps: + - label: ":windows: Auditbeat Windows 2019 Unit Tests" + key: "auditbeat-extended-win-2019-unit-tests" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Auditbeat Windows 10 Unit Tests" + key: "auditbeat-extended-win-10-unit-tests" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "auditbeat-extended-win-11-unit-tests" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - group: "Auditbeat Packaging" + key: "auditbeat-packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "auditbeat-mandatory-tests" + steps: + - label: ":ubuntu: Auditbeat/Packaging Linux X86" + key: "auditbeat-package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - label: ":linux: Auditbeat/Packaging Linux ARM" + key: "auditbeat-package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 9b1ea1984a74..d74254904216 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -2,6 +2,9 @@ env: BEATS_PROJECT_NAME: "heartbeat" + + ASDF_MAGE_VERSION: 1.15.0 + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" @@ -12,26 +15,22 @@ env: IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_MACOS_ARM: "generic-13-ventura-arm" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" - #Packaging - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - PACKAGING_ARM_PLATFORMS: "linux/arm64" +notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME" steps: - group: "Heartbeat Mandatory Testing" - key: "mandatory-tests" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" - + key: "heartbeat-mandatory-tests" steps: - label: ":ubuntu: Unit Tests" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: ubuntu/Unit Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -42,9 +41,6 @@ steps: - label: ":rhel:-9 Unit Tests" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: rhel-9/Unit Tests" agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -58,12 +54,9 @@ steps: command: | Set-Location -Path $BEATS_PROJECT_NAME mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 2016/Unit Tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2016}" + image: "${IMAGE_WIN_2016}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_type: "pd-ssd" artifact_paths: @@ -75,9 +68,6 @@ steps: command: | Set-Location -Path $BEATS_PROJECT_NAME mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 2022/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -88,10 +78,7 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Go Integration Tests" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" - notify: - - github_commit_status: - context: "Heartbeat: Go Integration Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -102,9 +89,6 @@ steps: - label: ":ubuntu: Python Integration Tests" command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" - notify: - - github_commit_status: - context: "Heartbeat: Python Integration Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -113,61 +97,41 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - group: "Extended Testing" - key: "extended-tests" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for extended support" - + - group: "Heartbeat ARM Tests" + key: "heartbeat-extended-tests-arm" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ steps: - label: ":linux: ARM64 Unit Tests" key: "arm-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: Unit Tests ARM" agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "heartbeat/build/*.xml" + - group: "Heartbeat Extended Testing MacOS" + key: "heartbeat-extended-tests-macos" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: - label: ":mac: MacOS Unit Tests" key: "macos-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: MacOS Unit Tests" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" artifact_paths: "heartbeat/build/*.xml" - - label: ":mac: MacOS ARM Unit Tests" - key: "macos-extended-arm" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: MacOS ARM Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: "heartbeat/build/*.xml" - - - group: "Windows Extended Testing" - key: "extended-tests-win" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + - group: "Heartbeat Windows Extended Testing" + key: "heartbeat-extended-tests-win" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ steps: - - label: ":windows:-2019 Unit Tests" - key: "windows-extended-2019" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 2019/Unit Tests" + - label: ":windows: Win 2019 Unit Tests" + key: "heartbeat-win-extended-2019" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -178,13 +142,10 @@ steps: - "heartbeat/build/*.json" - label: ":windows:-11 Unit Tests" - key: "windows-extended-11" + key: "heartbeat-windows-extended-11" command: | Set-Location -Path $BEATS_PROJECT_NAME mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 11/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -195,13 +156,10 @@ steps: - "heartbeat/build/*.json" - label: ":windows:-10 Unit Tests" - key: "windows-extended-10" + key: "heartbeat-windows-extended-10" command: | Set-Location -Path $BEATS_PROJECT_NAME mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 10/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -211,12 +169,31 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - group: "Packaging" - key: "packaging" + - group: "Heartbeat Packaging" + key: "heartbeat-packaging" if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: - - "mandatory-tests" - + - "heartbeat-mandatory-tests" steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" + - label: ":ubuntu: Packaging Linux X86" + key: "heartbeat-package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - label: ":linux: Packaging Linux ARM" + key: "heartbeat-package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 52bee3fb674c..01d56d54363d 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -65,6 +65,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi fi ENABLED_BEATS_PIPELINES_SLUGS=( + "beats" "auditbeat" "filebeat" "beats-metricbeat" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 84f725981fe8..8515ebc76527 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,7 +1,48 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json -env: - ASDF_MAGE_VERSION: '1.15.0' steps: - - label: "Generate dynamic pipeline" - command: ".buildkite/scripts/generate_pipeline.sh" + - label: "Trigger Auditbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only HEAD~1" + watch: + - path: + - auditbeat/ + - .buildkite/auditbeat/ + #OSS + - go.mod + - pytest.ini + - dev-tools/** + - libbeat/** + - testing/** + config: + trigger: "auditbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger Heartbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only HEAD~1" + watch: + - path: + - heartbeat/ + - .buildkite/heartbeat/ + #OSS + - go.mod + - pytest.ini + - dev-tools/** + - libbeat/** + - testing/** + config: + trigger: "heartbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 11fad2d18140..ba73203ec689 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -48,38 +48,6 @@ "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": [ "^metricbeat/.*", ".buildkite/metricbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, - { - "enabled": true, - "pipelineSlug": "auditbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", - "always_trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^auditbeat/.*", ".buildkite/auditbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, - { - "enabled": true, - "pipelineSlug": "heartbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", - "always_trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^heartbeat/.*", ".buildkite/heartbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, { "enabled": true, "pipelineSlug": "deploy-k8s", diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 64b36131caf6..49e6d794c263 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -12,7 +12,6 @@ ONLY_DOCS=${ONLY_DOCS:-"true"} OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat -[ -z "${run_auditbeat+x}" ] && run_auditbeat="$(buildkite-agent meta-data get run_auditbeat --default "false")" [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" @@ -29,7 +28,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat -[ -z "${run_auditbeat_arm_tests+x}" ] && run_auditbeat_arm_tests="$(buildkite-agent meta-data get run_auditbeat_arm_tests --default "false")" [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" [ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" @@ -39,7 +37,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" # define if needed run MacOS platform-specific tests for the particular beat -[ -z "${run_auditbeat_tests+x}" ] && run_auditbeat_macos_tests="$(buildkite-agent meta-data get run_auditbeat_macos_tests --default "false")" [ -z "${run_filebeat_tests+x}" ] && run_filebeat_macos_tests="$(buildkite-agent meta-data get run_filebeat_macos_tests --default "false")" [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" @@ -58,10 +55,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" [ -z "${run_xpack_filebeat_aws_tests+x}" ] && run_xpack_filebeat_aws_tests="$(buildkite-agent meta-data get run_xpack_filebeat_aws_tests --default "false")" -auditbeat_changeset=( - "^auditbeat/.*" - ) - filebeat_changeset=( "^filebeat/.*" ) @@ -149,10 +142,7 @@ packaging_changeset=( ".go-version" ) -case "${BUILDKITE_PIPELINE_SLUG}" in - "auditbeat") - BEAT_CHANGESET_REFERENCE=${auditbeat_changeset[@]} - ;; +case "${BUILDKITE_PIPELINE_SLUG}" in "filebeat") BEAT_CHANGESET_REFERENCE=${filebeat_changeset[@]} ;; @@ -434,7 +424,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -445,7 +435,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi @@ -456,7 +446,7 @@ are_conditions_met_macos_tests() { are_conditions_met_win_tests() { if are_conditions_met_mandatory_tests; then - if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_WIN_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_WIN_LABEL} || "${!TRIGGER_SPECIFIC_WIN_TESTS}" == "true" ]]; then return 0 fi diff --git a/.buildkite/scripts/generate_auditbeat_pipeline.sh b/.buildkite/scripts/generate_auditbeat_pipeline.sh deleted file mode 100644 index d514c5d79d85..000000000000 --- a/.buildkite/scripts/generate_auditbeat_pipeline.sh +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.auditbeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - group: "Mandatory Testing" - key: "mandatory-tests" - - steps: - - label: ":ubuntu: Ubuntu Unit Tests" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - - - label: ":rhel: RHEL Unit Tests" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - - - label: ":windows: Windows 2022 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - - - label: ":linux: Crosscompile" - command: "make -C $BEATS_PROJECT_NAME crosscompile" - env: - GOX_FLAGS: "-arch amd64" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Crosscompile" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "macos-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - - - label: ":mac: MacOS ARM Unit Tests" - key: "macos-arm64-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS ARM Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: Ubuntu ARM Unit Tests" - key: "extended-arm64-unit-test" - command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - group: "Windows Extended Testing" - key: "extended-tests-win" - steps: - - label: ":windows: Windows 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - group: "Packaging" - key: "packaging" - depends_on: - - "mandatory-tests" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" - - -YAML -fi - -echo "+++ Printing dynamic steps" -cat $pipelineName | yq . -P - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName \ No newline at end of file