From d186490cf4c997209b650d2e1cf42692181ee156 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 9 Apr 2024 21:43:41 +0200 Subject: [PATCH] Single pipeline entrypoint (#38783) * Single pipeline entrypoint * Fix indentations * Notifications (cherry picked from commit d0a3b457ec2423007bf55dbfa0234fad2bb5c20a) # Conflicts: # .buildkite/heartbeat/heartbeat-pipeline.yml # .buildkite/pipeline.yml # .buildkite/scripts/generate_auditbeat_pipeline.sh --- .buildkite/auditbeat/auditbeat-pipeline.yml | 227 +++++++++++++++----- .buildkite/heartbeat/heartbeat-pipeline.yml | 137 ++++++------ .buildkite/hooks/pre-command | 1 + .buildkite/pipeline.yml | 50 ++++- .buildkite/pull-requests.json | 32 --- .buildkite/scripts/common.sh | 18 +- 6 files changed, 292 insertions(+), 173 deletions(-) diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index edbb600e3852..c88d64519532 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -2,12 +2,17 @@ name: "beats-auditbeat" env: + BEATS_PROJECT_NAME: "auditbeat" + + ASDF_MAGE_VERSION: 1.15.0 + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - BEATS_PROJECT_NAME: "auditbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" @@ -22,56 +27,174 @@ env: PACKAGING_ARM_PLATFORMS: "linux/arm64" PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - #Deps - ASDF_MAGE_VERSION: 1.15.0 +notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME" steps: - - input: "Input Parameters" - key: "run_auditbeat" - fields: - - select: "auditbeat - run_auditbeat" - key: "run_auditbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "auditbeat - run_auditbeat_macos_tests" - key: "run_auditbeat_macos_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "auditbeat - run_auditbeat_arm_tests" - key: "run_auditbeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "auditbeat - run_auditbeat_win_tests" - key: "run_auditbeat_win_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - if: "build.source == 'ui'" - - - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic auditbeat pipeline" - key: "auditbeat-pipeline" - command: ".buildkite/scripts/generate_auditbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + - group: "Auditbeat Mandatory Testing" + key: "auditbeat-mandatory-tests" + + steps: + - label: ":ubuntu: Auditbeat Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":ubuntu: Auditbeat Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":rhel: Auditbeat Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows: Auditbeat Win-2016 Auditbeat Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows: Auditbeat Win-2022 Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":linux: Auditbeat Crosscompile" + command: "make -C $BEATS_PROJECT_NAME crosscompile" + env: + GOX_FLAGS: "-arch amd64" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - group: "Auditbeat ARM Tests" + key: "auditbeat-extended-tests-arm" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + steps: + - label: ":arm: ARM64 Unit Tests" + key: "auditbeat-extended-arm64-unit-tests" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - group: "Heartbeat MacOS Extended" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: + - label: ":mac: MacOS Unit Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":mac: MacOS ARM Unit Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - group: "Auditbeat Windows Extended Testing" + key: "auditbeat-extended-tests-win" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + steps: + - label: ":windows: Auditbeat Windows 2019 Unit Tests" + key: "auditbeat-extended-win-2019-unit-tests" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Auditbeat Windows 10 Unit Tests" + key: "auditbeat-extended-win-10-unit-tests" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "auditbeat-extended-win-11-unit-tests" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - group: "Auditbeat Packaging" + key: "auditbeat-packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "auditbeat-mandatory-tests" + steps: + - label: ":ubuntu: Auditbeat/Packaging Linux X86" + key: "auditbeat-package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - label: ":linux: Auditbeat/Packaging Linux ARM" + key: "auditbeat-package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 67fc7595fa17..6c1d12f0f347 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -1,7 +1,15 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: +<<<<<<< HEAD AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" +======= + BEATS_PROJECT_NAME: "heartbeat" + + ASDF_MAGE_VERSION: 1.15.0 + + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" +>>>>>>> d0a3b457ec (Single pipeline entrypoint (#38783)) AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" BEATS_PROJECT_NAME: "heartbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" @@ -16,22 +24,27 @@ env: IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" +<<<<<<< HEAD +======= + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" +>>>>>>> d0a3b457ec (Single pipeline entrypoint (#38783)) - #Packaging - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - PACKAGING_ARM_PLATFORMS: "linux/arm64" +notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME" steps: - group: "Heartbeat Mandatory Testing" - key: "mandatory-tests" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" - + key: "heartbeat-mandatory-tests" steps: - label: ":ubuntu: Unit Tests" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: ubuntu/Unit Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -42,9 +55,6 @@ steps: - label: ":rhel:-9 Unit Tests" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: rhel-9/Unit Tests" agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -58,12 +68,9 @@ steps: command: | Set-Location -Path $BEATS_PROJECT_NAME mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 2016/Unit Tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2016}" + image: "${IMAGE_WIN_2016}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_type: "pd-ssd" artifact_paths: @@ -75,9 +82,6 @@ steps: command: | Set-Location -Path $BEATS_PROJECT_NAME mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 2022/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -88,10 +92,7 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Go Integration Tests" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" - notify: - - github_commit_status: - context: "Heartbeat: Go Integration Tests" + command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -102,9 +103,6 @@ steps: - label: ":ubuntu: Python Integration Tests" command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" - notify: - - github_commit_status: - context: "Heartbeat: Python Integration Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -113,61 +111,41 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - group: "Extended Testing" - key: "extended-tests" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for extended support" - + - group: "Heartbeat ARM Tests" + key: "heartbeat-extended-tests-arm" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ steps: - label: ":linux: ARM64 Unit Tests" key: "arm-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: Unit Tests ARM" agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "heartbeat/build/*.xml" + - group: "Heartbeat Extended Testing MacOS" + key: "heartbeat-extended-tests-macos" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: - label: ":mac: MacOS Unit Tests" key: "macos-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: MacOS Unit Tests" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" artifact_paths: "heartbeat/build/*.xml" - - label: ":mac: MacOS ARM Unit Tests" - key: "macos-extended-arm" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" - notify: - - github_commit_status: - context: "Heartbeat: MacOS ARM Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: "heartbeat/build/*.xml" - - - group: "Windows Extended Testing" - key: "extended-tests-win" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + - group: "Heartbeat Windows Extended Testing" + key: "heartbeat-extended-tests-win" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ steps: - - label: ":windows:-2019 Unit Tests" - key: "windows-extended-2019" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 2019/Unit Tests" + - label: ":windows: Win 2019 Unit Tests" + key: "heartbeat-win-extended-2019" + command: "mage -d ${BEATS_PROJECT_NAME} unitTest" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -178,13 +156,10 @@ steps: - "heartbeat/build/*.json" - label: ":windows:-11 Unit Tests" - key: "windows-extended-11" + key: "heartbeat-windows-extended-11" command: | Set-Location -Path $BEATS_PROJECT_NAME mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 11/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -195,13 +170,10 @@ steps: - "heartbeat/build/*.json" - label: ":windows:-10 Unit Tests" - key: "windows-extended-10" + key: "heartbeat-windows-extended-10" command: | Set-Location -Path $BEATS_PROJECT_NAME mage build unitTest - notify: - - github_commit_status: - context: "Heartbeat: Windows 10/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -211,12 +183,31 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - group: "Packaging" - key: "packaging" + - group: "Heartbeat Packaging" + key: "heartbeat-packaging" if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: - - "mandatory-tests" - + - "heartbeat-mandatory-tests" steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" + - label: ":ubuntu: Packaging Linux X86" + key: "heartbeat-package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + + - label: ":linux: Packaging Linux ARM" + key: "heartbeat-package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 52bee3fb674c..01d56d54363d 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -65,6 +65,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi fi ENABLED_BEATS_PIPELINES_SLUGS=( + "beats" "auditbeat" "filebeat" "beats-metricbeat" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 50334d1b6195..2a1e4d2dc94e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,7 +1,53 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json -env: - ASDF_MAGE_VERSION: '1.15.0' steps: +<<<<<<< HEAD - label: "Generate dynamic pipeline" command: "echo Main pipeline TBD" +======= + - label: "Trigger Auditbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only HEAD~1" + watch: + - path: + - auditbeat/ + - .buildkite/auditbeat/ + #OSS + - go.mod + - pytest.ini + - dev-tools/** + - libbeat/** + - testing/** + config: + trigger: "auditbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger Heartbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only HEAD~1" + watch: + - path: + - heartbeat/ + - .buildkite/heartbeat/ + #OSS + - go.mod + - pytest.ini + - dev-tools/** + - libbeat/** + - testing/** + config: + trigger: "heartbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} +>>>>>>> d0a3b457ec (Single pipeline entrypoint (#38783)) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index abf9580c2086..93bdeba07c52 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -48,38 +48,6 @@ "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": [ "^metricbeat/.*", ".buildkite/metricbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, - { - "enabled": true, - "pipelineSlug": "auditbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", - "always_trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^auditbeat/.*", ".buildkite/auditbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, - { - "enabled": true, - "pipelineSlug": "heartbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", - "always_trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ "^heartbeat/.*", ".buildkite/heartbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, { "enabled": true, "pipelineSlug": "deploy-k8s", diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 3875fc5955d1..90fced3802ea 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -12,7 +12,6 @@ ONLY_DOCS=${ONLY_DOCS:-"true"} OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat -[ -z "${run_auditbeat+x}" ] && run_auditbeat="$(buildkite-agent meta-data get run_auditbeat --default "false")" [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" @@ -29,7 +28,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat -[ -z "${run_auditbeat_arm_tests+x}" ] && run_auditbeat_arm_tests="$(buildkite-agent meta-data get run_auditbeat_arm_tests --default "false")" [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" [ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" @@ -39,7 +37,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" # define if needed run MacOS platform-specific tests for the particular beat -[ -z "${run_auditbeat_tests+x}" ] && run_auditbeat_macos_tests="$(buildkite-agent meta-data get run_auditbeat_macos_tests --default "false")" [ -z "${run_filebeat_tests+x}" ] && run_filebeat_macos_tests="$(buildkite-agent meta-data get run_filebeat_macos_tests --default "false")" [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" @@ -58,10 +55,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" [ -z "${run_xpack_filebeat_aws_tests+x}" ] && run_xpack_filebeat_aws_tests="$(buildkite-agent meta-data get run_xpack_filebeat_aws_tests --default "false")" -auditbeat_changeset=( - "^auditbeat/.*" - ) - filebeat_changeset=( "^filebeat/.*" ) @@ -149,10 +142,7 @@ packaging_changeset=( ".go-version" ) -case "${BUILDKITE_PIPELINE_SLUG}" in - "auditbeat") - BEAT_CHANGESET_REFERENCE=${auditbeat_changeset[@]} - ;; +case "${BUILDKITE_PIPELINE_SLUG}" in "filebeat") BEAT_CHANGESET_REFERENCE=${filebeat_changeset[@]} ;; @@ -433,7 +423,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -444,7 +434,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi @@ -455,7 +445,7 @@ are_conditions_met_macos_tests() { are_conditions_met_win_tests() { if are_conditions_met_mandatory_tests; then - if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_WIN_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_WIN_LABEL} || "${!TRIGGER_SPECIFIC_WIN_TESTS}" == "true" ]]; then return 0 fi