diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index be03d7843b2..df5504f2fc0 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -1,145 +1,68 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-auditbeat" env: - BEATS_PROJECT_NAME: "auditbeat" - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + BEATS_PROJECT_NAME: "auditbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" steps: - - group: "Auditbeat Mandatory Testing" - key: "mandatory-tests" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" - - steps: - - label: ":ubuntu: Unit Tests" - command: - - ".buildkite/auditbeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Auditbeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - - - label: ":rhel: Unit Tests" - command: - - ".buildkite/auditbeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Auditbeat: rhel/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - - - label: ":windows:-{{matrix.image}} Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Auditbeat: windows/Unit Tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - - - label: ":linux: Crosscompile" - command: - - ".buildkite/auditbeat/scripts/crosscompile.sh" - env: - GOX_FLAGS: "-arch amd64" - notify: - - github_commit_status: - context: "Auditbeat: Crosscompile" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - - - group: "Extended Testing" - key: "extended-tests" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for extended support" - - steps: - - label: ":linux: ARM64 Unit Tests" - key: "arm-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: - - ".buildkite/auditbeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Auditbeat/Extended: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "auditbeat/build/*.xml" - - - label: ":mac: MacOS Unit Tests" - key: "macos-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: - - ".buildkite/auditbeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Auditbeat/Extended: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "auditbeat/build/*.xml" - - - group: "Windows Extended Testing" - key: "extended-tests-win" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ - - steps: - - label: ":windows: Win 2019 Unit Tests" - key: "win-extended-2019" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Auditbeat/Extended: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" + - input: "Input Parameters" + key: "run_auditbeat" + fields: + - select: "auditbeat - run_auditbeat" + key: "run_auditbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "auditbeat - run_auditbeat_macos_tests" + key: "run_auditbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "auditbeat - run_auditbeat_arm_tests" + key: "run_auditbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "auditbeat - run_auditbeat_win_tests" + key: "run_auditbeat_win_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" - - group: "Packaging" - key: "packaging" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - "mandatory-tests" + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" + - label: ":linux: Load dynamic auditbeat pipeline" + key: "auditbeat-pipeline" + command: ".buildkite/scripts/generate_auditbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index ae22629e6ec..e240e47af45 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -1,145 +1,67 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-filebeat" env: - BEATS_PROJECT_NAME: "filebeat" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + BEATS_PROJECT_NAME: "filebeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" steps: - - group: "Filebeat Mandatory Testing" - key: "mandatory-tests" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" - - steps: - - label: ":ubuntu: Unit Tests" - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Go Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-gotests.sh" - notify: - - github_commit_status: - context: "Filebeat: Go Integration Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Python Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-pytests.sh" - notify: - - github_commit_status: - context: "Filebeat: Python Integration Tests" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":windows:-{{matrix.image}} Unit Tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Filebeat: windows/Unit Tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - group: "Extended Testing" - key: "extended-tests" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support" - - steps: - - label: ":linux: ARM64 Unit Tests" - key: "arm-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat/Extended: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "filebeat/build/*.xml" - - - label: ":mac: MacOS Unit Tests" - key: "macos-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat/Extended: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "filebeat/build/*.xml" - - - group: "Windows Extended Testing" - key: "extended-tests-win" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ - - steps: - - label: ":windows: Win 2019 Unit Tests" - key: "win-extended-2019" - command: ".buildkite/scripts/win_unit_tests.ps1" - notify: - - github_commit_status: - context: "Filebeat/Extended: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" + - input: "Input Parameters" + key: "run_filebeat" + fields: + - select: "filebeat - run_filebeat" + key: "run_filebeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "filebeat - run_filebeat_macos_tests" + key: "run_filebeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "filebeat - run_filebeat_arm_tests" + key: "run_filebeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "filebeat - run_filebeat_win_tests" + key: "run_filebeat_win_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" - - group: "Packaging" - key: "packaging" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - "mandatory-tests" + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" + - label: ":linux: Load dynamic filebeat pipeline" + key: "filebeat-pipeline" + command: ".buildkite/scripts/generate_filebeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 2c69f322ad8..e03ade43af1 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -62,13 +62,31 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-dockerlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then - if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then - echo "Skipped pre-command when running the Upload pipeline" - exit 0 +ENABLED_BEATS_PIPELINES_SLUGS=( + "auditbeat" + "filebeat" + "beats-metricbeat" + "beats-libbeat" + "beats-packetbeat" + "beats-winlogbeat" + "beats-winlogbeat" + "beats-xpack-libbeat" + "beats-xpack-metricbeat" + "beats-xpack-packetbeat" + "beats-xpack-winlogbeat" + "beats-xpack-dockerlogbeat" + "beats-xpack-auditbeat" + "beats-xpack-filebeat" + "beats-xpack-heartbeat" + "beats-xpack-osquerybeat" + ) + +for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do + if [[ "$BUILDKITE_PIPELINE_SLUG" == "$slug" ]]; then + source .buildkite/scripts/setenv.sh + break fi - source .buildkite/scripts/setenv.sh -fi +done if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then if [[ "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then @@ -79,8 +97,6 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPE fi fi - - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh index 07ab6cf4c9a..29ffb389914 100755 --- a/.buildkite/hooks/scripts/util.sh +++ b/.buildkite/hooks/scripts/util.sh @@ -25,6 +25,7 @@ google_cloud_logout_active_account() { } cleanup() { + # TODO: disable for upload pipeline echo "Deleting temporary files..." if [[ -e "${BIN}/${TMP_FOLDER}" ]]; then rm -rf "${BIN}/${TMP_FOLDER}.*" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 201fa8a42c7..3c968af2088 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -12,6 +12,8 @@ ONLY_DOCS=${ONLY_DOCS:-"true"} OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat +[ -z "${run_auditbeat+x}" ] && run_auditbeat="$(buildkite-agent meta-data get run_auditbeat --default "false")" +[ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" @@ -26,6 +28,8 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat +[ -z "${run_auditbeat_arm_tests+x}" ] && run_auditbeat_arm_tests="$(buildkite-agent meta-data get run_auditbeat_arm_tests --default "false")" +[ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" [ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_xpack_auditbeat_arm_tests+x}" ] && run_xpack_auditbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_arm_tests --default "false")" @@ -34,6 +38,8 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" # define if needed run MacOS platform-specific tests for the particular beat +[ -z "${run_auditbeat_tests+x}" ] && run_auditbeat_macos_tests="$(buildkite-agent meta-data get run_auditbeat_macos_tests --default "false")" +[ -z "${run_filebeat_tests+x}" ] && run_filebeat_macos_tests="$(buildkite-agent meta-data get run_filebeat_macos_tests --default "false")" [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" [ -z "${run_xpack_auditbeat_macos_tests+x}" ] && run_xpack_auditbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_macos_tests --default "false")" @@ -43,10 +49,22 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_heartbeat_macos_tests+x}" ] && run_xpack_heartbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_heartbeat_macos_tests --default "false")" [ -z "${run_xpack_osquerybeat_macos_tests+x}" ] && run_xpack_osquerybeat_macos_tests="$(buildkite-agent meta-data get run_xpack_osquerybeat_macos_tests --default "false")" +# define if needed run Windows platform-specific tests for the particular beat +[ -z "${run_auditbeat_win_tests+x}" ] && run_auditbeat_win_tests="$(buildkite-agent meta-data get run_auditbeat_win_tests --default "false")" +[ -z "${run_filebeat_win_tests+x}" ] && run_filebeat_win_tests="$(buildkite-agent meta-data get run_filebeat_win_tests --default "false")" + # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" [ -z "${run_xpack_filebeat_aws_tests+x}" ] && run_xpack_filebeat_aws_tests="$(buildkite-agent meta-data get run_xpack_filebeat_aws_tests --default "false")" +auditbeat_changeset=( + "^auditbeat/.*" + ) + +filebeat_changeset=( + "^filebeat/.*" + ) + metricbeat_changeset=( "^metricbeat/.*" ) @@ -131,6 +149,12 @@ packaging_changeset=( ) case "${BUILDKITE_PIPELINE_SLUG}" in + "auditbeat") + BEAT_CHANGESET_REFERENCE=${auditbeat_changeset[@]} + ;; + "filebeat") + BEAT_CHANGESET_REFERENCE=${filebeat_changeset[@]} + ;; "beats-metricbeat") BEAT_CHANGESET_REFERENCE=${metricbeat_changeset[@]} ;; @@ -185,6 +209,7 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" + TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_win_tests" echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" @@ -197,6 +222,7 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" + TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_PROJECT_NAME}_win_tests" echo "Beats project name is $BEATS_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" @@ -208,9 +234,11 @@ check_and_set_beat_vars() { BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud" + BEATS_GH_WIN_COMMENT="${BEATS_GH_COMMENT} for windows" BEATS_GH_MACOS_LABEL="macOS" BEATS_GH_ARM_LABEL="arm" BEATS_GH_AWS_LABEL="aws" + BEATS_GH_WIN_LABEL="windows" } with_docker_compose() { @@ -417,7 +445,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -428,7 +456,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi @@ -437,6 +465,17 @@ are_conditions_met_macos_tests() { return 1 } +are_conditions_met_win_tests() { + if are_conditions_met_mandatory_tests; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_WIN_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_WIN_LABEL} || "${!TRIGGER_SPECIFIC_WIN_TESTS}" == "true" ]]; then + return 0 + fi + fi + fi + return 1 +} + are_conditions_met_aws_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then @@ -449,8 +488,8 @@ are_conditions_met_aws_tests() { } are_conditions_met_packaging() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then + if are_conditions_met_mandatory_tests; then + if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then return 0 fi fi diff --git a/.buildkite/scripts/generate_auditbeat_pipeline.sh b/.buildkite/scripts/generate_auditbeat_pipeline.sh new file mode 100644 index 00000000000..0ff26cda5d3 --- /dev/null +++ b/.buildkite/scripts/generate_auditbeat_pipeline.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.auditbeat-dynamic.yml" + +# TODO: steps: must be always included +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + - group: "Auditbeat Mandatory Testing" + key: "mandatory-tests" + + steps: + - label: ":ubuntu: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Auditbeat: linux/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":rhel: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Auditbeat: rhel/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows:-2016 Unit Tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + notify: + - github_commit_status: + context: "Auditbeat: windows 2016/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows:-2022 Unit Tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + notify: + - github_commit_status: + context: "Auditbeat: windows 2022/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":linux: Crosscompile" + command: + - ".buildkite/auditbeat/scripts/crosscompile.sh" + env: + GOX_FLAGS: "-arch amd64" + notify: + - github_commit_status: + context: "Auditbeat: Crosscompile" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - group: "Extended Tests" + key: "extended-tests-arm" + steps: + - label: ":arm: ARM64 Unit Tests" + key: "extended-arm64-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_ARM64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + notify: + - github_commit_status: + context: "Auditbeat: ARM Unit tests" +YAML +fi + +if are_conditions_met_win_tests; then + cat >> $pipelineName <<- YAML + - group: "Windows Extended Testing" + key: "extended-tests-win" + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + notify: + - github_commit_status: + context: "Auditbeat: Windows 2019 Unit Tests" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + notify: + - github_commit_status: + context: "Auditbeat: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + notify: + - github_commit_status: + context: "Auditbeat: Windows 11 Unit Tests" +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then +cat >> $pipelineName <<- YAML + - group: "Packaging" + key: "packaging" + depends_on: + - "mandatory-tests" + steps: + - label: Package pipeline + commands: ".buildkite/scripts/packaging/package-step.sh" + notify: + - github_commit_status: + context: "Auditbeat: Packaging" + + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_filebeat_pipeline.sh b/.buildkite/scripts/generate_filebeat_pipeline.sh new file mode 100644 index 00000000000..63108d5aacd --- /dev/null +++ b/.buildkite/scripts/generate_filebeat_pipeline.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.filebeat-dynamic.yml" + +# TODO: steps: must be always included +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + + +steps: + - group: "Filebeat Mandatory Testing" + key: "mandatory-tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" + + steps: + - label: ":ubuntu: Unit Tests" + command: + - ".buildkite/filebeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Filebeat: linux/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Go Integration Tests" + command: + - ".buildkite/filebeat/scripts/integration-gotests.sh" + notify: + - github_commit_status: + context: "Filebeat: Go Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Python Integration Tests" + command: + - ".buildkite/filebeat/scripts/integration-pytests.sh" + notify: + - github_commit_status: + context: "Filebeat: Python Integration Tests" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":windows:-2016 Unit Tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + notify: + - github_commit_status: + context: "Filebeat: windows/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":windows:-2022 Unit Tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + notify: + - github_commit_status: + context: "Filebeat: windows 2022/Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - group: "Extended Tests: ARM" + key: "extended-tests-arm" + steps: + - label: ":linux: ARM64 Unit Tests" + key: "arm-extended" + command: + - ".buildkite/filebeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Filebeat/Extended: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "filebeat/build/*.xml" + +YAML +fi + +if are_conditions_met_win_tests; then + cat >> $pipelineName <<- YAML + - group: "Windows Extended Testing" + key: "extended-tests-win" + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "win-extended-2019" + command: ".buildkite/scripts/win_unit_tests.ps1" + notify: + - github_commit_status: + context: "Filebeat/Extended: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then +cat >> $pipelineName <<- YAML + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + + steps: + - label: Package pipeline + commands: ".buildkite/scripts/packaging/package-step.sh" + notify: + - github_commit_status: + context: "Filebeat: Packaging" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName