diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 77fa47dbc80..3743c095041 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -18,9 +18,6 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 - # Integration Tests - K8S_VERSION: "v1.29.0" - ASDF_KIND_VERSION: "0.20.0" # Unit tests RACE_DETECTOR: "true" @@ -30,7 +27,7 @@ steps: - group: "Filebeat Mandatory Tests" key: "filebeat-mandatory-tests" steps: - - label: ":ubuntu: Ubuntu Unit Tests" + - label: ":ubuntu: Filebeat Unit Tests" command: | cd filebeat mage build unitTest @@ -46,9 +43,27 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Ubuntu Unit Tests" + context: "filebeat: Linux x86_64 Unit Tests" - - label: ":ubuntu: Ubuntu Go Integration Tests" + - label: ":linux: Filebeat arm64 Unit Tests" + command: | + cd filebeat + mage build unitTest + retry: + automatic: + - limit: 3 + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Linux arm64 Unit Tests" + + - label: ":ubuntu: Filebeat Go Integration Tests" command: | cd filebeat mage goIntegTest @@ -64,9 +79,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Ubuntu Go Integration Tests" + context: "filebeat: Filebeat Go Integration Tests" - - label: ":ubuntu: Ubuntu Python Integration Tests" + - label: ":ubuntu: Filebeat Python Integration Tests" command: | cd filebeat mage pythonIntegTest @@ -89,7 +104,7 @@ steps: env: STACK_ENVIRONMENT: "prev-minor" TESTING_FILEBEAT_SKIP_DIFF: 1 - PYTEST_ADDOPTS: "-k test_xpack_modules" + PYTEST_ADDOPTS: "-k test_modules" command: | set -euo pipefail source .buildkite/scripts/changesets.sh @@ -99,7 +114,7 @@ steps: mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -125,7 +140,7 @@ steps: mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -137,38 +152,17 @@ steps: - github_commit_status: context: "filebeat: Module compat tests / next major" - - label: ":windows: Windows 2016 Unit Tests" - key: "windows-2016-unit-tests" - command: | - Set-Location -Path filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - notify: - - github_commit_status: - context: "filebeat: Windows 2016 Unit Tests" - - - label: ":windows: Windows 2022 Unit Tests" - key: "windows-2022-unit-tests" + - label: ":windows: Filebeat Windows 2019 Unit Tests" + key: "windows-extended-2019" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" - image: "${IMAGE_WIN_2022}" + image: "${IMAGE_WIN_2019}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" @@ -177,10 +171,11 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Windows 2022 Unit Tests" + context: "filebeat: Windows 2019 Unit Tests" - group: "Filebeat Extended Tests" key: "filebeat-extended-tests" + steps: - label: ":mac: MacOS x64_64 Unit Tests" key: "macos-unit-tests-extended" @@ -201,85 +196,24 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended MacOS Unit Tests" - - - label: ":mac: MacOS arm64 Unit Tests" - key: "macos-arm64-unit-tests-extended" - if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - notify: - - github_commit_status: - context: "filebeat: Extended MacOS ARM Unit Tests" - - - label: ":linux: Ubuntu ARM Unit Tests" - key: "extended-arm64-unit-test" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: | - cd filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - notify: - - github_commit_status: - context: "filebeat: Extended Ubuntu ARM Unit Tests" + context: "filebeat: macOS x86_64 Unit Tests" - group: "Extended Windows Tests" key: "filebeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ - steps: - - label: ":windows: Windows 2019 Unit Tests" - key: "windows-extended-2019" - command: | - Set-Location -Path filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - notify: - - github_commit_status: - context: "filebeat: Extended Windows 2019 Unit Tests" - - label: ":windows: Windows 11 Unit Tests" - key: "windows-extended-11" + steps: + - label: ":windows: Filebeat Windows 2016 Unit Tests" + key: "windows-2016-unit-tests" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" - image: "${IMAGE_WIN_11}" + image: "${IMAGE_WIN_2016}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" @@ -288,9 +222,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 11 Unit Tests" + context: "filebeat: Windows 2016 Unit Tests" - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: Filebeat Windows 10 Unit Tests" key: "windows-extended-10" command: | Set-Location -Path filebeat @@ -309,7 +243,7 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 10 Unit Tests" + context: "filebeat: Windows 10 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -321,7 +255,7 @@ steps: - group: "Filebeat Packaging" key: "packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":linux: Filebeat Packaging Linux" key: "packaging-linux" command: | cd filebeat @@ -340,9 +274,9 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" notify: - github_commit_status: - context: "filebeat: Packaging" + context: "filebeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":linux: Filebeat Packaging arm64" key: "packaging-arm" command: | cd filebeat @@ -360,4 +294,4 @@ steps: PACKAGES: "docker" notify: - github_commit_status: - context: "filebeat: Packaging ARM" + context: "filebeat: Packaging arm64" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index af4762b03da..587d10688e9 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then source .buildkite/env-scripts/env.sh # Images with prefix "platform-ingest-beats-*" has Go pre-setup.