From ce0db387df02a573b74e36e094ca0e15069fe235 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Fri, 10 May 2024 12:48:25 +0300 Subject: [PATCH 1/5] Update filebeat pipeline to match Jenkins steps (#39261) * updated filebeat pipeline * updated pre-command * updated timeout for python tests * restored original timeout * pr fixes (cherry picked from commit a878b4927e9db6c56cde0ae4e700c432d6350ace) --- .buildkite/filebeat/filebeat-pipeline.yml | 51 ++++++++++++----------- .buildkite/hooks/pre-command | 2 +- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 70df2d140b63..16b78e03e1f5 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -26,7 +26,7 @@ steps: - group: "Filebeat Mandatory Tests" key: "filebeat-mandatory-tests" steps: - - label: ":ubuntu: Ubuntu Unit Tests" + - label: ":ubuntu: Filebeat Unit Tests" command: | cd filebeat mage build unitTest @@ -42,9 +42,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Ubuntu Unit Tests" + context: "filebeat: Linux x86_64 Unit Tests" - - label: ":ubuntu: Ubuntu Go Integration Tests" + - label: ":ubuntu: Filebeat Go Integration Tests" command: | cd filebeat mage goIntegTest @@ -60,9 +60,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Ubuntu Go Integration Tests" + context: "filebeat: Filebeat Go Integration Tests" - - label: ":ubuntu: Ubuntu Python Integration Tests" + - label: ":ubuntu: Filebeat Python Integration Tests" command: | cd filebeat mage pythonIntegTest @@ -80,7 +80,7 @@ steps: - github_commit_status: context: "filebeat: Python Integration Tests" - - label: ":windows: Windows 2016 Unit Tests" + - label: ":windows: Filebeat Windows 2016 Unit Tests" key: "windows-2016-unit-tests" command: | Set-Location -Path filebeat @@ -101,7 +101,7 @@ steps: - github_commit_status: context: "filebeat: Windows 2016 Unit Tests" - - label: ":windows: Windows 2022 Unit Tests" + - label: ":windows: Filebeat Windows 2022 Unit Tests" key: "windows-2022-unit-tests" command: | Set-Location -Path filebeat @@ -124,8 +124,10 @@ steps: - group: "Filebeat Extended Tests" key: "filebeat-extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ + steps: - - label: ":mac: MacOS x64_64 Unit Tests" + - label: ":mac: Filebeat macOS x86_64 Unit Tests" key: "macos-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | @@ -144,9 +146,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended MacOS Unit Tests" + context: "filebeat: macOS x86_64 Unit Tests" - - label: ":mac: MacOS arm64 Unit Tests" + - label: ":mac: Filebeat macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | @@ -165,9 +167,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended MacOS ARM Unit Tests" + context: "filebeat: macOS arm64 Unit Tests" - - label: ":linux: Ubuntu ARM Unit Tests" + - label: ":linux: Filebeat arm64 Unit Tests" key: "extended-arm64-unit-test" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ command: | @@ -185,13 +187,14 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Ubuntu ARM Unit Tests" + context: "filebeat: Linux arm64 Unit Tests" - - group: "Extended Windows Tests" + - group: "Filebeat Extended Windows Tests" key: "filebeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: Filebeat Windows 2019 Unit Tests" key: "windows-extended-2019" command: | Set-Location -Path filebeat @@ -210,9 +213,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 2019 Unit Tests" + context: "filebeat: Windows 2019 Unit Tests" - - label: ":windows: Windows 11 Unit Tests" + - label: ":windows: Filebeat Windows 11 Unit Tests" key: "windows-extended-11" command: | Set-Location -Path filebeat @@ -231,9 +234,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 11 Unit Tests" + context: "filebeat: Windows 11 Unit Tests" - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: Filebeat Windows 10 Unit Tests" key: "windows-extended-10" command: | Set-Location -Path filebeat @@ -252,7 +255,7 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Extended Windows 10 Unit Tests" + context: "filebeat: Windows 10 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -264,7 +267,7 @@ steps: - group: "Filebeat Packaging" key: "packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":linux: Filebeat Packaging Linux" key: "packaging-linux" command: | cd filebeat @@ -283,9 +286,9 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" notify: - github_commit_status: - context: "filebeat: Packaging" + context: "filebeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":linux: Filebeat Packaging arm64" key: "packaging-arm" command: | cd filebeat @@ -303,4 +306,4 @@ steps: PACKAGES: "docker" notify: - github_commit_status: - context: "filebeat: Packaging ARM" + context: "filebeat: Packaging arm64" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index af4762b03dad..587d10688e9f 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then source .buildkite/env-scripts/env.sh # Images with prefix "platform-ingest-beats-*" has Go pre-setup. From c4a88a1477cb98f35847bfbfc004ffcffbccdf9f Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Fri, 10 May 2024 13:52:17 +0300 Subject: [PATCH 2/5] adjusted to match steps from Jenkins 7.17 --- .buildkite/filebeat/filebeat-pipeline.yml | 109 ++++++++++++---------- 1 file changed, 59 insertions(+), 50 deletions(-) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 16b78e03e1f5..e8740a369364 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -80,17 +80,17 @@ steps: - github_commit_status: context: "filebeat: Python Integration Tests" - - label: ":windows: Filebeat Windows 2016 Unit Tests" - key: "windows-2016-unit-tests" + - label: ":windows: Filebeat Windows 2019 Unit Tests" + key: "windows-extended-2019" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" - image: "${IMAGE_WIN_2016}" + image: "${IMAGE_WIN_2019}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" @@ -99,7 +99,7 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Windows 2016 Unit Tests" + context: "filebeat: Windows 2019 Unit Tests" - label: ":windows: Filebeat Windows 2022 Unit Tests" key: "windows-2022-unit-tests" @@ -122,62 +122,66 @@ steps: - github_commit_status: context: "filebeat: Windows 2022 Unit Tests" - - group: "Filebeat Extended Tests" - key: "filebeat-extended-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ - - steps: - - label: ":mac: Filebeat macOS x86_64 Unit Tests" - key: "macos-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + - label: ":ubuntu: Filebeat Module Compatibility ITs (Previous Minor)" command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet filebeat + + # echo "~~~ Will run tests with env var MODULE=$$MODULE" cd filebeat - mage build unitTest + mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 + env: + STACK_ENVIRONMENT: prev-minor + TESTING_FILEBEAT_SKIP_DIFF: 1 + PYTEST_ADDOPTS: '-k test_modules' agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "filebeat/build/*.xml" - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: macOS x86_64 Unit Tests" + context: "filebeat: Module Compatibility ITs (Previous Minor)" - - label: ":mac: Filebeat macOS arm64 Unit Tests" - key: "macos-arm64-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + - label: ":ubuntu: Filebeat Module Compatibility ITs (Next Major)" command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet filebeat + + # echo "~~~ Will run tests with env var MODULE=$$MODULE" cd filebeat - mage build unitTest + mage pythonIntegTest retry: automatic: - - limit: 3 + - limit: 3 + env: + STACK_ENVIRONMENT: next-major + TESTING_FILEBEAT_SKIP_DIFF: 1 agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - "filebeat/build/*.xml" - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: macOS arm64 Unit Tests" + context: "filebeat: Module Compatibility ITs (Next Major)" - label: ":linux: Filebeat arm64 Unit Tests" - key: "extended-arm64-unit-test" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ command: | cd filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -189,43 +193,48 @@ steps: - github_commit_status: context: "filebeat: Linux arm64 Unit Tests" - - group: "Filebeat Extended Windows Tests" - key: "filebeat-extended-win-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + - group: "Filebeat Extended Tests" + key: "filebeat-extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ steps: - - label: ":windows: Filebeat Windows 2019 Unit Tests" - key: "windows-extended-2019" + - label: ":mac: Filebeat macOS x86_64 Unit Tests" + key: "macos-unit-tests-extended" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | - Set-Location -Path filebeat + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd filebeat mage build unitTest retry: automatic: - limit: 3 agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" artifact_paths: - "filebeat/build/*.xml" - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Windows 2019 Unit Tests" + context: "filebeat: macOS x86_64 Unit Tests" + + - group: "Filebeat Extended Windows Tests" + key: "filebeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ - - label: ":windows: Filebeat Windows 11 Unit Tests" - key: "windows-extended-11" + steps: + - label: ":windows: Filebeat Windows 2016 Unit Tests" + key: "windows-2016-unit-tests" command: | Set-Location -Path filebeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" - image: "${IMAGE_WIN_11}" + image: "${IMAGE_WIN_2016}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" @@ -234,7 +243,7 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Windows 11 Unit Tests" + context: "filebeat: Windows 2016 Unit Tests" - label: ":windows: Filebeat Windows 10 Unit Tests" key: "windows-extended-10" From 08aefae641cfc28340a093f4ba2d54fe4847c482 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Wed, 15 May 2024 13:47:54 +0300 Subject: [PATCH 3/5] updated pipeline based on comments --- .buildkite/filebeat/filebeat-pipeline.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index e6b768c07199..66b222431605 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -131,8 +131,6 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet filebeat - - # echo "~~~ Will run tests with env var MODULE=$$MODULE" cd filebeat mage pythonIntegTest retry: @@ -158,8 +156,6 @@ steps: # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet filebeat - - # echo "~~~ Will run tests with env var MODULE=$$MODULE" cd filebeat mage pythonIntegTest retry: From d68bcb7027062f6ff9a7fe1dffb41d72d3477601 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Wed, 15 May 2024 19:29:17 +0300 Subject: [PATCH 4/5] fixed addopts --- .buildkite/filebeat/filebeat-pipeline.yml | 28 ++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 06c57d7aa37d..4d849f4e5f99 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -104,7 +104,7 @@ steps: env: STACK_ENVIRONMENT: "prev-minor" TESTING_FILEBEAT_SKIP_DIFF: 1 - PYTEST_ADDOPTS: "-k test_xpack_modules" + PYTEST_ADDOPTS: "-k test_modules" command: | set -euo pipefail source .buildkite/scripts/changesets.sh @@ -126,6 +126,32 @@ steps: - github_commit_status: context: "filebeat: Module compat tests / previous minor" + - label: ":ubuntu: Filebeat Module Compatibility ITs (Previous Minor)" + command: | + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet filebeat + # echo "~~~ Will run tests with env var MODULE=$$MODULE" + cd filebeat + mage pythonIntegTest + retry: + automatic: + - limit: 3 + env: + STACK_ENVIRONMENT: prev-minor + TESTING_FILEBEAT_SKIP_DIFF: 1 + PYTEST_ADDOPTS: '-k test_modules' + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Module Compatibility ITs (Previous Minor)" + - label: "Module compat tests: next major" # Run module integration tests under next major of Elastic stack. env: From d4a6ce4b9a1f17792bce99090c30e3175efe4e9b Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Wed, 15 May 2024 19:53:03 +0300 Subject: [PATCH 5/5] removed duplicated step --- .buildkite/filebeat/filebeat-pipeline.yml | 26 ----------------------- 1 file changed, 26 deletions(-) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 4d849f4e5f99..3743c095041a 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -126,32 +126,6 @@ steps: - github_commit_status: context: "filebeat: Module compat tests / previous minor" - - label: ":ubuntu: Filebeat Module Compatibility ITs (Previous Minor)" - command: | - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet filebeat - # echo "~~~ Will run tests with env var MODULE=$$MODULE" - cd filebeat - mage pythonIntegTest - retry: - automatic: - - limit: 3 - env: - STACK_ENVIRONMENT: prev-minor - TESTING_FILEBEAT_SKIP_DIFF: 1 - PYTEST_ADDOPTS: '-k test_modules' - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - notify: - - github_commit_status: - context: "filebeat: Module Compatibility ITs (Previous Minor)" - - label: "Module compat tests: next major" # Run module integration tests under next major of Elastic stack. env: