[7.17](backport #39261) Update filebeat pipeline to match Jenkins steps

.buildkite/filebeat/filebeat-pipeline.yml

@@ -30,7 +30,7 @@ steps:
   - group: "Filebeat Mandatory Tests"
     key: "filebeat-mandatory-tests"
     steps:
-      - label: ":ubuntu: Ubuntu Unit Tests"
+      - label: ":ubuntu: Filebeat Unit Tests"
         command: |
           cd filebeat
           mage build unitTest
@@ -46,9 +46,9 @@ steps:
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Ubuntu Unit Tests"
+              context: "filebeat: Linux x86_64 Unit Tests"
 
-      - label: ":ubuntu: Ubuntu Go Integration Tests"
+      - label: ":ubuntu: Filebeat Go Integration Tests"
         command: |
           cd filebeat
           mage goIntegTest
@@ -64,9 +64,9 @@ steps:
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Ubuntu Go Integration Tests"
+              context: "filebeat: Filebeat Go Integration Tests"
 
-      - label: ":ubuntu: Ubuntu Python Integration Tests"
+      - label: ":ubuntu: Filebeat Python Integration Tests"
         command: |
           cd filebeat
           mage pythonIntegTest
@@ -84,17 +84,17 @@ steps:
           - github_commit_status:
               context: "filebeat: Python Integration Tests"
 
-      - label: ":windows: Windows 2016 Unit Tests"
-        key: "windows-2016-unit-tests"
+      - label: ":windows: Filebeat Windows 2019 Unit Tests"
+        key: "windows-extended-2019"
         command: |
           Set-Location -Path filebeat
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_2016}"
+          image: "${IMAGE_WIN_2019}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
@@ -103,9 +103,9 @@ steps:
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Windows 2016 Unit Tests"
+              context: "filebeat: Windows 2019 Unit Tests"
 
-      - label: ":windows: Windows 2022 Unit Tests"
+      - label: ":windows: Filebeat Windows 2022 Unit Tests"
         key: "windows-2022-unit-tests"
         command: |
           Set-Location -Path filebeat
@@ -126,60 +126,66 @@ steps:
           - github_commit_status:
               context: "filebeat: Windows 2022 Unit Tests"
 
-  - group: "Filebeat Extended Tests"
-    key: "filebeat-extended-tests"
-    steps:
-      - label: ":mac: MacOS x64_64 Unit Tests"
-        key: "macos-unit-tests-extended"
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+      - label: ":ubuntu: Filebeat Module Compatibility ITs (Previous Minor)"
         command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet filebeat
+
+          # echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd filebeat
-          mage build unitTest
+          mage pythonIntegTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
+        env:
+          STACK_ENVIRONMENT: prev-minor
+          TESTING_FILEBEAT_SKIP_DIFF: 1
+          PYTEST_ADDOPTS: '-k test_modules'
         agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_X86_64}"
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Extended MacOS Unit Tests"
+              context: "filebeat: Module Compatibility ITs (Previous Minor)"
 
-      - label: ":mac: MacOS arm64 Unit Tests"
-        key: "macos-arm64-unit-tests-extended"
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+      - label: ":ubuntu: Filebeat Module Compatibility ITs (Next Major)"
         command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet filebeat
+
+          # echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd filebeat
-          mage build unitTest
+          mage pythonIntegTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
+        env:
+          STACK_ENVIRONMENT: next-major
+          TESTING_FILEBEAT_SKIP_DIFF: 1
         agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_ARM}"
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Extended MacOS ARM Unit Tests"
+              context: "filebeat: Module Compatibility ITs (Next Major)"
 
-      - label: ":linux: Ubuntu ARM Unit Tests"
-        key: "extended-arm64-unit-test"
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+      - label: ":linux: Filebeat arm64 Unit Tests"
         command: |
           cd filebeat
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -189,44 +195,50 @@ steps:
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Extended Ubuntu ARM Unit Tests"
+              context: "filebeat: Linux arm64 Unit Tests"
+
+  - group: "Filebeat Extended Tests"
+    key: "filebeat-extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/
 
-  - group: "Extended Windows Tests"
-    key: "filebeat-extended-win-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
     steps:
-      - label: ":windows: Windows 2019 Unit Tests"
-        key: "windows-extended-2019"
+      - label: ":mac: Filebeat macOS x86_64 Unit Tests"
+        key: "macos-unit-tests-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
-          Set-Location -Path filebeat
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd filebeat
           mage build unitTest
         retry:
           automatic:
            - limit: 3
         agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 200
-          disk_type: "pd-ssd"
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Extended Windows 2019 Unit Tests"
+              context: "filebeat: macOS x86_64 Unit Tests"
 
-      - label: ":windows: Windows 11 Unit Tests"
-        key: "windows-extended-11"
+  - group: "Filebeat Extended Windows Tests"
+    key: "filebeat-extended-win-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
+    steps:
+      - label: ":windows: Filebeat Windows 2016 Unit Tests"
+        key: "windows-2016-unit-tests"
         command: |
           Set-Location -Path filebeat
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_11}"
+          image: "${IMAGE_WIN_2016}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
@@ -235,9 +247,9 @@ steps:
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Extended Windows 11 Unit Tests"
+              context: "filebeat: Windows 2016 Unit Tests"
 
-      - label: ":windows: Windows 10 Unit Tests"
+      - label: ":windows: Filebeat Windows 10 Unit Tests"
         key: "windows-extended-10"
         command: |
           Set-Location -Path filebeat
@@ -256,7 +268,7 @@ steps:
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Extended Windows 10 Unit Tests"
+              context: "filebeat: Windows 10 Unit Tests"
 
   - wait: ~
     # with PRs, we want to run packaging only if mandatory tests succeed
@@ -268,7 +280,7 @@ steps:
   - group: "Filebeat Packaging"
     key: "packaging"
     steps:
-      - label: ":linux: Packaging Linux"
+      - label: ":linux: Filebeat Packaging Linux"
         key: "packaging-linux"
         command: |
           cd filebeat
@@ -287,9 +299,9 @@ steps:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
         notify:
           - github_commit_status:
-              context: "filebeat: Packaging"
+              context: "filebeat: Packaging Linux"
 
-      - label: ":linux: Packaging ARM"
+      - label: ":linux: Filebeat Packaging arm64"
         key: "packaging-arm"
         command: |
           cd filebeat
@@ -307,4 +319,4 @@ steps:
           PACKAGES: "docker"
         notify:
           - github_commit_status:
-              context: "filebeat: Packaging ARM"
+              context: "filebeat: Packaging arm64"

.buildkite/hooks/pre-command

@@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then
   source .buildkite/env-scripts/env.sh
 
   # Images with prefix "platform-ingest-beats-*" has Go pre-setup.