Move to static BK pipeline for x-pack/metricbeat (elastic#38913)

This commit refactors the x-pack/metricbeat
pipeline to the pipeline selector approach,
as laid out in elastic#38783

Relates: elastic/ingest-dev#3072

.buildkite/hooks/pre-command

@@ -62,7 +62,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=(
   "beats-winlogbeat"
   "beats-winlogbeat"
   "beats-xpack-libbeat"
-  "beats-xpack-metricbeat"
   "beats-xpack-packetbeat"
   "beats-xpack-winlogbeat"
   "beats-xpack-dockerlogbeat"
@@ -80,8 +79,8 @@ for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do
   fi
 done
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == *"xpack-metricbeat"* || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == *"extended-cloud-test"* ]]; then
     BEATS_AWS_SECRET_KEY=$(retry_with_count 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export BEATS_AWS_SECRET_KEY
     BEATS_AWS_ACCESS_KEY=$(retry_with_count 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})

.buildkite/pipeline.yml

@@ -102,6 +102,33 @@ steps:
                     - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
                     - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
 
+  - label: "Trigger x-pack/metricbeat"
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - x-pack/metricbeat/
+                - x-pack/libbeat/common/aws
+                - .buildkite/x-pack/pipeline.xpack.metricbeat.yml
+                - .buildkite/scripts
+                - .buildkite/hooks/
+                #OSS
+                - go.mod
+                - pytest.ini
+                - dev-tools/
+                - libbeat/**
+                - testing/**
+              config:
+                trigger: "beats-xpack-metricbeat"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
   - label: "Trigger Xpack/Winlogbeat"
     plugins:
       - monorepo-diff#v1.0.1:

.buildkite/pull-requests.json

@@ -112,22 +112,6 @@
             "skip_ci_on_only_changed": [ ],
             "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
         },
-        {
-            "enabled": true,
-            "pipelineSlug": "beats-xpack-metricbeat",
-            "allow_org_users": true,
-            "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [ ],
-            "set_commit_status": true,
-            "build_on_commit": true,
-            "build_on_comment": true,
-            "trigger_comment_regex": "^/test x-pack/metricbeat$",
-            "always_trigger_comment_regex": "^/test x-pack/metricbeat$",
-            "skip_ci_labels": [  ],
-            "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
-            "always_require_ci_on_changed": ["^x-pack/metricbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
-        },
         {
             "enabled": true,
             "pipelineSlug": "beats-xpack-auditbeat",

.buildkite/scripts/cloud_tests.sh

@@ -2,9 +2,7 @@
 set -euo pipefail
 
 # What Terraform Module will run
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
-  export MODULE_DIR="x-pack/metricbeat/module/aws"
-elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then
   export MODULE_DIR="x-pack/filebeat/input/awss3/_meta/terraform"
 fi
 

.buildkite/scripts/common.sh

@@ -87,10 +87,6 @@ xpack_libbeat_changeset=(
   "^x-pack/libbeat/.*"
   )
 
-xpack_metricbeat_changeset=(
-  "^x-pack/metricbeat/.*"
-  )
-
 xpack_osquerybeat_changeset=(
   "^x-pack/osquerybeat/.*"
   )