Merge remote-tracking branch 'upstream/main' into feature/check-on-gh…

…-actions

* upstream/main: (313 commits)
  github-action: delete opentelemetry workflow (elastic#39559)
  updatecli: move to the .github folder and support for signed commits (elastic#39472)
  Osquerybeat: Add action responses data stream (elastic#39143)
  [winlogbeat] performance improvment; avoid rendering event message twice (elastic#39544)
  Fix the AWS SDK dependencies issue causing the "not found, ResolveEndpointV2" error (elastic#39454)
  x-pack/filebeat/input/cel: add http metrics collection (elastic#39503)
  build(deps): bump github.com/elastic/elastic-agent-libs from 0.9.4 to 0.9.7 (elastic#39424)
  Remove unused env vars from pipelines (elastic#39534)
  [BK] - Remove osx steps from branch execution (elastic#39552)
  [BK] - Remove certain steps from running for Branches (elastic#39533)
  Allow dependabot report BK status checks (elastic#39540)
  Remove hardcoded module definitions in CI (elastic#39506)
  Explicitly set DOCKER_PULL, RACE_DETECTOR and TEST_COVERAGE for pipelines (elastic#39510)
  Fixed pipelines formatting (elastic#39513)
  Update filebeat pipeline to match Jenkins steps (elastic#39261)
  Add error check to groupToEvents so we don't blindly add error values (elastic#39404)
  Remove fields not needed for session view in add_session_view processor (elastic#39500)
  `aws-s3` input: Split S3 poller and SQS reader into explicit input objects (elastic#39353)
  ci(jenkins): remove post-build notifications (elastic#39483)
  [DOCS] Add the `read_pipeline` cluster privilege for winlogbeat and the `auto_configure` index privilege to beats documentation (elastic#38534)
  ...

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -1,5 +1,115 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  MODULE: "kubernetes"
+
+  # Other deps
+  ASDF_KIND_VERSION: "0.20.0"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - group: "Deploy/K8S"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
+
+    steps:
+    - label: "Checks"
+      command: |
+        set -euo pipefail
+        make -C deploy/kubernetes all
+        make check-no-changes
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s checks"
+
+    - label: "K8S Test/K8S version: v1.29.0"
+      key: "k8s-test-129"
+      env:
+        K8S_VERSION: "v1.29.0"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s test v1.29.0"
+
+    - label: "K8S Test/K8S version: v1.28.0"
+      key: "k8s-test-128"
+      env:
+        K8S_VERSION: "v1.28.0"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s test v1.28.0"
+
+    - label: "K8S Test/K8S version: v1.27.3"
+      key: "k8s-test-1273"
+      env:
+        K8S_VERSION: "v1.27.3"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s test v1.27.3"
+
+    - label: "K8S Test/K8S version: v1.26.6"
+      key: "k8s-test-1266"
+      env:
+        K8S_VERSION: "v1.26.6"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s test v1.26.6"

.buildkite/deploy/kubernetes/scripts/kind-setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kind create cluster --image kindest/node:${K8S_VERSION}
+
+echo "Cluster info: "
+kubectl cluster-info

.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/util.sh
+
+export KUBECONFIG="${WORKSPACE}/kubecfg"
+export BIN="${WORKSPACE}/bin"
+add_bin_path
+
+echo "--- Installing kind & kubectl"
+asdf plugin add kind
+asdf install kind $ASDF_KIND_VERSION
+
+echo "~~~ Setting up kind"
+max_retries=3
+timeout=5
+retries=0
+
+while true; do
+  echo "Creating cluster"
+  script_output=$(.buildkite/deploy/kubernetes/scripts/kind-setup.sh 2>&1)
+  exit_code=$?
+
+  echo "Script Output: $script_output"
+
+  if [ $exit_code -eq 0 ]; then
+    break
+  else
+    retries=$((retries + 1))
+
+    if [ $retries -gt $max_retries ]; then
+      kind delete cluster
+      echo "Kind setup FAILED: $script_output"
+      exit 1
+    fi
+
+    kind delete cluster
+
+    sleep_time=$((timeout * retries))
+    echo "Retry #$retries failed. Retrying after ${sleep_time}s..."
+    sleep $sleep_time
+  fi
+done

.buildkite/env-scripts/env.sh

@@ -1,19 +1,59 @@
 #!/usr/bin/env bash
 
-SETUP_GVM_VERSION="v0.5.1"
+source .buildkite/env-scripts/util.sh
+
+DOCS_CHANGESET="^.*\.(asciidoc|md)$
+deploy/kubernetes/.*-kubernetes.yaml"
+PACKAGING_CHANGESET="^dev-tools/packaging/
+^.go-version"
+
+REPO="beats"
 WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
-REPO="beats"
-TMP_FOLDER="tmp.${REPO}"
+SNAPSHOT="true"
+PYTEST_ADDOPTS=""
+OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+
+SETUP_GVM_VERSION="v0.5.1"
+ASDF_MAGE_VERSION="1.14.0"
+SETUP_WIN_PYTHON_VERSION="3.11.0"
+
+# Docker & DockerHub
+DOCKER_COMPOSE_VERSION="1.21.0"
 DOCKER_REGISTRY="docker.elastic.co"
 
-export SETUP_GVM_VERSION
+ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET")
+PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET")
+GO_MOD_CHANGES=$(changeset_applies "^go.mod")
+
+KIND_VERSION="v0.20.0"
+KUBECONFIG="${WORKSPACE}/kubecfg"
+
+export REPO
 export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
-export REPO
-export TMP_FOLDER
+export SNAPSHOT
+export PYTEST_ADDOPTS
+export OSS_MODULE_PATTERN
+export XPACK_MODULE_PATTERN
+
+export SETUP_GVM_VERSION
+export ASDF_MAGE_VERSION
+export SETUP_WIN_PYTHON_VERSION
+
+export DOCKER_COMPOSE_VERSION
 export DOCKER_REGISTRY
+
+export ONLY_DOCS
+export PACKAGING_CHANGES
+export GO_MOD_CHANGES
+
+export KIND_VERSION
+export KUBECONFIG
+
+add_bin_path