Merge branch 'elastic:main' into main

.buildkite/aws-tests-pipeline.yml

@@ -0,0 +1,64 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+
+  #  TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476
+  TEST_TAGS: "aws"
+  MODULE: "aws"
+
+  # Other deps
+  ASDF_TERRAFORM_VERSION: 1.0.2
+  ASDF_MAGE_VERSION: 1.15.0
+
+  # Unit tests
+  RACE_DETECTOR: "true"
+  TEST_COVERAGE: "true"
+
+steps:
+  - label: ":ubuntu: x-pack/filebeat: AWS Tests"
+    key: "x-pack-filebeat-extended-cloud-test"
+    skip: "skipping:  elastic/ingest-dev#3467"
+    # Related issue: https://github.com/elastic/ingest-dev/issues/3467
+    env:
+      MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
+    command: |
+      set -euo pipefail
+      source .buildkite/scripts/initCloudEnv.sh
+      echo "~~~ Running tests"
+      cd x-pack/filebeat
+      mage build test goIntegTest
+    agents:
+      provider: "aws"
+      imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+      instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+    artifact_paths:
+      - "x-pack/filebeat/build/*.xml"
+      - "x-pack/filebeat/build/*.json"
+    notify:
+      - github_commit_status:
+          context: "x-pack/filebeat: AWS Tests"
+
+  - label: ":ubuntu: x-pack/metricbeat: AWS Tests"
+    key: "x-pack-metricbeat-extended-cloud-test"
+    env:
+      MODULE_DIR: "x-pack/metricbeat/module/aws"
+    command: |
+      set -euo pipefail
+      source .buildkite/scripts/initCloudEnv.sh
+      echo "~~~ Running tests"
+      cd x-pack/metricbeat
+      mage build test goIntegTest
+    agents:
+      provider: "aws"
+      imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+      instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+    artifact_paths:
+      - "x-pack/metricbeat/build/*.xml"
+      - "x-pack/metricbeat/build/*.json"
+    notify:
+      - github_commit_status:
+          context: "x-pack/metricbeat: AWS Tests"
+
+

.ci/jobs/docker-compose.yml → .buildkite/deploy/docker/docker-compose.yml

@@ -6,10 +6,10 @@ services:
     image: busybox
     depends_on:
       localstack: { condition: service_healthy }
-      
+
   localstack:
     container_name: "${localstack_integration_test_container}"
-    image: localstack/localstack:2.1.0 # Latest stable release
+    image: localstack/localstack:3.1.0 # Latest stable release
     ports:
       - "127.0.0.1:4566:4566"            # LocalStack Gateway
     environment:

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -6,13 +6,26 @@ env:
   MODULE: "kubernetes"
 
   # Other deps
-  ASDF_KIND_VERSION: "0.20.0"
+  ASDF_KIND_VERSION: "0.24.0"
+
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
 steps:
   - group: "Deploy/K8S"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
 
     steps:
+    - label: "Deploy/K8S: Run pre-commit"
+      command: "pre-commit run --all-files"
+      agents:
+        image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+        cpu: "2000m"
+        memory: "4Gi"
+        useCustomGlobalHooks: true
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s: pre-commit"
+
     - label: "Checks"
       command: |
         set -euo pipefail
@@ -26,10 +39,10 @@ steps:
         - github_commit_status:
             context: "deploy/k8s checks"
 
-    - label: "K8S Test/K8S version: v1.29.0"
-      key: "k8s-test-129"
+    - label: "K8S Test/K8S version: v1.31.0"
+      key: "k8s-test-131"
       env:
-        K8S_VERSION: "v1.29.0"
+        K8S_VERSION: "v1.31.0"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -39,19 +52,19 @@ steps:
         make -C deploy/kubernetes test
       retry:
         automatic:
-          - limit: 3
+          - limit: 1
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.29.0"
+            context: "deploy/k8s test v1.31.0"
 
-    - label: "K8S Test/K8S version: v1.28.0"
-      key: "k8s-test-128"
+    - label: "K8S Test/K8S version: v1.30.4"
+      key: "k8s-test-130"
       env:
-        K8S_VERSION: "v1.28.0"
+        K8S_VERSION: "v1.30.4"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -61,19 +74,19 @@ steps:
         make -C deploy/kubernetes test
       retry:
         automatic:
-          - limit: 3
+          - limit: 1
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.28.0"
+            context: "deploy/k8s test v1.30.4"
 
-    - label: "K8S Test/K8S version: v1.27.3"
-      key: "k8s-test-1273"
+    - label: "K8S Test/K8S version: v1.29.8"
+      key: "k8s-test-129"
       env:
-        K8S_VERSION: "v1.27.3"
+        K8S_VERSION: "v1.29.8"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -83,19 +96,19 @@ steps:
         make -C deploy/kubernetes test
       retry:
         automatic:
-          - limit: 3
+          - limit: 1
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.27.3"
+            context: "deploy/k8s test v1.29.8"
 
-    - label: "K8S Test/K8S version: v1.26.6"
-      key: "k8s-test-1266"
+    - label: "K8S Test/K8S version: v1.28.13"
+      key: "k8s-test-128"
       env:
-        K8S_VERSION: "v1.26.6"
+        K8S_VERSION: "v1.28.13"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -105,11 +118,11 @@ steps:
         make -C deploy/kubernetes test
       retry:
         automatic:
-          - limit: 3
+          - limit: 1
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.26.6"
+            context: "deploy/k8s test v1.28.13"

.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh

@@ -2,15 +2,19 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/util.sh
-
+WORKSPACE=$(pwd)
 export KUBECONFIG="${WORKSPACE}/kubecfg"
 export BIN="${WORKSPACE}/bin"
-add_bin_path
 
-echo "--- Installing kind & kubectl"
+echo "--- Add ${BIN} to PATH"
+if [[ ! -d "${BIN}" ]]; then
+  mkdir -p "${BIN}"
+fi
+export PATH="${PATH}:${BIN}"
+
+echo "~~~ Installing kind & kubectl"
 asdf plugin add kind
-asdf install kind $ASDF_KIND_VERSION
+asdf install kind "$ASDF_KIND_VERSION"
 
 echo "~~~ Setting up kind"
 max_retries=3