replace default images (#38583)

* replace default images -------------------------- Signed-off-by: Alexandros Sapranidis <[email protected]> Co-authored-by: Alexandros Sapranidis <[email protected]> (cherry picked from commit 80dab50) # Conflicts: # .buildkite/auditbeat/auditbeat-pipeline.yml # .buildkite/filebeat/filebeat-pipeline.yml # .buildkite/heartbeat/heartbeat-pipeline.yml # .buildkite/hooks/pre-command # .buildkite/scripts/generate_auditbeat_pipeline.sh # .buildkite/scripts/generate_filebeat_pipeline.sh # .buildkite/scripts/setenv.sh
elastic · Apr 5, 2024 · 9176040 · 9176040
1 parent ef48eb4
commit 9176040
Show file tree

Hide file tree

Showing 39 changed files with 1,658 additions and 665 deletions.
diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -2,21 +2,43 @@
 name: "beats-auditbeat"
 
 env:
+<<<<<<< HEAD
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+=======
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  BEATS_PROJECT_NAME: "auditbeat"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+>>>>>>> 80dab50f0c (replace default images (#38583))
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
   IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
+<<<<<<< HEAD
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   BEATS_PROJECT_NAME: "auditbeat"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
   AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+=======
+
+  #Packaging
+  PACKAGING_ARM_PLATFORMS: "linux/arm64"
+  PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+
+  #Deps
+  ASDF_MAGE_VERSION: 1.15.0
+>>>>>>> 80dab50f0c (replace default images (#38583))
 
 steps:
   - input: "Input Parameters"
@@ -63,6 +85,11 @@ steps:
   - label: ":linux: Load dynamic auditbeat pipeline"
     key: "auditbeat-pipeline"
     command: ".buildkite/scripts/generate_auditbeat_pipeline.sh"
+<<<<<<< HEAD
+=======
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest"
+>>>>>>> 80dab50f0c (replace default images (#38583))
     notify:
       - github_commit_status:
           context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -2,20 +2,42 @@
 name: "beats-filebeat"
 
 env:
+<<<<<<< HEAD
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
+=======
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  BEATS_PROJECT_NAME: "filebeat"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+>>>>>>> 80dab50f0c (replace default images (#38583))
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
   IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
+<<<<<<< HEAD
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   BEATS_PROJECT_NAME: "filebeat"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
   AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+=======
+
+  #Packaging
+  PACKAGING_ARM_PLATFORMS: "linux/arm64"
+  PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+
+  #Deps
+  ASDF_MAGE_VERSION: 1.15.0
+>>>>>>> 80dab50f0c (replace default images (#38583))
 
 steps:
   - input: "Input Parameters"
@@ -62,6 +84,11 @@ steps:
   - label: ":linux: Load dynamic filebeat pipeline"
     key: "filebeat-pipeline"
     command: ".buildkite/scripts/generate_filebeat_pipeline.sh"
+<<<<<<< HEAD
+=======
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest"
+>>>>>>> 80dab50f0c (replace default images (#38583))
     notify:
       - github_commit_status:
           context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -3,26 +3,45 @@
 env:
   BEATS_PROJECT_NAME: "heartbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+<<<<<<< HEAD
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
+=======
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
+  IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
+  IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
+  IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
+  IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
+  IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
+>>>>>>> 80dab50f0c (replace default images (#38583))
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
   AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
 
+  #Packaging
+  PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+  PACKAGING_ARM_PLATFORMS: "linux/arm64"
+
 steps:
   - group: "Heartbeat Mandatory Testing"
     key: "mandatory-tests"
     if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
 
     steps:
+<<<<<<< HEAD
       - label: ":linux: Unit Tests / {{matrix.image}}"
         command:
           - ".buildkite/heartbeat/scripts/unit-tests.sh"
+=======
+      - label: ":ubuntu: Unit Tests"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+>>>>>>> 80dab50f0c (replace default images (#38583))
         notify:
           - github_commit_status:
               context: "Heartbeat: linux/Unit Tests"
@@ -39,15 +58,57 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
+<<<<<<< HEAD
       - label: ":windows: Unit Tests / {{matrix.image}}"
         command:
           - ".buildkite/scripts/win_unit_tests.ps1"
+=======
+      - label: ":rhel:-9 Unit Tests"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+>>>>>>> 80dab50f0c (replace default images (#38583))
         notify:
           - github_commit_status:
               context: "Heartbeat: windows/Unit Tests"
         agents:
           provider: "gcp"
+<<<<<<< HEAD
           image: "{{matrix.image}}"
+=======
+          image: "${IMAGE_RHEL9}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":windows:-2016 Unit Test"
+        key: "windows-2016"
+        command: |
+          Set-Location -Path $BEATS_PROJECT_NAME
+          mage build unitTest
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Windows 2016/Unit Tests"
+        agents:
+          provider: "gcp"
+          image:  "${IMAGE_WIN_2016}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":windows:-2022 Unit Test"
+        key: "windows-2022"
+        command: |
+          Set-Location -Path $BEATS_PROJECT_NAME
+          mage build unitTest
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Windows 2022/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2022}"
+>>>>>>> 80dab50f0c (replace default images (#38583))
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_type: "pd-ssd"
         matrix:
@@ -60,8 +121,12 @@ steps:
           - "heartbeat/build/*.json"
 
       - label: ":ubuntu: Go Integration Tests"
+<<<<<<< HEAD
         command:
           - ".buildkite/heartbeat/scripts/integration-gotests.sh"
+=======
+        command:  "cd $BEATS_PROJECT_NAME && mage goIntegTest"
+>>>>>>> 80dab50f0c (replace default images (#38583))
         notify:
           - github_commit_status:
               context: "Heartbeat: Go Integration Tests"
@@ -74,8 +139,12 @@ steps:
           - "heartbeat/build/*.json"
 
       - label: ":ubuntu: Python Integration Tests"
+<<<<<<< HEAD
         command:
           - ".buildkite/heartbeat/scripts/integration-pytests.sh"
+=======
+        command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest"
+>>>>>>> 80dab50f0c (replace default images (#38583))
         notify:
           - github_commit_status:
               context: "Heartbeat: Python Integration Tests"
@@ -95,8 +164,12 @@ steps:
       - label: ":linux: ARM64 Unit Tests"
         key: "arm-extended"
         if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+<<<<<<< HEAD
         command:
           - ".buildkite/heartbeat/scripts/unit-tests.sh"
+=======
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+>>>>>>> 80dab50f0c (replace default images (#38583))
         notify:
           - github_commit_status:
               context: "Heartbeat/Extended: Unit Tests ARM"
@@ -109,8 +182,12 @@ steps:
       - label: ":mac: MacOS Unit Tests"
         key: "macos-extended"
         if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+<<<<<<< HEAD
         command:
           - ".buildkite/heartbeat/scripts/unit-tests.sh"
+=======
+        command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest"
+>>>>>>> 80dab50f0c (replace default images (#38583))
         notify:
           - github_commit_status:
               context: "Heartbeat/Extended: MacOS Unit Tests"
@@ -119,17 +196,43 @@ steps:
           imagePrefix: "${IMAGE_MACOS_X86_64}"
         artifact_paths: "heartbeat/build/*.xml"
 
+<<<<<<< HEAD
+=======
+      - label: ":mac: MacOS ARM Unit Tests"
+        key: "macos-extended-arm"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: MacOS ARM Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths: "heartbeat/build/*.xml"
+
+>>>>>>> 80dab50f0c (replace default images (#38583))
   - group: "Windows Extended Testing"
     key: "extended-tests-win"
     if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
 
     steps:
+<<<<<<< HEAD
       - label: ":windows: Win 2019 Unit Tests"
         key: "win-extended-2019"
         command: ".buildkite/scripts/win_unit_tests.ps1"
         notify:
           - github_commit_status:
               context: "Heartbeat/Extended: Win-2019 Unit Tests"
+=======
+      - label: ":windows:-2019 Unit Tests"
+        key: "windows-extended-2019"
+        command: |
+          Set-Location -Path $BEATS_PROJECT_NAME
+          mage build unitTest
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Windows 2019/Unit Tests"
+>>>>>>> 80dab50f0c (replace default images (#38583))
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -139,6 +242,43 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
+<<<<<<< HEAD
+=======
+      - label: ":windows:-11 Unit Tests"
+        key: "windows-extended-11"
+        command: |
+          Set-Location -Path $BEATS_PROJECT_NAME
+          mage build unitTest
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Windows 11/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":windows:-10 Unit Tests"
+        key: "windows-extended-10"
+        command: |
+          Set-Location -Path $BEATS_PROJECT_NAME
+          mage build unitTest
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Windows 10/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+>>>>>>> 80dab50f0c (replace default images (#38583))
   - group: "Packaging"
     key: "packaging"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -7,7 +7,10 @@ set -euo pipefail
 AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
+<<<<<<< HEAD
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials"
+=======
+>>>>>>> 80dab50f0c (replace default images (#38583))
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
 retry() {
@@ -32,8 +35,11 @@ retry() {
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
   source .buildkite/env-scripts/env.sh
+<<<<<<< HEAD
   source .buildkite/env-scripts/util.sh
 
+=======
+>>>>>>> 80dab50f0c (replace default images (#38583))
   if [[ -z "${GO_VERSION-""}" ]]; then
     export GO_VERSION=$(cat "${WORKSPACE}/.go-version")
   fi