From 271b27b18940e8af24dd793a4f4780f66b126c6d Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 4 Dec 2023 15:36:12 +0200
Subject: [PATCH 01/31] wip: getting GH message

---
 .buildkite/filebeat/arm-tests.sh | 83 ++++++++++++++++++++++++++++++++
 .buildkite/filebeat/common.sh    | 14 ++++++
 .buildkite/pipeline.yml          | 15 ++++++
 3 files changed, 112 insertions(+)
 create mode 100644 .buildkite/filebeat/arm-tests.sh
 create mode 100644 .buildkite/filebeat/common.sh

diff --git a/.buildkite/filebeat/arm-tests.sh b/.buildkite/filebeat/arm-tests.sh
new file mode 100644
index 00000000000..660e91de668
--- /dev/null
+++ b/.buildkite/filebeat/arm-tests.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+WORKSPACE="$(pwd)"
+BIN="${WORKSPACE}/bin"
+
+add_bin_path() {
+    echo "Adding PATH to the environment variables..."
+    create_bin
+    export PATH="${PATH}:${BIN}"
+}
+
+with_go() {
+    local go_version="${1}"
+    echo "Setting up the Go environment..."
+    create_bin
+    check_platform_architecture
+    retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
+    export PATH="${PATH}:${BIN}"
+    chmod +x ${BIN}/gvm
+    eval "$(gvm "$go_version")"
+    go version
+    which go
+    export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin"
+}
+
+with_mage() {
+    local install_packages=(
+            "github.com/magefile/mage"
+            "github.com/elastic/go-licenser"
+            "golang.org/x/tools/cmd/goimports"
+            "github.com/jstemmer/go-junit-report"
+            "gotest.tools/gotestsum"
+    )
+    create_bin
+    for pkg in "${install_packages[@]}"; do
+        go install "${pkg}@latest"
+    done
+}
+
+create_bin() {
+    if [[ ! -d "${BIN}" ]]; then
+    mkdir -p ${BIN}
+    fi
+}
+
+check_platform_architecture() {
+# for downloading the GVM and Terraform packages
+  case "${HW_TYPE}" in
+   "x86_64")
+        arch_type="amd64"
+        ;;
+    "aarch64")
+        arch_type="arm64"
+        ;;
+    "arm64")
+        arch_type="arm64"
+        ;;
+    *)
+    echo "The current platform/OS type is unsupported yet"
+    ;;
+  esac
+}
+
+retry() {
+    local retries=$1
+    shift
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}
diff --git a/.buildkite/filebeat/common.sh b/.buildkite/filebeat/common.sh
new file mode 100644
index 00000000000..0ac9279d8cd
--- /dev/null
+++ b/.buildkite/filebeat/common.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+are_files_changed() {
+  changeset=$1
+
+  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
+    return 0;
+  else
+    echo "WARN! No files changed in $changeset"
+    return 1;
+  fi
+}
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 34321b61161..c4984ba52ce 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -3,3 +3,18 @@
 steps:
   - label: "Example test"
     command: echo "Hello!"
+# ToDo: setup execution from UI when specific pipelines are finished
+#  - input: "Build parameters"
+#    if: build.source == "ui"
+#    fields:
+#      - select: "Select build parameters"
+#        key: "PIPELINE_SLUG"
+#        required: false
+#        options:
+#          - label: "Filebeat"
+#            value: "filebeat"
+#          - label: "Metricbeat"
+#            value: "metricbeat"
+#        hint: "Select what should be built"
+#  - wait
+#  - trigger: "${PIPELINE_SLUG}"

From 9f1e8a347edaeb3d4808edc702852eb66a9aa59f Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 7 Dec 2023 11:35:07 +0200
Subject: [PATCH 02/31] added test scripts

---
 .buildkite/filebeat/common.sh                 | 14 --------------
 .buildkite/filebeat/scripts/arm-tests.sh      |  3 +++
 .../{arm-tests.sh => scripts/common.sh}       |  8 +++++++-
 .../filebeat/scripts/integration-tests.sh     | 19 +++++++++++++++++++
 4 files changed, 29 insertions(+), 15 deletions(-)
 delete mode 100644 .buildkite/filebeat/common.sh
 create mode 100644 .buildkite/filebeat/scripts/arm-tests.sh
 rename .buildkite/filebeat/{arm-tests.sh => scripts/common.sh} (90%)
 create mode 100755 .buildkite/filebeat/scripts/integration-tests.sh

diff --git a/.buildkite/filebeat/common.sh b/.buildkite/filebeat/common.sh
deleted file mode 100644
index 0ac9279d8cd..00000000000
--- a/.buildkite/filebeat/common.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-are_files_changed() {
-  changeset=$1
-
-  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
-    return 0;
-  else
-    echo "WARN! No files changed in $changeset"
-    return 1;
-  fi
-}
diff --git a/.buildkite/filebeat/scripts/arm-tests.sh b/.buildkite/filebeat/scripts/arm-tests.sh
new file mode 100644
index 00000000000..5b2371c6ed4
--- /dev/null
+++ b/.buildkite/filebeat/scripts/arm-tests.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
diff --git a/.buildkite/filebeat/arm-tests.sh b/.buildkite/filebeat/scripts/common.sh
similarity index 90%
rename from .buildkite/filebeat/arm-tests.sh
rename to .buildkite/filebeat/scripts/common.sh
index 660e91de668..4d2fd3aba6f 100644
--- a/.buildkite/filebeat/arm-tests.sh
+++ b/.buildkite/filebeat/scripts/common.sh
@@ -4,6 +4,12 @@ set -euo pipefail
 
 WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
+HW_TYPE="$(uname -m)"
+PLATFORM_TYPE="$(uname)"
+
+if [[ -z "${GOLANG_VERSION-""}" ]]; then
+    export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
+fi
 
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
@@ -12,7 +18,7 @@ add_bin_path() {
 }
 
 with_go() {
-    local go_version="${1}"
+    local go_version="${GOLANG_VERSION}"
     echo "Setting up the Go environment..."
     create_bin
     check_platform_architecture
diff --git a/.buildkite/filebeat/scripts/integration-tests.sh b/.buildkite/filebeat/scripts/integration-tests.sh
new file mode 100755
index 00000000000..5060a18047f
--- /dev/null
+++ b/.buildkite/filebeat/scripts/integration-tests.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/filebeat/scripts/common.sh
+
+# ToDo - remove after Beats agent is created"
+echo ":: Setup Env ::"
+add_bin_path
+with_go
+with_mage
+# ToDo - end
+
+echo ":: Execute Integration Tests ::"
+sudo chmod -R go-w filebeat/
+
+cd filebeat
+umask 0022
+mage goIntegTest

From b7f40a366194d8fdb34f21370233678828ecf054 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 13 Dec 2023 04:59:27 +0200
Subject: [PATCH 03/31] added windows tests

---
 .buildkite/filebeat/scripts/common.sh     |  8 ++++----
 .buildkite/filebeat/scripts/pre-common.sh | 10 ++++++++++
 filebeat/Jenkinsfile.yml                  |  2 +-
 3 files changed, 15 insertions(+), 5 deletions(-)
 create mode 100644 .buildkite/filebeat/scripts/pre-common.sh

diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/filebeat/scripts/common.sh
index 4d2fd3aba6f..39c445b6acc 100644
--- a/.buildkite/filebeat/scripts/common.sh
+++ b/.buildkite/filebeat/scripts/common.sh
@@ -2,14 +2,14 @@
 
 set -euo pipefail
 
-WORKSPACE="$(pwd)"
+#WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
 
-if [[ -z "${GOLANG_VERSION-""}" ]]; then
-    export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
-fi
+#if [[ -z "${GOLANG_VERSION-""}" ]]; then
+#    export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
+#fi
 
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
diff --git a/.buildkite/filebeat/scripts/pre-common.sh b/.buildkite/filebeat/scripts/pre-common.sh
new file mode 100644
index 00000000000..12c69b771bc
--- /dev/null
+++ b/.buildkite/filebeat/scripts/pre-common.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+prepare_win() {
+  local os
+  os="$(uname)"
+  if [[ $os = MINGW* ]]; then
+    choco install mingw -y
+    choco install python --version=3.11.0 -y
+  fi
+}
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
index 127d82cecfd..4588823bd3c 100644
--- a/filebeat/Jenkinsfile.yml
+++ b/filebeat/Jenkinsfile.yml
@@ -42,7 +42,7 @@ stages:
           - "macos12 && x86_64"
         when:                  ## Override the top-level when.
             comments:
-                - "/test filebeat for macos"
+                  - "/test filebeat for macos"
             labels:
                 - "macOS"
             parameters:

From 292b0530e2b7722b66e434bcbffcc3a32735a5b6 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 3 Jan 2024 13:13:26 +0200
Subject: [PATCH 04/31] added packaging step

---
 .buildkite/filebeat/scripts/pre-common.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.buildkite/filebeat/scripts/pre-common.sh b/.buildkite/filebeat/scripts/pre-common.sh
index 12c69b771bc..f1a2bf5840e 100644
--- a/.buildkite/filebeat/scripts/pre-common.sh
+++ b/.buildkite/filebeat/scripts/pre-common.sh
@@ -8,3 +8,13 @@ prepare_win() {
     choco install python --version=3.11.0 -y
   fi
 }
+
+check_filebeat_changes() {
+  changeset=$1
+
+  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
+    export FILEBEAT_CHANGESET=true
+  else
+    export FILEBEAT_CHANGESET=false
+  fi
+}

From a42085b40214f07bb9820329930bcb487bb35129 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 4 Jan 2024 10:14:00 +0200
Subject: [PATCH 05/31] updated packaging execution conditions

---
 .buildkite/filebeat/scripts/common.sh     | 26 ++++++++++++++++++-----
 .buildkite/filebeat/scripts/pre-common.sh | 20 -----------------
 2 files changed, 21 insertions(+), 25 deletions(-)
 delete mode 100644 .buildkite/filebeat/scripts/pre-common.sh

diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/filebeat/scripts/common.sh
index 39c445b6acc..943d3b98a6f 100644
--- a/.buildkite/filebeat/scripts/common.sh
+++ b/.buildkite/filebeat/scripts/common.sh
@@ -2,15 +2,11 @@
 
 set -euo pipefail
 
-#WORKSPACE="$(pwd)"
+WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
 
-#if [[ -z "${GOLANG_VERSION-""}" ]]; then
-#    export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
-#fi
-
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
     create_bin
@@ -87,3 +83,23 @@ retry() {
     done
     return 0
 }
+
+are_files_changed() {
+  changeset=$1
+
+  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
+    return 0;
+  else
+    echo "WARN! No files changed in $changeset"
+    return 1;
+  fi
+}
+
+prepare_win() {
+  local os
+  os="$(uname)"
+  if [[ $os = MINGW* ]]; then
+    choco install mingw -y
+    choco install python --version=3.11.0 -y
+  fi
+}
diff --git a/.buildkite/filebeat/scripts/pre-common.sh b/.buildkite/filebeat/scripts/pre-common.sh
deleted file mode 100644
index f1a2bf5840e..00000000000
--- a/.buildkite/filebeat/scripts/pre-common.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-prepare_win() {
-  local os
-  os="$(uname)"
-  if [[ $os = MINGW* ]]; then
-    choco install mingw -y
-    choco install python --version=3.11.0 -y
-  fi
-}
-
-check_filebeat_changes() {
-  changeset=$1
-
-  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
-    export FILEBEAT_CHANGESET=true
-  else
-    export FILEBEAT_CHANGESET=false
-  fi
-}

From 8df62da829c4b2e77c6b71548b277a94f810b87e Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 9 Jan 2024 22:24:32 +0200
Subject: [PATCH 06/31] win-test failure: updated artifact path

---
 .buildkite/filebeat/scripts/arm-tests.sh | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 .buildkite/filebeat/scripts/arm-tests.sh

diff --git a/.buildkite/filebeat/scripts/arm-tests.sh b/.buildkite/filebeat/scripts/arm-tests.sh
deleted file mode 100644
index 5b2371c6ed4..00000000000
--- a/.buildkite/filebeat/scripts/arm-tests.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail

From b8967a3d4a812ef50f599629b56e06d24205b053 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 15 Jan 2024 11:58:12 +0200
Subject: [PATCH 07/31] pr fixes

---
 .../filebeat/scripts/integration-tests.sh     | 19 -------------------
 .buildkite/pipeline.yml                       | 15 ---------------
 filebeat/Jenkinsfile.yml                      |  2 +-
 3 files changed, 1 insertion(+), 35 deletions(-)
 delete mode 100755 .buildkite/filebeat/scripts/integration-tests.sh

diff --git a/.buildkite/filebeat/scripts/integration-tests.sh b/.buildkite/filebeat/scripts/integration-tests.sh
deleted file mode 100755
index 5060a18047f..00000000000
--- a/.buildkite/filebeat/scripts/integration-tests.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/filebeat/scripts/common.sh
-
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
-
-echo ":: Execute Integration Tests ::"
-sudo chmod -R go-w filebeat/
-
-cd filebeat
-umask 0022
-mage goIntegTest
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index c4984ba52ce..34321b61161 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -3,18 +3,3 @@
 steps:
   - label: "Example test"
     command: echo "Hello!"
-# ToDo: setup execution from UI when specific pipelines are finished
-#  - input: "Build parameters"
-#    if: build.source == "ui"
-#    fields:
-#      - select: "Select build parameters"
-#        key: "PIPELINE_SLUG"
-#        required: false
-#        options:
-#          - label: "Filebeat"
-#            value: "filebeat"
-#          - label: "Metricbeat"
-#            value: "metricbeat"
-#        hint: "Select what should be built"
-#  - wait
-#  - trigger: "${PIPELINE_SLUG}"
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
index 4588823bd3c..127d82cecfd 100644
--- a/filebeat/Jenkinsfile.yml
+++ b/filebeat/Jenkinsfile.yml
@@ -42,7 +42,7 @@ stages:
           - "macos12 && x86_64"
         when:                  ## Override the top-level when.
             comments:
-                  - "/test filebeat for macos"
+                - "/test filebeat for macos"
             labels:
                 - "macOS"
             parameters:

From 22c1a506b1c1ca08e67fb5c0c0dfcb3c5996f658 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 16 Jan 2024 16:58:22 +0200
Subject: [PATCH 08/31] changed group_test.go

---
 .buildkite/hooks/pre-command | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 7be1f965230..d5c827cc2e3 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -32,6 +32,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   if [[ -z "${GOLANG_VERSION-""}" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
   fi
+
+  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
+    ulimit -Sn 30000
+  fi
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-dockerlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then

From 02376271750366eb221d321fad5f717f8a3dc15d Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 17 Jan 2024 12:34:39 +0200
Subject: [PATCH 09/31] moved env setup to separate script

---
 .buildkite/filebeat/scripts/common.sh | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/filebeat/scripts/common.sh
index 943d3b98a6f..efbb81e66aa 100644
--- a/.buildkite/filebeat/scripts/common.sh
+++ b/.buildkite/filebeat/scripts/common.sh
@@ -2,10 +2,10 @@
 
 set -euo pipefail
 
-WORKSPACE="$(pwd)"
-BIN="${WORKSPACE}/bin"
-HW_TYPE="$(uname -m)"
-PLATFORM_TYPE="$(uname)"
+#WORKSPACE="$(pwd)"
+#BIN="${WORKSPACE}/bin"
+#HW_TYPE="$(uname -m)"
+#PLATFORM_TYPE="$(uname)"
 
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
@@ -85,7 +85,7 @@ retry() {
 }
 
 are_files_changed() {
-  changeset=$1
+  local changeset=$1
 
   if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
     return 0;
@@ -95,11 +95,11 @@ are_files_changed() {
   fi
 }
 
-prepare_win() {
-  local os
-  os="$(uname)"
-  if [[ $os = MINGW* ]]; then
-    choco install mingw -y
-    choco install python --version=3.11.0 -y
-  fi
-}
+#prepare_win() {
+##  local os
+##  os="$(uname)"
+#  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
+#    choco install mingw -y
+#    choco install python --version=3.11.0 -y
+#  fi
+#}

From fec7219fde064e6964286833018ef214c9da3e73 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 17 Jan 2024 15:48:22 +0200
Subject: [PATCH 10/31] added dynamic step for packaging

---
 .buildkite/filebeat/scripts/common.sh | 105 --------------------------
 1 file changed, 105 deletions(-)
 delete mode 100644 .buildkite/filebeat/scripts/common.sh

diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/filebeat/scripts/common.sh
deleted file mode 100644
index efbb81e66aa..00000000000
--- a/.buildkite/filebeat/scripts/common.sh
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-#WORKSPACE="$(pwd)"
-#BIN="${WORKSPACE}/bin"
-#HW_TYPE="$(uname -m)"
-#PLATFORM_TYPE="$(uname)"
-
-add_bin_path() {
-    echo "Adding PATH to the environment variables..."
-    create_bin
-    export PATH="${PATH}:${BIN}"
-}
-
-with_go() {
-    local go_version="${GOLANG_VERSION}"
-    echo "Setting up the Go environment..."
-    create_bin
-    check_platform_architecture
-    retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
-    export PATH="${PATH}:${BIN}"
-    chmod +x ${BIN}/gvm
-    eval "$(gvm "$go_version")"
-    go version
-    which go
-    export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin"
-}
-
-with_mage() {
-    local install_packages=(
-            "github.com/magefile/mage"
-            "github.com/elastic/go-licenser"
-            "golang.org/x/tools/cmd/goimports"
-            "github.com/jstemmer/go-junit-report"
-            "gotest.tools/gotestsum"
-    )
-    create_bin
-    for pkg in "${install_packages[@]}"; do
-        go install "${pkg}@latest"
-    done
-}
-
-create_bin() {
-    if [[ ! -d "${BIN}" ]]; then
-    mkdir -p ${BIN}
-    fi
-}
-
-check_platform_architecture() {
-# for downloading the GVM and Terraform packages
-  case "${HW_TYPE}" in
-   "x86_64")
-        arch_type="amd64"
-        ;;
-    "aarch64")
-        arch_type="arm64"
-        ;;
-    "arm64")
-        arch_type="arm64"
-        ;;
-    *)
-    echo "The current platform/OS type is unsupported yet"
-    ;;
-  esac
-}
-
-retry() {
-    local retries=$1
-    shift
-    local count=0
-    until "$@"; do
-        exit=$?
-        wait=$((2 ** count))
-        count=$((count + 1))
-        if [ $count -lt "$retries" ]; then
-            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
-            sleep $wait
-        else
-            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
-            return $exit
-        fi
-    done
-    return 0
-}
-
-are_files_changed() {
-  local changeset=$1
-
-  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
-    return 0;
-  else
-    echo "WARN! No files changed in $changeset"
-    return 1;
-  fi
-}
-
-#prepare_win() {
-##  local os
-##  os="$(uname)"
-#  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
-#    choco install mingw -y
-#    choco install python --version=3.11.0 -y
-#  fi
-#}

From 8d373375e68ed8f6c32fb3c31b34953187051fe4 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 1 Feb 2024 11:33:38 +0200
Subject: [PATCH 11/31] added image tag and push

---
 .buildkite/env-scripts/env.sh               |  1 +
 .buildkite/env-scripts/macos-env.sh         |  0
 .buildkite/env-scripts/packaging-env.sh     | 24 +++++++++++++
 .buildkite/env-scripts/util.sh              | 37 +++++++++++++++++++++
 .buildkite/filebeat/filebeat-pipeline.yml   | 10 +++---
 .buildkite/filebeat/scripts/package-step.sh |  5 +--
 .buildkite/filebeat/scripts/package.sh      |  8 ++---
 .buildkite/hooks/pre-exit                   | 15 +++++++++
 8 files changed, 89 insertions(+), 11 deletions(-)
 create mode 100644 .buildkite/env-scripts/macos-env.sh
 create mode 100644 .buildkite/env-scripts/packaging-env.sh
 create mode 100644 .buildkite/hooks/pre-exit

diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
index 4dfc01bafc3..f1c0d76d2bb 100644
--- a/.buildkite/env-scripts/env.sh
+++ b/.buildkite/env-scripts/env.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+REPO="beats"
 SETUP_GVM_VERSION="v0.5.1"
 WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.buildkite/env-scripts/packaging-env.sh b/.buildkite/env-scripts/packaging-env.sh
new file mode 100644
index 00000000000..94ccb9296d3
--- /dev/null
+++ b/.buildkite/env-scripts/packaging-env.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+source .buildkite/env-scripts/util.sh
+
+docs_changeset="^.*\.(asciidoc|md)$
+deploy/kubernetes/.*-kubernetes.yaml"
+packaging_changeset="^dev-tools/packaging/
+^.go-version"
+
+# Env vars for Packaging stage
+DOCKER_REGISTRY="docker.elastic.co"
+SNAPSHOT=true
+VERSION=$(make get-version | tr -d '\n')
+ONLY_DOCS=$(changeset_applies "$docs_changeset")
+PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
+GO_MOD_CHANGES=$(changeset_applies "^go.mod")
+
+export PACKAGING_CHANGES
+export ONLY_DOCS
+export GO_MOD_CHANGES
+export DOCKER_REGISTRY
+export SNAPSHOT
+export VERSION
+export REPO
diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
index 6a5c36bcd04..c08e12ec756 100644
--- a/.buildkite/env-scripts/util.sh
+++ b/.buildkite/env-scripts/util.sh
@@ -102,3 +102,40 @@ unset_secrets () {
     fi
   done
 }
+
+changeset_applies() {
+  local changeset=$1
+  if are_files_changed "$changeset"; then
+    echo true
+  else
+    echo false
+  fi
+}
+
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
+      unset "$var"
+    fi
+  done
+}
+
+google_cloud_logout_active_account() {
+  local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
+  if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
+    echo "Logging out from GCP for active account"
+    gcloud auth revoke $active_account > /dev/null 2>&1
+  else
+    echo "No active GCP accounts found."
+  fi
+  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+    unset GOOGLE_APPLICATION_CREDENTIALS
+    cleanup
+  fi
+}
+
+cleanup() {
+  echo "Deleting temporary files..."
+  rm -rf ${BIN}/${TMP_FOLDER}.*
+  echo "Done."
+}
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index e811d286953..8faedc401ab 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -1,12 +1,14 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-eng-prod-base-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  SETUP_MAGE_VERSION: "1.14.0"
+  ASDF_MAGE_VERSION: "1.14.0"
 
 steps:
   - group: "Filebeat Mandatory Testing"
@@ -23,7 +25,6 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "c2-standard-16"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
@@ -37,7 +38,6 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "c2-standard-16"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
@@ -49,9 +49,9 @@ steps:
           - github_commit_status:
               context: "Filebeat: Python Integration Tests"
         agents:
-          provider: "gcp"
+          provider: gcp
+          imageProject: elastic-images-qa
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "c2-standard-16"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh
index f8fa02db81d..069299602f4 100755
--- a/.buildkite/filebeat/scripts/package-step.sh
+++ b/.buildkite/filebeat/scripts/package-step.sh
@@ -25,8 +25,9 @@ if are_files_changed "$changeset"; then
           - github_commit_status:
               context: "Filebeat/Packaging: Linux X86"
         agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
+          provider: gcp
+          imageProject: elastic-images-qa
+          image: family/platform-ingest-eng-prod-base-ubuntu-2204
 
       - label: ":linux: Packaging Linux ARM"
         key: "package-linux-arm"
diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh
index 0bb03250348..a3a1a02af93 100755
--- a/.buildkite/filebeat/scripts/package.sh
+++ b/.buildkite/filebeat/scripts/package.sh
@@ -4,7 +4,7 @@ set -euo pipefail
 
 source .buildkite/env-scripts/linux-env.sh
 
-echo "--- Start Packaging"
-cd filebeat
-umask 0022
-mage package
+echo ":: Start Packaging ::"
+#cd filebeat
+#umask 0022
+mage -d filebeat package
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
new file mode 100644
index 00000000000..6c759aaa267
--- /dev/null
+++ b/.buildkite/hooks/pre-exit
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/util.sh
+
+unset_secrets
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" && "$BUILDKITE_STEP_KEY" == package* ]]; then
+    google_cloud_logout_active_account
+    docker logout "${DOCKER_REGISTRY}"
+fi
+
+# Ensure that any temporal files created during any step are removed
+cleanup

From 5c177baab35170f519fa1cca6c3969a71581bda5 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 6 Feb 2024 12:21:19 +0200
Subject: [PATCH 12/31] changed ubuntu agent

---
 .buildkite/env-scripts/env.sh                 | 12 ++++--
 .buildkite/env-scripts/macos-env.sh           |  8 ++++
 .buildkite/env-scripts/util.sh                | 42 -------------------
 .buildkite/filebeat/scripts/package.sh        | 10 -----
 .../scripts/{ => packaging}/package-step.sh   |  9 ++--
 .../filebeat/scripts/packaging/package.sh     | 29 +++++++++++++
 .../scripts/packaging}/packaging-env.sh       | 11 ++++-
 .buildkite/hooks/pre-command                  |  5 ++-
 .buildkite/hooks/pre-exit                     |  2 +-
 .buildkite/hooks/scripts/util.sh              | 31 ++++++++++++++
 10 files changed, 95 insertions(+), 64 deletions(-)
 delete mode 100755 .buildkite/filebeat/scripts/package.sh
 rename .buildkite/filebeat/scripts/{ => packaging}/package-step.sh (82%)
 create mode 100755 .buildkite/filebeat/scripts/packaging/package.sh
 rename .buildkite/{env-scripts => filebeat/scripts/packaging}/packaging-env.sh (70%)
 mode change 100644 => 100755
 create mode 100644 .buildkite/hooks/scripts/util.sh

diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
index f1c0d76d2bb..9abee5554ef 100644
--- a/.buildkite/env-scripts/env.sh
+++ b/.buildkite/env-scripts/env.sh
@@ -6,9 +6,10 @@ WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
-REPO="beats"
 TMP_FOLDER="tmp.${REPO}"
-DOCKER_REGISTRY="docker.elastic.co"
+ASDF_MAGE_VERSION="1.14.0"
+SETUP_MAGE_VERSION="1.14.0"
+DEBIAN_FRONTEND="noninteractive"
 
 export SETUP_GVM_VERSION
 export WORKSPACE
@@ -17,4 +18,9 @@ export HW_TYPE
 export PLATFORM_TYPE
 export REPO
 export TMP_FOLDER
-export DOCKER_REGISTRY
+export ASDF_MAGE_VERSION
+export SETUP_MAGE_VERSION
+
+if grep -q 'Ubuntu' /etc/*release; then
+  export DEBIAN_FRONTEND
+fi
diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh
index e69de29bb2d..ac1486b64fd 100644
--- a/.buildkite/env-scripts/macos-env.sh
+++ b/.buildkite/env-scripts/macos-env.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+if [[ $PLATFORM_TYPE == Darwin* ]]; then
+  echo ":: Setting larger ulimit on MacOS ::"
+  # To bypass file descriptor errors like "Too many open files error" on MacOS
+  ulimit -Sn 50000
+  echo ":: ULIMIT :: $(ulimit -n)"
+fi
diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
index c08e12ec756..2076a0af7da 100644
--- a/.buildkite/env-scripts/util.sh
+++ b/.buildkite/env-scripts/util.sh
@@ -89,20 +89,6 @@ are_files_changed() {
   fi
 }
 
-cleanup() {
-  echo "Deleting temporary files..."
-  rm -rf ${BIN}/${TMP_FOLDER}.*
-  echo "Done."
-}
-
-unset_secrets () {
-  for var in $(printenv | sed 's;=.*;;' | sort); do
-    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
-      unset "$var"
-    fi
-  done
-}
-
 changeset_applies() {
   local changeset=$1
   if are_files_changed "$changeset"; then
@@ -111,31 +97,3 @@ changeset_applies() {
     echo false
   fi
 }
-
-unset_secrets () {
-  for var in $(printenv | sed 's;=.*;;' | sort); do
-    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
-      unset "$var"
-    fi
-  done
-}
-
-google_cloud_logout_active_account() {
-  local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
-  if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
-    echo "Logging out from GCP for active account"
-    gcloud auth revoke $active_account > /dev/null 2>&1
-  else
-    echo "No active GCP accounts found."
-  fi
-  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
-    unset GOOGLE_APPLICATION_CREDENTIALS
-    cleanup
-  fi
-}
-
-cleanup() {
-  echo "Deleting temporary files..."
-  rm -rf ${BIN}/${TMP_FOLDER}.*
-  echo "Done."
-}
diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh
deleted file mode 100755
index a3a1a02af93..00000000000
--- a/.buildkite/filebeat/scripts/package.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/env-scripts/linux-env.sh
-
-echo ":: Start Packaging ::"
-#cd filebeat
-#umask 0022
-mage -d filebeat package
diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/packaging/package-step.sh
similarity index 82%
rename from .buildkite/filebeat/scripts/package-step.sh
rename to .buildkite/filebeat/scripts/packaging/package-step.sh
index 069299602f4..bc6d33e5910 100755
--- a/.buildkite/filebeat/scripts/package-step.sh
+++ b/.buildkite/filebeat/scripts/packaging/package-step.sh
@@ -19,23 +19,20 @@ if are_files_changed "$changeset"; then
         key: "package-linux-x86"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-        command:
-          - ".buildkite/filebeat/scripts/package.sh"
+        command: ".buildkite/filebeat/scripts/package.sh"
         notify:
           - github_commit_status:
               context: "Filebeat/Packaging: Linux X86"
         agents:
           provider: gcp
-          imageProject: elastic-images-qa
-          image: family/platform-ingest-eng-prod-base-ubuntu-2204
+          image: "${IMAGE_UBUNTU_X86_64}"
 
       - label: ":linux: Packaging Linux ARM"
         key: "package-linux-arm"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
-        command:
-          - ".buildkite/filebeat/scripts/package.sh"
+        command: ".buildkite/filebeat/scripts/package.sh"
         notify:
           - github_commit_status:
               context: "Filebeat/Packaging: ARM"
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
new file mode 100755
index 00000000000..9cce3b1dc66
--- /dev/null
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo ":: Start Packaging ::"
+mage -d filebeat package
+
+calculate_tags() {
+  if [ "$snapshot" = true ]; then
+    aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
+    sourceTag+=${IMG_POSTFIX}
+  fi
+
+  local tags="${BUILDKITE_COMMIT}"
+
+#  if isPR; then
+#    tags+=("pr-${CHANGE_ID}")
+#  else
+#    tags+=("${sourceTag}")
+#  fi
+#
+#  if ! isPR && [ -n "$(aliasVersion)" ]; then
+#    tags+=("${aliasVersion}")
+#  fi
+#
+#  echo "${tags[@]}"
+}
+
+#buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
diff --git a/.buildkite/env-scripts/packaging-env.sh b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
old mode 100644
new mode 100755
similarity index 70%
rename from .buildkite/env-scripts/packaging-env.sh
rename to .buildkite/filebeat/scripts/packaging/packaging-env.sh
index 94ccb9296d3..ad4957bd911
--- a/.buildkite/env-scripts/packaging-env.sh
+++ b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
@@ -7,13 +7,14 @@ deploy/kubernetes/.*-kubernetes.yaml"
 packaging_changeset="^dev-tools/packaging/
 ^.go-version"
 
-# Env vars for Packaging stage
 DOCKER_REGISTRY="docker.elastic.co"
 SNAPSHOT=true
 VERSION=$(make get-version | tr -d '\n')
 ONLY_DOCS=$(changeset_applies "$docs_changeset")
 PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
 GO_MOD_CHANGES=$(changeset_applies "^go.mod")
+# Change the postfix to -SNAPSHOT, once Jenkins is disabled
+IMG_POSTFIX="-BK-SNAPSHOT"
 
 export PACKAGING_CHANGES
 export ONLY_DOCS
@@ -22,3 +23,11 @@ export DOCKER_REGISTRY
 export SNAPSHOT
 export VERSION
 export REPO
+export IMG_POSTFIX
+
+set_git_config() {
+  git config user.name "${GITHUB_USERNAME_SECRET}"
+  git config user.email "${GITHUB_EMAIL_SECRET}"
+}
+
+set_git_config
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index d5c827cc2e3..ce0fb84043f 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -26,7 +26,6 @@ retry() {
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
   source .buildkite/env-scripts/env.sh
-  source .buildkite/env-scripts/util.sh
   source .buildkite/env-scripts/win-env.sh
 
   if [[ -z "${GOLANG_VERSION-""}" ]]; then
@@ -36,6 +35,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
     ulimit -Sn 30000
   fi
+
+  if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
+    source .buildkite/filebeat/scripts/packaging/packaging-env.sh
+  fi
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-dockerlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 6c759aaa267..4d1dd4ef5b3 100644
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/util.sh
+source .buildkite/hooks/scripts/util.sh
 
 unset_secrets
 
diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh
new file mode 100644
index 00000000000..669bfbbfc8b
--- /dev/null
+++ b/.buildkite/hooks/scripts/util.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
+      unset "$var"
+    fi
+  done
+}
+
+google_cloud_logout_active_account() {
+  local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
+  if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
+    echo "Logging out from GCP for active account"
+    gcloud auth revoke $active_account > /dev/null 2>&1
+  else
+    echo "No active GCP accounts found."
+  fi
+  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+    unset GOOGLE_APPLICATION_CREDENTIALS
+    cleanup
+  fi
+}
+
+cleanup() {
+  echo "Deleting temporary files..."
+  rm -rf ${BIN}/${TMP_FOLDER}.*
+  echo "Done."
+}

From f24bd02c6143e0aa1ee4f810f1b1a46f97649a23 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 6 Feb 2024 20:21:45 +0200
Subject: [PATCH 13/31] added secrets

---
 .../filebeat/scripts/packaging/package-step.sh   |  4 ++--
 .buildkite/filebeat/scripts/packaging/package.sh |  7 +++++++
 .../filebeat/scripts/packaging/packaging-env.sh  |  7 -------
 .buildkite/hooks/pre-command                     | 16 ++++++++++++++++
 .buildkite/hooks/scripts/util.sh                 |  0
 5 files changed, 25 insertions(+), 9 deletions(-)
 mode change 100644 => 100755 .buildkite/hooks/scripts/util.sh

diff --git a/.buildkite/filebeat/scripts/packaging/package-step.sh b/.buildkite/filebeat/scripts/packaging/package-step.sh
index bc6d33e5910..a06b973e1fa 100755
--- a/.buildkite/filebeat/scripts/packaging/package-step.sh
+++ b/.buildkite/filebeat/scripts/packaging/package-step.sh
@@ -19,7 +19,7 @@ if are_files_changed "$changeset"; then
         key: "package-linux-x86"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-        command: ".buildkite/filebeat/scripts/package.sh"
+        command: ".buildkite/filebeat/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
               context: "Filebeat/Packaging: Linux X86"
@@ -32,7 +32,7 @@ if are_files_changed "$changeset"; then
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
-        command: ".buildkite/filebeat/scripts/package.sh"
+        command: ".buildkite/filebeat/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
               context: "Filebeat/Packaging: ARM"
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index 9cce3b1dc66..29faa449519 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -27,3 +27,10 @@ calculate_tags() {
 }
 
 #buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
+
+#set_git_config() {
+#  git config user.name "${GITHUB_USERNAME_SECRET}"
+#  git config user.email "${GITHUB_EMAIL_SECRET}"
+#}
+#
+#set_git_config
diff --git a/.buildkite/filebeat/scripts/packaging/packaging-env.sh b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
index ad4957bd911..ab8f8f68257 100755
--- a/.buildkite/filebeat/scripts/packaging/packaging-env.sh
+++ b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
@@ -24,10 +24,3 @@ export SNAPSHOT
 export VERSION
 export REPO
 export IMG_POSTFIX
-
-set_git_config() {
-  git config user.name "${GITHUB_USERNAME_SECRET}"
-  git config user.email "${GITHUB_EMAIL_SECRET}"
-}
-
-set_git_config
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index ce0fb84043f..5c696fd7a95 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -4,6 +4,9 @@ set -euo pipefail
 
 AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
+DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
+PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials"
+GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
 retry() {
   local retries=$1
@@ -27,6 +30,7 @@ retry() {
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
   source .buildkite/env-scripts/env.sh
   source .buildkite/env-scripts/win-env.sh
+  source .buildkite/env-scripts/util.sh
 
   if [[ -z "${GOLANG_VERSION-""}" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
@@ -38,6 +42,18 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
 
   if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
     source .buildkite/filebeat/scripts/packaging/packaging-env.sh
+
+    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field=data -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+    export GITHUB_TOKEN_SECRET=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+
+    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
+
+    git config user.name "${GITHUB_USERNAME_SECRET}"
+    git config user.email "${GITHUB_EMAIL_SECRET}"
   fi
 fi
 
diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh
old mode 100644
new mode 100755

From efa94e85bd6d7a424ad4227fe8525df93a30e5ec Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 6 Feb 2024 21:07:57 +0200
Subject: [PATCH 14/31] added tag definition

---
 .buildkite/env-scripts/macos-env.sh           |  8 --
 .buildkite/env-scripts/util.sh                | 13 ++++
 .../filebeat/scripts/packaging/package.sh     | 73 +++++++++++++------
 .buildkite/hooks/pre-command                  |  2 +
 4 files changed, 65 insertions(+), 31 deletions(-)
 delete mode 100644 .buildkite/env-scripts/macos-env.sh

diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh
deleted file mode 100644
index ac1486b64fd..00000000000
--- a/.buildkite/env-scripts/macos-env.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ $PLATFORM_TYPE == Darwin* ]]; then
-  echo ":: Setting larger ulimit on MacOS ::"
-  # To bypass file descriptor errors like "Too many open files error" on MacOS
-  ulimit -Sn 50000
-  echo ":: ULIMIT :: $(ulimit -n)"
-fi
diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
index 2076a0af7da..185b5c9ba2d 100644
--- a/.buildkite/env-scripts/util.sh
+++ b/.buildkite/env-scripts/util.sh
@@ -97,3 +97,16 @@ changeset_applies() {
     echo false
   fi
 }
+
+is_pr() {
+  if [[ $BUILDKITE_PULL_REQUEST != false ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+set_git_config() {
+  git config user.name "${GITHUB_USERNAME_SECRET}"
+  git config user.email "${GITHUB_EMAIL_SECRET}"
+}
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index 29faa449519..6546db82df3 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -2,35 +2,62 @@
 
 set -euo pipefail
 
-echo ":: Start Packaging ::"
-mage -d filebeat package
+#source .buildkite/env-scripts/linux-env.sh
+source .buildkite/env-scripts/util.sh
 
-calculate_tags() {
-  if [ "$snapshot" = true ]; then
+#echo "--- Creating package"
+#mage -d filebeat package
+
+
+TAGS=("${BUILDKITE_COMMIT}")
+
+VARIANTS=("" "-ubi" "-oss")
+
+# IMAGES
+SOURCE_NAMESPACE="beats"
+BEAT_NAME="filebeat"
+SOURCE="beats/${BEAT_NAME}${variant}"
+TARGET="observability-ci/${BEAT_NAME}"
+
+# ARGS
+REGISTRY="docker.elastic.co"
+SNAPSHOT=true
+VERSION="$(make get-version)",
+images: images
+
+#VARIANTS -> foreach = IMAGES: image map with SOURCE, TARGET, ARCH
+
+#IMAGES -> foreach = tag and push
+# registry: ${REGISTRY},
+# sourceTag: calculate_tags->sourceTag,
+# targetTag: "${tag}" (non arm) // ${tag}-${image.arch} (arm) --> foreach $TAGS
+# source: ${SOURCE},
+# target: ${TARGET}
+
+
+define_tags() {
+echo "--- Defined tags"
+  if [ "$SNAPSHOT" = true ]; then
     aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
-    sourceTag+=${IMG_POSTFIX}
+    sourceTag+="${VERSION}${IMG_POSTFIX}"
   fi
 
-  local tags="${BUILDKITE_COMMIT}"
-
-#  if isPR; then
-#    tags+=("pr-${CHANGE_ID}")
-#  else
-#    tags+=("${sourceTag}")
-#  fi
-#
-#  if ! isPR && [ -n "$(aliasVersion)" ]; then
-#    tags+=("${aliasVersion}")
-#  fi
-#
-#  echo "${tags[@]}"
+  if is_pr; then
+    TAGS+=("pr-${GITHUB_PR_NUMBER}")
+  else
+    TAGS+=("${sourceTag}" "${aliasVersion}")
+  fi
+
+  local tag=""
+  for tag in "${TAGS[@]}"; do
+    echo "$tag"
+  done
 }
 
 #buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
 
-#set_git_config() {
-#  git config user.name "${GITHUB_USERNAME_SECRET}"
-#  git config user.email "${GITHUB_EMAIL_SECRET}"
-#}
-#
+echo "--- Calculating tags"
+daefine_tags
+
+#echo "--- Setting git config"
 #set_git_config
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 5c696fd7a95..a9887880c63 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+# Secrets must be redacted
+# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"

From 6c5b30009003ebb961e629f0a0fa4b2a1a9fb5de Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 9 Feb 2024 12:06:18 +0200
Subject: [PATCH 15/31] added docker img params

---
 .buildkite/auditbeat/auditbeat-pipeline.yml   |   2 +-
 .buildkite/env-scripts/env.sh                 |   4 +
 .buildkite/env-scripts/linux-env.sh           |  27 +-
 .buildkite/filebeat/filebeat-pipeline.yml     | 242 +++++++++---------
 .../filebeat/scripts/packaging/package.sh     |  79 +++---
 .../scripts/packaging/packaging-env.sh        |  17 +-
 .buildkite/heartbeat/heartbeat-pipeline.yml   |   2 +-
 .buildkite/hooks/pre-command                  |   6 +-
 .buildkite/hooks/pre-exit                     |  13 +-
 .buildkite/hooks/scripts/util.sh              |   4 +-
 10 files changed, 208 insertions(+), 188 deletions(-)

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index 798939bbf32..70dcc1067a6 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
index 9abee5554ef..1677274cdfe 100644
--- a/.buildkite/env-scripts/env.sh
+++ b/.buildkite/env-scripts/env.sh
@@ -11,6 +11,8 @@ ASDF_MAGE_VERSION="1.14.0"
 SETUP_MAGE_VERSION="1.14.0"
 DEBIAN_FRONTEND="noninteractive"
 
+DOCKER_REGISTRY="docker.elastic.co"
+
 export SETUP_GVM_VERSION
 export WORKSPACE
 export BIN
@@ -21,6 +23,8 @@ export TMP_FOLDER
 export ASDF_MAGE_VERSION
 export SETUP_MAGE_VERSION
 
+export DOCKER_REGISTRY
+
 if grep -q 'Ubuntu' /etc/*release; then
   export DEBIAN_FRONTEND
 fi
diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh
index 1365aaace4a..773a75c7614 100644
--- a/.buildkite/env-scripts/linux-env.sh
+++ b/.buildkite/env-scripts/linux-env.sh
@@ -6,8 +6,15 @@ source .buildkite/env-scripts/util.sh
 
 DEBIAN_FRONTEND="noninteractive"
 
+set_env() {
+  echo "--- Setting up environment"
+  add_bin_path
+  with_go
+  with_mage
+}
+
 sudo mkdir -p /etc/needrestart
-echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
+echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf >/dev/null
 
 if [[ $PLATFORM_TYPE == "Linux" ]]; then
   # Remove this code once beats specific agent is set up
@@ -19,6 +26,8 @@ if [[ $PLATFORM_TYPE == "Linux" ]]; then
     sudo apt-get install -y libsystemd-dev
     sudo apt install -y python3-pip
     sudo apt-get install -y python3-venv
+
+    set_env
   fi
 
   # Remove this code once beats specific agent is set up
@@ -29,17 +38,15 @@ if [[ $PLATFORM_TYPE == "Linux" ]]; then
     sudo yum install -y python3-pip
     sudo yum install -y python3
     pip3 install virtualenv
+
+    set_env
   fi
 fi
 
 if [[ $PLATFORM_TYPE == Darwin* ]]; then
-  echo "--- Setting larger ulimit on MacOS"
-  # To bypass file descriptor errors like "Too many open files error" on MacOS
-  ulimit -Sn 50000
-  echo "--- ULIMIT: $(ulimit -n)"
-fi
+  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
+    ulimit -Sn 30000
+  fi
 
-echo "--- Setting up environment"
-add_bin_path
-with_go
-with_mage
+  set_env
+fi
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 8faedc401ab..971ee09529b 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  IMAGE_UBUNTU_X86_64: "family/platform-ingest-eng-prod-base-ubuntu-2204"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
@@ -11,130 +11,130 @@ env:
   ASDF_MAGE_VERSION: "1.14.0"
 
 steps:
-  - group: "Filebeat Mandatory Testing"
-    key: "mandatory-tests"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
-
-    steps:
-      - label: ":ubuntu: Unit Tests"
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: linux/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":ubuntu: Go Integration Tests"
-        command:
-          - ".buildkite/filebeat/scripts/integration-gotests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Go Integration Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":ubuntu: Python Integration Tests"
-        command:
-          - ".buildkite/filebeat/scripts/integration-pytests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Python Integration Tests"
-        agents:
-          provider: gcp
-          imageProject: elastic-images-qa
-          image: "${IMAGE_UBUNTU_X86_64}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":windows:-{{matrix.image}} Unit Tests"
-        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: windows/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "{{matrix.image}}"
-          machine_type: "n2-standard-8"
-          disk_size: 200
-          disk_type: "pd-ssd"
-        matrix:
-          setup:
-            image:
-              - "${IMAGE_WIN_2016}"
-              - "${IMAGE_WIN_2022}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-  - group: "Extended Testing"
-    key: "extended-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support"
-
-    steps:
-      - label: ":linux: ARM64 Unit Tests"
-        key: "arm-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: Unit Tests ARM"
-        agents:
-          provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
-        artifact_paths: "filebeat/build/*.xml"
-
-      - label: ":mac: MacOS Unit Tests"
-        key: "macos-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: MacOS Unit Tests"
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths: "filebeat/build/*.xml"
-
-  - group: "Windows Extended Testing"
-    key: "extended-tests-win"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
-
-    steps:
-      - label: ":windows: Win 2019 Unit Tests"
-        key: "win-extended-2019"
-        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: Win-2019 Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machine_type: "n2-standard-8"
-          disk_size: 200
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
+#  - group: "Filebeat Mandatory Testing"
+#    key: "mandatory-tests"
+#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
+#
+#    steps:
+#      - label: ":ubuntu: Unit Tests"
+#        command:
+#          - ".buildkite/filebeat/scripts/unit-tests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat: linux/Unit Tests"
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
+#
+#      - label: ":ubuntu: Go Integration Tests"
+#        command:
+#          - ".buildkite/filebeat/scripts/integration-gotests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat: Go Integration Tests"
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
+#
+#      - label: ":ubuntu: Python Integration Tests"
+#        command:
+#          - ".buildkite/filebeat/scripts/integration-pytests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat: Python Integration Tests"
+#        agents:
+#          provider: gcp
+#          imageProject: elastic-images-qa
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
+#
+#      - label: ":windows:-{{matrix.image}} Unit Tests"
+#        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat: windows/Unit Tests"
+#        agents:
+#          provider: "gcp"
+#          image: "{{matrix.image}}"
+#          machine_type: "n2-standard-8"
+#          disk_size: 200
+#          disk_type: "pd-ssd"
+#        matrix:
+#          setup:
+#            image:
+#              - "${IMAGE_WIN_2016}"
+#              - "${IMAGE_WIN_2022}"
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
+#
+#  - group: "Extended Testing"
+#    key: "extended-tests"
+#    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support"
+#
+#    steps:
+#      - label: ":linux: ARM64 Unit Tests"
+#        key: "arm-extended"
+#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+#        command:
+#          - ".buildkite/filebeat/scripts/unit-tests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat/Extended: Unit Tests ARM"
+#        agents:
+#          provider: "aws"
+#          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+#          instanceType: "t4g.large"
+#        artifact_paths: "filebeat/build/*.xml"
+#
+#      - label: ":mac: MacOS Unit Tests"
+#        key: "macos-extended"
+#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+#        command:
+#          - ".buildkite/filebeat/scripts/unit-tests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat/Extended: MacOS Unit Tests"
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_X86_64}"
+#        artifact_paths: "filebeat/build/*.xml"
+#
+#  - group: "Windows Extended Testing"
+#    key: "extended-tests-win"
+#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+#
+#    steps:
+#      - label: ":windows: Win 2019 Unit Tests"
+#        key: "win-extended-2019"
+#        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat/Extended: Win-2019 Unit Tests"
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2019}"
+#          machine_type: "n2-standard-8"
+#          disk_size: 200
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
 
   - group: "Packaging"
     key: "packaging"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "mandatory-tests"
+#    depends_on:
+#      - "mandatory-tests"
 
     steps:
       - label: Package pipeline
-        commands: ".buildkite/filebeat/scripts/package-step.sh"
+        commands: ".buildkite/filebeat/scripts/packaging/package-step.sh"
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index 6546db82df3..55a3159d8cb 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -5,59 +5,58 @@ set -euo pipefail
 #source .buildkite/env-scripts/linux-env.sh
 source .buildkite/env-scripts/util.sh
 
-#echo "--- Creating package"
-#mage -d filebeat package
-
-
-TAGS=("${BUILDKITE_COMMIT}")
-
+IMG_POSTFIX="-BK-SNAPSHOT"
 VARIANTS=("" "-ubi" "-oss")
-
-# IMAGES
-SOURCE_NAMESPACE="beats"
+VERSION="$(make get-version)"
+SOURCE_TAG+="${VERSION}${IMG_POSTFIX}"
 BEAT_NAME="filebeat"
-SOURCE="beats/${BEAT_NAME}${variant}"
 TARGET="observability-ci/${BEAT_NAME}"
 
-# ARGS
-REGISTRY="docker.elastic.co"
-SNAPSHOT=true
-VERSION="$(make get-version)",
-images: images
-
-#VARIANTS -> foreach = IMAGES: image map with SOURCE, TARGET, ARCH
-
-#IMAGES -> foreach = tag and push
-# registry: ${REGISTRY},
-# sourceTag: calculate_tags->sourceTag,
-# targetTag: "${tag}" (non arm) // ${tag}-${image.arch} (arm) --> foreach $TAGS
-# source: ${SOURCE},
-# target: ${TARGET}
-
-
 define_tags() {
-echo "--- Defined tags"
-  if [ "$SNAPSHOT" = true ]; then
-    aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
-    sourceTag+="${VERSION}${IMG_POSTFIX}"
-  fi
+  aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
+  tags=("${BUILDKITE_COMMIT}")
 
   if is_pr; then
-    TAGS+=("pr-${GITHUB_PR_NUMBER}")
+    tags+=("pr-${GITHUB_PR_NUMBER}")
   else
-    TAGS+=("${sourceTag}" "${aliasVersion}")
+    tags+=("${SOURCE_TAG}" "${aliasVersion}")
   fi
+}
 
-  local tag=""
-  for tag in "${TAGS[@]}"; do
-    echo "$tag"
-  done
+check_is_arm() {
+  if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
+    is_arm="-arm"
+  else
+    is_arm=""
+  fi
 }
 
-#buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
+define_tags
+
+for variant in "${VARIANTS[@]}"; do
+  echo "--- PARAMS for variant: $variant"
+
+  check_is_arm
+  registry=${DOCKER_REGISTRY}
+  sourceTag=$SOURCE_TAG
+  source="beats/${BEAT_NAME}${variant}"
+  target=$TARGET
+
+  echo "Registry: $registry"
+  echo "Source: $source"
+  echo "Source tag: $sourceTag"
+  echo "Target: $target"
 
-echo "--- Calculating tags"
-daefine_tags
+  for tag in "${tags[@]}"; do
+    targetTag=$tag${is_arm}
+    echo "Target tag: $targetTag"
+  done
+done
+
+#echo "--- Creating package"
+#mage -d filebeat package
 
 #echo "--- Setting git config"
 #set_git_config
+
+#buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'
diff --git a/.buildkite/filebeat/scripts/packaging/packaging-env.sh b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
index ab8f8f68257..02c942361b4 100755
--- a/.buildkite/filebeat/scripts/packaging/packaging-env.sh
+++ b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
@@ -7,20 +7,27 @@ deploy/kubernetes/.*-kubernetes.yaml"
 packaging_changeset="^dev-tools/packaging/
 ^.go-version"
 
-DOCKER_REGISTRY="docker.elastic.co"
-SNAPSHOT=true
 VERSION=$(make get-version | tr -d '\n')
 ONLY_DOCS=$(changeset_applies "$docs_changeset")
 PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
 GO_MOD_CHANGES=$(changeset_applies "^go.mod")
 # Change the postfix to -SNAPSHOT, once Jenkins is disabled
-IMG_POSTFIX="-BK-SNAPSHOT"
+
 
 export PACKAGING_CHANGES
 export ONLY_DOCS
 export GO_MOD_CHANGES
 export DOCKER_REGISTRY
-export SNAPSHOT
 export VERSION
 export REPO
-export IMG_POSTFIX
+#export IMG_POSTFIX
+
+
+#VARIANTS -> foreach = IMAGES: image map with SOURCE, TARGET, ARCH
+
+#IMAGES -> foreach = tag and push
+# registry: ${REGISTRY},
+# sourceTag: calculate_tags->sourceTag,
+# targetTag: "${tag}" (non arm) // ${tag}-${image.arch} (arm) --> foreach $TAGS
+# source: ${SOURCE},
+# target: ${TARGET}
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index bf645a2b295..81c67134f06 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index a9887880c63..d3602c39fa2 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -29,16 +29,16 @@ retry() {
   return 0
 }
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
   source .buildkite/env-scripts/env.sh
   source .buildkite/env-scripts/win-env.sh
-  source .buildkite/env-scripts/util.sh
+#  source .buildkite/env-scripts/util.sh
 
   if [[ -z "${GOLANG_VERSION-""}" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
   fi
 
-  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
     ulimit -Sn 30000
   fi
 
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 4d1dd4ef5b3..007966a97a4 100644
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -4,12 +4,13 @@ set -euo pipefail
 
 source .buildkite/hooks/scripts/util.sh
 
-unset_secrets
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" && "$BUILDKITE_STEP_KEY" == package* ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
     google_cloud_logout_active_account
     docker logout "${DOCKER_REGISTRY}"
-fi
+  fi
 
-# Ensure that any temporal files created during any step are removed
-cleanup
+  # Ensure that any temporal files created during any step are removed
+  cleanup
+  unset_secrets
+fi
diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh
index 669bfbbfc8b..07ab6cf4c9a 100755
--- a/.buildkite/hooks/scripts/util.sh
+++ b/.buildkite/hooks/scripts/util.sh
@@ -26,6 +26,8 @@ google_cloud_logout_active_account() {
 
 cleanup() {
   echo "Deleting temporary files..."
-  rm -rf ${BIN}/${TMP_FOLDER}.*
+  if [[ -e "${BIN}/${TMP_FOLDER}" ]]; then
+    rm -rf "${BIN}/${TMP_FOLDER}.*"
+  fi
   echo "Done."
 }

From 0fade9e440e5a9cfedaaa6e064be9bb1f3bb60ec Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 12 Feb 2024 13:42:42 +0200
Subject: [PATCH 16/31] added tag and push

---
 .buildkite/env-scripts/util.sh                |  8 ---
 .../scripts/packaging/package-step.sh         |  2 +
 .../scripts/packaging/package-util.sh         | 30 ++++++++
 .../filebeat/scripts/packaging/package.sh     | 71 +++++++++----------
 4 files changed, 64 insertions(+), 47 deletions(-)
 create mode 100755 .buildkite/filebeat/scripts/packaging/package-util.sh

diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
index 185b5c9ba2d..962cb970bf1 100644
--- a/.buildkite/env-scripts/util.sh
+++ b/.buildkite/env-scripts/util.sh
@@ -98,14 +98,6 @@ changeset_applies() {
   fi
 }
 
-is_pr() {
-  if [[ $BUILDKITE_PULL_REQUEST != false ]]; then
-    return 0
-  else
-    return 1
-  fi
-}
-
 set_git_config() {
   git config user.name "${GITHUB_USERNAME_SECRET}"
   git config user.email "${GITHUB_EMAIL_SECRET}"
diff --git a/.buildkite/filebeat/scripts/packaging/package-step.sh b/.buildkite/filebeat/scripts/packaging/package-step.sh
index a06b973e1fa..739924dc057 100755
--- a/.buildkite/filebeat/scripts/packaging/package-step.sh
+++ b/.buildkite/filebeat/scripts/packaging/package-step.sh
@@ -19,6 +19,7 @@ if are_files_changed "$changeset"; then
         key: "package-linux-x86"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: true
         command: ".buildkite/filebeat/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
@@ -32,6 +33,7 @@ if are_files_changed "$changeset"; then
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
+          SNAPSHOT: true
         command: ".buildkite/filebeat/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
diff --git a/.buildkite/filebeat/scripts/packaging/package-util.sh b/.buildkite/filebeat/scripts/packaging/package-util.sh
new file mode 100755
index 00000000000..cb409e8d484
--- /dev/null
+++ b/.buildkite/filebeat/scripts/packaging/package-util.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+is_pr() {
+  if [[ $BUILDKITE_PULL_REQUEST != false ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+define_tags() {
+  aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
+  tags=("${BUILDKITE_COMMIT}")
+
+  if is_pr; then
+    tags+=("pr-${GITHUB_PR_NUMBER}")
+  else
+    tags+=("${SOURCE_TAG}" "${aliasVersion}")
+  fi
+}
+
+check_is_arm() {
+  if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
+    is_arm="-arm"
+  else
+    is_arm=""
+  fi
+}
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index 55a3159d8cb..d7ec456f615 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -2,61 +2,54 @@
 
 set -euo pipefail
 
-#source .buildkite/env-scripts/linux-env.sh
-source .buildkite/env-scripts/util.sh
+source .buildkite/env-scripts/linux-env.sh
+source .buildkite/filebeat/scripts/packaging/package-util.sh
 
-IMG_POSTFIX="-BK-SNAPSHOT"
+IMG_POSTFIX="-SNAPSHOT"
 VARIANTS=("" "-ubi" "-oss")
 VERSION="$(make get-version)"
 SOURCE_TAG+="${VERSION}${IMG_POSTFIX}"
 BEAT_NAME="filebeat"
 TARGET="observability-ci/${BEAT_NAME}"
+# Remove following once beats fully migrated
+BK_IMG_POSTFIX="-BK-SNAPSHOT"
+BK_SOURCE_TAG+="${VERSION}${BK_IMG_POSTFIX}"
 
-define_tags() {
-  aliasVersion="${VERSION%.*}${IMG_POSTFIX}"
-  tags=("${BUILDKITE_COMMIT}")
+echo "--- Creating package"
+mage -d filebeat package
 
-  if is_pr; then
-    tags+=("pr-${GITHUB_PR_NUMBER}")
-  else
-    tags+=("${SOURCE_TAG}" "${aliasVersion}")
-  fi
-}
+echo "--- Distribution list"
+ls -la filebeat/build/distributions
 
-check_is_arm() {
-  if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
-    is_arm="-arm"
-  else
-    is_arm=""
-  fi
-}
+echo "--- Docker image list"
+docker images
 
 define_tags
+check_is_arm
 
 for variant in "${VARIANTS[@]}"; do
-  echo "--- PARAMS for variant: $variant"
 
-  check_is_arm
-  registry=${DOCKER_REGISTRY}
-  sourceTag=$SOURCE_TAG
   source="beats/${BEAT_NAME}${variant}"
-  target=$TARGET
-
-  echo "Registry: $registry"
-  echo "Source: $source"
-  echo "Source tag: $sourceTag"
-  echo "Target: $target"
 
   for tag in "${tags[@]}"; do
-    targetTag=$tag${is_arm}
-    echo "Target tag: $targetTag"
+  targetTag=$tag${is_arm}
+
+  sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
+  targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
+
+  if docker image inspect "${sourceName}" &>/dev/null; then
+    echo "--- Tag & Push"
+    echo "Source name: $sourceName"
+    echo "Target name: $targetName"
+
+    # Remove following lines once beats fully migrated
+    bkSourceName="${DOCKER_REGISTRY}/${source}:${BK_SOURCE_TAG}"
+    docker tag "$sourceName" "$bkSourceName"
+    # Replace bkSourceName to sourceName once beats fully migrated
+    docker tag "${bkSourceName}" "${targetName}"
+#    docker push "${targetName}"
+  else
+    echo "Docker image ${sourceName} does not exist"
+  fi
   done
 done
-
-#echo "--- Creating package"
-#mage -d filebeat package
-
-#echo "--- Setting git config"
-#set_git_config
-
-#buildkite-agent annotate "Tag '$TAG' has been created." --style 'success' --context 'ctx-success'

From e7c65840b91100450d5c818f801796f9d0cec4c3 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 14 Feb 2024 12:28:16 +0200
Subject: [PATCH 17/31] added distribution upload

---
 .../filebeat/scripts/packaging/package.sh     | 26 +++++++------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
index d7ec456f615..44e91044576 100755
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ b/.buildkite/filebeat/scripts/packaging/package.sh
@@ -11,15 +11,13 @@ VERSION="$(make get-version)"
 SOURCE_TAG+="${VERSION}${IMG_POSTFIX}"
 BEAT_NAME="filebeat"
 TARGET="observability-ci/${BEAT_NAME}"
-# Remove following once beats fully migrated
-BK_IMG_POSTFIX="-BK-SNAPSHOT"
-BK_SOURCE_TAG+="${VERSION}${BK_IMG_POSTFIX}"
 
 echo "--- Creating package"
 mage -d filebeat package
 
 echo "--- Distribution list"
-ls -la filebeat/build/distributions
+dir="filebeat/build/distributions"
+buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
 
 echo "--- Docker image list"
 docker images
@@ -27,8 +25,8 @@ docker images
 define_tags
 check_is_arm
 
+echo "--- Tag & Push"
 for variant in "${VARIANTS[@]}"; do
-
   source="beats/${BEAT_NAME}${variant}"
 
   for tag in "${tags[@]}"; do
@@ -36,20 +34,16 @@ for variant in "${VARIANTS[@]}"; do
 
   sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
   targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
+  # Remove following lines once beats fully migrated
+  targetName="${targetName}-buildkite"
 
   if docker image inspect "${sourceName}" &>/dev/null; then
-    echo "--- Tag & Push"
-    echo "Source name: $sourceName"
-    echo "Target name: $targetName"
-
-    # Remove following lines once beats fully migrated
-    bkSourceName="${DOCKER_REGISTRY}/${source}:${BK_SOURCE_TAG}"
-    docker tag "$sourceName" "$bkSourceName"
-    # Replace bkSourceName to sourceName once beats fully migrated
-    docker tag "${bkSourceName}" "${targetName}"
-#    docker push "${targetName}"
+    echo "Source name: $sourceName Target name: $targetName"
+    docker tag "$sourceName" "$targetName"
+#    docker push "$targetName"
+
   else
     echo "Docker image ${sourceName} does not exist"
   fi
-  done
+done
 done

From 6708eeddbdd4dd353d4a9cc2ceec0d93e33ef1dc Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 15 Feb 2024 16:22:27 +0200
Subject: [PATCH 18/31] made packaging reusable

---
 .buildkite/env-scripts/env.sh                 | 37 +++++++++++---
 .buildkite/env-scripts/linux-env.sh           | 11 +++--
 .buildkite/filebeat/filebeat-pipeline.yml     |  3 +-
 .../filebeat/scripts/packaging/package.sh     | 49 -------------------
 .../scripts/packaging/packaging-env.sh        | 33 -------------
 .buildkite/hooks/pre-exit                     |  1 -
 .../scripts/packaging/package-step.sh         | 16 +++---
 .../scripts/packaging/package-util.sh         |  2 +-
 .buildkite/scripts/packaging/package.sh       | 47 ++++++++++++++++++
 9 files changed, 95 insertions(+), 104 deletions(-)
 delete mode 100755 .buildkite/filebeat/scripts/packaging/package.sh
 delete mode 100755 .buildkite/filebeat/scripts/packaging/packaging-env.sh
 rename .buildkite/{filebeat => }/scripts/packaging/package-step.sh (69%)
 rename .buildkite/{filebeat => }/scripts/packaging/package-util.sh (95%)
 create mode 100755 .buildkite/scripts/packaging/package.sh

diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
index 1677274cdfe..79a520cc2d5 100644
--- a/.buildkite/env-scripts/env.sh
+++ b/.buildkite/env-scripts/env.sh
@@ -1,30 +1,53 @@
 #!/usr/bin/env bash
 
+source .buildkite/env-scripts/util.sh
+
+DOCS_CHANGESET="^.*\.(asciidoc|md)$
+deploy/kubernetes/.*-kubernetes.yaml"
+PACKAGING_CHANGESET="^dev-tools/packaging/
+^.go-version"
+
 REPO="beats"
-SETUP_GVM_VERSION="v0.5.1"
 WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
 TMP_FOLDER="tmp.${REPO}"
+SNAPSHOT="true"
+PYTEST_ADDOPTS=""
+OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+
+SETUP_GVM_VERSION="v0.5.1"
 ASDF_MAGE_VERSION="1.14.0"
 SETUP_MAGE_VERSION="1.14.0"
-DEBIAN_FRONTEND="noninteractive"
 
+# Docker & DockerHub
+DOCKER_COMPOSE_VERSION="1.21.0"
 DOCKER_REGISTRY="docker.elastic.co"
 
-export SETUP_GVM_VERSION
+ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET")
+PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET")
+GO_MOD_CHANGES=$(changeset_applies "^go.mod")
+
+export REPO
 export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
-export REPO
 export TMP_FOLDER
+export SNAPSHOT
+export PYTEST_ADDOPTS
+export OSS_MODULE_PATTERN
+export XPACK_MODULE_PATTERN
+
+export SETUP_GVM_VERSION
 export ASDF_MAGE_VERSION
 export SETUP_MAGE_VERSION
 
+export DOCKER_COMPOSE_VERSION
 export DOCKER_REGISTRY
 
-if grep -q 'Ubuntu' /etc/*release; then
-  export DEBIAN_FRONTEND
-fi
+export ONLY_DOCS
+export PACKAGING_CHANGES
+export GO_MOD_CHANGES
diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh
index 773a75c7614..4fd5fef8928 100644
--- a/.buildkite/env-scripts/linux-env.sh
+++ b/.buildkite/env-scripts/linux-env.sh
@@ -13,14 +13,17 @@ set_env() {
   with_mage
 }
 
-sudo mkdir -p /etc/needrestart
-echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf >/dev/null
-
 if [[ $PLATFORM_TYPE == "Linux" ]]; then
+  check_platform_architecture
+  echo "ARCH: ${arch_type}"
+
   # Remove this code once beats specific agent is set up
-  if grep -q 'Ubuntu' /etc/*release; then
+  if grep -q 'Ubuntu' /etc/*release && [ "${arch_type}" == "arm64" ]; then
     export DEBIAN_FRONTEND
 
+    sudo mkdir -p /etc/needrestart
+    echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf >/dev/null
+
     echo "--- Ubuntu - Installing libs"
     sudo apt-get update
     sudo apt-get install -y libsystemd-dev
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 971ee09529b..148c49c7e03 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -1,6 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
+  BEATS_PROJECT_NAME: "filebeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
@@ -137,4 +138,4 @@ steps:
 
     steps:
       - label: Package pipeline
-        commands: ".buildkite/filebeat/scripts/packaging/package-step.sh"
+        commands: ".buildkite/scripts/packaging/package-step.sh"
diff --git a/.buildkite/filebeat/scripts/packaging/package.sh b/.buildkite/filebeat/scripts/packaging/package.sh
deleted file mode 100755
index 44e91044576..00000000000
--- a/.buildkite/filebeat/scripts/packaging/package.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/env-scripts/linux-env.sh
-source .buildkite/filebeat/scripts/packaging/package-util.sh
-
-IMG_POSTFIX="-SNAPSHOT"
-VARIANTS=("" "-ubi" "-oss")
-VERSION="$(make get-version)"
-SOURCE_TAG+="${VERSION}${IMG_POSTFIX}"
-BEAT_NAME="filebeat"
-TARGET="observability-ci/${BEAT_NAME}"
-
-echo "--- Creating package"
-mage -d filebeat package
-
-echo "--- Distribution list"
-dir="filebeat/build/distributions"
-buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
-
-echo "--- Docker image list"
-docker images
-
-define_tags
-check_is_arm
-
-echo "--- Tag & Push"
-for variant in "${VARIANTS[@]}"; do
-  source="beats/${BEAT_NAME}${variant}"
-
-  for tag in "${tags[@]}"; do
-  targetTag=$tag${is_arm}
-
-  sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
-  targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
-  # Remove following lines once beats fully migrated
-  targetName="${targetName}-buildkite"
-
-  if docker image inspect "${sourceName}" &>/dev/null; then
-    echo "Source name: $sourceName Target name: $targetName"
-    docker tag "$sourceName" "$targetName"
-#    docker push "$targetName"
-
-  else
-    echo "Docker image ${sourceName} does not exist"
-  fi
-done
-done
diff --git a/.buildkite/filebeat/scripts/packaging/packaging-env.sh b/.buildkite/filebeat/scripts/packaging/packaging-env.sh
deleted file mode 100755
index 02c942361b4..00000000000
--- a/.buildkite/filebeat/scripts/packaging/packaging-env.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-source .buildkite/env-scripts/util.sh
-
-docs_changeset="^.*\.(asciidoc|md)$
-deploy/kubernetes/.*-kubernetes.yaml"
-packaging_changeset="^dev-tools/packaging/
-^.go-version"
-
-VERSION=$(make get-version | tr -d '\n')
-ONLY_DOCS=$(changeset_applies "$docs_changeset")
-PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
-GO_MOD_CHANGES=$(changeset_applies "^go.mod")
-# Change the postfix to -SNAPSHOT, once Jenkins is disabled
-
-
-export PACKAGING_CHANGES
-export ONLY_DOCS
-export GO_MOD_CHANGES
-export DOCKER_REGISTRY
-export VERSION
-export REPO
-#export IMG_POSTFIX
-
-
-#VARIANTS -> foreach = IMAGES: image map with SOURCE, TARGET, ARCH
-
-#IMAGES -> foreach = tag and push
-# registry: ${REGISTRY},
-# sourceTag: calculate_tags->sourceTag,
-# targetTag: "${tag}" (non arm) // ${tag}-${image.arch} (arm) --> foreach $TAGS
-# source: ${SOURCE},
-# target: ${TARGET}
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 007966a97a4..d1ff6e0ac1c 100644
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -6,7 +6,6 @@ source .buildkite/hooks/scripts/util.sh
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
-    google_cloud_logout_active_account
     docker logout "${DOCKER_REGISTRY}"
   fi
 
diff --git a/.buildkite/filebeat/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh
similarity index 69%
rename from .buildkite/filebeat/scripts/packaging/package-step.sh
rename to .buildkite/scripts/packaging/package-step.sh
index 739924dc057..6d17e0e9440 100755
--- a/.buildkite/filebeat/scripts/packaging/package-step.sh
+++ b/.buildkite/scripts/packaging/package-step.sh
@@ -4,40 +4,40 @@ set -euo pipefail
 
 source .buildkite/env-scripts/util.sh
 
-changeset="^filebeat/
+changeset="^${BEATS_PROJECT_NAME}/
 ^go.mod
 ^pytest.ini
 ^dev-tools/
 ^libbeat/
 ^testing/
-^\.buildkite/filebeat/"
+^\.buildkite/${BEATS_PROJECT_NAME}/"
 
 if are_files_changed "$changeset"; then
   bk_pipeline=$(cat <<-YAML
     steps:
-      - label: ":ubuntu: Packaging Linux X86"
+      - label: ":ubuntu: ${BEATS_PROJECT_NAME}/Packaging Linux X86"
         key: "package-linux-x86"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
-        command: ".buildkite/filebeat/scripts/packaging/package.sh"
+        command: ".buildkite/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
-              context: "Filebeat/Packaging: Linux X86"
+              context: "${BEATS_PROJECT_NAME}/Packaging: Linux X86"
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
 
-      - label: ":linux: Packaging Linux ARM"
+      - label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM"
         key: "package-linux-arm"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
-        command: ".buildkite/filebeat/scripts/packaging/package.sh"
+        command: ".buildkite/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
-              context: "Filebeat/Packaging: ARM"
+              context: "${BEATS_PROJECT_NAME}/Packaging: ARM"
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
diff --git a/.buildkite/filebeat/scripts/packaging/package-util.sh b/.buildkite/scripts/packaging/package-util.sh
similarity index 95%
rename from .buildkite/filebeat/scripts/packaging/package-util.sh
rename to .buildkite/scripts/packaging/package-util.sh
index cb409e8d484..6b91fd55759 100755
--- a/.buildkite/filebeat/scripts/packaging/package-util.sh
+++ b/.buildkite/scripts/packaging/package-util.sh
@@ -23,7 +23,7 @@ define_tags() {
 
 check_is_arm() {
   if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
-    is_arm="-arm"
+    is_arm="-arm64"
   else
     is_arm=""
   fi
diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh
new file mode 100755
index 00000000000..efba93ca873
--- /dev/null
+++ b/.buildkite/scripts/packaging/package.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+source .buildkite/scripts/packaging/package-util.sh
+
+IMG_POSTFIX="-SNAPSHOT"
+VARIANTS=("" "-ubi" "-oss")
+VERSION="$(make get-version)"
+SOURCE_TAG+="${VERSION}${IMG_POSTFIX}"
+TARGET="observability-ci/${BEATS_PROJECT_NAME}"
+
+echo "--- Creating package"
+mage -d "${BEATS_PROJECT_NAME}" package
+
+echo "--- Distribution list"
+dir="${BEATS_PROJECT_NAME}/build/distributions"
+buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
+
+echo "--- Docker image list"
+docker images
+
+define_tags
+check_is_arm
+
+echo "--- Tag & Push"
+for variant in "${VARIANTS[@]}"; do
+  source="beats/${BEATS_PROJECT_NAME}${variant}"
+
+  for tag in "${tags[@]}"; do
+    targetTag=$tag${is_arm}
+
+    sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
+    targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
+    # Remove following line once beats fully migrated
+    targetName="${targetName}-buildkite"
+
+    if docker image inspect "${sourceName}" &>/dev/null; then
+      echo "Source name: $sourceName Target name: $targetName"
+      docker tag "$sourceName" "$targetName"
+      docker push "$targetName"
+    else
+      echo "Docker image ${sourceName} does not exist"
+    fi
+  done
+done

From 55e308d18cd7fdba44283aa446eab3f6a961da7a Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 15 Feb 2024 17:47:29 +0200
Subject: [PATCH 19/31] added packaging to heartbeat and auditbeat

---
 .buildkite/auditbeat/auditbeat-pipeline.yml |   3 +-
 .buildkite/filebeat/filebeat-pipeline.yml   | 240 ++++++++++----------
 .buildkite/heartbeat/heartbeat-pipeline.yml |   3 +-
 .buildkite/scripts/packaging/package.sh     |   4 +-
 4 files changed, 125 insertions(+), 125 deletions(-)

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index 70dcc1067a6..2d4899c6162 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -1,6 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
+  BEATS_PROJECT_NAME: "auditbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
@@ -134,4 +135,4 @@ steps:
 
     steps:
       - label: Package pipeline
-        commands: ".buildkite/auditbeat/scripts/package-step.sh"
+        commands: ".buildkite/scripts/packaging/package-step.sh"
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 148c49c7e03..3fac4325887 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -8,133 +8,131 @@ env:
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
-  SETUP_MAGE_VERSION: "1.14.0"
-  ASDF_MAGE_VERSION: "1.14.0"
 
 steps:
-#  - group: "Filebeat Mandatory Testing"
-#    key: "mandatory-tests"
-#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
-#
-#    steps:
-#      - label: ":ubuntu: Unit Tests"
-#        command:
-#          - ".buildkite/filebeat/scripts/unit-tests.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "Filebeat: linux/Unit Tests"
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_UBUNTU_X86_64}"
-#        artifact_paths:
-#          - "filebeat/build/*.xml"
-#          - "filebeat/build/*.json"
-#
-#      - label: ":ubuntu: Go Integration Tests"
-#        command:
-#          - ".buildkite/filebeat/scripts/integration-gotests.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "Filebeat: Go Integration Tests"
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_UBUNTU_X86_64}"
-#        artifact_paths:
-#          - "filebeat/build/*.xml"
-#          - "filebeat/build/*.json"
-#
-#      - label: ":ubuntu: Python Integration Tests"
-#        command:
-#          - ".buildkite/filebeat/scripts/integration-pytests.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "Filebeat: Python Integration Tests"
-#        agents:
-#          provider: gcp
-#          imageProject: elastic-images-qa
-#          image: "${IMAGE_UBUNTU_X86_64}"
-#        artifact_paths:
-#          - "filebeat/build/*.xml"
-#          - "filebeat/build/*.json"
-#
-#      - label: ":windows:-{{matrix.image}} Unit Tests"
-#        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
-#        notify:
-#          - github_commit_status:
-#              context: "Filebeat: windows/Unit Tests"
-#        agents:
-#          provider: "gcp"
-#          image: "{{matrix.image}}"
-#          machine_type: "n2-standard-8"
-#          disk_size: 200
-#          disk_type: "pd-ssd"
-#        matrix:
-#          setup:
-#            image:
-#              - "${IMAGE_WIN_2016}"
-#              - "${IMAGE_WIN_2022}"
-#        artifact_paths:
-#          - "filebeat/build/*.xml"
-#          - "filebeat/build/*.json"
-#
-#  - group: "Extended Testing"
-#    key: "extended-tests"
-#    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support"
-#
-#    steps:
-#      - label: ":linux: ARM64 Unit Tests"
-#        key: "arm-extended"
-#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-#        command:
-#          - ".buildkite/filebeat/scripts/unit-tests.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "Filebeat/Extended: Unit Tests ARM"
-#        agents:
-#          provider: "aws"
-#          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-#          instanceType: "t4g.large"
-#        artifact_paths: "filebeat/build/*.xml"
-#
-#      - label: ":mac: MacOS Unit Tests"
-#        key: "macos-extended"
-#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-#        command:
-#          - ".buildkite/filebeat/scripts/unit-tests.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "Filebeat/Extended: MacOS Unit Tests"
-#        agents:
-#          provider: "orka"
-#          imagePrefix: "${IMAGE_MACOS_X86_64}"
-#        artifact_paths: "filebeat/build/*.xml"
-#
-#  - group: "Windows Extended Testing"
-#    key: "extended-tests-win"
-#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
-#
-#    steps:
-#      - label: ":windows: Win 2019 Unit Tests"
-#        key: "win-extended-2019"
-#        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
-#        notify:
-#          - github_commit_status:
-#              context: "Filebeat/Extended: Win-2019 Unit Tests"
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_WIN_2019}"
-#          machine_type: "n2-standard-8"
-#          disk_size: 200
-#          disk_type: "pd-ssd"
-#        artifact_paths:
-#          - "filebeat/build/*.xml"
-#          - "filebeat/build/*.json"
+  - group: "Filebeat Mandatory Testing"
+    key: "mandatory-tests"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
+
+    steps:
+      - label: ":ubuntu: Unit Tests"
+        command:
+          - ".buildkite/filebeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: linux/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+
+      - label: ":ubuntu: Go Integration Tests"
+        command:
+          - ".buildkite/filebeat/scripts/integration-gotests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Go Integration Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+
+      - label: ":ubuntu: Python Integration Tests"
+        command:
+          - ".buildkite/filebeat/scripts/integration-pytests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Python Integration Tests"
+        agents:
+          provider: gcp
+          imageProject: elastic-images-qa
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+
+      - label: ":windows:-{{matrix.image}} Unit Tests"
+        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: windows/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+
+  - group: "Extended Testing"
+    key: "extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support"
+
+    steps:
+      - label: ":linux: ARM64 Unit Tests"
+        key: "arm-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        command:
+          - ".buildkite/filebeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat/Extended: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+        artifact_paths: "filebeat/build/*.xml"
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command:
+          - ".buildkite/filebeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat/Extended: MacOS Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "filebeat/build/*.xml"
+
+  - group: "Windows Extended Testing"
+    key: "extended-tests-win"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+
+    steps:
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "win-extended-2019"
+        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "Filebeat/Extended: Win-2019 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
 
   - group: "Packaging"
     key: "packaging"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-#    depends_on:
-#      - "mandatory-tests"
+    depends_on:
+      - "mandatory-tests"
 
     steps:
       - label: Package pipeline
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 81c67134f06..8aa0a8c22de 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -1,6 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
+  BEATS_PROJECT_NAME: "heartbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
@@ -139,4 +140,4 @@ steps:
 
     steps:
       - label: Package pipeline
-        commands: ".buildkite/heartbeat/scripts/package-step.sh"
+        commands: ".buildkite/scripts/packaging/package-step.sh"
diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh
index efba93ca873..e1f36238d00 100755
--- a/.buildkite/scripts/packaging/package.sh
+++ b/.buildkite/scripts/packaging/package.sh
@@ -24,7 +24,6 @@ docker images
 define_tags
 check_is_arm
 
-echo "--- Tag & Push"
 for variant in "${VARIANTS[@]}"; do
   source="beats/${BEATS_PROJECT_NAME}${variant}"
 
@@ -37,7 +36,8 @@ for variant in "${VARIANTS[@]}"; do
     targetName="${targetName}-buildkite"
 
     if docker image inspect "${sourceName}" &>/dev/null; then
-      echo "Source name: $sourceName Target name: $targetName"
+      echo "--- Tag & Push with target: $targetName"
+      echo "Source name: $sourceName"
       docker tag "$sourceName" "$targetName"
       docker push "$targetName"
     else

From 2488a59eebc04b9719e09ddee26e1083d4bf4166 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 26 Feb 2024 14:36:50 +0200
Subject: [PATCH 20/31] changed arm agent

---
 .buildkite/auditbeat/auditbeat-pipeline.yml |  2 +-
 .buildkite/env-scripts/linux-env.sh         | 64 ++++++++++-----------
 .buildkite/filebeat/filebeat-pipeline.yml   |  2 +-
 .buildkite/heartbeat/heartbeat-pipeline.yml |  2 +-
 4 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index 2d4899c6162..2d4bc23eb2b 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "auditbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh
index 4fd5fef8928..7b0f989a55c 100644
--- a/.buildkite/env-scripts/linux-env.sh
+++ b/.buildkite/env-scripts/linux-env.sh
@@ -13,38 +13,38 @@ set_env() {
   with_mage
 }
 
-if [[ $PLATFORM_TYPE == "Linux" ]]; then
-  check_platform_architecture
-  echo "ARCH: ${arch_type}"
-
-  # Remove this code once beats specific agent is set up
-  if grep -q 'Ubuntu' /etc/*release && [ "${arch_type}" == "arm64" ]; then
-    export DEBIAN_FRONTEND
-
-    sudo mkdir -p /etc/needrestart
-    echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf >/dev/null
-
-    echo "--- Ubuntu - Installing libs"
-    sudo apt-get update
-    sudo apt-get install -y libsystemd-dev
-    sudo apt install -y python3-pip
-    sudo apt-get install -y python3-venv
-
-    set_env
-  fi
-
-  # Remove this code once beats specific agent is set up
-  if grep -q 'Red Hat' /etc/*release; then
-    echo "--- RHL - Installing libs"
-    sudo yum update -y
-    sudo yum install -y systemd-devel
-    sudo yum install -y python3-pip
-    sudo yum install -y python3
-    pip3 install virtualenv
-
-    set_env
-  fi
-fi
+#if [[ $PLATFORM_TYPE == "Linux" ]]; then
+#  check_platform_architecture
+#  echo "ARCH: ${arch_type}"
+#
+#  # Remove this code once beats specific agent is set up
+#  if grep -q 'Ubuntu' /etc/*release && [ "${arch_type}" == "arm64" ]; then
+#    export DEBIAN_FRONTEND
+#
+#    sudo mkdir -p /etc/needrestart
+#    echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf >/dev/null
+#
+#    echo "--- Ubuntu - Installing libs"
+#    sudo apt-get update
+#    sudo apt-get install -y libsystemd-dev
+#    sudo apt install -y python3-pip
+#    sudo apt-get install -y python3-venv
+#
+#    set_env
+#  fi
+#
+#  # Remove this code once beats specific agent is set up
+#  if grep -q 'Red Hat' /etc/*release; then
+#    echo "--- RHL - Installing libs"
+#    sudo yum update -y
+#    sudo yum install -y systemd-devel
+#    sudo yum install -y python3-pip
+#    sudo yum install -y python3
+#    pip3 install virtualenv
+#
+#    set_env
+#  fi
+#fi
 
 if [[ $PLATFORM_TYPE == Darwin* ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 3fac4325887..4ef9b680533 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "filebeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 8aa0a8c22de..9be695cb8ca 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "heartbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"

From 6e5ab28ef8b3e554200b8d51b1d44aba6458b325 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 26 Feb 2024 14:49:35 +0200
Subject: [PATCH 21/31] changed rhel9 agent

---
 .buildkite/auditbeat/auditbeat-pipeline.yml | 4 ++--
 .buildkite/filebeat/filebeat-pipeline.yml   | 2 +-
 .buildkite/heartbeat/heartbeat-pipeline.yml | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index 2d4bc23eb2b..c160b973c3a 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -3,11 +3,11 @@
 env:
   BEATS_PROJECT_NAME: "auditbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2004-aarch64"
+  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
-  IMAGE_RHEL9: "family/core-rhel-9"
+  IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
 
 steps:
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 4ef9b680533..77a2b8e7dbd 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "filebeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2004-aarch64"
+  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 9be695cb8ca..720bfb6070f 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -3,11 +3,11 @@
 env:
   BEATS_PROJECT_NAME: "heartbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2004-aarch64"
+  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
-  IMAGE_RHEL9: "family/core-rhel-9"
+  IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
 
 steps:

From 1a25aec41a3d3dc049f409e9e1f6c73d06f5c77c Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 26 Feb 2024 15:47:14 +0200
Subject: [PATCH 22/31] fixed arm agent config

---
 .buildkite/auditbeat/auditbeat-pipeline.yml  | 2 +-
 .buildkite/filebeat/filebeat-pipeline.yml    | 2 +-
 .buildkite/heartbeat/heartbeat-pipeline.yml  | 2 +-
 .buildkite/scripts/packaging/package-step.sh | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index c160b973c3a..a4a67e6a506 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "auditbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2204-aarch64"
+  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 77a2b8e7dbd..b6b9a3e4af5 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "filebeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2204-aarch64"
+  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 720bfb6070f..d6bb3b29494 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "heartbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "family/platform-ingest-beats-ubuntu-2204-aarch64"
+  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh
index 6d17e0e9440..db0f38f5c6b 100755
--- a/.buildkite/scripts/packaging/package-step.sh
+++ b/.buildkite/scripts/packaging/package-step.sh
@@ -41,7 +41,7 @@ if are_files_changed "$changeset"; then
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
+          instanceType: "t4g.xlarge"
 YAML
 )
   echo "${bk_pipeline}" | buildkite-agent pipeline upload

From 5c4a3e3132ecb3fcb4897013f2a83ebbb2fb53e2 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 28 Feb 2024 11:58:02 +0200
Subject: [PATCH 23/31] fixed PR according to comments

---
 .buildkite/auditbeat/auditbeat-pipeline.yml   |  4 +-
 .buildkite/auditbeat/scripts/crosscompile.sh  |  2 -
 .buildkite/auditbeat/scripts/package-step.sh  | 51 -----------------
 .buildkite/auditbeat/scripts/package.sh       | 13 -----
 .buildkite/auditbeat/scripts/unit-tests.sh    |  2 -
 .buildkite/env-scripts/linux-env.sh           | 55 -------------------
 .buildkite/env-scripts/util.sh                |  4 +-
 .buildkite/filebeat/filebeat-pipeline.yml     | 10 ++--
 .../filebeat/scripts/integration-gotests.sh   |  2 -
 .../filebeat/scripts/integration-pytests.sh   |  2 -
 .buildkite/filebeat/scripts/unit-tests.sh     |  2 -
 .buildkite/heartbeat/heartbeat-pipeline.yml   |  4 +-
 .../heartbeat/scripts/integration-gotests.sh  |  3 -
 .../heartbeat/scripts/integration-pytests.sh  |  3 -
 .buildkite/heartbeat/scripts/package-step.sh  | 51 -----------------
 .buildkite/heartbeat/scripts/package.sh       | 13 -----
 .buildkite/heartbeat/scripts/unit-tests.sh    |  3 -
 .buildkite/hooks/pre-command                  | 22 ++++----
 .buildkite/scripts/packaging/package-step.sh  |  2 +-
 .buildkite/scripts/packaging/package.sh       |  1 -
 20 files changed, 24 insertions(+), 225 deletions(-)
 delete mode 100755 .buildkite/auditbeat/scripts/package-step.sh
 delete mode 100755 .buildkite/auditbeat/scripts/package.sh
 delete mode 100644 .buildkite/env-scripts/linux-env.sh
 delete mode 100755 .buildkite/heartbeat/scripts/package-step.sh
 delete mode 100755 .buildkite/heartbeat/scripts/package.sh

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index a4a67e6a506..ab98c077fdf 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "auditbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
@@ -89,7 +89,7 @@ steps:
               context: "Auditbeat/Extended: Unit Tests ARM"
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.large"
         artifact_paths: "auditbeat/build/*.xml"
 
diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh
index 866d6be4223..da8452d5380 100755
--- a/.buildkite/auditbeat/scripts/crosscompile.sh
+++ b/.buildkite/auditbeat/scripts/crosscompile.sh
@@ -2,7 +2,5 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Executing Crosscompile"
 make -C auditbeat crosscompile
diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/auditbeat/scripts/package-step.sh
deleted file mode 100755
index cb06895879a..00000000000
--- a/.buildkite/auditbeat/scripts/package-step.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/env-scripts/util.sh
-
-changeset="^auditbeat/
-^go.mod
-^pytest.ini
-^dev-tools/
-^libbeat/
-^testing/
-^\.buildkite/auditbeat/"
-
-if are_files_changed "$changeset"; then
-  bk_pipeline=$(cat <<-YAML
-    steps:
-      - label: ":ubuntu: Packaging Linux X86"
-        key: "package-linux-x86"
-        env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-        command:
-          - ".buildkite/auditbeat/scripts/package.sh"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat/Packaging: Linux X86"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-
-      - label: ":linux: Packaging Linux ARM"
-        key: "package-linux-arm"
-        env:
-          PLATFORMS: "linux/arm64"
-          PACKAGES: "docker"
-        command:
-          - ".buildkite/auditbeat/scripts/package.sh"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat/Packaging: ARM"
-        agents:
-          provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
-YAML
-)
-  echo "${bk_pipeline}" | buildkite-agent pipeline upload
-else
-  buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
-  exit 0
-fi
diff --git a/.buildkite/auditbeat/scripts/package.sh b/.buildkite/auditbeat/scripts/package.sh
deleted file mode 100755
index 71872ca15a3..00000000000
--- a/.buildkite/auditbeat/scripts/package.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/env-scripts/linux-env.sh
-
-echo "--- Docker Version: $(docker --version)"
-
-echo "--- Start Packaging"
-cd auditbeat
-umask 0022
-mage package
-
diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh
index c1f5685c77f..8a7514d747a 100755
--- a/.buildkite/auditbeat/scripts/unit-tests.sh
+++ b/.buildkite/auditbeat/scripts/unit-tests.sh
@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Running Unit Tests"
 sudo chmod -R go-w auditbeat/
 
diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh
deleted file mode 100644
index 7b0f989a55c..00000000000
--- a/.buildkite/env-scripts/linux-env.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/env-scripts/util.sh
-
-DEBIAN_FRONTEND="noninteractive"
-
-set_env() {
-  echo "--- Setting up environment"
-  add_bin_path
-  with_go
-  with_mage
-}
-
-#if [[ $PLATFORM_TYPE == "Linux" ]]; then
-#  check_platform_architecture
-#  echo "ARCH: ${arch_type}"
-#
-#  # Remove this code once beats specific agent is set up
-#  if grep -q 'Ubuntu' /etc/*release && [ "${arch_type}" == "arm64" ]; then
-#    export DEBIAN_FRONTEND
-#
-#    sudo mkdir -p /etc/needrestart
-#    echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf >/dev/null
-#
-#    echo "--- Ubuntu - Installing libs"
-#    sudo apt-get update
-#    sudo apt-get install -y libsystemd-dev
-#    sudo apt install -y python3-pip
-#    sudo apt-get install -y python3-venv
-#
-#    set_env
-#  fi
-#
-#  # Remove this code once beats specific agent is set up
-#  if grep -q 'Red Hat' /etc/*release; then
-#    echo "--- RHL - Installing libs"
-#    sudo yum update -y
-#    sudo yum install -y systemd-devel
-#    sudo yum install -y python3-pip
-#    sudo yum install -y python3
-#    pip3 install virtualenv
-#
-#    set_env
-#  fi
-#fi
-
-if [[ $PLATFORM_TYPE == Darwin* ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
-    ulimit -Sn 30000
-  fi
-
-  set_env
-fi
diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
index 962cb970bf1..8df36296aa2 100644
--- a/.buildkite/env-scripts/util.sh
+++ b/.buildkite/env-scripts/util.sh
@@ -13,7 +13,7 @@ with_go() {
     echo "Setting up the Go environment..."
     create_bin
     check_platform_architecture
-    retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
+    retry_with_count 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
     export PATH="${PATH}:${BIN}"
     chmod +x ${BIN}/gvm
     eval "$(gvm "$go_version")"
@@ -60,7 +60,7 @@ check_platform_architecture() {
   esac
 }
 
-retry() {
+retry_with_count() {
     local retries=$1
     shift
     local count=0
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index b6b9a3e4af5..56c81f9a5b5 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "filebeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
@@ -82,7 +82,7 @@ steps:
     steps:
       - label: ":linux: ARM64 Unit Tests"
         key: "arm-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command:
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
@@ -90,13 +90,13 @@ steps:
               context: "Filebeat/Extended: Unit Tests ARM"
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.large"
         artifact_paths: "filebeat/build/*.xml"
 
       - label: ":mac: MacOS Unit Tests"
         key: "macos-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command:
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
@@ -109,7 +109,7 @@ steps:
 
   - group: "Windows Extended Testing"
     key: "extended-tests-win"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
 
     steps:
       - label: ":windows: Win 2019 Unit Tests"
diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh
index d64ce7c98eb..6de39ff8817 100755
--- a/.buildkite/filebeat/scripts/integration-gotests.sh
+++ b/.buildkite/filebeat/scripts/integration-gotests.sh
@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Executing Integration Tests"
 sudo chmod -R go-w filebeat/
 
diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh
index b51e8ae18a6..9aff8695c35 100755
--- a/.buildkite/filebeat/scripts/integration-pytests.sh
+++ b/.buildkite/filebeat/scripts/integration-pytests.sh
@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Executing Integration Tests"
 sudo chmod -R go-w filebeat/
 
diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh
index 08ce9d4ea1c..2efb6b1ff8e 100755
--- a/.buildkite/filebeat/scripts/unit-tests.sh
+++ b/.buildkite/filebeat/scripts/unit-tests.sh
@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Executing Unit Tests"
 sudo chmod -R go-w filebeat/
 
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index d6bb3b29494..0ed47e8c395 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "heartbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
@@ -95,7 +95,7 @@ steps:
               context: "Heartbeat/Extended: Unit Tests ARM"
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.large"
         artifact_paths: "heartbeat/build/*.xml"
 
diff --git a/.buildkite/heartbeat/scripts/integration-gotests.sh b/.buildkite/heartbeat/scripts/integration-gotests.sh
index 8eab0e8b5d8..c50dbf45347 100755
--- a/.buildkite/heartbeat/scripts/integration-gotests.sh
+++ b/.buildkite/heartbeat/scripts/integration-gotests.sh
@@ -2,9 +2,6 @@
 
 set -euo pipefail
 
-# Remove when custom image is set up
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Executing Integration Tests"
 # Remove when custom image is set up
 sudo chmod -R go-w heartbeat/
diff --git a/.buildkite/heartbeat/scripts/integration-pytests.sh b/.buildkite/heartbeat/scripts/integration-pytests.sh
index 729df5ae6f6..5875b5460ed 100755
--- a/.buildkite/heartbeat/scripts/integration-pytests.sh
+++ b/.buildkite/heartbeat/scripts/integration-pytests.sh
@@ -2,9 +2,6 @@
 
 set -euo pipefail
 
-# Remove when custom image is set up
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Executing Integration Tests"
 # Remove when custom image is set up
 sudo chmod -R go-w heartbeat/
diff --git a/.buildkite/heartbeat/scripts/package-step.sh b/.buildkite/heartbeat/scripts/package-step.sh
deleted file mode 100755
index 03790edfa5f..00000000000
--- a/.buildkite/heartbeat/scripts/package-step.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/env-scripts/util.sh
-
-changeset="^heartbeat/
-^go.mod
-^pytest.ini
-^dev-tools/
-^libbeat/
-^testing/
-^\.buildkite/heartbeat/"
-
-if are_files_changed "$changeset"; then
-  bk_pipeline=$(cat <<-YAML
-    steps:
-      - label: ":ubuntu: Packaging Linux X86"
-        key: "package-linux-x86"
-        env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-        command:
-          - ".buildkite/heartbeat/scripts/package.sh"
-        notify:
-          - github_commit_status:
-              context: "heartbeat/Packaging: Linux X86"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-
-      - label: ":linux: Packaging Linux ARM"
-        key: "package-linux-arm"
-        env:
-          PLATFORMS: "linux/arm64"
-          PACKAGES: "docker"
-        command:
-          - ".buildkite/heartbeat/scripts/package.sh"
-        notify:
-          - github_commit_status:
-              context: "heartbeat/Packaging: ARM"
-        agents:
-          provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
-YAML
-)
-  echo "${bk_pipeline}" | buildkite-agent pipeline upload
-else
-  buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
-  exit 0
-fi
diff --git a/.buildkite/heartbeat/scripts/package.sh b/.buildkite/heartbeat/scripts/package.sh
deleted file mode 100755
index 7f51a6b5ca1..00000000000
--- a/.buildkite/heartbeat/scripts/package.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/env-scripts/linux-env.sh
-
-echo "--- Docker Version: $(docker --version)"
-
-echo "--- Start Packaging"
-cd heartbeat
-umask 0022
-mage package
-
diff --git a/.buildkite/heartbeat/scripts/unit-tests.sh b/.buildkite/heartbeat/scripts/unit-tests.sh
index 4b746da2d57..1d8de945788 100755
--- a/.buildkite/heartbeat/scripts/unit-tests.sh
+++ b/.buildkite/heartbeat/scripts/unit-tests.sh
@@ -2,9 +2,6 @@
 
 set -euo pipefail
 
-# Remove when custom image is set up
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Running Unit Tests"
 # Remove when custom image is set up
 sudo chmod -R go-w heartbeat/
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index d3602c39fa2..7d88b555023 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -32,25 +32,27 @@ retry() {
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
   source .buildkite/env-scripts/env.sh
   source .buildkite/env-scripts/win-env.sh
-#  source .buildkite/env-scripts/util.sh
+  source .buildkite/env-scripts/util.sh
 
   if [[ -z "${GOLANG_VERSION-""}" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
   fi
 
-    if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
     ulimit -Sn 30000
+
+    echo "--- Setting up environment"
+    add_bin_path
+    with_go
+    with_mage
   fi
 
   if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
-    source .buildkite/filebeat/scripts/packaging/packaging-env.sh
-
-    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field=data -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
-    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    export GITHUB_TOKEN_SECRET=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
-    export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
-    export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+    export DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+    export DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+    export GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_EMAIL_SECRET=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
 
     docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
 
diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh
index db0f38f5c6b..79c4d9ee3f3 100755
--- a/.buildkite/scripts/packaging/package-step.sh
+++ b/.buildkite/scripts/packaging/package-step.sh
@@ -40,7 +40,7 @@ if are_files_changed "$changeset"; then
               context: "${BEATS_PROJECT_NAME}/Packaging: ARM"
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.xlarge"
 YAML
 )
diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh
index e1f36238d00..2f60ffa8682 100755
--- a/.buildkite/scripts/packaging/package.sh
+++ b/.buildkite/scripts/packaging/package.sh
@@ -2,7 +2,6 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
 source .buildkite/scripts/packaging/package-util.sh
 
 IMG_POSTFIX="-SNAPSHOT"

From 3cbb8bf66619bacd34cfea5d2bd5b9c9c5811ed2 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 29 Feb 2024 11:56:10 +0200
Subject: [PATCH 24/31] updated machine types and added go flag

---
 .buildkite/auditbeat/auditbeat-pipeline.yml  | 13 ++++++++++---
 .buildkite/filebeat/filebeat-pipeline.yml    | 20 +++++++++++++-------
 .buildkite/heartbeat/heartbeat-pipeline.yml  | 13 ++++++++++---
 .buildkite/hooks/pre-command                 |  9 +++++----
 .buildkite/scripts/packaging/package-step.sh |  5 ++++-
 .buildkite/scripts/packaging/package.sh      |  8 ++++++--
 6 files changed, 48 insertions(+), 20 deletions(-)

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index ab98c077fdf..1d260652e59 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -9,6 +9,10 @@ env:
   IMAGE_WIN_2022: "family/core-windows-2022"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
 
 steps:
   - group: "Auditbeat Mandatory Testing"
@@ -25,6 +29,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
@@ -38,6 +43,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
@@ -50,7 +56,7 @@ steps:
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"
-          machine_type: "n2-standard-8"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
         matrix:
@@ -73,6 +79,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
   - group: "Extended Testing"
     key: "extended-tests"
@@ -90,7 +97,7 @@ steps:
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths: "auditbeat/build/*.xml"
 
       - label: ":mac: MacOS Unit Tests"
@@ -120,7 +127,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
-          machine_type: "n2-standard-8"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
         artifact_paths:
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 56c81f9a5b5..2d79b5c525e 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -8,6 +8,10 @@ env:
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
 
 steps:
   - group: "Filebeat Mandatory Testing"
@@ -24,6 +28,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
@@ -37,6 +42,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
@@ -49,8 +55,8 @@ steps:
               context: "Filebeat: Python Integration Tests"
         agents:
           provider: gcp
-          imageProject: elastic-images-qa
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
@@ -63,7 +69,7 @@ steps:
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"
-          machine_type: "n2-standard-8"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
         matrix:
@@ -82,7 +88,7 @@ steps:
     steps:
       - label: ":linux: ARM64 Unit Tests"
         key: "arm-extended"
-#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command:
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
@@ -91,12 +97,12 @@ steps:
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths: "filebeat/build/*.xml"
 
       - label: ":mac: MacOS Unit Tests"
         key: "macos-extended"
-#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command:
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
@@ -109,7 +115,7 @@ steps:
 
   - group: "Windows Extended Testing"
     key: "extended-tests-win"
-#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
 
     steps:
       - label: ":windows: Win 2019 Unit Tests"
@@ -121,7 +127,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
-          machine_type: "n2-standard-8"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
         artifact_paths:
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 0ed47e8c395..be3c198854a 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -9,6 +9,10 @@ env:
   IMAGE_WIN_2022: "family/core-windows-2022"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
 
 steps:
   - group: "Heartbeat Mandatory Testing"
@@ -25,6 +29,7 @@ steps:
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         matrix:
           setup:
             image:
@@ -43,7 +48,7 @@ steps:
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"
-          machine_type: "n2-standard-8"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_type: "pd-ssd"
         matrix:
           setup:
@@ -63,6 +68,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
@@ -76,6 +82,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
@@ -96,7 +103,7 @@ steps:
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths: "heartbeat/build/*.xml"
 
       - label: ":mac: MacOS Unit Tests"
@@ -126,7 +133,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
-          machine_type: "n2-standard-8"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_type: "pd-ssd"
         artifact_paths:
           - "heartbeat/build/*.xml"
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 7d88b555023..ebbafc024c7 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -51,13 +51,14 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
     export DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
     export DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
     export GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
-    export GITHUB_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
-    export GITHUB_EMAIL_SECRET=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
 
     docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
 
-    git config user.name "${GITHUB_USERNAME_SECRET}"
-    git config user.email "${GITHUB_EMAIL_SECRET}"
+    github_username=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    github_email=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+
+    git config user.name "$github_username"
+    git config user.email "$github_email"
   fi
 fi
 
diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh
index 79c4d9ee3f3..cd00c7f25b1 100755
--- a/.buildkite/scripts/packaging/package-step.sh
+++ b/.buildkite/scripts/packaging/package-step.sh
@@ -20,6 +20,7 @@ if are_files_changed "$changeset"; then
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
+          GOFLAGS: “-buildvcs=false”
         command: ".buildkite/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
@@ -27,6 +28,7 @@ if are_files_changed "$changeset"; then
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
       - label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM"
         key: "package-linux-arm"
@@ -34,6 +36,7 @@ if are_files_changed "$changeset"; then
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
+          GOFLAGS: “-buildvcs=false”
         command: ".buildkite/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
@@ -41,7 +44,7 @@ if are_files_changed "$changeset"; then
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.xlarge"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
 YAML
 )
   echo "${bk_pipeline}" | buildkite-agent pipeline upload
diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh
index 2f60ffa8682..6aaadb744e0 100755
--- a/.buildkite/scripts/packaging/package.sh
+++ b/.buildkite/scripts/packaging/package.sh
@@ -21,13 +21,17 @@ echo "--- Docker image list"
 docker images
 
 define_tags
-check_is_arm
+
+targetSuffix=""
+if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
+  targetSuffix="-arm64"
+fi
 
 for variant in "${VARIANTS[@]}"; do
   source="beats/${BEATS_PROJECT_NAME}${variant}"
 
   for tag in "${tags[@]}"; do
-    targetTag=$tag${is_arm}
+    targetTag=$tag${targetSuffix}
 
     sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
     targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"

From 34e09217d996118183d6cc7fb78091ef149f6f2b Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 29 Feb 2024 12:22:47 +0200
Subject: [PATCH 25/31] updated goflag

---
 .buildkite/scripts/packaging/package-step.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh
index cd00c7f25b1..d87212ef1c3 100755
--- a/.buildkite/scripts/packaging/package-step.sh
+++ b/.buildkite/scripts/packaging/package-step.sh
@@ -20,7 +20,7 @@ if are_files_changed "$changeset"; then
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
-          GOFLAGS: “-buildvcs=false”
+          GOFLAGS: -buildvcs=false
         command: ".buildkite/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
@@ -36,7 +36,7 @@ if are_files_changed "$changeset"; then
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
-          GOFLAGS: “-buildvcs=false”
+          GOFLAGS: -buildvcs=false
         command: ".buildkite/scripts/packaging/package.sh"
         notify:
           - github_commit_status:

From 3c38d134c46e7cc7a89429b996e36d586560ec0f Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 11 Mar 2024 13:25:52 +0200
Subject: [PATCH 26/31] removed flags

---
 .buildkite/scripts/packaging/package-step.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh
index d87212ef1c3..9eddfafcfba 100755
--- a/.buildkite/scripts/packaging/package-step.sh
+++ b/.buildkite/scripts/packaging/package-step.sh
@@ -20,7 +20,6 @@ if are_files_changed "$changeset"; then
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
-          GOFLAGS: -buildvcs=false
         command: ".buildkite/scripts/packaging/package.sh"
         notify:
           - github_commit_status:
@@ -36,7 +35,6 @@ if are_files_changed "$changeset"; then
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
-          GOFLAGS: -buildvcs=false
         command: ".buildkite/scripts/packaging/package.sh"
         notify:
           - github_commit_status:

From e169556da9bf1e90dde13cce6771de5b49703332 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 11 Mar 2024 14:45:03 +0200
Subject: [PATCH 27/31] changed ps1 script paths

---
 .buildkite/auditbeat/auditbeat-pipeline.yml |  4 ++--
 .buildkite/env-scripts/env.sh               |  2 ++
 .buildkite/filebeat/filebeat-pipeline.yml   |  4 ++--
 .buildkite/heartbeat/heartbeat-pipeline.yml |  4 ++--
 .buildkite/hooks/post-checkout              | 12 +++++++-----
 .buildkite/hooks/pre-command                |  5 ++---
 6 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index 1d260652e59..be03d7843b2 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -49,7 +49,7 @@ steps:
           - "auditbeat/build/*.json"
 
       - label: ":windows:-{{matrix.image}} Unit Tests"
-        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
         notify:
           - github_commit_status:
               context: "Auditbeat: windows/Unit Tests"
@@ -120,7 +120,7 @@ steps:
     steps:
       - label: ":windows: Win 2019 Unit Tests"
         key: "win-extended-2019"
-        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
         notify:
           - github_commit_status:
               context: "Auditbeat/Extended: Win-2019 Unit Tests"
diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
index 79a520cc2d5..fd5da02f79d 100644
--- a/.buildkite/env-scripts/env.sh
+++ b/.buildkite/env-scripts/env.sh
@@ -21,6 +21,7 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 SETUP_GVM_VERSION="v0.5.1"
 ASDF_MAGE_VERSION="1.14.0"
 SETUP_MAGE_VERSION="1.14.0"
+SETUP_WIN_PYTHON_VERSION="3.11.0"
 
 # Docker & DockerHub
 DOCKER_COMPOSE_VERSION="1.21.0"
@@ -44,6 +45,7 @@ export XPACK_MODULE_PATTERN
 export SETUP_GVM_VERSION
 export ASDF_MAGE_VERSION
 export SETUP_MAGE_VERSION
+export SETUP_WIN_PYTHON_VERSION
 
 export DOCKER_COMPOSE_VERSION
 export DOCKER_REGISTRY
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 2d79b5c525e..329794de818 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -62,7 +62,7 @@ steps:
           - "filebeat/build/*.json"
 
       - label: ":windows:-{{matrix.image}} Unit Tests"
-        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
         notify:
           - github_commit_status:
               context: "Filebeat: windows/Unit Tests"
@@ -120,7 +120,7 @@ steps:
     steps:
       - label: ":windows: Win 2019 Unit Tests"
         key: "win-extended-2019"
-        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
         notify:
           - github_commit_status:
               context: "Filebeat/Extended: Win-2019 Unit Tests"
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index be3c198854a..2b5f6195f45 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -41,7 +41,7 @@ steps:
 
       - label: ":windows: Unit Tests / {{matrix.image}}"
         command:
-          - ".buildkite/heartbeat/scripts/unit-tests-win.ps1"
+          - ".buildkite/scripts/win_unit_tests.ps1"
         notify:
           - github_commit_status:
               context: "Heartbeat: windows/Unit Tests"
@@ -126,7 +126,7 @@ steps:
     steps:
       - label: ":windows: Win 2019 Unit Tests"
         key: "win-extended-2019"
-        command: ".buildkite/heartbeat/scripts/unit-tests-win.ps1"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
         notify:
           - github_commit_status:
               context: "Heartbeat/Extended: Win-2019 Unit Tests"
diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
index b6cc7ad60bd..d33dd796328 100644
--- a/.buildkite/hooks/post-checkout
+++ b/.buildkite/hooks/post-checkout
@@ -8,7 +8,7 @@ checkout_merge() {
     local merge_branch=$3
 
     if [[ -z "${target_branch}" ]]; then
-        echo "No pull request target branch"
+        echo "--- No pull request target branch"
         exit 1
     fi
 
@@ -24,18 +24,20 @@ checkout_merge() {
     git config user.name "github-merged-pr-post-checkout"
     git config user.email "auto-merge@buildkite"
 
-    git merge --no-edit "${BUILDKITE_COMMIT}" || {
+    git merge --no-edit "${pr_commit}" || {
         local merge_result=$?
-        echo "Merge failed: ${merge_result}"
+        echo "--- Merge failed: ${merge_result}"
         git merge --abort
         exit ${merge_result}
     }
+
+    git reset --hard "${pr_commit}"
 }
 
 pull_request="${BUILDKITE_PULL_REQUEST:-false}"
 
 if [[ "${pull_request}" == "false" ]]; then
-    echo "Not a pull request, skipping"
+    echo "--- Not a pull request, skipping"
     exit 0
 fi
 
@@ -46,7 +48,7 @@ MERGE_BRANCH="pr_merge_${PR_ID}"
 
 checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
 
-echo "Commit information"
+echo "--- Commit information"
 git --no-pager log --format=%B -n 1
 
 # Ensure buildkite groups are rendered
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index ebbafc024c7..9df1bed50a7 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -31,11 +31,10 @@ retry() {
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
   source .buildkite/env-scripts/env.sh
-  source .buildkite/env-scripts/win-env.sh
   source .buildkite/env-scripts/util.sh
 
-  if [[ -z "${GOLANG_VERSION-""}" ]]; then
-    export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
+  if [[ -z "${GO_VERSION-""}" ]]; then
+    export GO_VERSION=$(cat "${WORKSPACE}/.go-version")
   fi
 
   if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then

From 23d34eb9ea7aa68c8c0505808c86d84f539b0d22 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 14 Mar 2024 15:10:12 +0200
Subject: [PATCH 28/31] changed aws arm instance type and updated post checkout

---
 .buildkite/filebeat/filebeat-pipeline.yml | 2 +-
 .buildkite/hooks/post-checkout            | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 329794de818..38fa896d1ce 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -11,7 +11,7 @@ env:
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
 
 steps:
   - group: "Filebeat Mandatory Testing"
diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
index d33dd796328..43881f6e2d8 100644
--- a/.buildkite/hooks/post-checkout
+++ b/.buildkite/hooks/post-checkout
@@ -30,8 +30,6 @@ checkout_merge() {
         git merge --abort
         exit ${merge_result}
     }
-
-    git reset --hard "${pr_commit}"
 }
 
 pull_request="${BUILDKITE_PULL_REQUEST:-false}"

From 0de3efdf8cc9f0800834217de862b52c7480746f Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 14 Mar 2024 16:24:43 +0200
Subject: [PATCH 29/31] updated arm agent to v22

---
 .buildkite/env-scripts/env.sh             |  2 -
 .buildkite/filebeat/filebeat-pipeline.yml |  2 +-
 .buildkite/scripts/packaging/package.sh   | 72 +++++++++++------------
 3 files changed, 37 insertions(+), 39 deletions(-)

diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
index fd5da02f79d..f28658a107d 100644
--- a/.buildkite/env-scripts/env.sh
+++ b/.buildkite/env-scripts/env.sh
@@ -20,7 +20,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 
 SETUP_GVM_VERSION="v0.5.1"
 ASDF_MAGE_VERSION="1.14.0"
-SETUP_MAGE_VERSION="1.14.0"
 SETUP_WIN_PYTHON_VERSION="3.11.0"
 
 # Docker & DockerHub
@@ -44,7 +43,6 @@ export XPACK_MODULE_PATTERN
 
 export SETUP_GVM_VERSION
 export ASDF_MAGE_VERSION
-export SETUP_MAGE_VERSION
 export SETUP_WIN_PYTHON_VERSION
 
 export DOCKER_COMPOSE_VERSION
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index 38fa896d1ce..ae22629e6ec 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -3,7 +3,7 @@
 env:
   BEATS_PROJECT_NAME: "filebeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh
index 6aaadb744e0..9318d100171 100755
--- a/.buildkite/scripts/packaging/package.sh
+++ b/.buildkite/scripts/packaging/package.sh
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/packaging/package-util.sh
+#source .buildkite/scripts/packaging/package-util.sh
 
 IMG_POSTFIX="-SNAPSHOT"
 VARIANTS=("" "-ubi" "-oss")
@@ -13,38 +13,38 @@ TARGET="observability-ci/${BEATS_PROJECT_NAME}"
 echo "--- Creating package"
 mage -d "${BEATS_PROJECT_NAME}" package
 
-echo "--- Distribution list"
-dir="${BEATS_PROJECT_NAME}/build/distributions"
-buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
-
-echo "--- Docker image list"
-docker images
-
-define_tags
-
-targetSuffix=""
-if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
-  targetSuffix="-arm64"
-fi
-
-for variant in "${VARIANTS[@]}"; do
-  source="beats/${BEATS_PROJECT_NAME}${variant}"
-
-  for tag in "${tags[@]}"; do
-    targetTag=$tag${targetSuffix}
-
-    sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
-    targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
-    # Remove following line once beats fully migrated
-    targetName="${targetName}-buildkite"
-
-    if docker image inspect "${sourceName}" &>/dev/null; then
-      echo "--- Tag & Push with target: $targetName"
-      echo "Source name: $sourceName"
-      docker tag "$sourceName" "$targetName"
-      docker push "$targetName"
-    else
-      echo "Docker image ${sourceName} does not exist"
-    fi
-  done
-done
+#echo "--- Distribution list"
+#dir="${BEATS_PROJECT_NAME}/build/distributions"
+#buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
+#
+#echo "--- Docker image list"
+#docker images
+#
+#define_tags
+#
+#targetSuffix=""
+#if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
+#  targetSuffix="-arm64"
+#fi
+#
+#for variant in "${VARIANTS[@]}"; do
+#  source="beats/${BEATS_PROJECT_NAME}${variant}"
+#
+#  for tag in "${tags[@]}"; do
+#    targetTag=$tag${targetSuffix}
+#
+#    sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
+#    targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
+#    # Remove following line once beats fully migrated
+#    targetName="${targetName}-buildkite"
+#
+#    if docker image inspect "${sourceName}" &>/dev/null; then
+#      echo "--- Tag & Push with target: $targetName"
+#      echo "Source name: $sourceName"
+#      docker tag "$sourceName" "$targetName"
+#      docker push "$targetName"
+#    else
+#      echo "Docker image ${sourceName} does not exist"
+#    fi
+#  done
+#done

From c08a01765f864538820789877dffb6aba48a2ddb Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 15 Mar 2024 19:11:49 +0200
Subject: [PATCH 30/31] disabled metricset tests on BK win

---
 auditbeat/module/file_integrity/metricset_test.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go
index 4ad58aa89fa..2a6c33e1798 100644
--- a/auditbeat/module/file_integrity/metricset_test.go
+++ b/auditbeat/module/file_integrity/metricset_test.go
@@ -62,6 +62,7 @@ func TestData(t *testing.T) {
 
 func TestActions(t *testing.T) {
 	skipOnCIForDarwinAMD64(t)
+	skipOnBuildkiteWindows(t)
 
 	defer abtest.SetupDataDir(t)()
 
@@ -154,6 +155,7 @@ func TestActions(t *testing.T) {
 
 func TestExcludedFiles(t *testing.T) {
 	skipOnCIForDarwinAMD64(t)
+	skipOnBuildkiteWindows(t)
 
 	defer abtest.SetupDataDir(t)()
 
@@ -201,6 +203,7 @@ func TestExcludedFiles(t *testing.T) {
 
 func TestIncludedExcludedFiles(t *testing.T) {
 	skipOnCIForDarwinAMD64(t)
+	skipOnBuildkiteWindows(t)
 
 	defer abtest.SetupDataDir(t)()
 
@@ -949,3 +952,9 @@ func skipOnCIForDarwinAMD64(t testing.TB) {
 		t.Skip("Skip test on CI for darwin/amd64")
 	}
 }
+
+func skipOnBuildkiteWindows(t testing.TB) {
+	if os.Getenv("BUILDKITE") == "true" && runtime.GOOS == "windows" {
+		t.Skip("Skip on Buildkite Windows: Shortened TMP problem")
+	}
+}

From 650fac7748e6de1ef5e2a7a64f19f7cf8cd7943a Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 19 Mar 2024 12:57:14 +0200
Subject: [PATCH 31/31] updated pr according to comments

---
 .../auditbeat/scripts/unit-tests-win.ps1      | 51 -------------
 .buildkite/env-scripts/util.sh                |  2 +-
 .../filebeat/scripts/unit-tests-win.ps1       | 51 -------------
 .../heartbeat/scripts/unit-tests-win.ps1      | 51 -------------
 .buildkite/scripts/packaging/package-util.sh  |  8 --
 .buildkite/scripts/packaging/package.sh       | 73 ++++++++++---------
 6 files changed, 38 insertions(+), 198 deletions(-)
 delete mode 100644 .buildkite/auditbeat/scripts/unit-tests-win.ps1
 delete mode 100644 .buildkite/filebeat/scripts/unit-tests-win.ps1
 delete mode 100644 .buildkite/heartbeat/scripts/unit-tests-win.ps1

diff --git a/.buildkite/auditbeat/scripts/unit-tests-win.ps1 b/.buildkite/auditbeat/scripts/unit-tests-win.ps1
deleted file mode 100644
index 200627d518f..00000000000
--- a/.buildkite/auditbeat/scripts/unit-tests-win.ps1
+++ /dev/null
@@ -1,51 +0,0 @@
-$ErrorActionPreference = "Stop" # set -e
-$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually
-
-# Forcing to checkout again all the files with a correct autocrlf.
-# Doing this here because we cannot set git clone options before.
-function fixCRLF() {
-    Write-Host "--- Fixing CRLF in git checkout --"
-    git config core.autocrlf false
-    git rm --quiet --cached -r .
-    git reset --quiet --hard
-}
-
-function withGolang() {
-    Write-Host "--- Install golang $GoVersion --"
-    choco install golang -y --version $GoVersion
-
-    $choco = Convert-Path "$((Get-Command choco).Path)\..\.."
-    Import-Module "$choco\helpers\chocolateyProfile.psm1"
-    refreshenv
-    go version
-    go env
-}
-
-function installGoDependencies() {
-    $installPackages = @(
-        "github.com/magefile/mage"
-        "github.com/elastic/go-licenser"
-        "golang.org/x/tools/cmd/goimports"
-        "github.com/jstemmer/go-junit-report"
-        "github.com/tebeka/go2xunit"
-    )
-    foreach ($pkg in $installPackages) {
-        go install "$pkg"
-    }
-}
-
-fixCRLF
-
-$ErrorActionPreference = "Continue" # set +e
-
-Set-Location -Path auditbeat
-New-Item -ItemType Directory -Force -Path "build"
-withGolang
-installGoDependencies
-
-mage build unitTest
-
-$EXITCODE=$LASTEXITCODE
-$ErrorActionPreference = "Stop"
-
-Exit $EXITCODE
diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
index 8df36296aa2..68fd08a75df 100644
--- a/.buildkite/env-scripts/util.sh
+++ b/.buildkite/env-scripts/util.sh
@@ -9,7 +9,7 @@ add_bin_path() {
 }
 
 with_go() {
-    local go_version="${GOLANG_VERSION}"
+    local go_version="${GO_VERSION}"
     echo "Setting up the Go environment..."
     create_bin
     check_platform_architecture
diff --git a/.buildkite/filebeat/scripts/unit-tests-win.ps1 b/.buildkite/filebeat/scripts/unit-tests-win.ps1
deleted file mode 100644
index 8990eb30a09..00000000000
--- a/.buildkite/filebeat/scripts/unit-tests-win.ps1
+++ /dev/null
@@ -1,51 +0,0 @@
-$ErrorActionPreference = "Stop" # set -e
-$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually
-
-# Forcing to checkout again all the files with a correct autocrlf.
-# Doing this here because we cannot set git clone options before.
-function fixCRLF() {
-    Write-Host "-- Fixing CRLF in git checkout --"
-    git config core.autocrlf false
-    git rm --quiet --cached -r .
-    git reset --quiet --hard
-}
-
-function withGolang() {
-    Write-Host "-- Install golang $GoVersion --"
-    choco install golang -y --version $GoVersion
-
-    $choco = Convert-Path "$((Get-Command choco).Path)\..\.."
-    Import-Module "$choco\helpers\chocolateyProfile.psm1"
-    refreshenv
-    go version
-    go env
-}
-
-function installGoDependencies() {
-    $installPackages = @(
-        "github.com/magefile/mage"
-        "github.com/elastic/go-licenser"
-        "golang.org/x/tools/cmd/goimports"
-        "github.com/jstemmer/go-junit-report"
-        "github.com/tebeka/go2xunit"
-    )
-    foreach ($pkg in $installPackages) {
-        go install "$pkg"
-    }
-}
-
-fixCRLF
-
-$ErrorActionPreference = "Continue" # set +e
-
-Set-Location -Path filebeat
-New-Item -ItemType Directory -Force -Path "build"
-withGolang
-installGoDependencies
-
-mage build unitTest
-
-$EXITCODE=$LASTEXITCODE
-$ErrorActionPreference = "Stop"
-
-Exit $EXITCODE
diff --git a/.buildkite/heartbeat/scripts/unit-tests-win.ps1 b/.buildkite/heartbeat/scripts/unit-tests-win.ps1
deleted file mode 100644
index 17282813e13..00000000000
--- a/.buildkite/heartbeat/scripts/unit-tests-win.ps1
+++ /dev/null
@@ -1,51 +0,0 @@
-$ErrorActionPreference = "Stop" # set -e
-$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually
-
-# Forcing to checkout again all the files with a correct autocrlf.
-# Doing this here because we cannot set git clone options before.
-function fixCRLF() {
-    Write-Host "--- Fixing CRLF in git checkout --"
-    git config core.autocrlf false
-    git rm --quiet --cached -r .
-    git reset --quiet --hard
-}
-
-function withGolang() {
-    Write-Host "--- Install golang $GoVersion --"
-    choco install golang -y --version $GoVersion
-
-    $choco = Convert-Path "$((Get-Command choco).Path)\..\.."
-    Import-Module "$choco\helpers\chocolateyProfile.psm1"
-    refreshenv
-    go version
-    go env
-}
-
-function installGoDependencies() {
-    $installPackages = @(
-        "github.com/magefile/mage"
-        "github.com/elastic/go-licenser"
-        "golang.org/x/tools/cmd/goimports"
-        "github.com/jstemmer/go-junit-report"
-        "github.com/tebeka/go2xunit"
-    )
-    foreach ($pkg in $installPackages) {
-        go install "$pkg"
-    }
-}
-
-fixCRLF
-
-$ErrorActionPreference = "Continue" # set +e
-
-Set-Location -Path heartbeat
-New-Item -ItemType Directory -Force -Path "build"
-withGolang
-installGoDependencies
-
-mage build unitTest
-
-$EXITCODE=$LASTEXITCODE
-$ErrorActionPreference = "Stop"
-
-Exit $EXITCODE
diff --git a/.buildkite/scripts/packaging/package-util.sh b/.buildkite/scripts/packaging/package-util.sh
index 6b91fd55759..4a50457cc9c 100755
--- a/.buildkite/scripts/packaging/package-util.sh
+++ b/.buildkite/scripts/packaging/package-util.sh
@@ -20,11 +20,3 @@ define_tags() {
     tags+=("${SOURCE_TAG}" "${aliasVersion}")
   fi
 }
-
-check_is_arm() {
-  if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
-    is_arm="-arm64"
-  else
-    is_arm=""
-  fi
-}
diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh
index 9318d100171..5744ee0776b 100755
--- a/.buildkite/scripts/packaging/package.sh
+++ b/.buildkite/scripts/packaging/package.sh
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-#source .buildkite/scripts/packaging/package-util.sh
+source .buildkite/scripts/packaging/package-util.sh
 
 IMG_POSTFIX="-SNAPSHOT"
 VARIANTS=("" "-ubi" "-oss")
@@ -13,38 +13,39 @@ TARGET="observability-ci/${BEATS_PROJECT_NAME}"
 echo "--- Creating package"
 mage -d "${BEATS_PROJECT_NAME}" package
 
-#echo "--- Distribution list"
-#dir="${BEATS_PROJECT_NAME}/build/distributions"
-#buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
-#
-#echo "--- Docker image list"
-#docker images
-#
-#define_tags
-#
-#targetSuffix=""
-#if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
-#  targetSuffix="-arm64"
-#fi
-#
-#for variant in "${VARIANTS[@]}"; do
-#  source="beats/${BEATS_PROJECT_NAME}${variant}"
-#
-#  for tag in "${tags[@]}"; do
-#    targetTag=$tag${targetSuffix}
-#
-#    sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
-#    targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
-#    # Remove following line once beats fully migrated
-#    targetName="${targetName}-buildkite"
-#
-#    if docker image inspect "${sourceName}" &>/dev/null; then
-#      echo "--- Tag & Push with target: $targetName"
-#      echo "Source name: $sourceName"
-#      docker tag "$sourceName" "$targetName"
-#      docker push "$targetName"
-#    else
-#      echo "Docker image ${sourceName} does not exist"
-#    fi
-#  done
-#done
+echo "--- Distribution list"
+dir="${BEATS_PROJECT_NAME}/build/distributions"
+buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512"
+
+echo "--- Docker image list"
+docker images
+
+define_tags
+
+targetSuffix=""
+if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then
+  targetSuffix="-arm64"
+fi
+
+for variant in "${VARIANTS[@]}"; do
+  source="beats/${BEATS_PROJECT_NAME}${variant}"
+
+  for tag in "${tags[@]}"; do
+    targetTag=$tag${targetSuffix}
+
+    sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}"
+    targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}"
+    #TODO Remove following line once beats fully migrated to Buildkite and Jenkins builds will be disabled
+    #Avoid clashing with the Jenkins produced images
+    targetName="${targetName}-buildkite"
+
+    if docker image inspect "${sourceName}" &>/dev/null; then
+      echo "--- Tag & Push with target: $targetName"
+      echo "Source name: $sourceName"
+      docker tag "$sourceName" "$targetName"
+      docker push "$targetName"
+    else
+      echo "Docker image ${sourceName} does not exist"
+    fi
+  done
+done