migrate libbeat pipeline (#37725)

.buildkite/hooks/pre-command

@@ -12,7 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
   source .buildkite/scripts/setenv.sh
   if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
     echo "Skipped pre-command when running the Upload pipeline"

.buildkite/libbeat/pipeline.libbeat.yml

@@ -1,5 +1,45 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  BEATS_PROJECT_NAME: "libbeat"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+
+  - input: "Input Parameters"
+    key: "input-run-all-stages"
+    fields:
+    - select: "Libbeat - runLibbeat"
+      key: "runLibbeat"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Libbeat - runLibBeatArmTest"
+      key: "runLibBeatArmTest"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":linux: Load dynamic Libbeat pipeline"
+    key: "libbeat-pipeline"
+    command: ".buildkite/scripts/generate_libbeat_pipeline.sh"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_DEFAULT_MACHINE_TYPE}"

.buildkite/metricbeat/pipeline.yml

@@ -10,22 +10,27 @@ env:
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   GO_AGENT_IMAGE: "golang:${GO_VERSION}"
   BEATS_PROJECT_NAME: "metricbeat"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+
 
 steps:
 
   - input: "Input Parameters"
-    key: "input-run-all-stages"
+    key: "runMetricbeat"
     fields:
-    - select: "Metricbeat - runAllStages"
-      key: "runAllStages"
+    - select: "Metricbeat - runMetricbeat"
+      key: "runMetricbeat"
       options:
         - label: "True"
           value: "true"
         - label: "False"
           value: "false"
       default: "false"
-    - select: "Metricbeat - runMacOsTests"
-      key: "UI_MACOS_TESTS"
+    - select: "Metricbeat - runMetricbeatMacOsTests"
+      key: "runMetricbeatMacOsTests"
       options:
         - label: "True"
           value: "true"
@@ -44,3 +49,4 @@ steps:
     agents:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_DEFAULT_MACHINE_TYPE}"

.buildkite/scripts/common.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euo pipefail
 
 WORKSPACE=${WORKSPACE:-"$(pwd)"}
@@ -8,8 +8,11 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]')
 arch_type="$(uname -m)"
 GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
 ONLY_DOCS=${ONLY_DOCS:-"true"}
-UI_MACOS_TESTS="$(buildkite-agent meta-data get UI_MACOS_TESTS --default ${UI_MACOS_TESTS:-"false"})"
-runAllStages="$(buildkite-agent meta-data get runAllStages --default ${runAllStages:-"false"})"
+runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
+runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})"
+runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
+runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})"
+
 metricbeat_changeset=(
   "^metricbeat/.*"
   "^go.mod"
@@ -175,64 +178,54 @@ are_changed_only_paths() {
   local changed_files=$(git diff --name-only HEAD@{1} HEAD)
   if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then
     return 0
-  else
-    return 1
   fi
+  return 1
 }
 
 are_conditions_met_mandatory_tests() {
-  if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then     # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137
-    if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
-      return 0
-    else
-      return 1
+  if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
+        return 0
+      fi
+    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
+        return 0
+      fi
     fi
-  else
-    return 1
   fi
+  return 1
 }
 
-are_conditions_met_extended_tests() {
+are_conditions_met_libbeat_arm_tests() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
-    return 0
-  else
-    return 1
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then
+        return 0
+      fi
+    fi
   fi
+  return 1
 }
 
-are_conditions_met_macos_tests() {
-  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
-    if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
-      return 0
-    else
-      return 1
-    fi
-  else
-    return 1
+are_conditions_met_metricbeat_macos_tests() {
+  if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+    return 0
   fi
+  return 1
 }
 
-are_conditions_met_extended_windows_tests() {
-  if [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]] || [[ "${runAllStages}" == "true" ]]; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
-    if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
-      return 0
-    else
-      return 1
-    fi
-  else
-    return 1
+are_conditions_met_packaging() {
+  if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
+    return 0
   fi
+  return 1
 }
 
-are_conditions_met_packaging() {
-  if are_conditions_met_extended_windows_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
-    if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
-      return 0
-    else
-      return 1
-    fi
-  else
-    return 1
+config_git() {
+  if [ -z "$(git config --get user.email)" ]; then
+    git config --global user.email "[email protected]"
+    git config --global user.name "beatsmachine"
   fi
 }
 
@@ -249,4 +242,4 @@ fi
 
 if are_paths_changed "${packaging_changeset[@]}" ; then
   PACKAGING_CHANGES="true"
-fi
+fi

.buildkite/scripts/generate_libbeat_pipeline.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+pipelineName="pipeline.libbeat-dynamic.yml"
+
+echo "Add the mandatory and extended tests without additional conditions into the pipeline"
+if are_conditions_met_mandatory_tests; then
+  cat > $pipelineName <<- YAML
+
+steps:
+
+  - group: "Mandatory Tests"
+    key: "mandatory-tests"
+    steps:
+      - label: ":linux: Ubuntu Unit Tests"
+        key: "mandatory-linux-unit-test"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":go: Go Integration Tests"
+        key: "mandatory-int-test"
+        command: ".buildkite/scripts/go_int_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":python: Python Integration Tests"
+        key: "mandatory-python-int-test"
+        command: ".buildkite/scripts/py_int_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":negative_squared_cross_mark: Cross compile"
+        key: "mandatory-cross-compile"
+        command: ".buildkite/scripts/crosscompile.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+        artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":testengine: Stress Tests"
+        key: "mandatory-stress-test"
+        command: ".buildkite/scripts/stress_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml"
+
+YAML
+fi
+
+echo "Check and add the Extended Tests into the pipeline"
+if are_conditions_met_libbeat_arm_tests; then
+  cat >> $pipelineName <<- YAML
+
+  - group: "Extended Tests"
+    key: "extended-tests"
+    steps:
+      - label: ":linux: Arm64 Unit Tests"
+        key: "extended-arm64-unit-tests"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+YAML
+fi
+
+echo "--- Printing dynamic steps"     #TODO: remove if the pipeline is public
+cat $pipelineName
+
+echo "--- Loading dynamic steps"
+buildkite-agent pipeline upload $pipelineName