Skip to content

Commit

Permalink
Merge branch 'main' into add-aks-cluster-name-id
Browse files Browse the repository at this point in the history
  • Loading branch information
MichaelKatsoulis authored Mar 11, 2024
2 parents 665f9d5 + 6d8882b commit 785135f
Show file tree
Hide file tree
Showing 91 changed files with 1,993 additions and 1,024 deletions.
2 changes: 1 addition & 1 deletion .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
fi
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then
source .buildkite/scripts/setenv.sh
if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
echo "Skipped pre-command when running the Upload pipeline"
Expand Down
84 changes: 67 additions & 17 deletions .buildkite/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ arch_type="$(uname -m)"
GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""}
ONLY_DOCS=${ONLY_DOCS:-"true"}

[ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")"
[ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")"
[ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")"
Expand All @@ -18,9 +17,6 @@ ONLY_DOCS=${ONLY_DOCS:-"true"}
[ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")"
[ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")"
[ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")"
trigger_specific_beat="run_${BEATS_PROJECT_NAME}"
trigger_specific_arm_tests="run_${BEATS_PROJECT_NAME}_arm_tests"
trigger_specific_macos_tests="run_${BEATS_PROJECT_NAME}_macos_tests"

metricbeat_changeset=(
"^metricbeat/.*"
Expand All @@ -38,6 +34,30 @@ winlogbeat_changeset=(
"^winlogbeat/.*"
)

xpack_libbeat_changeset=(
"^x-pack/libbeat/.*"
)

xpack_metricbeat_changeset=(
"^x-pack/metricbeat/.*"
)

xpack_packetbeat_changeset=(
"^x-pack/packetbeat/.*"
)

xpack_winlogbeat_changeset=(
"^x-pack/winlogbeat/.*"
)

ci_changeset=(
"^.buildkite/.*"
)

go_mod_changeset=(
"^go.mod"
)

oss_changeset=(
"^go.mod"
"^pytest.ini"
Expand All @@ -46,14 +66,11 @@ oss_changeset=(
"^testing/.*"
)

ci_changeset=(
"^.buildkite/.*"
xpack_changeset=(
"${xpack_libbeat_changeset[@]}"
"${oss_changeset[@]}"
)

go_mod_changeset=(
"^go.mod"
)

docs_changeset=(
".*\\.(asciidoc|md)"
"deploy/kubernetes/.*-kubernetes\\.yaml"
Expand All @@ -64,6 +81,41 @@ packaging_changeset=(
".go-version"
)

check_and_set_beat_vars() {
if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then
BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove -
BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_} #replace / to _
BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-} #replace / to - for labels
BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME}
TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}"
TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests"
TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests"
declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset"
echo "Beats project name is $BEATS_XPACK_PROJECT_NAME"
mandatory_changeset=(
"${BEAT_CHANGESET_REFERENCE[@]}"
"${xpack_changeset[@]}"
"${ci_changeset[@]}"
)
else
BEATS_GH_LABEL=${BEATS_PROJECT_NAME}
TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}"
TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests"
TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests"
declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset"
echo "Beats project name is $BEATS_PROJECT_NAME"
mandatory_changeset=(
"${BEAT_CHANGESET_REFERENCE[@]}"
"${oss_changeset[@]}"
"${ci_changeset[@]}"
)
fi
BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}"
BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos"
BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm"
BAETS_GH_MACOS_LABEL="macOS"
BAETS_GH_ARM_LABEL="arm"
}

with_docker_compose() {
local version=$1
Expand Down Expand Up @@ -240,11 +292,7 @@ are_changed_only_paths() {
}

are_conditions_met_mandatory_tests() {
declare -n beat_changeset_reference="${BEATS_PROJECT_NAME}_changeset"
if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
return 0
fi
if are_paths_changed "${beat_changeset_reference[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_PROJECT_NAME}/ || "${!trigger_specific_beat}" == "true" ]]; then
if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then
return 0
fi
return 1
Expand All @@ -253,7 +301,7 @@ are_conditions_met_mandatory_tests() {
are_conditions_met_arm_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME} for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${!trigger_specific_arm_tests}" == "true" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
return 0
fi
fi
Expand All @@ -264,7 +312,7 @@ are_conditions_met_arm_tests() {
are_conditions_met_macos_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME} for macos" || "${GITHUB_PR_LABELS}" =~ macOS || "${!trigger_specific_macos_tests}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
return 0
fi
fi
Expand Down Expand Up @@ -302,3 +350,5 @@ fi
if are_paths_changed "${packaging_changeset[@]}" ; then
PACKAGING_CHANGES="true"
fi

check_and_set_beat_vars
127 changes: 127 additions & 0 deletions .buildkite/scripts/generate_xpack_libbeat_pipeline.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
#!/usr/bin/env bash

source .buildkite/scripts/common.sh

set -euo pipefail

pipelineName="pipeline.libbeat-dynamic.yml"

echo "Add the mandatory and extended tests without additional conditions into the pipeline"
if are_conditions_met_mandatory_tests; then
cat > $pipelineName <<- YAML
steps:
- group: "Mandatory Tests"
key: "mandatory-tests"
steps:
- label: ":linux: Ubuntu Unit Tests"
key: "mandatory-linux-unit-test"
command: ".buildkite/scripts/unit_tests.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
- label: ":go: Go Integration Tests"
key: "mandatory-int-test"
command: ".buildkite/scripts/go_int_tests.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
- label: ":python: Python Integration Tests"
key: "mandatory-python-int-test"
command: ".buildkite/scripts/py_int_tests.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
- label: ":windows: Windows Unit Tests - {{matrix.image}}"
command: ".buildkite/scripts/win_unit_tests.ps1"
key: "mandatory-win-unit-tests"
agents:
provider: "gcp"
image: "{{matrix.image}}"
machineType: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
matrix:
setup:
image:
- "${IMAGE_WIN_2016}"
- "${IMAGE_WIN_2022}"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h
- group: "Extended Windows Tests"
key: "extended-win-tests"
steps:
- label: ":windows: Win 2019 Unit Tests"
key: "extended-win-2019-unit-tests"
command: ".buildkite/scripts/win_unit_tests.ps1"
agents:
provider: "gcp"
image: "${IMAGE_WIN_2019}"
machineType: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
- label: ":windows: Windows 10 Unit Tests"
key: "extended-win-10-unit-tests"
command: ".buildkite/scripts/win_unit_tests.ps1"
agents:
provider: "gcp"
image: "${IMAGE_WIN_10}"
machineType: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
- label: ":windows: Windows 11 Unit Tests"
key: "extended-win-11-unit-tests"
command: ".buildkite/scripts/win_unit_tests.ps1"
agents:
provider: "gcp"
image: "${IMAGE_WIN_11}"
machineType: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
YAML
else
echo "The conditions don't match to requirements for generating pipeline steps."
exit 0
fi

echo "Check and add the Extended Tests into the pipeline"
if are_conditions_met_arm_tests; then
cat >> $pipelineName <<- YAML
- group: "Extended Tests"
key: "extended-tests"
steps:
- label: ":linux: Arm64 Unit Tests"
key: "extended-arm64-unit-tests"
command: ".buildkite/scripts/unit_tests.sh"
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
YAML
fi

echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public
cat $pipelineName

echo "--- Loading dynamic steps"
buildkite-agent pipeline upload $pipelineName
3 changes: 2 additions & 1 deletion .buildkite/scripts/setenv.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@ SETUP_GVM_VERSION="v0.5.1"
DOCKER_COMPOSE_VERSION="1.21.0"
DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0"
SETUP_WIN_PYTHON_VERSION="3.11.0"
NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI)
GO_VERSION=$(cat .go-version)


export SETUP_GVM_VERSION
export DOCKER_COMPOSE_VERSION
export DOCKER_COMPOSE_VERSION_AARCH64
export SETUP_WIN_PYTHON_VERSION
export NMAP_WIN_VERSION
export GO_VERSION

exportVars() {
Expand Down
Loading

0 comments on commit 785135f

Please sign in to comment.