Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Snapshot packaging stage migration #37827

Merged
merged 31 commits into from
Mar 20, 2024
Merged

Snapshot packaging stage migration #37827

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
271b27b
wip: getting GH message
oakrizan Dec 4, 2023
9f1e8a3
added test scripts
oakrizan Dec 7, 2023
b7f40a3
added windows tests
oakrizan Dec 13, 2023
292b053
added packaging step
oakrizan Jan 3, 2024
a42085b
updated packaging execution conditions
oakrizan Jan 4, 2024
8df62da
win-test failure: updated artifact path
oakrizan Jan 9, 2024
b8967a3
pr fixes
oakrizan Jan 15, 2024
22c1a50
changed group_test.go
oakrizan Jan 16, 2024
0237627
moved env setup to separate script
oakrizan Jan 17, 2024
fec7219
added dynamic step for packaging
oakrizan Jan 17, 2024
8d37337
added image tag and push
oakrizan Feb 1, 2024
5c177ba
changed ubuntu agent
oakrizan Feb 6, 2024
f24bd02
added secrets
oakrizan Feb 6, 2024
efa94e8
added tag definition
oakrizan Feb 6, 2024
6c5b300
added docker img params
oakrizan Feb 9, 2024
0fade9e
added tag and push
oakrizan Feb 12, 2024
e7c6584
added distribution upload
oakrizan Feb 14, 2024
6708eed
made packaging reusable
oakrizan Feb 15, 2024
55e308d
added packaging to heartbeat and auditbeat
oakrizan Feb 15, 2024
2488a59
changed arm agent
oakrizan Feb 26, 2024
6e5ab28
changed rhel9 agent
oakrizan Feb 26, 2024
1a25aec
fixed arm agent config
oakrizan Feb 26, 2024
5c4a3e3
fixed PR according to comments
oakrizan Feb 28, 2024
3cbb8bf
updated machine types and added go flag
oakrizan Feb 29, 2024
34e0921
updated goflag
oakrizan Feb 29, 2024
3c38d13
removed flags
oakrizan Mar 11, 2024
e169556
changed ps1 script paths
oakrizan Mar 11, 2024
23d34eb
changed aws arm instance type and updated post checkout
oakrizan Mar 14, 2024
0de3efd
updated arm agent to v22
oakrizan Mar 14, 2024
c08a017
disabled metricset tests on BK win
oakrizan Mar 15, 2024
650fac7
updated pr according to comments
oakrizan Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions .buildkite/auditbeat/auditbeat-pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json

env:
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
BEATS_PROJECT_NAME: "auditbeat"
IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
IMAGE_WIN_2016: "family/core-windows-2016"
IMAGE_WIN_2019: "family/core-windows-2019"
IMAGE_WIN_2022: "family/core-windows-2022"
IMAGE_RHEL9: "family/core-rhel-9"
IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
GCP_WIN_MACHINE_TYPE: "n2-standard-8"
AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"

steps:
- group: "Auditbeat Mandatory Testing"
Expand All @@ -24,6 +29,7 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths:
- "auditbeat/build/*.xml"
- "auditbeat/build/*.json"
Expand All @@ -37,19 +43,20 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_RHEL9}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths:
- "auditbeat/build/*.xml"
- "auditbeat/build/*.json"

- label: ":windows:-{{matrix.image}} Unit Tests"
command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
command: ".buildkite/scripts/win_unit_tests.ps1"
notify:
- github_commit_status:
context: "Auditbeat: windows/Unit Tests"
agents:
provider: "gcp"
image: "{{matrix.image}}"
machine_type: "n2-standard-8"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 200
disk_type: "pd-ssd"
matrix:
Expand All @@ -72,6 +79,7 @@ steps:
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"

- group: "Extended Testing"
key: "extended-tests"
Expand All @@ -88,8 +96,8 @@ steps:
context: "Auditbeat/Extended: Unit Tests ARM"
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "t4g.large"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths: "auditbeat/build/*.xml"

- label: ":mac: MacOS Unit Tests"
Expand All @@ -112,14 +120,14 @@ steps:
steps:
- label: ":windows: Win 2019 Unit Tests"
key: "win-extended-2019"
command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
command: ".buildkite/scripts/win_unit_tests.ps1"
notify:
- github_commit_status:
context: "Auditbeat/Extended: Win-2019 Unit Tests"
agents:
provider: "gcp"
image: "${IMAGE_WIN_2019}"
machine_type: "n2-standard-8"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 200
disk_type: "pd-ssd"
artifact_paths:
Expand All @@ -134,4 +142,4 @@ steps:

steps:
- label: Package pipeline
commands: ".buildkite/auditbeat/scripts/package-step.sh"
commands: ".buildkite/scripts/packaging/package-step.sh"
2 changes: 0 additions & 2 deletions .buildkite/auditbeat/scripts/crosscompile.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,5 @@

set -euo pipefail

source .buildkite/env-scripts/linux-env.sh

echo "--- Executing Crosscompile"
make -C auditbeat crosscompile
13 changes: 0 additions & 13 deletions .buildkite/auditbeat/scripts/package.sh
View file
Open in desktop

This file was deleted.

51 changes: 0 additions & 51 deletions .buildkite/auditbeat/scripts/unit-tests-win.ps1
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions .buildkite/auditbeat/scripts/unit-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

set -euo pipefail

source .buildkite/env-scripts/linux-env.sh

echo "--- Running Unit Tests"
sudo chmod -R go-w auditbeat/

Expand Down
42 changes: 38 additions & 4 deletions .buildkite/env-scripts/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,53 @@
#!/usr/bin/env bash

SETUP_GVM_VERSION="v0.5.1"
source .buildkite/env-scripts/util.sh

DOCS_CHANGESET="^.*\.(asciidoc|md)$
deploy/kubernetes/.*-kubernetes.yaml"
PACKAGING_CHANGESET="^dev-tools/packaging/
^.go-version"

REPO="beats"
WORKSPACE="$(pwd)"
BIN="${WORKSPACE}/bin"
HW_TYPE="$(uname -m)"
PLATFORM_TYPE="$(uname)"
REPO="beats"
TMP_FOLDER="tmp.${REPO}"
SNAPSHOT="true"
PYTEST_ADDOPTS=""
OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"

SETUP_GVM_VERSION="v0.5.1"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we still need this after using our custom vm images? I suppose it's for modules that haven't been switched to the new approach yet, right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's needed for MacOS for the current not refactored version of env setup scripts.

ASDF_MAGE_VERSION="1.14.0"
SETUP_WIN_PYTHON_VERSION="3.11.0"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same question here, when we start using the custom windows vm images, this shouldn't be required.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could be removed once we will switch to Beats win images. I thought it would be better to have win image change in a separate PR, since the current one is already overwhelmed with the changes. WDYT?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if things work in the current hybrid state i.e. some tests using the vanilla images and some others the custom ones, I am good for us to address in the near future.


# Docker & DockerHub
DOCKER_COMPOSE_VERSION="1.21.0"
DOCKER_REGISTRY="docker.elastic.co"

export SETUP_GVM_VERSION
ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET")
PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET")
GO_MOD_CHANGES=$(changeset_applies "^go.mod")

export REPO
export WORKSPACE
export BIN
export HW_TYPE
export PLATFORM_TYPE
export REPO
export TMP_FOLDER
export SNAPSHOT
export PYTEST_ADDOPTS
export OSS_MODULE_PATTERN
export XPACK_MODULE_PATTERN

export SETUP_GVM_VERSION
export ASDF_MAGE_VERSION
export SETUP_WIN_PYTHON_VERSION

export DOCKER_COMPOSE_VERSION
export DOCKER_REGISTRY

export ONLY_DOCS
export PACKAGING_CHANGES
export GO_MOD_CHANGES
45 changes: 0 additions & 45 deletions .buildkite/env-scripts/linux-env.sh
View file
Open in desktop

This file was deleted.

26 changes: 13 additions & 13 deletions .buildkite/env-scripts/util.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ add_bin_path() {
}

with_go() {
local go_version="${GOLANG_VERSION}"
local go_version="${GO_VERSION}"
echo "Setting up the Go environment..."
create_bin
check_platform_architecture
retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
retry_with_count 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
export PATH="${PATH}:${BIN}"
chmod +x ${BIN}/gvm
eval "$(gvm "$go_version")"
Expand Down Expand Up @@ -60,7 +60,7 @@ check_platform_architecture() {
esac
}

retry() {
retry_with_count() {
v1v marked this conversation as resolved.
Show resolved Hide resolved
local retries=$1
shift
local count=0
Expand Down Expand Up @@ -89,16 +89,16 @@ are_files_changed() {
fi
}

cleanup() {
echo "Deleting temporary files..."
rm -rf ${BIN}/${TMP_FOLDER}.*
echo "Done."
changeset_applies() {
local changeset=$1
if are_files_changed "$changeset"; then
echo true
else
echo false
fi
}

unset_secrets () {
for var in $(printenv | sed 's;=.*;;' | sort); do
if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
unset "$var"
fi
done
set_git_config() {
git config user.name "${GITHUB_USERNAME_SECRET}"
git config user.email "${GITHUB_EMAIL_SECRET}"
}
Loading
Loading