diff --git a/.buildkite/packaging.pipeline.md b/.buildkite/packaging.pipeline.md new file mode 100644 index 00000000000..577310d0104 --- /dev/null +++ b/.buildkite/packaging.pipeline.md @@ -0,0 +1,23 @@ +### Beats Packaging pipeline +[Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches. +The pipeline definition is located in the `.buildkite/packaging.pipeline.yml` + +### Triggers +Staging packaging DRA is triggered for release branches only. +Snapshot is triggered for `main` and release branches. + +### Pipeline steps + +#### Beats dashboard +Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. This is required by the release-manager configuration. + +#### Packaging snapshot/staging + +- Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`) +- Copies artifacts `build/distributions//` directory and adds it as an artifact, where `` is the corresponding beat name. +- x-pack artifacts are also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. It's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle). + +#### DRA publish +Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry. + + diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml new file mode 100644 index 00000000000..a07168c235d --- /dev/null +++ b/.buildkite/packaging.pipeline.yml @@ -0,0 +1,249 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +# TODO: Pre-cache beats-dev/golang-crossbuild container image + +env: + ASDF_MAGE_VERSION: 1.15.0 + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS_ARM: "linux/arm64" + +steps: + # we use concurrency gates (https://buildkite.com/blog/concurrency-gates) + # to implement two FIFO queues for DRA-snapshot and DRA-staging + # this prevents parallel builds and possibility of publishing out of order DRA artifacts if the first job takes longer than the second + + - name: Start of concurrency group for DRA Snapshot + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + command: echo "--> Start of concurrency gate dra-snapshot" + concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH" + concurrency: 1 + key: start-gate-snapshot + + - name: Start of concurrency group for DRA Staging + if: build.branch =~ /^\d+\.\d+$$/ + command: echo "--> Start of concurrency gate dra-staging" + concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH" + concurrency: 1 + key: start-gate-staging + + - wait + + - group: Beats dashboards + key: dashboards + steps: + - label: Snapshot dashboards + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + depends_on: start-gate-snapshot + key: dashboards-snapshot + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + env: + SNAPSHOT: true + DEV: true + artifact_paths: + - build/distributions/**/* + + - label: Staging dashboards + if: build.branch =~ /^\d+\.\d+$$/ + depends_on: start-gate-staging + key: dashboards-staging + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + env: + SNAPSHOT: false + DEV: false + artifact_paths: + - build/distributions/**/* + + - group: Packaging snapshot + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + key: packaging-snapshot + depends_on: start-gate-snapshot + steps: + - label: "SNAPSHOT: {{matrix}}" + env: + PLATFORMS: "${PLATFORMS}" + SNAPSHOT: true + DEV: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: true + DEV: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + - group: Packaging staging + + key: packaging-staging + depends_on: start-gate-staging + ## Only for release + if: build.branch =~ /^\d+\.\d+$$/ + steps: + - label: "STAGING: {{matrix}}" + env: + PLATFORMS: "${PLATFORMS}" + SNAPSHOT: false + DEV: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + - label: "STAGING: {{matrix}} docker Linux/arm64" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: false + DEV: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + - group: DRA publish + key: dra + steps: + - label: DRA Snapshot + ## Only for release branches and main + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + key: dra-snapshot + env: + DRA_WORKFLOW: snapshot + depends_on: + - start-gate-snapshot + - packaging-snapshot + - dashboards-snapshot + command: | + buildkite-agent artifact download "build/**/*" . + .buildkite/scripts/packaging/prepare-release-manager.sh snapshot + .buildkite/scripts/dra.sh + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + + - label: DRA Staging + ## Only for release branches + if: build.branch =~ /^\d+\.\d+$$/ + key: dra-staging + env: + DRA_WORKFLOW: staging + depends_on: + - start-gate-staging + - packaging-staging + - dashboards-staging + command: | + buildkite-agent artifact download "build/**" . + .buildkite/scripts/packaging/prepare-release-manager.sh staging + .buildkite/scripts/dra.sh + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + + - wait + + - command: echo "End of concurrency gate dra-snapshot <--" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH" + concurrency: 1 + key: end-gate-snapshot + + - command: echo "End of concurrency gate dra-staging <--" + if: build.branch =~ /^\d+\.\d+$$/ + concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH" + concurrency: 1 + key: end-gate-staging diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh new file mode 100755 index 00000000000..aeee80908a7 --- /dev/null +++ b/.buildkite/scripts/dra.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# TODO: uncomment out below when Jenkins packaging has been stopped +# if [[ "$DRY_RUN" == "true" ]]; then +# echo "~~~ Running in dry-run mode -- will NOT publish artifacts" +# DRY_RUN="--dry-run" +# else +# echo "~~~ Running in publish mode" +# DRY_RUN="" +# fi + +# TODO: delete the conditional below (and replace it with the above, uncommented out, section) after Jenkins packaging has been stopped +if [[ "$DRY_RUN" == "false" ]]; then + echo "~~~ Running in publish mode" + DRY_RUN="" +else + echo "~~~ Running in dry-run mode -- will NOT publish artifacts" + DRY_RUN="--dry-run" +fi + +set -euo pipefail + +# DRA_BRANCH can be used for manually testing packaging with PRs +# e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build +BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" + +BEAT_VERSION=$(make get-version) + +CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" + +function release_manager_login { + DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') + VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') + VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') + export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET +} + +set +x +release_manager_login + +# required by the release-manager docker image, otherwise we hit: +# > java.io.FileNotFoundException: /artifacts/build/distributions/agentbeat/agentbeat-8.15.0-SNAPSHOT-darwin-x86_64.tar.gz.sha512 (Permission denied) +chmod -R a+r build/* +chmod -R a+w build + +echo "+++ :clipboard: Listing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH] and workflow [$DRA_WORKFLOW]" +set +x +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli list \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" + +echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]" + +set +x +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" \ + ${DRY_RUN} | tee rm-output.txt + + +if [[ "$DRY_RUN" != "--dry-run" ]]; then + # extract the summary URL from a release manager output line like: + # Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html + SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') + rm rm-output.txt + + # and make it easily clickable as a Builkite annotation + printf "**${DRA_WORKFLOW} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append +fi diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh new file mode 100755 index 00000000000..10d5425da36 --- /dev/null +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -ueo pipefail +BEAT_DIR=${1:-""} + +if [ -z "$BEAT_DIR" ]; then + echo "Error: Beat directory must be specified." + exit 1 +fi + +echo "~~~ Packaging : $BEAT_DIR" + +WORKSPACE=$(pwd) +BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') + +cd $BEAT_DIR +mage package +mage ironbank + +mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG +cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ +cd $WORKSPACE \ No newline at end of file diff --git a/.buildkite/scripts/packaging/prepare-release-manager.sh b/.buildkite/scripts/packaging/prepare-release-manager.sh new file mode 100755 index 00000000000..2727fd986cc --- /dev/null +++ b/.buildkite/scripts/packaging/prepare-release-manager.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# This script is executed by the DRA stage. +# It prepares the required files to be consumed by the release-manager +# It can be published as snapshot or staging, for such you use +# the paramater $0 "snapshot" or $0 "staging" +# +set -ueo pipefail + +readonly TYPE=${1:-snapshot} + +# rename dependencies.csv to the name expected by release-manager. +VERSION=$(make get-version) +FINAL_VERSION=$VERSION-SNAPSHOT +if [ "$TYPE" != "snapshot" ] ; then + FINAL_VERSION=$VERSION +fi +echo "Rename dependencies to $FINAL_VERSION" +mv build/distributions/dependencies.csv \ + build/distributions/dependencies-"$FINAL_VERSION".csv + +# rename docker files to support the unified release format. +# TODO: this could be supported by the package system itself +# or the unified release process the one to do the transformation +# See https://github.com/elastic/beats/pull/30895 +find build/distributions -name '*linux-arm64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-arm64.docker.tar.gz/docker-image-linux-arm64.tar.gz}" + done + +find build/distributions -name '*linux-amd64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-amd64.docker.tar.gz/docker-image-linux-amd64.tar.gz}" + done + +echo 'List all the files' +find build/distributions -type f -ls || true