Beats DRA pipeline

.buildkite/packaging.pipeline.md

@@ -0,0 +1,23 @@
+### Beats Packaging pipeline
+[Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches.
+The pipeline definition is located in the `.buildkite/packaging.pipeline.yml`
+
+### Triggers
+Staging packaging DRA is triggered for release branches only.
+Snapshot is triggered for `main` and release branches.
+
+### Pipeline steps
+
+#### Beats dashboard
+Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. This is required by the release-manager configuration.
+
+#### Packaging snapshot/staging
+
+- Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`)
+- Copies artifacts `build/distributions/<beat>/` directory and adds it as an artifact, where `<beat>` is the corresponding beat name.
+- x-pack artifacts are also copied to `build/distributions/<beat>/` directory, where `<beat>` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. It's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle).
+
+#### DRA publish
+Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry.
+
+

.buildkite/packaging.pipeline.yml

@@ -0,0 +1,228 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+# TODO: Pre-cache beats-dev/golang-crossbuild container image
+
+env:
+  ASDF_MAGE_VERSION: 1.15.0
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+
+steps:
+  - group: Beats dashboards
+    key: dashboards
+    steps:
+      - label: Snapshot dashboards
+        # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'"
+        key: dashboards-snapshot
+        # TODO: container with go and make
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        commands:
+          - make build/distributions/dependencies.csv
+          - make beats-dashboards
+        env:
+          SNAPSHOT: true
+        artifact_paths:
+          - build/distributions/**/*
+
+      - label: Staging dashboards
+        if: "build.branch =~ /\\d+\\.\\d+/"
+        key: dashboards-staging
+        # TODO: container with go and make
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        commands:
+          - make build/distributions/dependencies.csv
+          - make beats-dashboards
+        artifact_paths:
+          - build/distributions/**/*
+
+  - group: Packaging snapshot
+    # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'"
+    key: packaging-snapshot
+    steps:
+      # x86
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: true
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - winlogbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/functionbeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/osquerybeat
+          - x-pack/packetbeat
+          - x-pack/winlogbeat
+
+      ## ARM
+      - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+          SNAPSHOT: true
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
+
+      ## Agentbeat needs more CPUs because it builds many other beats
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: true
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - x-pack/agentbeat
+
+  - group: Packaging staging
+
+    key: packaging-staging
+    ## Only for release
+    if: "build.branch =~ /\\d+\\.\\d+/"
+    steps:
+      # x86
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Staging"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: false
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - winlogbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/functionbeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/osquerybeat
+          - x-pack/packetbeat
+          - x-pack/winlogbeat
+
+      ## ARM
+      - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+          SNAPSHOT: false
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - build/distributions/**
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
+
+        ## Agentbeat needs more CPUs because it builds many other beats
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: true
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - x-pack/agentbeat
+
+  - group: DRA publish
+    key: dra
+    steps:
+      - label: DRA Snapshot
+        ## Only for release branches and main
+        # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'"
+        key: dra-snapshot
+        env:
+          DRA_WORKFLOW: snapshot
+        depends_on:
+          - packaging-snapshot
+          - dashboards-snapshot
+        command: |
+          buildkite-agent artifact download "build/**/*" . 
+          .buildkite/scripts/packaging/prepare-release-manager.sh
+          .buildkite/scripts/dra.sh
+        agents:
+          provider: "gcp"
+
+      - label: DRA Staging
+        ## Only for release branches
+        if: "build.branch =~ /\\d+\\.\\d+/"
+        key: dra-staging
+        env:
+          DRA_WORKFLOW: staging
+        depends_on:
+          - packaging-staging
+          - dashboards-staging
+        command: |
+          buildkite-agent artifact download "build/**" .
+          .buildkite/scripts/packaging/prepare-release-manager.sh
+          .buildkite/scripts/dra.sh
+        agents:
+          provider: "gcp"

.buildkite/pull-requests.json

@@ -31,22 +31,6 @@
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": ["^x-pack/elastic-agent/README.md", "^x-pack/elastic-agent/docs/.*", "^x-pack/elastic-agent/devtools/.*" ],
             "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
-        },
-        {
-            "enabled": true,
-            "pipelineSlug": "beats-winlogbeat",
-            "allow_org_users": true,
-            "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [ ],
-            "set_commit_status": true,
-            "build_on_commit": true,
-            "build_on_comment": true,
-            "trigger_comment_regex": "^/test winlogbeat$",
-            "always_trigger_comment_regex": "^/test winlogbeat$",
-            "skip_ci_labels": [  ],
-            "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
-            "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
         }
     ]
 }

.buildkite/scripts/dra.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+## TODO: Set to empty string when Jenkins is disabled
+DRY_RUN="--dry-run"
+BRANCH="${BUILDKITE_BRANCH}"
+
+if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "ci_packaging_pipieline" ]]; then
+    BRANCH=main
+    DRY_RUN="--dry-run"
+    echo "+++ Running in PR and setting branch main and --dry-run"
+fi
+
+BEAT_VERSION=$(make get-version)
+
+CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
+
+function release_manager_login {
+  DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
+  VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
+  VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
+  VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
+  export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET
+}
+
+release_manager_login
+
+echo "+++ Changing permissions for the BK API commands"
+sudo chown -R :1000 build/distributions/
+
+echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..."
+docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli collect \
+        --project "beats" \
+        --branch "${BRANCH}" \
+        --commit "${BUILDKITE_COMMIT}" \
+        --workflow "${DRA_WORKFLOW}" \
+        --version "${BEAT_VERSION}" \
+        --artifact-set "main" \
+        ${DRY_RUN}

.buildkite/scripts/packaging/package-dra.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -ueo pipefail
+BEAT_DIR=${1:-""}
+
+if [ -z "$BEAT_DIR" ]; then
+    echo "Error: Beat directory must be specified."
+    exit 1
+fi
+
+echo "~~~Packaging : $BEAT_DIR"
+
+WORKSPACE=$(pwd)
+BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g')
+
+cd $BEAT_DIR
+mage package
+mage ironbank
+
+mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG
+cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/
+cd $WORKSPACE

.buildkite/scripts/packaging/prepare-release-manager.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# This script is executed by the DRA stage.
+# It prepares the required files to be consumed by the release-manager
+# It can be published as snapshot or staging, for such you use
+# the paramater $0 "snapshot" or $0 "staging"
+#
+set -ueo pipefail
+
+readonly TYPE=${1:-snapshot}
+
+# rename dependencies.csv to the name expected by release-manager.
+VERSION=$(make get-version)
+FINAL_VERSION=$VERSION-SNAPSHOT
+if [ "$TYPE" != "snapshot" ] ; then
+  FINAL_VERSION=$VERSION
+fi
+echo "Rename dependencies to $FINAL_VERSION"
+mv build/distributions/dependencies.csv \
+   build/distributions/dependencies-"$FINAL_VERSION".csv
+
+# rename docker files to support the unified release format.
+# TODO: this could be supported by the package system itself
+#       or the unified release process the one to do the transformation
+#       See https://github.com/elastic/beats/pull/30895
+find build/distributions -name '*linux-arm64.docker.tar.gz*' -print0 |
+  while IFS= read -r -d '' file
+  do
+    echo "Rename file $file"
+    mv "$file" "${file/linux-arm64.docker.tar.gz/docker-image-linux-arm64.tar.gz}"
+  done
+
+find build/distributions -name '*linux-amd64.docker.tar.gz*' -print0 |
+  while IFS= read -r -d '' file
+  do
+    echo "Rename file $file"
+    mv "$file" "${file/linux-amd64.docker.tar.gz/docker-image-linux-amd64.tar.gz}"
+  done
+
+echo 'List all the files'
+find build/distributions -type f -ls || true