From e2eccd187c3483088498afaade920cafb2b1f0e5 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 19 Apr 2024 16:16:06 +0200 Subject: [PATCH 01/34] Packaging pipieline resource --- catalog-info.yaml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index c9fe5b4f05c3..e41b1ebfb578 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -983,11 +983,11 @@ spec: apiVersion: backstage.io/v1alpha1 kind: Resource metadata: - name: beats-xpack-agentbeat-package - description: Buildkite pipeline for packaging and publishing agentbeat + name: beats-packaging-pipeline + description: Buildkite pipeline for packaging and publishing to DRA links: - title: Pipeline - url: https://buildkite.com/elastic/beats-xpack-agentbeat-package + url: https://buildkite.com/elastic/beats-packaging-pipeline spec: type: buildkite-pipeline owner: group:ingest-fp @@ -996,19 +996,16 @@ spec: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - name: beats-xpack-agentbeat-package - description: Buildkite pipeline for packaging and publishing agentbeat + name: beats-packaging-pipeline + description: Pipeline for Beats packaging and publishing spec: repository: elastic/beats - pipeline_file: ".buildkite/x-pack/agentbeat/pipeline.xpack.agentbeat.package.yml" - # todo release branched must be 8.14+ - branch_configuration: "main 8.14" + pipeline_file: ".buildkite/packaging.pipeline.yml" + branch_configuration: "main 8.*" cancel_intermediate_builds: false skip_intermediate_builds: false provider_settings: - trigger_mode: code - build_pull_requests: false - build_branches: true + trigger_mode: none teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ From cbaba3ef83a10cfa13757574b65e74248de27460 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 22 Apr 2024 16:16:36 +0200 Subject: [PATCH 02/34] Revetred agentbeat pipeline deletion --- .buildkite/packaging.pipeline.yml | 120 ++++++++++++++++++++++++++++++ catalog-info.yaml | 41 +++++++++- 2 files changed, 160 insertions(+), 1 deletion(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index e69de29bb2d1..5cab334e50aa 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -0,0 +1,120 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +# Generate steps +# Linux step (Parallel) +# +# ARM Step (Parallel) +# beats-dashboards (Parallel) +# + +env: + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + +steps: + - group: Beats dashboards + key: dashboards + steps: + - label: Snapshot dashboards + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:0.1" + cpu: "8" + memory: "16Gi" + ephemeralStorage: "200Gi" + commands: + - make build/distributions/dependencies.csv + - beats-dashboards + env: + SNAPSHOT: true + artifact_paths: + - build/distributions/** + + - label: Staging dashboards + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:0.1" + cpu: "8" + memory: "16Gi" + ephemeralStorage: "200Gi" + commands: + - make build/distributions/dependencies.csv + - beats-dashboards + artifact_paths: + - build/distributions/** + + - group: Packaging snapshot + key: packaging-snapshot + steps: + - label: ":ubuntu: Auditbeat/Packaging Linux" + key: "auditbeat-package-linux-x86-snapshot" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: | + cd auditbeat + mage package + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/** + + - label: ":linux: Auditbeat/Packaging Linux arm64" + key: "auditbeat-package-linux-arm64" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: | + cd auditbeat + mage package + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + + - group: Packaging staging + key: packaging-staging + steps: + - label: ":ubuntu: Auditbeat/Packaging Linux" + key: "auditbeat-package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: | + cd auditbeat + mage package + mage ironbank + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/** + + - label: ":linux: Auditbeat/Packaging Linux arm64" + key: "auditbeat-package-linux-arm64" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: | + cd auditbeat + mage package + mage ironbank + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + + - group: DRA publish + key: dra + steps: + - label: DRA Snapshot + key: dra-snapshot + depends_on: + - packaging-snapshot diff --git a/catalog-info.yaml b/catalog-info.yaml index e41b1ebfb578..0bb9eeb88e1e 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -978,6 +978,45 @@ spec: everyone: access_level: READ_ONLY +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: beats-xpack-agentbeat-package + description: Buildkite pipeline for packaging and publishing agentbeat + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-xpack-agentbeat-package +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-xpack-agentbeat-package + description: Buildkite pipeline for packaging and publishing agentbeat + spec: + repository: elastic/beats + pipeline_file: ".buildkite/x-pack/agentbeat/pipeline.xpack.agentbeat.package.yml" + # todo release branched must be 8.14+ + branch_configuration: "main 8.14" + cancel_intermediate_builds: false + skip_intermediate_builds: false + provider_settings: + trigger_mode: code + build_pull_requests: false + build_branches: true + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + release-eng: + access_level: BUILD_AND_READ + everyone: + access_level: READ_ONLY + --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 @@ -997,7 +1036,7 @@ spec: kind: Pipeline metadata: name: beats-packaging-pipeline - description: Pipeline for Beats packaging and publishing + description: Pipeline for Beats packaging and publishing DRA artifacts spec: repository: elastic/beats pipeline_file: ".buildkite/packaging.pipeline.yml" From 7dc47b37265460ae30a86302e1a3351cdf4409b4 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 22 Apr 2024 16:19:01 +0200 Subject: [PATCH 03/34] Cleanup --- .buildkite/packaging.pipeline.yml | 120 ------------------------------ catalog-info.yaml | 2 +- 2 files changed, 1 insertion(+), 121 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 5cab334e50aa..e69de29bb2d1 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -1,120 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json - -# Generate steps -# Linux step (Parallel) -# -# ARM Step (Parallel) -# beats-dashboards (Parallel) -# - -env: - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - -steps: - - group: Beats dashboards - key: dashboards - steps: - - label: Snapshot dashboards - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:0.1" - cpu: "8" - memory: "16Gi" - ephemeralStorage: "200Gi" - commands: - - make build/distributions/dependencies.csv - - beats-dashboards - env: - SNAPSHOT: true - artifact_paths: - - build/distributions/** - - - label: Staging dashboards - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:0.1" - cpu: "8" - memory: "16Gi" - ephemeralStorage: "200Gi" - commands: - - make build/distributions/dependencies.csv - - beats-dashboards - artifact_paths: - - build/distributions/** - - - group: Packaging snapshot - key: packaging-snapshot - steps: - - label: ":ubuntu: Auditbeat/Packaging Linux" - key: "auditbeat-package-linux-x86-snapshot" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true - command: | - cd auditbeat - mage package - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - build/distributions/** - - - label: ":linux: Auditbeat/Packaging Linux arm64" - key: "auditbeat-package-linux-arm64" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - SNAPSHOT: true - command: | - cd auditbeat - mage package - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - build/distributions/** - - - group: Packaging staging - key: packaging-staging - steps: - - label: ":ubuntu: Auditbeat/Packaging Linux" - key: "auditbeat-package-linux-x86" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true - command: | - cd auditbeat - mage package - mage ironbank - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - build/distributions/** - - - label: ":linux: Auditbeat/Packaging Linux arm64" - key: "auditbeat-package-linux-arm64" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - SNAPSHOT: true - command: | - cd auditbeat - mage package - mage ironbank - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - build/distributions/** - - - group: DRA publish - key: dra - steps: - - label: DRA Snapshot - key: dra-snapshot - depends_on: - - packaging-snapshot diff --git a/catalog-info.yaml b/catalog-info.yaml index 0bb9eeb88e1e..32856d53c35b 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -1040,7 +1040,7 @@ spec: spec: repository: elastic/beats pipeline_file: ".buildkite/packaging.pipeline.yml" - branch_configuration: "main 8.*" + branch_configuration: "main 8.* 7.17" cancel_intermediate_builds: false skip_intermediate_builds: false provider_settings: From ed599da94b9ad69b355744819712b68088d1d207 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 22 Apr 2024 16:48:44 +0200 Subject: [PATCH 04/34] Test auditbeat packaging pipeline --- .buildkite/packaging.pipeline.yml | 130 ++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index e69de29bb2d1..9613772d003f 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -0,0 +1,130 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +# Generate steps +# Linux step (Parallel) +# +# ARM Step (Parallel) +# beats-dashboards (Parallel) +# + +env: + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + +steps: + - group: Beats dashboards + key: dashboards + steps: + - label: Snapshot dashboards + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:0.1" + cpu: "8" + memory: "16Gi" + ephemeralStorage: "200Gi" + commands: + - make build/distributions/dependencies.csv + - beats-dashboards + env: + SNAPSHOT: true + artifact_paths: + - build/distributions/** + + - label: Staging dashboards + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:0.1" + cpu: "8" + memory: "16Gi" + ephemeralStorage: "200Gi" + commands: + - make build/distributions/dependencies.csv + - beats-dashboards + artifact_paths: + - build/distributions/** + + - group: Packaging snapshot + key: packaging-snapshot + steps: + - label: ":ubuntu: Auditbeat/Packaging Linux" + key: "auditbeat-package-linux-x86-snapshot" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: | + cd auditbeat + mage package + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/** + + - label: ":linux: Auditbeat/Packaging Linux arm64" + key: "auditbeat-package-linux-arm64" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: | + cd auditbeat + mage package + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + + - group: Packaging staging + key: packaging-staging + steps: + - label: ":ubuntu: Auditbeat/Packaging Linux" + key: "auditbeat-package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: | + cd auditbeat + mage package + mage ironbank + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/** + + - label: ":linux: Auditbeat/Packaging Linux arm64" + key: "auditbeat-package-linux-arm64" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: | + cd auditbeat + mage package + mage ironbank + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + + - group: DRA publish + key: dra + steps: + - label: DRA Snapshot + key: dra-snapshot + depends_on: + - packaging-snapshot + command: | + buildkite-agent artifact download "build/**" . + ls - lah build/distributions + - label: DRA Staging + key: dra-snapshot + depends_on: + - packaging-staging + command: | + buildkite-agent artifact download "build/**" . + ls - lah build/distributions From 881fd97be153da5ea177365f222aeaa467f7cdc0 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 22 Apr 2024 16:53:14 +0200 Subject: [PATCH 05/34] Fix steps keys --- .buildkite/packaging.pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 9613772d003f..3e600f1749c8 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -60,7 +60,7 @@ steps: - build/distributions/** - label: ":linux: Auditbeat/Packaging Linux arm64" - key: "auditbeat-package-linux-arm64" + key: "auditbeat-package-linux-arm64-snapshot" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" @@ -79,7 +79,7 @@ steps: key: packaging-staging steps: - label: ":ubuntu: Auditbeat/Packaging Linux" - key: "auditbeat-package-linux-x86" + key: "auditbeat-package-linux-x86-staging" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true @@ -95,7 +95,7 @@ steps: - build/distributions/** - label: ":linux: Auditbeat/Packaging Linux arm64" - key: "auditbeat-package-linux-arm64" + key: "auditbeat-package-linux-arm64-staging" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" From 75b4db9d38d276de4914d131be2f2616dde11eef Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 22 Apr 2024 16:54:53 +0200 Subject: [PATCH 06/34] Fix steps keys --- .buildkite/packaging.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 3e600f1749c8..a1e30781610c 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -122,7 +122,7 @@ steps: buildkite-agent artifact download "build/**" . ls - lah build/distributions - label: DRA Staging - key: dra-snapshot + key: dra-staging depends_on: - packaging-staging command: | From 720e1295f13b7bef4f535e54c6e8149dd504d7dd Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 22 Apr 2024 17:32:41 +0200 Subject: [PATCH 07/34] Fix env vars --- .buildkite/packaging.pipeline.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index a1e30781610c..dae3f42d173f 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -8,6 +8,9 @@ # env: + ASDF_MAGE_VERSION: 1.15.0 + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" @@ -16,11 +19,11 @@ steps: key: dashboards steps: - label: Snapshot dashboards + # TODO: container with go and make agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:0.1" - cpu: "8" - memory: "16Gi" - ephemeralStorage: "200Gi" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - beats-dashboards @@ -30,11 +33,11 @@ steps: - build/distributions/** - label: Staging dashboards + # TODO: container with go and make agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:0.1" - cpu: "8" - memory: "16Gi" - ephemeralStorage: "200Gi" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - beats-dashboards From 75e445bce047fa192e1383aca8603e55e2df08ed Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 22 Apr 2024 17:53:26 +0200 Subject: [PATCH 08/34] Fix env vars --- .buildkite/packaging.pipeline.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index dae3f42d173f..27c847ae0952 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -6,6 +6,7 @@ # ARM Step (Parallel) # beats-dashboards (Parallel) # +# TODO: Pre-cache beats-dev/golang-crossbuild container image env: ASDF_MAGE_VERSION: 1.15.0 @@ -26,7 +27,7 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - - beats-dashboards + - make beats-dashboards env: SNAPSHOT: true artifact_paths: @@ -40,7 +41,7 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - - beats-dashboards + - make beats-dashboards artifact_paths: - build/distributions/** @@ -60,7 +61,7 @@ steps: image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - - build/distributions/** + - auditbeat/build/distributions/** - label: ":linux: Auditbeat/Packaging Linux arm64" key: "auditbeat-package-linux-arm64-snapshot" @@ -76,7 +77,7 @@ steps: imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - - build/distributions/** + - auditbeat/build/distributions/** - group: Packaging staging key: packaging-staging @@ -95,7 +96,7 @@ steps: image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - - build/distributions/** + - auditbeat/build/distributions/** - label: ":linux: Auditbeat/Packaging Linux arm64" key: "auditbeat-package-linux-arm64-staging" @@ -112,7 +113,7 @@ steps: imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - - build/distributions/** + - auditbeat/build/distributions/** - group: DRA publish key: dra @@ -122,12 +123,14 @@ steps: depends_on: - packaging-snapshot command: | - buildkite-agent artifact download "build/**" . - ls - lah build/distributions + buildkite-agent artifact download "**build/**" . + ls -lah + ls -lah build/distributions - label: DRA Staging key: dra-staging depends_on: - packaging-staging command: | - buildkite-agent artifact download "build/**" . - ls - lah build/distributions + buildkite-agent artifact download ""**build/**"" . + ls -lah + ls -lah build/distributions From 5790718799b93cbaca92b6593abf1628b75c5d22 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 22 Apr 2024 18:24:33 +0200 Subject: [PATCH 09/34] Fix env vars --- .buildkite/packaging.pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 27c847ae0952..64221bdfd8fc 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -86,7 +86,7 @@ steps: key: "auditbeat-package-linux-x86-staging" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true + SNAPSHOT: false command: | cd auditbeat mage package @@ -103,7 +103,7 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - SNAPSHOT: true + SNAPSHOT: false command: | cd auditbeat mage package From fefe6745ca9d6f240df8a2c2f0de4d117f7bfaa0 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 12:56:43 +0200 Subject: [PATCH 10/34] Unified artifacts dir --- .buildkite/packaging.pipeline.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 64221bdfd8fc..f1f71a504499 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -2,7 +2,6 @@ # Generate steps # Linux step (Parallel) -# # ARM Step (Parallel) # beats-dashboards (Parallel) # @@ -56,12 +55,14 @@ steps: command: | cd auditbeat mage package + mage ironbank + mv auditbeat/build build agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - - auditbeat/build/distributions/** + - build/distributions/** - label: ":linux: Auditbeat/Packaging Linux arm64" key: "auditbeat-package-linux-arm64-snapshot" @@ -72,12 +73,14 @@ steps: command: | cd auditbeat mage package + mage ironbank + mv auditbeat/build build agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - - auditbeat/build/distributions/** + - build/distributions/** - group: Packaging staging key: packaging-staging @@ -91,12 +94,13 @@ steps: cd auditbeat mage package mage ironbank + mv auditbeat/build build agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - - auditbeat/build/distributions/** + - build/distributions/** - label: ":linux: Auditbeat/Packaging Linux arm64" key: "auditbeat-package-linux-arm64-staging" @@ -108,12 +112,13 @@ steps: cd auditbeat mage package mage ironbank + mv auditbeat/build build agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - - auditbeat/build/distributions/** + - build/distributions/** - group: DRA publish key: dra @@ -123,14 +128,15 @@ steps: depends_on: - packaging-snapshot command: | - buildkite-agent artifact download "**build/**" . - ls -lah + buildkite-agent artifact download "build/**" . + ls -lah build ls -lah build/distributions + - label: DRA Staging key: dra-staging depends_on: - packaging-staging command: | - buildkite-agent artifact download ""**build/**"" . - ls -lah + buildkite-agent artifact download "build/**" . + ls -lah build ls -lah build/distributions From 3e312b69673d56bec6ff02e4997a04a24f539e15 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 14:09:27 +0200 Subject: [PATCH 11/34] Implemented DRA steps --- .buildkite/packaging.pipeline.yml | 6 +++ .buildkite/scripts/dra.sh | 61 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 .buildkite/scripts/dra.sh diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index f1f71a504499..2ee30eb4f376 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -125,18 +125,24 @@ steps: steps: - label: DRA Snapshot key: dra-snapshot + env: + DRA_WORKFLOW: snapshot depends_on: - packaging-snapshot command: | buildkite-agent artifact download "build/**" . ls -lah build ls -lah build/distributions + .buildkite/scripts/dra.sh - label: DRA Staging key: dra-staging + env: + DRA_WORKFLOW: staging depends_on: - packaging-staging command: | buildkite-agent artifact download "build/**" . ls -lah build ls -lah build/distributions + .buildkite/scripts/dra.sh diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh new file mode 100644 index 000000000000..c03663be4373 --- /dev/null +++ b/.buildkite/scripts/dra.sh @@ -0,0 +1,61 @@ +set -uo pipefail + +DRY_RUN="" +BRANCH="${BUILDKITE_BRANCH}" + +if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "ci_packaging_pipieline" ]]; then + BRANCH=main + DRY_RUN="--dry-run" + echo "+++ Running in PR and setting branch main and --dry-run" +fi + +BEAT_VERSION=$(make get-version) + +CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" + +# TODO use common function +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +function release_manager_login { + DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') + VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') + VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') + export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET +} + +release_manager_login + +echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" \ + ${DRY_RUN} \ No newline at end of file From 20b724b9655e1adbc44ddbe51863bc8ba6bec49c Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 14:25:30 +0200 Subject: [PATCH 12/34] Test filebeat --- .buildkite/packaging.pipeline.yml | 80 +++++++++++++++++++++++++++++-- .buildkite/scripts/dra.sh | 3 ++ 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 2ee30eb4f376..42338bd2e1a6 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -55,8 +55,8 @@ steps: command: | cd auditbeat mage package - mage ironbank - mv auditbeat/build build + mage ironbank + mv build ../build agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -74,7 +74,42 @@ steps: cd auditbeat mage package mage ironbank - mv auditbeat/build build + mv build ../build + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + + - label: ":ubuntu: Filebeat/Packaging Linux" + key: "filebeat-package-linux-x86-snapshot" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: | + cd filebeat + mage package + mage ironbank + mv build ../build + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/** + + - label: ":linux: Filebeat/Packaging Linux arm64" + key: "filebeat-package-linux-arm64-snapshot" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: | + cd filebeat + mage package + mage ironbank + mv build ../build agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -94,7 +129,7 @@ steps: cd auditbeat mage package mage ironbank - mv auditbeat/build build + mv build ../build agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -112,7 +147,42 @@ steps: cd auditbeat mage package mage ironbank - mv auditbeat/build build + mv build ../build + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + + - label: ":ubuntu: Filebeat/Packaging Linux" + key: "filebeat-package-linux-x86-staging" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: false + command: | + cd filebeat + mage package + mage ironbank + mv build ../build + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/** + + - label: ":linux: Filebeat/Packaging Linux arm64" + key: "filebeat-package-linux-arm64-staging" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: false + command: | + cd filebeat + mage package + mage ironbank + mv build ../build agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index c03663be4373..e05dbd7d8724 100644 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -43,6 +43,9 @@ function release_manager_login { release_manager_login +echo "+++ Changing permissions for the BK API commands" +sudo chown -R :1000 build/distributions/ + echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." docker run --rm \ --name release-manager \ From 1065e14339f8cd712fb668fac6daf24709fe9fb1 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 15:07:54 +0200 Subject: [PATCH 13/34] Test matrix --- .buildkite/packaging.pipeline.yml | 166 ++++++++++++++++++------------ .buildkite/scripts/dra.sh | 0 2 files changed, 98 insertions(+), 68 deletions(-) mode change 100644 => 100755 .buildkite/scripts/dra.sh diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 42338bd2e1a6..24b169564944 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -1,10 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json -# Generate steps -# Linux step (Parallel) -# ARM Step (Parallel) -# beats-dashboards (Parallel) -# # TODO: Pre-cache beats-dev/golang-crossbuild container image env: @@ -47,148 +42,183 @@ steps: - group: Packaging snapshot key: packaging-snapshot steps: - - label: ":ubuntu: Auditbeat/Packaging Linux" - key: "auditbeat-package-linux-x86-snapshot" + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true command: | - cd auditbeat + WORKSPACE=$$PWD + cd {{matrix}} mage package mage ironbank - mv build ../build + mv build $$WORKSPACE agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - build/distributions/** + matrix: + - auditbeat + - filebeat + - heartbeat + # - metricbeat + # - packetbeat + # - winlogbeat + - x-pack/auditbeat + # - x-pack/dockerlogbeat + # - x-pack/filebeat + # - x-pack/functionbeat + # - x-pack/heartbeat + # - x-pack/metricbeat + # - x-pack/packetbeat + - x-pack/winlogbeat - - label: ":linux: Auditbeat/Packaging Linux arm64" - key: "auditbeat-package-linux-arm64-snapshot" + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" SNAPSHOT: true command: | - cd auditbeat + WORKSPACE=$$PWD + cd {{matrix}} mage package mage ironbank - mv build ../build + mv build $$WORKSPACE agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - build/distributions/** + matrix: + - auditbeat + # - filebeat + # - heartbeat + # - metricbeat + # - packetbeat + - x-pack/auditbeat + # - x-pack/dockerlogbeat + # - x-pack/filebeat + # - x-pack/heartbeat + # - x-pack/metricbeat + - x-pack/packetbeat - - label: ":ubuntu: Filebeat/Packaging Linux" - key: "filebeat-package-linux-x86-snapshot" + ## Osquerybeat and agentbeat require msitools to be installed + ## TODO: pre-install msitools to the VM and delete this exclusion + - label: ":ubuntu: {{matrix}}/Packaging Linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true command: | - cd filebeat + WORKSPACE=$$PWD + .buildkite/scripts/install-msitools.sh + cd {{matrix}} mage package - mage ironbank - mv build ../build + mage ironbank + mv build $$WORKSPACE agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - build/distributions/** - - - label: ":linux: Filebeat/Packaging Linux arm64" - key: "filebeat-package-linux-arm64-snapshot" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - SNAPSHOT: true - command: | - cd filebeat - mage package - mage ironbank - mv build ../build - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - build/distributions/** + matrix: + - x-pack/agentbeat + - x-pack/osquerybeat - group: Packaging staging key: packaging-staging + steps: - - label: ":ubuntu: Auditbeat/Packaging Linux" - key: "auditbeat-package-linux-x86-staging" + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: false command: | - cd auditbeat + WORKSPACE=$$PWD + cd {{matrix}} mage package - mage ironbank - mv build ../build + mage ironbank + mv build $$WORKSPACE agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - build/distributions/** + matrix: + - auditbeat + - filebeat + - heartbeat + # - metricbeat + # - packetbeat + # - winlogbeat + - x-pack/auditbeat + # - x-pack/dockerlogbeat + # - x-pack/filebeat + # - x-pack/functionbeat + # - x-pack/heartbeat + # - x-pack/metricbeat + # - x-pack/packetbeat + - x-pack/winlogbeat - - label: ":linux: Auditbeat/Packaging Linux arm64" - key: "auditbeat-package-linux-arm64-staging" + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" SNAPSHOT: false command: | - cd auditbeat + WORKSPACE=$$PWD + cd {{matrix}} mage package mage ironbank - mv build ../build + mv build $$WORKSPACE agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - build/distributions/** + matrix: + - auditbeat + # - filebeat + # - heartbeat + # - metricbeat + # - packetbeat + - x-pack/auditbeat + # - x-pack/dockerlogbeat + # - x-pack/filebeat + # - x-pack/heartbeat + # - x-pack/metricbeat + - x-pack/packetbeat - - label: ":ubuntu: Filebeat/Packaging Linux" - key: "filebeat-package-linux-x86-staging" + ## Osquerybeat and agentbeat require msitools to be installed + ## TODO: pre-install msitools to the VM and delete this exclusion + - label: ":ubuntu: {{matrix}}/Packaging Linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: false command: | - cd filebeat + WORKSPACE=$$PWD + .buildkite/scripts/install-msitools.sh + cd {{matrix}} mage package - mage ironbank - mv build ../build + mage ironbank + mv build $$WORKSPACE agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - build/distributions/** - - - label: ":linux: Filebeat/Packaging Linux arm64" - key: "filebeat-package-linux-arm64-staging" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - SNAPSHOT: false - command: | - cd filebeat - mage package - mage ironbank - mv build ../build - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - build/distributions/** + matrix: + - x-pack/agentbeat + - x-pack/osquerybeat - group: DRA publish key: dra diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh old mode 100644 new mode 100755 From f35efd331ae1c60f85e097f68bc414b90bfb0a0a Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 22:27:25 +0200 Subject: [PATCH 14/34] Aligned artifacts directories --- .buildkite/packaging.pipeline.yml | 32 ++++++--------------- .buildkite/scripts/packaging/package-dra.sh | 20 +++++++++++++ 2 files changed, 28 insertions(+), 24 deletions(-) create mode 100755 .buildkite/scripts/packaging/package-dra.sh diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 24b169564944..d70e6ef0899b 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -47,12 +47,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true - command: | - WORKSPACE=$$PWD - cd {{matrix}} - mage package - mage ironbank - mv build $$WORKSPACE + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -81,12 +76,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" SNAPSHOT: true - command: | - WORKSPACE=$$PWD - cd {{matrix}} - mage package - mage ironbank - mv build $$WORKSPACE + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -113,12 +103,8 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true command: | - WORKSPACE=$$PWD .buildkite/scripts/install-msitools.sh - cd {{matrix}} - mage package - mage ironbank - mv build $$WORKSPACE + .buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -131,19 +117,15 @@ steps: - group: Packaging staging key: packaging-staging - + ## Only for release and main + # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" steps: # x86 - label: ":ubuntu: {{matrix}}/Packaging Linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: false - command: | - WORKSPACE=$$PWD - cd {{matrix}} - mage package - mage ironbank - mv build $$WORKSPACE + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -236,6 +218,8 @@ steps: .buildkite/scripts/dra.sh - label: DRA Staging + ## Only for release and main + # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: dra-staging env: DRA_WORKFLOW: staging diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh new file mode 100755 index 000000000000..b95a5e63a2af --- /dev/null +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +BEAT_DIR=$1 + +if [ -z "$BEAT_DIR" ]; then + echo "Error: Beat directory must be specified." + exit 1 +fi + +echo ">>> Packaging : $BEAT_DIR" + +WORKSPACE=$(pwd) + +BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') + +cd $BEAT_DIR +mage package +mage ironbank + +cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ +ls -l $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ \ No newline at end of file From af251f70572bffb48f77cf40b6e90f38ac672b77 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 22:42:35 +0200 Subject: [PATCH 15/34] Aligned artifacts directories --- .buildkite/scripts/packaging/package-dra.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh index b95a5e63a2af..83fd182ca8b0 100755 --- a/.buildkite/scripts/packaging/package-dra.sh +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -16,5 +16,6 @@ cd $BEAT_DIR mage package mage ironbank +mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ ls -l $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ \ No newline at end of file From 660decdc108e2c1684b5b90f5a77dcdba26cf074 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 23:12:17 +0200 Subject: [PATCH 16/34] Aligned artifacts directories --- .buildkite/scripts/packaging/package-dra.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh index 83fd182ca8b0..4499c1328d16 100755 --- a/.buildkite/scripts/packaging/package-dra.sh +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -6,10 +6,9 @@ if [ -z "$BEAT_DIR" ]; then exit 1 fi -echo ">>> Packaging : $BEAT_DIR" +echo "Packaging : $BEAT_DIR" WORKSPACE=$(pwd) - BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') cd $BEAT_DIR @@ -18,4 +17,5 @@ mage ironbank mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ -ls -l $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ \ No newline at end of file +ls -l $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ +cd $WORKSPACE \ No newline at end of file From cd8a604e00689709e338573ed0e0e090bdae0435 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 23:13:55 +0200 Subject: [PATCH 17/34] Aligned artifacts directories --- .buildkite/packaging.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index d70e6ef0899b..a4b8c6b608db 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -104,7 +104,7 @@ steps: SNAPSHOT: true command: | .buildkite/scripts/install-msitools.sh - .buildkite/scripts/packaging/package-dra.sh {{matrix}}" + .buildkite/scripts/packaging/package-dra.sh {{matrix}} agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" From 7fd5c7a861f1e5ca5f007b2bd1f92d76d517c7f6 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 23:29:23 +0200 Subject: [PATCH 18/34] Debug --- .buildkite/packaging.pipeline.md | 20 ++++++++++++++++++++ .buildkite/packaging.pipeline.yml | 27 +++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 .buildkite/packaging.pipeline.md diff --git a/.buildkite/packaging.pipeline.md b/.buildkite/packaging.pipeline.md new file mode 100644 index 000000000000..3ad48cc0d602 --- /dev/null +++ b/.buildkite/packaging.pipeline.md @@ -0,0 +1,20 @@ +### Beats Packaging pipeline +[Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches. +The pipeline definition is located in the `.buildkite/packaging.pipeline.yml` + +### Pipeline steps + +#### Beats dashboards + +Generates `build/distributions/dependencies.csv` and adds it to the `beats-dashboards` artifact. The `dependencies.csv` is required by the release-manager configuration + +#### Packaging snapshot/staging + +- Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`) +- Copies artifacts `build/distributions//` directory and adds it as an artifact. Where `` is the name of the beat +- x-pack artifacts a also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat` + +#### DRA publish +Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry. + + diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index a4b8c6b608db..dc79ff6e1ecd 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -47,7 +47,16 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + command: | + .buildkite/scripts/packaging/package-dra.sh {{matrix}} + echo 'build/distributions/' + ls -lah build/distributions/ + echo '{{matrix}}/build/distributions/' + ls -lah {{matrix}}/build/distributions/ + echo 'build/distributions/{{matrix}}' + ls -lah build/distributions/{{matrix}} + echo '{{matrix}}/build/distributions/{{matrix}}' + ls -lah {{matrix}}/build/distributions/{{matrix}} agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -76,7 +85,16 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" SNAPSHOT: true - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + command: | + .buildkite/scripts/packaging/package-dra.sh {{matrix}} + echo 'build/distributions/' + ls -lah build/distributions/ + echo '{{matrix}}/build/distributions/' + ls -lah {{matrix}}/build/distributions/ + echo 'build/distributions/{{matrix}}' + ls -lah build/distributions/{{matrix}} + echo '{{matrix}}/build/distributions/{{matrix}}' + ls -lah {{matrix}}/build/distributions/{{matrix}} agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -116,9 +134,10 @@ steps: - x-pack/osquerybeat - group: Packaging staging + key: packaging-staging ## Only for release and main - # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" steps: # x86 - label: ":ubuntu: {{matrix}}/Packaging Linux" @@ -219,7 +238,7 @@ steps: - label: DRA Staging ## Only for release and main - # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: dra-staging env: DRA_WORKFLOW: staging From 0d1da08ee58eaf5bf45680fb30befe28e5a08cd1 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 23 Apr 2024 23:56:36 +0200 Subject: [PATCH 19/34] Debug --- .buildkite/packaging.pipeline.yml | 106 +++++++++++++++--------------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index dc79ff6e1ecd..2ada849be906 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -55,14 +55,12 @@ steps: ls -lah {{matrix}}/build/distributions/ echo 'build/distributions/{{matrix}}' ls -lah build/distributions/{{matrix}} - echo '{{matrix}}/build/distributions/{{matrix}}' - ls -lah {{matrix}}/build/distributions/{{matrix}} agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - - build/distributions/** + - build/distributions/**/* matrix: - auditbeat - filebeat @@ -80,58 +78,58 @@ steps: - x-pack/winlogbeat ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - SNAPSHOT: true - command: | - .buildkite/scripts/packaging/package-dra.sh {{matrix}} - echo 'build/distributions/' - ls -lah build/distributions/ - echo '{{matrix}}/build/distributions/' - ls -lah {{matrix}}/build/distributions/ - echo 'build/distributions/{{matrix}}' - ls -lah build/distributions/{{matrix}} - echo '{{matrix}}/build/distributions/{{matrix}}' - ls -lah {{matrix}}/build/distributions/{{matrix}} - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - build/distributions/** - matrix: - - auditbeat - # - filebeat - # - heartbeat - # - metricbeat - # - packetbeat - - x-pack/auditbeat - # - x-pack/dockerlogbeat - # - x-pack/filebeat - # - x-pack/heartbeat - # - x-pack/metricbeat - - x-pack/packetbeat + # - label: ":linux: {{matrix}}/Packaging Linux arm64" + # env: + # PLATFORMS: "linux/arm64" + # PACKAGES: "docker" + # SNAPSHOT: true + # command: | + # .buildkite/scripts/packaging/package-dra.sh {{matrix}} + # echo 'build/distributions/' + # ls -lah build/distributions/ + # echo '{{matrix}}/build/distributions/' + # ls -lah {{matrix}}/build/distributions/ + # echo 'build/distributions/{{matrix}}' + # ls -lah build/distributions/{{matrix}} + # echo '{{matrix}}/build/distributions/{{matrix}}' + # ls -lah {{matrix}}/build/distributions/{{matrix}} + # agents: + # provider: "aws" + # imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + # instanceType: "${AWS_ARM_INSTANCE_TYPE}" + # artifact_paths: + # - build/distributions/** + # matrix: + # - auditbeat + # # - filebeat + # # - heartbeat + # # - metricbeat + # # - packetbeat + # - x-pack/auditbeat + # # - x-pack/dockerlogbeat + # # - x-pack/filebeat + # # - x-pack/heartbeat + # # - x-pack/metricbeat + # - x-pack/packetbeat - ## Osquerybeat and agentbeat require msitools to be installed - ## TODO: pre-install msitools to the VM and delete this exclusion - - label: ":ubuntu: {{matrix}}/Packaging Linux" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true - command: | - .buildkite/scripts/install-msitools.sh - .buildkite/scripts/packaging/package-dra.sh {{matrix}} - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - build/distributions/** - matrix: - - x-pack/agentbeat - - x-pack/osquerybeat + # ## Osquerybeat and agentbeat require msitools to be installed + # ## TODO: pre-install msitools to the VM and delete this exclusion + # - label: ":ubuntu: {{matrix}}/Packaging Linux" + # env: + # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + # SNAPSHOT: true + # command: | + # .buildkite/scripts/install-msitools.sh + # .buildkite/scripts/packaging/package-dra.sh {{matrix}} + # agents: + # provider: gcp + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # artifact_paths: + # - build/distributions/** + # matrix: + # - x-pack/agentbeat + # - x-pack/osquerybeat - group: Packaging staging From 5671e376b7c1910ac081ac35df347145cdd58504 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 00:19:44 +0200 Subject: [PATCH 20/34] Package all beats --- .buildkite/packaging.pipeline.yml | 180 +++++++++++++----------------- 1 file changed, 77 insertions(+), 103 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 2ada849be906..05b2a0140bdc 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -1,5 +1,4 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json - # TODO: Pre-cache beats-dev/golang-crossbuild container image env: @@ -43,18 +42,12 @@ steps: key: packaging-snapshot steps: # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux" + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true command: | .buildkite/scripts/packaging/package-dra.sh {{matrix}} - echo 'build/distributions/' - ls -lah build/distributions/ - echo '{{matrix}}/build/distributions/' - ls -lah {{matrix}}/build/distributions/ - echo 'build/distributions/{{matrix}}' - ls -lah build/distributions/{{matrix}} agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -65,71 +58,63 @@ steps: - auditbeat - filebeat - heartbeat - # - metricbeat - # - packetbeat - # - winlogbeat + - metricbeat + - packetbeat + - winlogbeat - x-pack/auditbeat - # - x-pack/dockerlogbeat - # - x-pack/filebeat - # - x-pack/functionbeat - # - x-pack/heartbeat - # - x-pack/metricbeat - # - x-pack/packetbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat - x-pack/winlogbeat ## ARM - # - label: ":linux: {{matrix}}/Packaging Linux arm64" - # env: - # PLATFORMS: "linux/arm64" - # PACKAGES: "docker" - # SNAPSHOT: true - # command: | - # .buildkite/scripts/packaging/package-dra.sh {{matrix}} - # echo 'build/distributions/' - # ls -lah build/distributions/ - # echo '{{matrix}}/build/distributions/' - # ls -lah {{matrix}}/build/distributions/ - # echo 'build/distributions/{{matrix}}' - # ls -lah build/distributions/{{matrix}} - # echo '{{matrix}}/build/distributions/{{matrix}}' - # ls -lah {{matrix}}/build/distributions/{{matrix}} - # agents: - # provider: "aws" - # imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - # instanceType: "${AWS_ARM_INSTANCE_TYPE}" - # artifact_paths: - # - build/distributions/** - # matrix: - # - auditbeat - # # - filebeat - # # - heartbeat - # # - metricbeat - # # - packetbeat - # - x-pack/auditbeat - # # - x-pack/dockerlogbeat - # # - x-pack/filebeat - # # - x-pack/heartbeat - # # - x-pack/metricbeat - # - x-pack/packetbeat + - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: | + .buildkite/scripts/packaging/package-dra.sh {{matrix}} + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat - # ## Osquerybeat and agentbeat require msitools to be installed - # ## TODO: pre-install msitools to the VM and delete this exclusion - # - label: ":ubuntu: {{matrix}}/Packaging Linux" - # env: - # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - # SNAPSHOT: true - # command: | - # .buildkite/scripts/install-msitools.sh - # .buildkite/scripts/packaging/package-dra.sh {{matrix}} - # agents: - # provider: gcp - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - # artifact_paths: - # - build/distributions/** - # matrix: - # - x-pack/agentbeat - # - x-pack/osquerybeat + ## Osquerybeat and agentbeat require msitools to be installed + ## TODO: pre-install msitools to the VM and delete this exclusion + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: | + .buildkite/scripts/install-msitools.sh + .buildkite/scripts/packaging/package-dra.sh {{matrix}} + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat + - x-pack/osquerybeat - group: Packaging staging @@ -138,7 +123,7 @@ steps: if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" steps: # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux" + - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: false @@ -148,35 +133,30 @@ steps: image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - - build/distributions/** + - build/distributions/**/* matrix: - auditbeat - filebeat - heartbeat - # - metricbeat - # - packetbeat - # - winlogbeat + - metricbeat + - packetbeat + - winlogbeat - x-pack/auditbeat - # - x-pack/dockerlogbeat - # - x-pack/filebeat - # - x-pack/functionbeat - # - x-pack/heartbeat - # - x-pack/metricbeat - # - x-pack/packetbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat - x-pack/winlogbeat ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64" + - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" SNAPSHOT: false - command: | - WORKSPACE=$$PWD - cd {{matrix}} - mage package - mage ironbank - mv build $$WORKSPACE + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -185,30 +165,24 @@ steps: - build/distributions/** matrix: - auditbeat - # - filebeat - # - heartbeat - # - metricbeat - # - packetbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat - x-pack/auditbeat - # - x-pack/dockerlogbeat - # - x-pack/filebeat - # - x-pack/heartbeat - # - x-pack/metricbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat - x-pack/packetbeat ## Osquerybeat and agentbeat require msitools to be installed ## TODO: pre-install msitools to the VM and delete this exclusion - - label: ":ubuntu: {{matrix}}/Packaging Linux" + - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: false - command: | - WORKSPACE=$$PWD - .buildkite/scripts/install-msitools.sh - cd {{matrix}} - mage package - mage ironbank - mv build $$WORKSPACE + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -229,7 +203,7 @@ steps: depends_on: - packaging-snapshot command: | - buildkite-agent artifact download "build/**" . + buildkite-agent artifact download "build/**/*" . ls -lah build ls -lah build/distributions .buildkite/scripts/dra.sh From e583f3f5be168ffa63d1d3cdc99afd859f72a21c Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 01:17:05 +0200 Subject: [PATCH 21/34] Test DRA snapshot --- .buildkite/packaging.pipeline.yml | 300 +++++++++++++++--------------- 1 file changed, 149 insertions(+), 151 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 05b2a0140bdc..f22c6a4db86f 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -38,160 +38,158 @@ steps: artifact_paths: - build/distributions/** - - group: Packaging snapshot - key: packaging-snapshot - steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true - command: | - .buildkite/scripts/packaging/package-dra.sh {{matrix}} - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - build/distributions/**/* - matrix: - - auditbeat - - filebeat - - heartbeat - - metricbeat - - packetbeat - - winlogbeat - - x-pack/auditbeat - - x-pack/dockerlogbeat - - x-pack/filebeat - - x-pack/functionbeat - - x-pack/heartbeat - - x-pack/metricbeat - - x-pack/packetbeat - - x-pack/winlogbeat + # - group: Packaging snapshot + # key: packaging-snapshot + # steps: + # # x86 + # - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + # env: + # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + # SNAPSHOT: true + # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + # agents: + # provider: gcp + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # artifact_paths: + # - build/distributions/**/* + # matrix: + # - auditbeat + # - filebeat + # - heartbeat + # - metricbeat + # - packetbeat + # - winlogbeat + # - x-pack/auditbeat + # - x-pack/dockerlogbeat + # - x-pack/filebeat + # - x-pack/functionbeat + # - x-pack/heartbeat + # - x-pack/metricbeat + # - x-pack/packetbeat + # - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - SNAPSHOT: true - command: | - .buildkite/scripts/packaging/package-dra.sh {{matrix}} - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - build/distributions/**/* - matrix: - - auditbeat - - filebeat - - heartbeat - - metricbeat - - packetbeat - - x-pack/auditbeat - - x-pack/dockerlogbeat - - x-pack/filebeat - - x-pack/heartbeat - - x-pack/metricbeat - - x-pack/packetbeat + # ## ARM + # - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + # env: + # PLATFORMS: "linux/arm64" + # PACKAGES: "docker" + # SNAPSHOT: true + # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + # agents: + # provider: "aws" + # imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + # instanceType: "${AWS_ARM_INSTANCE_TYPE}" + # artifact_paths: + # - build/distributions/**/* + # matrix: + # - auditbeat + # - filebeat + # - heartbeat + # - metricbeat + # - packetbeat + # - x-pack/auditbeat + # - x-pack/dockerlogbeat + # - x-pack/filebeat + # - x-pack/heartbeat + # - x-pack/metricbeat + # - x-pack/packetbeat - ## Osquerybeat and agentbeat require msitools to be installed - ## TODO: pre-install msitools to the VM and delete this exclusion - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true - command: | - .buildkite/scripts/install-msitools.sh - .buildkite/scripts/packaging/package-dra.sh {{matrix}} - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - build/distributions/**/* - matrix: - - x-pack/agentbeat - - x-pack/osquerybeat + # ## Osquerybeat and agentbeat require msitools to be installed + # ## TODO: pre-install msitools to the VM and delete this exclusion + # - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + # env: + # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + # SNAPSHOT: true + # command: | + # .buildkite/scripts/install-msitools.sh + # .buildkite/scripts/packaging/package-dra.sh {{matrix}} + # agents: + # provider: gcp + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # artifact_paths: + # - build/distributions/**/* + # matrix: + # - x-pack/agentbeat + # - x-pack/osquerybeat - - group: Packaging staging + # - group: Packaging staging - key: packaging-staging - ## Only for release and main - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" - steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: false - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - build/distributions/**/* - matrix: - - auditbeat - - filebeat - - heartbeat - - metricbeat - - packetbeat - - winlogbeat - - x-pack/auditbeat - - x-pack/dockerlogbeat - - x-pack/filebeat - - x-pack/functionbeat - - x-pack/heartbeat - - x-pack/metricbeat - - x-pack/packetbeat - - x-pack/winlogbeat + # key: packaging-staging + # ## Only for release and main + # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + # steps: + # # x86 + # - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + # env: + # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + # SNAPSHOT: false + # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + # agents: + # provider: gcp + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # artifact_paths: + # - build/distributions/**/* + # matrix: + # - auditbeat + # - filebeat + # - heartbeat + # - metricbeat + # - packetbeat + # - winlogbeat + # - x-pack/auditbeat + # - x-pack/dockerlogbeat + # - x-pack/filebeat + # - x-pack/functionbeat + # - x-pack/heartbeat + # - x-pack/metricbeat + # - x-pack/packetbeat + # - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - SNAPSHOT: false - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - build/distributions/** - matrix: - - auditbeat - - filebeat - - heartbeat - - metricbeat - - packetbeat - - x-pack/auditbeat - - x-pack/dockerlogbeat - - x-pack/filebeat - - x-pack/heartbeat - - x-pack/metricbeat - - x-pack/packetbeat + # ## ARM + # - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" + # env: + # PLATFORMS: "linux/arm64" + # PACKAGES: "docker" + # SNAPSHOT: false + # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + # agents: + # provider: "aws" + # imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + # instanceType: "${AWS_ARM_INSTANCE_TYPE}" + # artifact_paths: + # - build/distributions/** + # matrix: + # - auditbeat + # - filebeat + # - heartbeat + # - metricbeat + # - packetbeat + # - x-pack/auditbeat + # - x-pack/dockerlogbeat + # - x-pack/filebeat + # - x-pack/heartbeat + # - x-pack/metricbeat + # - x-pack/packetbeat - ## Osquerybeat and agentbeat require msitools to be installed - ## TODO: pre-install msitools to the VM and delete this exclusion - - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: false - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - build/distributions/** - matrix: - - x-pack/agentbeat - - x-pack/osquerybeat + # ## Osquerybeat and agentbeat require msitools to be installed + # ## TODO: pre-install msitools to the VM and delete this exclusion + # - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + # env: + # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + # SNAPSHOT: false + # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + # agents: + # provider: gcp + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # artifact_paths: + # - build/distributions/** + # matrix: + # - x-pack/agentbeat + # - x-pack/osquerybeat - group: DRA publish key: dra @@ -200,10 +198,10 @@ steps: key: dra-snapshot env: DRA_WORKFLOW: snapshot - depends_on: - - packaging-snapshot + # depends_on: + # - packaging-snapshot command: | - buildkite-agent artifact download "build/**/*" . + buildkite-agent artifact download "build/**/*" . --build 018f0d0c-5a6e-497d-9e96-e35747d1be87 ls -lah build ls -lah build/distributions .buildkite/scripts/dra.sh From 1377e5bbefc42a4e836ad1d4a44690d120b71516 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 01:20:16 +0200 Subject: [PATCH 22/34] Test DRA snapshot --- .buildkite/packaging.pipeline.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index f22c6a4db86f..cfc8f24a8baf 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -205,6 +205,8 @@ steps: ls -lah build ls -lah build/distributions .buildkite/scripts/dra.sh + agents: + provider: "gcp" - label: DRA Staging ## Only for release and main @@ -219,3 +221,5 @@ steps: ls -lah build ls -lah build/distributions .buildkite/scripts/dra.sh + agents: + provider: "gcp" From 0be23b8fee1c308a6059865a2869637b6d4eadf1 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 01:33:11 +0200 Subject: [PATCH 23/34] Rename artifacts --- .buildkite/packaging.pipeline.yml | 5 ++- .../packaging/prepare-release-manager.sh | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100755 .buildkite/scripts/packaging/prepare-release-manager.sh diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index cfc8f24a8baf..4caba08087a9 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -96,6 +96,7 @@ steps: # - x-pack/packetbeat # ## Osquerybeat and agentbeat require msitools to be installed + # ## Agentbeat needs more CPUs because it builds many other beats # ## TODO: pre-install msitools to the VM and delete this exclusion # - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" # env: @@ -107,7 +108,7 @@ steps: # agents: # provider: gcp # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # machineType: "c2-standard-16" # artifact_paths: # - build/distributions/**/* # matrix: @@ -204,6 +205,7 @@ steps: buildkite-agent artifact download "build/**/*" . --build 018f0d0c-5a6e-497d-9e96-e35747d1be87 ls -lah build ls -lah build/distributions + .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: provider: "gcp" @@ -220,6 +222,7 @@ steps: buildkite-agent artifact download "build/**" . ls -lah build ls -lah build/distributions + .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: provider: "gcp" diff --git a/.buildkite/scripts/packaging/prepare-release-manager.sh b/.buildkite/scripts/packaging/prepare-release-manager.sh new file mode 100755 index 000000000000..2727fd986cc0 --- /dev/null +++ b/.buildkite/scripts/packaging/prepare-release-manager.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# This script is executed by the DRA stage. +# It prepares the required files to be consumed by the release-manager +# It can be published as snapshot or staging, for such you use +# the paramater $0 "snapshot" or $0 "staging" +# +set -ueo pipefail + +readonly TYPE=${1:-snapshot} + +# rename dependencies.csv to the name expected by release-manager. +VERSION=$(make get-version) +FINAL_VERSION=$VERSION-SNAPSHOT +if [ "$TYPE" != "snapshot" ] ; then + FINAL_VERSION=$VERSION +fi +echo "Rename dependencies to $FINAL_VERSION" +mv build/distributions/dependencies.csv \ + build/distributions/dependencies-"$FINAL_VERSION".csv + +# rename docker files to support the unified release format. +# TODO: this could be supported by the package system itself +# or the unified release process the one to do the transformation +# See https://github.com/elastic/beats/pull/30895 +find build/distributions -name '*linux-arm64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-arm64.docker.tar.gz/docker-image-linux-arm64.tar.gz}" + done + +find build/distributions -name '*linux-amd64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-amd64.docker.tar.gz/docker-image-linux-amd64.tar.gz}" + done + +echo 'List all the files' +find build/distributions -type f -ls || true From a48c62d8973d538298a662e4ac9be37944cda77c Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 01:44:50 +0200 Subject: [PATCH 24/34] fix dashboards artifacts --- .buildkite/packaging.pipeline.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 4caba08087a9..ae16170250e3 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -13,6 +13,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards + key: dashboards-snapshot # TODO: container with go and make agents: provider: gcp @@ -24,9 +25,10 @@ steps: env: SNAPSHOT: true artifact_paths: - - build/distributions/** + - build/distributions/**/* - label: Staging dashboards + key: dashboards-staging # TODO: container with go and make agents: provider: gcp @@ -36,7 +38,7 @@ steps: - make build/distributions/dependencies.csv - make beats-dashboards artifact_paths: - - build/distributions/** + - build/distributions/**/* # - group: Packaging snapshot # key: packaging-snapshot @@ -199,10 +201,12 @@ steps: key: dra-snapshot env: DRA_WORKFLOW: snapshot - # depends_on: - # - packaging-snapshot + depends_on: + # - packaging-snapshot + - dashboards-snapshot command: | buildkite-agent artifact download "build/**/*" . --build 018f0d0c-5a6e-497d-9e96-e35747d1be87 + buildkite-agent artifact download "build/**/*" . ls -lah build ls -lah build/distributions .buildkite/scripts/packaging/prepare-release-manager.sh @@ -218,6 +222,7 @@ steps: DRA_WORKFLOW: staging depends_on: - packaging-staging + - dashboards-staging command: | buildkite-agent artifact download "build/**" . ls -lah build From 5c036c1b8adba01fe28505590285fe9b75120a64 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 02:20:55 +0200 Subject: [PATCH 25/34] Cleanup --- .buildkite/packaging.pipeline.md | 4 + .buildkite/packaging.pipeline.yml | 302 ++++++++++---------- .buildkite/pull-requests.json | 14 + .buildkite/scripts/packaging/package-dra.sh | 1 - 4 files changed, 167 insertions(+), 154 deletions(-) diff --git a/.buildkite/packaging.pipeline.md b/.buildkite/packaging.pipeline.md index 3ad48cc0d602..df39db0e9feb 100644 --- a/.buildkite/packaging.pipeline.md +++ b/.buildkite/packaging.pipeline.md @@ -2,6 +2,10 @@ [Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches. The pipeline definition is located in the `.buildkite/packaging.pipeline.yml` +### Triggers +Staging packaging DRA is triggered for the `main` and release branches. +Snapshot can be triggered for any branch by the `/package` comment in the PR. The release-manager dry-run will be used for PR builds. + ### Pipeline steps #### Beats dashboards diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index ae16170250e3..b14474c3f4f9 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -28,6 +28,7 @@ steps: - build/distributions/**/* - label: Staging dashboards + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: dashboards-staging # TODO: container with go and make agents: @@ -40,159 +41,159 @@ steps: artifact_paths: - build/distributions/**/* - # - group: Packaging snapshot - # key: packaging-snapshot - # steps: - # # x86 - # - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" - # env: - # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - # SNAPSHOT: true - # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - # agents: - # provider: gcp - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - # artifact_paths: - # - build/distributions/**/* - # matrix: - # - auditbeat - # - filebeat - # - heartbeat - # - metricbeat - # - packetbeat - # - winlogbeat - # - x-pack/auditbeat - # - x-pack/dockerlogbeat - # - x-pack/filebeat - # - x-pack/functionbeat - # - x-pack/heartbeat - # - x-pack/metricbeat - # - x-pack/packetbeat - # - x-pack/winlogbeat + - group: Packaging snapshot + key: packaging-snapshot + steps: + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + - x-pack/winlogbeat - # ## ARM - # - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" - # env: - # PLATFORMS: "linux/arm64" - # PACKAGES: "docker" - # SNAPSHOT: true - # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - # agents: - # provider: "aws" - # imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - # instanceType: "${AWS_ARM_INSTANCE_TYPE}" - # artifact_paths: - # - build/distributions/**/* - # matrix: - # - auditbeat - # - filebeat - # - heartbeat - # - metricbeat - # - packetbeat - # - x-pack/auditbeat - # - x-pack/dockerlogbeat - # - x-pack/filebeat - # - x-pack/heartbeat - # - x-pack/metricbeat - # - x-pack/packetbeat + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat - # ## Osquerybeat and agentbeat require msitools to be installed - # ## Agentbeat needs more CPUs because it builds many other beats - # ## TODO: pre-install msitools to the VM and delete this exclusion - # - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" - # env: - # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - # SNAPSHOT: true - # command: | - # .buildkite/scripts/install-msitools.sh - # .buildkite/scripts/packaging/package-dra.sh {{matrix}} - # agents: - # provider: gcp - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: - # - build/distributions/**/* - # matrix: - # - x-pack/agentbeat - # - x-pack/osquerybeat + ## Osquerybeat and agentbeat require msitools to be installed + ## Agentbeat needs more CPUs because it builds many other beats + ## TODO: pre-install msitools to the VM and delete this exclusion + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: | + .buildkite/scripts/install-msitools.sh + .buildkite/scripts/packaging/package-dra.sh {{matrix}} + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat + - x-pack/osquerybeat - # - group: Packaging staging + - group: Packaging staging - # key: packaging-staging - # ## Only for release and main - # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" - # steps: - # # x86 - # - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" - # env: - # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - # SNAPSHOT: false - # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - # agents: - # provider: gcp - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - # artifact_paths: - # - build/distributions/**/* - # matrix: - # - auditbeat - # - filebeat - # - heartbeat - # - metricbeat - # - packetbeat - # - winlogbeat - # - x-pack/auditbeat - # - x-pack/dockerlogbeat - # - x-pack/filebeat - # - x-pack/functionbeat - # - x-pack/heartbeat - # - x-pack/metricbeat - # - x-pack/packetbeat - # - x-pack/winlogbeat + key: packaging-staging + ## Only for release and main + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + steps: + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + - x-pack/winlogbeat - # ## ARM - # - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" - # env: - # PLATFORMS: "linux/arm64" - # PACKAGES: "docker" - # SNAPSHOT: false - # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - # agents: - # provider: "aws" - # imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - # instanceType: "${AWS_ARM_INSTANCE_TYPE}" - # artifact_paths: - # - build/distributions/** - # matrix: - # - auditbeat - # - filebeat - # - heartbeat - # - metricbeat - # - packetbeat - # - x-pack/auditbeat - # - x-pack/dockerlogbeat - # - x-pack/filebeat - # - x-pack/heartbeat - # - x-pack/metricbeat - # - x-pack/packetbeat + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat - # ## Osquerybeat and agentbeat require msitools to be installed - # ## TODO: pre-install msitools to the VM and delete this exclusion - # - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" - # env: - # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - # SNAPSHOT: false - # command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - # agents: - # provider: gcp - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - # artifact_paths: - # - build/distributions/** - # matrix: - # - x-pack/agentbeat - # - x-pack/osquerybeat + ## Osquerybeat and agentbeat require msitools to be installed + ## TODO: pre-install msitools to the VM and delete this exclusion + - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/** + matrix: + - x-pack/agentbeat + - x-pack/osquerybeat - group: DRA publish key: dra @@ -202,13 +203,10 @@ steps: env: DRA_WORKFLOW: snapshot depends_on: - # - packaging-snapshot + - packaging-snapshot - dashboards-snapshot command: | - buildkite-agent artifact download "build/**/*" . --build 018f0d0c-5a6e-497d-9e96-e35747d1be87 buildkite-agent artifact download "build/**/*" . - ls -lah build - ls -lah build/distributions .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: @@ -225,8 +223,6 @@ steps: - dashboards-staging command: | buildkite-agent artifact download "build/**" . - ls -lah build - ls -lah build/distributions .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index c98c1f07578d..18c22f41a8fb 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -63,6 +63,20 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-packaging-pipeline", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": false, + "build_on_commit": false, + "build_on_comment": true, + "trigger_comment_regex": "^/package", + "always_trigger_comment_regex": "^/package", + "skip_ci_labels": [ ], + "skip_target_branches": [ ] } ] } \ No newline at end of file diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh index 4499c1328d16..9383c88fe40a 100755 --- a/.buildkite/scripts/packaging/package-dra.sh +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -17,5 +17,4 @@ mage ironbank mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ -ls -l $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ cd $WORKSPACE \ No newline at end of file From 29d2028d68846d0aadc5c385f85591ff424889b6 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 14:34:23 +0200 Subject: [PATCH 26/34] cleanup --- .buildkite/packaging.pipeline.md | 11 +++---- .buildkite/packaging.pipeline.yml | 14 +++++--- .buildkite/pull-requests.json | 14 -------- .buildkite/scripts/dra.sh | 28 +++------------- .buildkite/scripts/packaging/package-dra.sh | 5 +-- catalog-info.yaml | 36 --------------------- 6 files changed, 22 insertions(+), 86 deletions(-) diff --git a/.buildkite/packaging.pipeline.md b/.buildkite/packaging.pipeline.md index df39db0e9feb..6da3fc7095d5 100644 --- a/.buildkite/packaging.pipeline.md +++ b/.buildkite/packaging.pipeline.md @@ -3,20 +3,19 @@ The pipeline definition is located in the `.buildkite/packaging.pipeline.yml` ### Triggers -Staging packaging DRA is triggered for the `main` and release branches. -Snapshot can be triggered for any branch by the `/package` comment in the PR. The release-manager dry-run will be used for PR builds. +Staging packaging DRA is triggered for the and release branches. +Snapshot is triggered for `main` and release branches ### Pipeline steps -#### Beats dashboards - -Generates `build/distributions/dependencies.csv` and adds it to the `beats-dashboards` artifact. The `dependencies.csv` is required by the release-manager configuration +#### Beats dashboard +Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. That is required by the release-manager configuration #### Packaging snapshot/staging - Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`) - Copies artifacts `build/distributions//` directory and adds it as an artifact. Where `` is the name of the beat -- x-pack artifacts a also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat` +- x-pack artifacts a also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. That's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle) #### DRA publish Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry. diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index b14474c3f4f9..41b697d3bc30 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -13,6 +13,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: dashboards-snapshot # TODO: container with go and make agents: @@ -28,7 +29,7 @@ steps: - build/distributions/**/* - label: Staging dashboards - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + if: "build.branch =~ /\\d+\\.\\d+/" key: dashboards-staging # TODO: container with go and make agents: @@ -42,6 +43,7 @@ steps: - build/distributions/**/* - group: Packaging snapshot + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: packaging-snapshot steps: # x86 @@ -121,8 +123,8 @@ steps: - group: Packaging staging key: packaging-staging - ## Only for release and main - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + ## Only for release + if: "build.branch =~ /\\d+\\.\\d+/" steps: # x86 - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" @@ -199,6 +201,8 @@ steps: key: dra steps: - label: DRA Snapshot + ## Only for release branches and main + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: dra-snapshot env: DRA_WORKFLOW: snapshot @@ -213,8 +217,8 @@ steps: provider: "gcp" - label: DRA Staging - ## Only for release and main - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + ## Only for release branches + if: "build.branch =~ /\\d+\\.\\d+/" key: dra-staging env: DRA_WORKFLOW: staging diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 18c22f41a8fb..c98c1f07578d 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -63,20 +63,6 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] - }, - { - "enabled": true, - "pipelineSlug": "beats-packaging-pipeline", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": false, - "build_on_commit": false, - "build_on_comment": true, - "trigger_comment_regex": "^/package", - "always_trigger_comment_regex": "^/package", - "skip_ci_labels": [ ], - "skip_target_branches": [ ] } ] } \ No newline at end of file diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index e05dbd7d8724..07cc9319617a 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -1,9 +1,11 @@ +#!/usr/bin/env bash set -uo pipefail -DRY_RUN="" +## TODO: Set to empty string when Jenkins is disabled +DRY_RUN="--dry-run" BRANCH="${BUILDKITE_BRANCH}" -if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "ci_packaging_pipieline" ]]; then +if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then BRANCH=main DRY_RUN="--dry-run" echo "+++ Running in PR and setting branch main and --dry-run" @@ -13,28 +15,8 @@ BEAT_VERSION=$(make get-version) CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" -# TODO use common function -retry() { - local retries=$1 - shift - local count=0 - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - if [ $count -lt "$retries" ]; then - >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." - sleep $wait - else - >&2 echo "Retry $count/$retries exited $exit, no more retries left." - return $exit - fi - done - return 0 -} - function release_manager_login { - DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh index 9383c88fe40a..a70bd4552785 100755 --- a/.buildkite/scripts/packaging/package-dra.sh +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash -BEAT_DIR=$1 +set -ueo pipefail +BEAT_DIR=${1:-""} if [ -z "$BEAT_DIR" ]; then echo "Error: Beat directory must be specified." exit 1 fi -echo "Packaging : $BEAT_DIR" +echo "~~~Packaging : $BEAT_DIR" WORKSPACE=$(pwd) BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') diff --git a/catalog-info.yaml b/catalog-info.yaml index 32856d53c35b..c9fe5b4f05c3 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -1052,39 +1052,3 @@ spec: access_level: BUILD_AND_READ everyone: access_level: READ_ONLY - ---- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json -apiVersion: backstage.io/v1alpha1 -kind: Resource -metadata: - name: beats-packaging-pipeline - description: Buildkite pipeline for packaging and publishing to DRA - links: - - title: Pipeline - url: https://buildkite.com/elastic/beats-packaging-pipeline -spec: - type: buildkite-pipeline - owner: group:ingest-fp - system: buildkite - implementation: - apiVersion: buildkite.elastic.dev/v1 - kind: Pipeline - metadata: - name: beats-packaging-pipeline - description: Pipeline for Beats packaging and publishing DRA artifacts - spec: - repository: elastic/beats - pipeline_file: ".buildkite/packaging.pipeline.yml" - branch_configuration: "main 8.* 7.17" - cancel_intermediate_builds: false - skip_intermediate_builds: false - provider_settings: - trigger_mode: none - teams: - ingest-fp: - access_level: MANAGE_BUILD_AND_READ - release-eng: - access_level: BUILD_AND_READ - everyone: - access_level: READ_ONLY From be94e627d1a08d476a84bc7671aa95922b3ba1e7 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 15:21:44 +0200 Subject: [PATCH 27/34] No need to install msi tools --- .buildkite/packaging.pipeline.yml | 27 +++++++++++--------------- .buildkite/pull-requests.json | 32 ------------------------------- .buildkite/scripts/dra.sh | 2 +- 3 files changed, 12 insertions(+), 49 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 41b697d3bc30..7d2a3119d3b8 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -13,7 +13,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: dashboards-snapshot # TODO: container with go and make agents: @@ -43,7 +43,7 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: packaging-snapshot steps: # x86 @@ -71,6 +71,7 @@ steps: - x-pack/functionbeat - x-pack/heartbeat - x-pack/metricbeat + - x-pack/osquerybeat - x-pack/packetbeat - x-pack/winlogbeat @@ -100,16 +101,12 @@ steps: - x-pack/metricbeat - x-pack/packetbeat - ## Osquerybeat and agentbeat require msitools to be installed ## Agentbeat needs more CPUs because it builds many other beats - ## TODO: pre-install msitools to the VM and delete this exclusion - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true - command: | - .buildkite/scripts/install-msitools.sh - .buildkite/scripts/packaging/package-dra.sh {{matrix}} + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" @@ -118,7 +115,6 @@ steps: - build/distributions/**/* matrix: - x-pack/agentbeat - - x-pack/osquerybeat - group: Packaging staging @@ -151,6 +147,7 @@ steps: - x-pack/functionbeat - x-pack/heartbeat - x-pack/metricbeat + - x-pack/osquerybeat - x-pack/packetbeat - x-pack/winlogbeat @@ -180,29 +177,27 @@ steps: - x-pack/metricbeat - x-pack/packetbeat - ## Osquerybeat and agentbeat require msitools to be installed - ## TODO: pre-install msitools to the VM and delete this exclusion - - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + ## Agentbeat needs more CPUs because it builds many other beats + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: false + SNAPSHOT: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "c2-standard-16" artifact_paths: - - build/distributions/** + - build/distributions/**/* matrix: - x-pack/agentbeat - - x-pack/osquerybeat - group: DRA publish key: dra steps: - label: DRA Snapshot ## Only for release branches and main - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: dra-snapshot env: DRA_WORKFLOW: snapshot diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index c98c1f07578d..2ceab6f2374b 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -31,38 +31,6 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": ["^x-pack/elastic-agent/README.md", "^x-pack/elastic-agent/docs/.*", "^x-pack/elastic-agent/devtools/.*" ], "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, - { - "enabled": true, - "pipelineSlug": "beats-winlogbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test winlogbeat$", - "always_trigger_comment_regex": "^/test winlogbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, - { - "enabled": true, - "pipelineSlug": "beats-xpack-packetbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test x-pack/packetbeat$", - "always_trigger_comment_regex": "^/test x-pack/packetbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] } ] } \ No newline at end of file diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 07cc9319617a..702d0c1b3ae0 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -5,7 +5,7 @@ set -uo pipefail DRY_RUN="--dry-run" BRANCH="${BUILDKITE_BRANCH}" -if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then +if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "ci_packaging_pipieline" ]]; then BRANCH=main DRY_RUN="--dry-run" echo "+++ Running in PR and setting branch main and --dry-run" From dd3ecb55b7674958ded02a8acae595ab0765c4ec Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 15:51:14 +0200 Subject: [PATCH 28/34] Apply suggestions from code review Co-authored-by: Dimitrios Liappis --- .buildkite/packaging.pipeline.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/packaging.pipeline.md b/.buildkite/packaging.pipeline.md index 6da3fc7095d5..577310d0104e 100644 --- a/.buildkite/packaging.pipeline.md +++ b/.buildkite/packaging.pipeline.md @@ -3,19 +3,19 @@ The pipeline definition is located in the `.buildkite/packaging.pipeline.yml` ### Triggers -Staging packaging DRA is triggered for the and release branches. -Snapshot is triggered for `main` and release branches +Staging packaging DRA is triggered for release branches only. +Snapshot is triggered for `main` and release branches. ### Pipeline steps #### Beats dashboard -Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. That is required by the release-manager configuration +Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. This is required by the release-manager configuration. #### Packaging snapshot/staging - Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`) -- Copies artifacts `build/distributions//` directory and adds it as an artifact. Where `` is the name of the beat -- x-pack artifacts a also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. That's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle) +- Copies artifacts `build/distributions//` directory and adds it as an artifact, where `` is the corresponding beat name. +- x-pack artifacts are also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. It's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle). #### DRA publish Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry. From 3e47581c6bd3c00a31fdc25ab41a5bfef8ae6e15 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 15:55:27 +0200 Subject: [PATCH 29/34] Extraced platforms variables --- .buildkite/packaging.pipeline.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 7d2a3119d3b8..c4c7157bfa62 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -8,6 +8,9 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS_ARM: "linux/arm64" + steps: - group: Beats dashboards key: dashboards @@ -49,7 +52,7 @@ steps: # x86 - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PLATFORMS_AMD}" SNAPSHOT: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: @@ -78,7 +81,7 @@ steps: ## ARM - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" SNAPSHOT: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" @@ -104,7 +107,7 @@ steps: ## Agentbeat needs more CPUs because it builds many other beats - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PLATFORMS_AMD}" SNAPSHOT: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: @@ -125,7 +128,7 @@ steps: # x86 - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PLATFORMS_AMD}" SNAPSHOT: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: @@ -154,7 +157,7 @@ steps: ## ARM - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" SNAPSHOT: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" @@ -180,7 +183,7 @@ steps: ## Agentbeat needs more CPUs because it builds many other beats - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PLATFORMS_AMD}" SNAPSHOT: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: From e850d1de991f268f894ccfc0ec03641f7d498684 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 15:56:14 +0200 Subject: [PATCH 30/34] Cleanup --- .buildkite/packaging.pipeline.yml | 6 +++--- .buildkite/scripts/dra.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index c4c7157bfa62..289ac2ba3d80 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -16,7 +16,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards - # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: dashboards-snapshot # TODO: container with go and make agents: @@ -46,7 +46,7 @@ steps: - build/distributions/**/* - group: Packaging snapshot - # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: packaging-snapshot steps: # x86 @@ -200,7 +200,7 @@ steps: steps: - label: DRA Snapshot ## Only for release branches and main - # if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" key: dra-snapshot env: DRA_WORKFLOW: snapshot diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 702d0c1b3ae0..07cc9319617a 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -5,7 +5,7 @@ set -uo pipefail DRY_RUN="--dry-run" BRANCH="${BUILDKITE_BRANCH}" -if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "ci_packaging_pipieline" ]]; then +if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then BRANCH=main DRY_RUN="--dry-run" echo "+++ Running in PR and setting branch main and --dry-run" From 21793d9de42d3e5f1e9714c5de04a5b55d3611d6 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 16:04:25 +0200 Subject: [PATCH 31/34] Added RUN_SNAPSHOT condition --- .buildkite/packaging.pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 289ac2ba3d80..fb396328d8e1 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -16,7 +16,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" key: dashboards-snapshot # TODO: container with go and make agents: @@ -46,7 +46,7 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" key: packaging-snapshot steps: # x86 @@ -200,7 +200,7 @@ steps: steps: - label: DRA Snapshot ## Only for release branches and main - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'" + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" key: dra-snapshot env: DRA_WORKFLOW: snapshot From cc3712128aae61d27352e2896194f864f35022be Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 17:09:34 +0200 Subject: [PATCH 32/34] Cleanup --- .buildkite/scripts/dra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 07cc9319617a..8a67a8bf81fe 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -uo pipefail +set -euo pipefail ## TODO: Set to empty string when Jenkins is disabled DRY_RUN="--dry-run" From 15a04ca2595cab97c4b0a61bcf652e4b2023ae36 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 17:24:14 +0200 Subject: [PATCH 33/34] Boolean dry-run --- .buildkite/scripts/dra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 8a67a8bf81fe..9405bc1bc0bb 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -2,7 +2,7 @@ set -euo pipefail ## TODO: Set to empty string when Jenkins is disabled -DRY_RUN="--dry-run" +if [[ "$DRY_RUN" == "false" ]]; then echo "--- Running in publish mode"; DRY_RUN=""; else echo "--- Running in dry-run mode"; DRY_RUN="--dry-run"; fi BRANCH="${BUILDKITE_BRANCH}" if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then From d13c0c54ecce972b7709734083ad2d74efbf3489 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 24 Apr 2024 17:26:51 +0200 Subject: [PATCH 34/34] Boolean dry-run --- .buildkite/scripts/dra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index 9405bc1bc0bb..ee0f4b286f00 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -set -euo pipefail ## TODO: Set to empty string when Jenkins is disabled if [[ "$DRY_RUN" == "false" ]]; then echo "--- Running in publish mode"; DRY_RUN=""; else echo "--- Running in dry-run mode"; DRY_RUN="--dry-run"; fi +set -euo pipefail BRANCH="${BUILDKITE_BRANCH}" if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then