[artifacts] Publish (#129499) (#130171)

* [artifacts] Publish release artifacts

This adds the publish step to the release pipeline, which uses the
release-manager CLI to upload a list Kibana artifacts.

* cleanup

* add missing artifacts

* enable rm

* fix syntax

* fix artifact

* source utils

* fix path

* fix version

* skip steps

* re-add pipeline steps

* set permissions

* Update .buildkite/scripts/steps/artifacts/publish.sh

Co-authored-by: Chris <[email protected]>

* Revert "Update .buildkite/scripts/steps/artifacts/publish.sh"

This reverts commit a6228f5.

* export

* support version qualifier

* setup env

* cleanup

* consistency

* \n

* comment

* unneccessary fallback

* +x

* hoist env args

* source

* link to artifacts

* fix artifacts summary

* relax docker glob

* \n

* fix summary

* fix variable

Co-authored-by: Chris <[email protected]>
(cherry picked from commit ab1bec3)

Co-authored-by: Jonathan Budzenski <[email protected]>

.buildkite/pipelines/artifacts.yml

@@ -8,25 +8,33 @@ steps:
   - wait
 
   - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for deb
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
 
   - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for rpm
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
 
   - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for docker
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
 
   - command: .buildkite/scripts/steps/artifacts/docker_context.sh
-    label: 'Docker Build Context'
+    label: 'Docker Context Verification'
     agents:
       queue: n2-2
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+
+  - wait
+
+  - command: .buildkite/scripts/steps/artifacts/publish.sh
+    label: 'Publish Kibana Artifacts'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30

.buildkite/scripts/steps/artifacts/build.sh

@@ -4,44 +4,14 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  VERSION="$(jq -r '.version' package.json)"
-  RELEASE_ARG="--release"
-else
-  VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
-  RELEASE_ARG=""
-fi
+source .buildkite/scripts/steps/artifacts/env.sh
 
-echo "--- Build Kibana Distribution"
-node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile --skip-docker-cloud
-
-echo "--- Build dependencies report"
-node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv"
-
-# Release verification
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  echo "--- Build and push Kibana Cloud Distribution"
-  # This doesn't meet the requirements for a release image, implementation TBD
-  # Beats artifacts will need to match a specific commit sha that matches other stack iamges
-  # For now this is a placeholder step that will allow us to run automated Cloud tests
-  # against a best guess approximation of a release image
-  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-  trap 'docker logout docker.elastic.co' EXIT
-
-  node scripts/build \
-    "$RELEASE_ARG" \
-    --skip-initialize \
-    --skip-generic-folders \
-    --skip-platform-folders \
-    --skip-archives \
-    --docker-images \
-    --docker-tag-qualifier="$GIT_COMMIT" \
-    --docker-push \
-    --skip-docker-ubi \
-    --skip-docker-ubuntu \
-    --skip-docker-contexts
-fi
+echo "--- Build Kibana artifacts"
+node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS")
 
+echo "--- Build and upload dependencies report"
+node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv"
 cd target
+sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt"
 buildkite-agent artifact upload "*"
-cd -
+cd -

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -4,25 +4,19 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  VERSION="$(jq -r '.version' package.json)"
-  RELEASE_ARG="--release"
-else
-  VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
-  RELEASE_ARG=""
-fi
+source .buildkite/scripts/steps/artifacts/env.sh
 
 echo "--- Create contexts"
 mkdir -p target
-node scripts/build "$RELEASE_ARG" --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact
+node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact $(echo "$BUILD_ARGS")
 
 echo "--- Setup default context"
 DOCKER_BUILD_FOLDER=$(mktemp -d)
 
 tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER"
 cd $DOCKER_BUILD_FOLDER
 
-buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 echo "--- Build context"
 docker build .

.buildkite/scripts/steps/artifacts/env.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+RELEASE_BUILD="${RELEASE_BUILD:="false"}"
+VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
+
+BASE_VERSION="$(jq -r '.version' package.json)"
+
+if [[ "$VERSION_QUALIFIER" == "" ]]; then
+  QUALIFIER_VERSION="$BASE_VERSION"
+else
+  QUALIFIER_VERSION="$BASE_VERSION-$VERSION_QUALIFIER"
+fi
+
+if [[ "$RELEASE_BUILD" == "true" ]]; then
+  FULL_VERSION="$QUALIFIER_VERSION"
+
+  # Beats artifacts will need to match a specific commit sha that matches other stack images
+  # for release builds.  For now we are skipping Cloud builds until there's a pointer.
+  BUILD_ARGS="--release --skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER"
+  WORKFLOW="staging"
+else
+  FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT"
+  BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER"
+  WORKFLOW="snapshot"
+fi
+
+export VERSION_QUALIFIER
+export BASE_VERSION
+export QUALIFIER_VERSION
+export FULL_VERSION
+export BUILD_ARGS
+export WORKFLOW

.buildkite/scripts/steps/artifacts/publish.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/artifacts/env.sh
+
+echo "--- Download and verify artifacts"
+function download {
+  buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  sha512sum -c "$1.sha512.txt"
+  rm "$1.sha512.txt"
+}
+
+mkdir -p target
+cd target
+
+download "kibana-$FULL_VERSION-docker-image.tar.gz"
+download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz"
+download "kibana-ubi8-$FULL_VERSION-docker-image.tar.gz"
+
+download "kibana-$FULL_VERSION-arm64.deb"
+download "kibana-$FULL_VERSION-amd64.deb"
+download "kibana-$FULL_VERSION-x86_64.rpm"
+download "kibana-$FULL_VERSION-aarch64.rpm"
+
+download "kibana-$FULL_VERSION-docker-build-context.tar.gz"
+download "kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz"
+download "kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
+
+download "kibana-$FULL_VERSION-linux-aarch64.tar.gz"
+download "kibana-$FULL_VERSION-linux-x86_64.tar.gz"
+
+download "kibana-$FULL_VERSION-darwin-x86_64.tar.gz"
+download "kibana-$FULL_VERSION-darwin-aarch64.tar.gz"
+
+download "kibana-$FULL_VERSION-windows-x86_64.zip"
+
+download "dependencies-$FULL_VERSION.csv"
+
+cd - 
+
+echo "--- Set artifact permissions"
+chmod -R a+r target/*
+chmod -R a+w target
+
+echo "--- Pull latest Release Manager CLI"
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+trap 'docker logout docker.elastic.co' EXIT
+docker pull docker.elastic.co/infra/release-manager:latest
+
+echo "--- Publish artifacts"
+export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
+export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+export VAULT_ADDR="https://secrets.elastic.co:8200"
+docker run --rm \
+  --name release-manager \
+  -e VAULT_ADDR \
+  -e VAULT_ROLE_ID \
+  -e VAULT_SECRET_ID \
+  --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
+  docker.elastic.co/infra/release-manager:latest \
+    cli collect \
+      --project kibana \
+      --branch "$KIBANA_BASE_BRANCH" \
+      --commit "$GIT_COMMIT" \
+      --workflow "$WORKFLOW" \
+      --version "$BASE_VERSION" \
+      --qualifier "$VERSION_QUALIFIER" \
+      --artifact-set main
+
+ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
+ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url')
+
+cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
+  ### Artifacts Summary
+
+  $ARTIFACTS_SUMMARY
+EOF

.buildkite/scripts/steps/package_testing/test.sh

@@ -15,7 +15,7 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
   buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.6"
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
-  buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.7"
 fi
 cd ..