[7.17] [artifacts] Publish (#129499) (#130173)

* [artifacts] Publish (#129499) * [artifacts] Publish release artifacts This adds the publish step to the release pipeline, which uses the release-manager CLI to upload a list Kibana artifacts. * cleanup * add missing artifacts * enable rm * fix syntax * fix artifact * source utils * fix path * fix version * skip steps * re-add pipeline steps * set permissions * Update .buildkite/scripts/steps/artifacts/publish.sh Co-authored-by: Chris <[email protected]> * Revert "Update .buildkite/scripts/steps/artifacts/publish.sh" This reverts commit a6228f5. * export * support version qualifier * setup env * cleanup * consistency * \n * comment * unneccessary fallback * +x * hoist env args * source * link to artifacts * fix artifacts summary * relax docker glob * \n * fix summary * fix variable Co-authored-by: Chris <[email protected]> * backport docker env variables Co-authored-by: Chris <[email protected]>
elastic · Apr 13, 2022 · 476c7b9 · 476c7b9
1 parent 68ba36e
commit 476c7b9
Show file tree

Hide file tree

Showing 7 changed files with 143 additions and 30 deletions.
diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml
@@ -8,25 +8,33 @@ steps:
   - wait
 
   - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for deb
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
 
   - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for rpm
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
 
   - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for docker
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
 
   - command: .buildkite/scripts/steps/artifacts/docker_context.sh
-    label: 'Docker Build Context'
+    label: 'Docker Context Verification'
     agents:
       queue: n2-2
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+
+  - wait
+
+  - command: .buildkite/scripts/steps/artifacts/publish.sh
+    label: 'Publish Kibana Artifacts'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh
@@ -92,6 +92,12 @@ EOF
 GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)
 export GITHUB_TOKEN
 
+KIBANA_DOCKER_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/container-registry)"
+export KIBANA_DOCKER_USERNAME
+
+KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
+export KIBANA_DOCKER_PASSWORD
+
 KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
 export KIBANA_CI_REPORTER_KEY
 

diff --git a/.buildkite/scripts/steps/artifacts/build.sh b/.buildkite/scripts/steps/artifacts/build.sh
@@ -4,20 +4,14 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  VERSION="$(jq -r '.version' package.json)"
-  RELEASE_ARG="--release"
-else
-  VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
-  RELEASE_ARG=""
-fi
+source .buildkite/scripts/steps/artifacts/env.sh
 
-echo "--- Build Kibana Distribution"
-node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile
-
-echo "--- Build dependencies report"
-node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv"
+echo "--- Build Kibana artifacts"
+node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS")
 
+echo "--- Build and upload dependencies report"
+node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv"
 cd target
+sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt"
 buildkite-agent artifact upload "*"
 cd -
diff --git a/.buildkite/scripts/steps/artifacts/docker_context.sh b/.buildkite/scripts/steps/artifacts/docker_context.sh
@@ -4,25 +4,19 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  VERSION="$(jq -r '.version' package.json)"
-  RELEASE_ARG="--release"
-else
-  VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
-  RELEASE_ARG=""
-fi
+source .buildkite/scripts/steps/artifacts/env.sh
 
 echo "--- Create contexts"
 mkdir -p target
-node scripts/build "$RELEASE_ARG" --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact
+node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact $(echo "$BUILD_ARGS")
 
 echo "--- Setup default context"
 DOCKER_BUILD_FOLDER=$(mktemp -d)
 
 tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER"
 cd $DOCKER_BUILD_FOLDER
 
-buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 echo "--- Build context"
 docker build .
diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+RELEASE_BUILD="${RELEASE_BUILD:="false"}"
+VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
+
+BASE_VERSION="$(jq -r '.version' package.json)"
+
+if [[ "$VERSION_QUALIFIER" == "" ]]; then
+  QUALIFIER_VERSION="$BASE_VERSION"
+else
+  QUALIFIER_VERSION="$BASE_VERSION-$VERSION_QUALIFIER"
+fi
+
+if [[ "$RELEASE_BUILD" == "true" ]]; then
+  FULL_VERSION="$QUALIFIER_VERSION"
+  BUILD_ARGS="--release --version-qualifier=$VERSION_QUALIFIER"
+  WORKFLOW="staging"
+else
+  FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT"
+  BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER"
+  WORKFLOW="snapshot"
+fi
+
+export VERSION_QUALIFIER
+export BASE_VERSION
+export QUALIFIER_VERSION
+export FULL_VERSION
+export BUILD_ARGS
+export WORKFLOW
diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/artifacts/env.sh
+
+echo "--- Download and verify artifacts"
+function download {
+  buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  sha512sum -c "$1.sha512.txt"
+  rm "$1.sha512.txt"
+}
+
+mkdir -p target
+cd target
+
+download "kibana-$FULL_VERSION-docker-image.tar.gz"
+download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz"
+download "kibana-ubi8-$FULL_VERSION-docker-image.tar.gz"
+
+download "kibana-$FULL_VERSION-arm64.deb"
+download "kibana-$FULL_VERSION-amd64.deb"
+download "kibana-$FULL_VERSION-x86_64.rpm"
+download "kibana-$FULL_VERSION-aarch64.rpm"
+
+download "kibana-$FULL_VERSION-docker-build-context.tar.gz"
+download "kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz"
+download "kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
+
+download "kibana-$FULL_VERSION-linux-aarch64.tar.gz"
+download "kibana-$FULL_VERSION-linux-x86_64.tar.gz"
+
+download "kibana-$FULL_VERSION-darwin-x86_64.tar.gz"
+download "kibana-$FULL_VERSION-darwin-aarch64.tar.gz"
+
+download "kibana-$FULL_VERSION-windows-x86_64.zip"
+
+download "dependencies-$FULL_VERSION.csv"
+
+cd - 
+
+echo "--- Set artifact permissions"
+chmod -R a+r target/*
+chmod -R a+w target
+
+echo "--- Pull latest Release Manager CLI"
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+trap 'docker logout docker.elastic.co' EXIT
+docker pull docker.elastic.co/infra/release-manager:latest
+
+echo "--- Publish artifacts"
+export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
+export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+export VAULT_ADDR="https://secrets.elastic.co:8200"
+docker run --rm \
+  --name release-manager \
+  -e VAULT_ADDR \
+  -e VAULT_ROLE_ID \
+  -e VAULT_SECRET_ID \
+  --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
+  docker.elastic.co/infra/release-manager:latest \
+    cli collect \
+      --project kibana \
+      --branch "$KIBANA_BASE_BRANCH" \
+      --commit "$GIT_COMMIT" \
+      --workflow "$WORKFLOW" \
+      --version "$BASE_VERSION" \
+      --qualifier "$VERSION_QUALIFIER" \
+      --artifact-set main
+
+ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
+ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url')
+
+cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
+  ### Artifacts Summary
+
+  $ARTIFACTS_SUMMARY
+EOF
diff --git a/.buildkite/scripts/steps/package_testing/test.sh b/.buildkite/scripts/steps/package_testing/test.sh
@@ -15,7 +15,7 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
   buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.6"
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
-  buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.7"
 fi
 cd ..