From 178a63497d5eaa90f7b69fbe20ea48ddfebdd39c Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 13 Apr 2022 13:06:40 -0500 Subject: [PATCH 1/2] [artifacts] Publish (#129499) * [artifacts] Publish release artifacts This adds the publish step to the release pipeline, which uses the release-manager CLI to upload a list Kibana artifacts. * cleanup * add missing artifacts * enable rm * fix syntax * fix artifact * source utils * fix path * fix version * skip steps * re-add pipeline steps * set permissions * Update .buildkite/scripts/steps/artifacts/publish.sh Co-authored-by: Chris * Revert "Update .buildkite/scripts/steps/artifacts/publish.sh" This reverts commit a6228f52cd75297f67a6d5505129c5257934f8ad. * export * support version qualifier * setup env * cleanup * consistency * \n * comment * unneccessary fallback * +x * hoist env args * source * link to artifacts * fix artifacts summary * relax docker glob * \n * fix summary * fix variable Co-authored-by: Chris --- .buildkite/pipelines/artifacts.yml | 24 ++++-- .buildkite/scripts/steps/artifacts/build.sh | 18 ++--- .../scripts/steps/artifacts/docker_context.sh | 12 +-- .buildkite/scripts/steps/artifacts/env.sh | 31 +++++++ .buildkite/scripts/steps/artifacts/publish.sh | 80 +++++++++++++++++++ .../scripts/steps/package_testing/test.sh | 2 +- 6 files changed, 137 insertions(+), 30 deletions(-) create mode 100755 .buildkite/scripts/steps/artifacts/env.sh create mode 100644 .buildkite/scripts/steps/artifacts/publish.sh diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index 773672c5f05f5..647f2c317569c 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -8,25 +8,33 @@ steps: - wait - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for deb + label: Artifact Testing agents: queue: n2-4-virt - timeout_in_minutes: 20 + timeout_in_minutes: 30 - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for rpm + label: Artifact Testing agents: queue: n2-4-virt - timeout_in_minutes: 20 + timeout_in_minutes: 30 - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for docker + label: Artifact Testing agents: queue: n2-4-virt - timeout_in_minutes: 20 + timeout_in_minutes: 30 - command: .buildkite/scripts/steps/artifacts/docker_context.sh - label: 'Docker Build Context' + label: 'Docker Context Verification' agents: queue: n2-2 - timeout_in_minutes: 20 + timeout_in_minutes: 30 + + - wait + + - command: .buildkite/scripts/steps/artifacts/publish.sh + label: 'Publish Kibana Artifacts' + agents: + queue: n2-2 + timeout_in_minutes: 30 diff --git a/.buildkite/scripts/steps/artifacts/build.sh b/.buildkite/scripts/steps/artifacts/build.sh index f8c67dcd1d98e..152b3ea9bf72b 100644 --- a/.buildkite/scripts/steps/artifacts/build.sh +++ b/.buildkite/scripts/steps/artifacts/build.sh @@ -4,20 +4,14 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - VERSION="$(jq -r '.version' package.json)" - RELEASE_ARG="--release" -else - VERSION="$(jq -r '.version' package.json)-SNAPSHOT" - RELEASE_ARG="" -fi +source .buildkite/scripts/steps/artifacts/env.sh -echo "--- Build Kibana Distribution" -node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile - -echo "--- Build dependencies report" -node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv" +echo "--- Build Kibana artifacts" +node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS") +echo "--- Build and upload dependencies report" +node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv" cd target +sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt" buildkite-agent artifact upload "*" cd - diff --git a/.buildkite/scripts/steps/artifacts/docker_context.sh b/.buildkite/scripts/steps/artifacts/docker_context.sh index 787f5d278febc..a20544de18fd9 100644 --- a/.buildkite/scripts/steps/artifacts/docker_context.sh +++ b/.buildkite/scripts/steps/artifacts/docker_context.sh @@ -4,17 +4,11 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - VERSION="$(jq -r '.version' package.json)" - RELEASE_ARG="--release" -else - VERSION="$(jq -r '.version' package.json)-SNAPSHOT" - RELEASE_ARG="" -fi +source .buildkite/scripts/steps/artifacts/env.sh echo "--- Create contexts" mkdir -p target -node scripts/build "$RELEASE_ARG" --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact +node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact $(echo "$BUILD_ARGS") echo "--- Setup default context" DOCKER_BUILD_FOLDER=$(mktemp -d) @@ -22,7 +16,7 @@ DOCKER_BUILD_FOLDER=$(mktemp -d) tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER" cd $DOCKER_BUILD_FOLDER -buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" +buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" echo "--- Build context" docker build . diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh new file mode 100755 index 0000000000000..bab1e6321b797 --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/env.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -euo pipefail + +RELEASE_BUILD="${RELEASE_BUILD:="false"}" +VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}" + +BASE_VERSION="$(jq -r '.version' package.json)" + +if [[ "$VERSION_QUALIFIER" == "" ]]; then + QUALIFIER_VERSION="$BASE_VERSION" +else + QUALIFIER_VERSION="$BASE_VERSION-$VERSION_QUALIFIER" +fi + +if [[ "$RELEASE_BUILD" == "true" ]]; then + FULL_VERSION="$QUALIFIER_VERSION" + BUILD_ARGS="--release --version-qualifier=$VERSION_QUALIFIER" + WORKFLOW="staging" +else + FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT" + BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER" + WORKFLOW="snapshot" +fi + +export VERSION_QUALIFIER +export BASE_VERSION +export QUALIFIER_VERSION +export FULL_VERSION +export BUILD_ARGS +export WORKFLOW diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh new file mode 100644 index 0000000000000..edb82d6f83ca0 --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh +source .buildkite/scripts/steps/artifacts/env.sh + +echo "--- Download and verify artifacts" +function download { + buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + sha512sum -c "$1.sha512.txt" + rm "$1.sha512.txt" +} + +mkdir -p target +cd target + +download "kibana-$FULL_VERSION-docker-image.tar.gz" +download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz" +download "kibana-ubi8-$FULL_VERSION-docker-image.tar.gz" + +download "kibana-$FULL_VERSION-arm64.deb" +download "kibana-$FULL_VERSION-amd64.deb" +download "kibana-$FULL_VERSION-x86_64.rpm" +download "kibana-$FULL_VERSION-aarch64.rpm" + +download "kibana-$FULL_VERSION-docker-build-context.tar.gz" +download "kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz" +download "kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz" + +download "kibana-$FULL_VERSION-linux-aarch64.tar.gz" +download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" + +download "kibana-$FULL_VERSION-darwin-x86_64.tar.gz" +download "kibana-$FULL_VERSION-darwin-aarch64.tar.gz" + +download "kibana-$FULL_VERSION-windows-x86_64.zip" + +download "dependencies-$FULL_VERSION.csv" + +cd - + +echo "--- Set artifact permissions" +chmod -R a+r target/* +chmod -R a+w target + +echo "--- Pull latest Release Manager CLI" +echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co +trap 'docker logout docker.elastic.co' EXIT +docker pull docker.elastic.co/infra/release-manager:latest + +echo "--- Publish artifacts" +export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" +export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" +export VAULT_ADDR="https://secrets.elastic.co:8200" +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project kibana \ + --branch "$KIBANA_BASE_BRANCH" \ + --commit "$GIT_COMMIT" \ + --workflow "$WORKFLOW" \ + --version "$BASE_VERSION" \ + --qualifier "$VERSION_QUALIFIER" \ + --artifact-set main + +ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW" +ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url') + +cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary + ### Artifacts Summary + + $ARTIFACTS_SUMMARY +EOF diff --git a/.buildkite/scripts/steps/package_testing/test.sh b/.buildkite/scripts/steps/package_testing/test.sh index a9a46502d5b3b..4c8bac026af84 100755 --- a/.buildkite/scripts/steps/package_testing/test.sh +++ b/.buildkite/scripts/steps/package_testing/test.sh @@ -15,7 +15,7 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" KIBANA_IP_ADDRESS="192.168.56.6" elif [[ "$TEST_PACKAGE" == "docker" ]]; then - buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" KIBANA_IP_ADDRESS="192.168.56.7" fi cd .. From 0defd683fe4e7a8ff2624c04f7e8d798def7a9d0 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 13 Apr 2022 14:10:26 -0500 Subject: [PATCH 2/2] backport docker env variables --- .buildkite/scripts/lifecycle/pre_command.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh index 71d1a92c1cb13..43d1f192ad3d5 100755 --- a/.buildkite/scripts/lifecycle/pre_command.sh +++ b/.buildkite/scripts/lifecycle/pre_command.sh @@ -92,6 +92,12 @@ EOF GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine) export GITHUB_TOKEN +KIBANA_DOCKER_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/container-registry)" +export KIBANA_DOCKER_USERNAME + +KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)" +export KIBANA_DOCKER_PASSWORD + KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter) export KIBANA_CI_REPORTER_KEY