diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index 773672c5f05f5..647f2c317569c 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -8,25 +8,33 @@ steps: - wait - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for deb + label: Artifact Testing agents: queue: n2-4-virt - timeout_in_minutes: 20 + timeout_in_minutes: 30 - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for rpm + label: Artifact Testing agents: queue: n2-4-virt - timeout_in_minutes: 20 + timeout_in_minutes: 30 - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for docker + label: Artifact Testing agents: queue: n2-4-virt - timeout_in_minutes: 20 + timeout_in_minutes: 30 - command: .buildkite/scripts/steps/artifacts/docker_context.sh - label: 'Docker Build Context' + label: 'Docker Context Verification' agents: queue: n2-2 - timeout_in_minutes: 20 + timeout_in_minutes: 30 + + - wait + + - command: .buildkite/scripts/steps/artifacts/publish.sh + label: 'Publish Kibana Artifacts' + agents: + queue: n2-2 + timeout_in_minutes: 30 diff --git a/.buildkite/scripts/steps/artifacts/build.sh b/.buildkite/scripts/steps/artifacts/build.sh index 9d40a713d4b0a..152b3ea9bf72b 100644 --- a/.buildkite/scripts/steps/artifacts/build.sh +++ b/.buildkite/scripts/steps/artifacts/build.sh @@ -4,44 +4,14 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - VERSION="$(jq -r '.version' package.json)" - RELEASE_ARG="--release" -else - VERSION="$(jq -r '.version' package.json)-SNAPSHOT" - RELEASE_ARG="" -fi +source .buildkite/scripts/steps/artifacts/env.sh -echo "--- Build Kibana Distribution" -node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile --skip-docker-cloud - -echo "--- Build dependencies report" -node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv" - -# Release verification -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - echo "--- Build and push Kibana Cloud Distribution" - # This doesn't meet the requirements for a release image, implementation TBD - # Beats artifacts will need to match a specific commit sha that matches other stack iamges - # For now this is a placeholder step that will allow us to run automated Cloud tests - # against a best guess approximation of a release image - echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co - trap 'docker logout docker.elastic.co' EXIT - - node scripts/build \ - "$RELEASE_ARG" \ - --skip-initialize \ - --skip-generic-folders \ - --skip-platform-folders \ - --skip-archives \ - --docker-images \ - --docker-tag-qualifier="$GIT_COMMIT" \ - --docker-push \ - --skip-docker-ubi \ - --skip-docker-ubuntu \ - --skip-docker-contexts -fi +echo "--- Build Kibana artifacts" +node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS") +echo "--- Build and upload dependencies report" +node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv" cd target +sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt" buildkite-agent artifact upload "*" -cd - \ No newline at end of file +cd - diff --git a/.buildkite/scripts/steps/artifacts/docker_context.sh b/.buildkite/scripts/steps/artifacts/docker_context.sh index 787f5d278febc..a20544de18fd9 100644 --- a/.buildkite/scripts/steps/artifacts/docker_context.sh +++ b/.buildkite/scripts/steps/artifacts/docker_context.sh @@ -4,17 +4,11 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - VERSION="$(jq -r '.version' package.json)" - RELEASE_ARG="--release" -else - VERSION="$(jq -r '.version' package.json)-SNAPSHOT" - RELEASE_ARG="" -fi +source .buildkite/scripts/steps/artifacts/env.sh echo "--- Create contexts" mkdir -p target -node scripts/build "$RELEASE_ARG" --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact +node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact $(echo "$BUILD_ARGS") echo "--- Setup default context" DOCKER_BUILD_FOLDER=$(mktemp -d) @@ -22,7 +16,7 @@ DOCKER_BUILD_FOLDER=$(mktemp -d) tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER" cd $DOCKER_BUILD_FOLDER -buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" +buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" echo "--- Build context" docker build . diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh new file mode 100755 index 0000000000000..ea2850ff9a38a --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/env.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +RELEASE_BUILD="${RELEASE_BUILD:="false"}" +VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}" + +BASE_VERSION="$(jq -r '.version' package.json)" + +if [[ "$VERSION_QUALIFIER" == "" ]]; then + QUALIFIER_VERSION="$BASE_VERSION" +else + QUALIFIER_VERSION="$BASE_VERSION-$VERSION_QUALIFIER" +fi + +if [[ "$RELEASE_BUILD" == "true" ]]; then + FULL_VERSION="$QUALIFIER_VERSION" + + # Beats artifacts will need to match a specific commit sha that matches other stack images + # for release builds. For now we are skipping Cloud builds until there's a pointer. + BUILD_ARGS="--release --skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER" + WORKFLOW="staging" +else + FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT" + BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER" + WORKFLOW="snapshot" +fi + +export VERSION_QUALIFIER +export BASE_VERSION +export QUALIFIER_VERSION +export FULL_VERSION +export BUILD_ARGS +export WORKFLOW diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh new file mode 100644 index 0000000000000..edb82d6f83ca0 --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh +source .buildkite/scripts/steps/artifacts/env.sh + +echo "--- Download and verify artifacts" +function download { + buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + sha512sum -c "$1.sha512.txt" + rm "$1.sha512.txt" +} + +mkdir -p target +cd target + +download "kibana-$FULL_VERSION-docker-image.tar.gz" +download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz" +download "kibana-ubi8-$FULL_VERSION-docker-image.tar.gz" + +download "kibana-$FULL_VERSION-arm64.deb" +download "kibana-$FULL_VERSION-amd64.deb" +download "kibana-$FULL_VERSION-x86_64.rpm" +download "kibana-$FULL_VERSION-aarch64.rpm" + +download "kibana-$FULL_VERSION-docker-build-context.tar.gz" +download "kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz" +download "kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz" + +download "kibana-$FULL_VERSION-linux-aarch64.tar.gz" +download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" + +download "kibana-$FULL_VERSION-darwin-x86_64.tar.gz" +download "kibana-$FULL_VERSION-darwin-aarch64.tar.gz" + +download "kibana-$FULL_VERSION-windows-x86_64.zip" + +download "dependencies-$FULL_VERSION.csv" + +cd - + +echo "--- Set artifact permissions" +chmod -R a+r target/* +chmod -R a+w target + +echo "--- Pull latest Release Manager CLI" +echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co +trap 'docker logout docker.elastic.co' EXIT +docker pull docker.elastic.co/infra/release-manager:latest + +echo "--- Publish artifacts" +export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" +export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" +export VAULT_ADDR="https://secrets.elastic.co:8200" +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project kibana \ + --branch "$KIBANA_BASE_BRANCH" \ + --commit "$GIT_COMMIT" \ + --workflow "$WORKFLOW" \ + --version "$BASE_VERSION" \ + --qualifier "$VERSION_QUALIFIER" \ + --artifact-set main + +ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW" +ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url') + +cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary + ### Artifacts Summary + + $ARTIFACTS_SUMMARY +EOF diff --git a/.buildkite/scripts/steps/package_testing/test.sh b/.buildkite/scripts/steps/package_testing/test.sh index a9a46502d5b3b..4c8bac026af84 100755 --- a/.buildkite/scripts/steps/package_testing/test.sh +++ b/.buildkite/scripts/steps/package_testing/test.sh @@ -15,7 +15,7 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" KIBANA_IP_ADDRESS="192.168.56.6" elif [[ "$TEST_PACKAGE" == "docker" ]]; then - buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" KIBANA_IP_ADDRESS="192.168.56.7" fi cd ..