From 7a26474de657ed1adf3d4bf06e390ba07bf1b8b1 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 5 Apr 2022 14:21:21 -0500 Subject: [PATCH 01/48] [artifacts] Publish release artifacts This adds the publish step to the release pipeline, which uses the release-manager CLI to upload a list Kibana artifacts. --- .buildkite/pipelines/artifacts.yml | 16 +++-- .buildkite/scripts/steps/artifacts/build.sh | 3 +- .buildkite/scripts/steps/artifacts/publish.sh | 70 +++++++++++++++++++ 3 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 .buildkite/scripts/steps/artifacts/publish.sh diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index 773672c5f05f5..f472a7b77943b 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -11,22 +11,30 @@ steps: label: Package testing for deb agents: queue: n2-4-virt - timeout_in_minutes: 20 + timeout_in_minutes: 30 - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh label: Package testing for rpm agents: queue: n2-4-virt - timeout_in_minutes: 20 + timeout_in_minutes: 30 - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh label: Package testing for docker agents: queue: n2-4-virt - timeout_in_minutes: 20 + timeout_in_minutes: 30 - command: .buildkite/scripts/steps/artifacts/docker_context.sh label: 'Docker Build Context' agents: queue: n2-2 - timeout_in_minutes: 20 + timeout_in_minutes: 30 + + - wait + + - command: .buildkite/scripts/steps/artifacts/publish.sh + label: 'Publish artifacts' + agents: + queue: n2-2 + timeout_in_minutes: 30 diff --git a/.buildkite/scripts/steps/artifacts/build.sh b/.buildkite/scripts/steps/artifacts/build.sh index 9d40a713d4b0a..83527f227cc3b 100644 --- a/.buildkite/scripts/steps/artifacts/build.sh +++ b/.buildkite/scripts/steps/artifacts/build.sh @@ -43,5 +43,6 @@ if [[ "${RELEASE_BUILD:-}" == "true" ]]; then fi cd target +sha512sum "dependencies-$VERSION.csv" > "dependencies-$VERSION.csv.sha512.txt" buildkite-agent artifact upload "*" -cd - \ No newline at end of file +cd - diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh new file mode 100644 index 0000000000000..0640f33408d0c --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "${RELEASE_BUILD:-}" == "true" ]]; then + VERSION="$(jq -r '.version' package.json)" + WORKFLOW="staging" +else + VERSION="$(jq -r '.version' package.json)-SNAPSHOT" + WORKFLOW="snapshot" +fi + +echo "--- Download and verify artifacts" +function download { + buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + sha512sum -c "$1.sha512.txt" +} + +mkdir artifacts +cd artifacts + +download "kibana-$VERSION-docker-image.tar.gz" +download "kibana-$VERSION-docker-image-aarch64.tar.gz" +download "kibana-ubi8-$VERSION-docker-image.tar.gz" + +download "kibana-$VERSION-arm64.deb" +download "kibana-$VERSION-amd64.deb" +download "kibana-$VERSION-x86_64.rpm" +download "kibana-$VERSION-aarch64.rpm" + +download "kibana-$VERSION-docker-build-context.tar.gz" +download "kibana-ironbank-$VERSION-docker-build-context.tar.gz" + +download "kibana-$VERSION-linux-aarch64.tar.gz" +download "kibana-$VERSION-linux-x86_64.tar.gz" + +download "kibana-$VERSION-darwin-x86_64.tar.gz" + +download "kibana-$VERSION-windows-x86_64.zip" + +download "dependencies-$VERSION.csv" + +cd - + +echo "--- Pull latest release manager CLI" +echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co +trap 'docker logout docker.elastic.co' EXIT +docker pull docker.elastic.co/infra/release-manager:latest + + +# echo "--- Publish artifacts" +# VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" +# VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" +# VAULT_ADDR="https://secrets.elastic.co:8200" +# docker run --rm \ +# --name release-manager \ +# -e VAULT_ADDR \ +# -e VAULT_ROLE_ID \ +# -e VAULT_SECRET_ID \ +# --mount type=bind,readonly=false,src="$PWD/artifacts",target=/artifacts \ +# docker.elastic.co/infra/release-manager:latest \ +# cli collect \ +# --project kibana \ +# --branch "$BRANCH" \ +# --commit "$GIT_COMMIT" \ +# --workflow "$WORKFLOW" \ +# --version "$VERSION" +# --qualifier "$QUALIFIER" +# --artifact-set main From 0b3590912344c55d7aeb6f8fddb88ece8037e16c Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 5 Apr 2022 14:29:04 -0500 Subject: [PATCH 02/48] cleanup --- .buildkite/scripts/steps/artifacts/publish.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 0640f33408d0c..d881d4afd6f6b 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -17,8 +17,8 @@ function download { sha512sum -c "$1.sha512.txt" } -mkdir artifacts -cd artifacts +mkdir -p target +cd target download "kibana-$VERSION-docker-image.tar.gz" download "kibana-$VERSION-docker-image-aarch64.tar.gz" @@ -48,21 +48,21 @@ echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --pas trap 'docker logout docker.elastic.co' EXIT docker pull docker.elastic.co/infra/release-manager:latest - # echo "--- Publish artifacts" # VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" # VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" # VAULT_ADDR="https://secrets.elastic.co:8200" +# QUALIFIER="" # docker run --rm \ # --name release-manager \ # -e VAULT_ADDR \ # -e VAULT_ROLE_ID \ # -e VAULT_SECRET_ID \ -# --mount type=bind,readonly=false,src="$PWD/artifacts",target=/artifacts \ +# --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts \ # docker.elastic.co/infra/release-manager:latest \ # cli collect \ # --project kibana \ -# --branch "$BRANCH" \ +# --branch "$KIBANA_BASE_BRANCH" \ # --commit "$GIT_COMMIT" \ # --workflow "$WORKFLOW" \ # --version "$VERSION" From e9967581a9f836648984a54e7215ba8da9b0617d Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 6 Apr 2022 13:53:46 -0500 Subject: [PATCH 03/48] [artifacts] Test cloud image --- .buildkite/pipelines/artifacts.yml | 6 ++ .buildkite/scripts/steps/artifacts/cloud.sh | 77 +++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 .buildkite/scripts/steps/artifacts/cloud.sh diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index 773672c5f05f5..510ed733e4bdf 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -30,3 +30,9 @@ steps: agents: queue: n2-2 timeout_in_minutes: 20 + + - command: .buildkite/scripts/steps/artifacts/cloud.sh + label: 'FTR on Cloud' + agents: + queue: n2-2 + timeout_in_minutes: 60 diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh new file mode 100644 index 0000000000000..8d4c6bd8b587b --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +set -euo pipefail + +.buildkite/scripts/bootstrap.sh + +if [[ "${RELEASE_BUILD:-}" == "true" ]]; then + VERSION="$(jq -r '.version' package.json)" + RELEASE_ARG="--release" +else + VERSION="$(jq -r '.version' package.json)-SNAPSHOT" + RELEASE_ARG="" +fi + +echo "--- Publish Cloud image" +mkdir -p target +cd target + +buildkite-agent artifact download "kibana-cloud-$VERSION-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" +docker load --input kibana-cloud-$VERSION-docker-image.tar.gz + +TAG="$VERSION-$GIT_COMMIT" +KIBANA_BASE_IMAGE="docker.elastic.co/kibana/kibana-cloud:$VERSION" +KIBANA_TEST_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$TAG" + +docker tag "$KIBANA_BASE_IMAGE" "KIBANA_TEST_IMAGE" + +echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co +trap 'docker logout docker.elastic.co' EXIT + +docker push "KIBANA_TEST_IMAGE" +docker logout docker.elastic.co + +cd - + +echo "--- Create deployment" +CLOUD_DEPLOYMENT_NAME="kibana-release-$TAG" + +LOGS=$(mktemp --suffix ".json") +DEPLOYMENT_SPEC=$(mktemp --suffix ".json") + +jq ' + .name = "'$CLOUD_DEPLOYMENT_NAME'" | + .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_TEST_IMAGE'" | + .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$VERSION'" | + .resources.kibana[0].plan.kibana.version = "'$VERSION'" | + .resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'" | + .resources.enterprise_search[0].plan.enterprise_search.version = "'$VERSION'" | + .resources.integrations_server[0].plan.integrations_server.version = "'$VERSION'" + ' .buildkite/scripts/steps/cloud/deploy.json > "$DEPLOYMENT_SPEC" + +ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" &> "$LOGS" + +CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS") +CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS") +CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS") +CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$LOGS") + +CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url') +CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url') + +echo "--- Run functional tests" +# TODO +# TEST_KIBANA_PROTOCOL=https +# TEST_KIBANA_HOSTNAME=my-kibana-instance.internal.net +# TEST_KIBANA_PORT=443 +# TEST_KIBANA_USER=kibana +# TEST_KIBANA_PASS= + +# TEST_ES_PROTOCOL=https +# TEST_ES_HOSTNAME=my-es-cluster.internal.net +# TEST_ES_PORT=9200 +# TEST_ES_USER="$CLOUD_DEPLOYMENT_USERNAME" +# TEST_ES_PASS="$CLOUD_DEPLOYMENT_PASSWORD" + +echo "--- Shutdown deployment" +ecctl deployment shutdown --force --track --output json &> "$LOGS" From b941a5a293315094eef0ac8cba3eb7ad00d3478a Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 6 Apr 2022 13:55:25 -0500 Subject: [PATCH 04/48] tmp skip unrelated steps --- .buildkite/pipelines/artifacts.yml | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index 510ed733e4bdf..f0d381c9b9166 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -7,29 +7,29 @@ steps: - wait - - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for deb - agents: - queue: n2-4-virt - timeout_in_minutes: 20 + # - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh + # label: Package testing for deb + # agents: + # queue: n2-4-virt + # timeout_in_minutes: 20 - - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for rpm - agents: - queue: n2-4-virt - timeout_in_minutes: 20 + # - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh + # label: Package testing for rpm + # agents: + # queue: n2-4-virt + # timeout_in_minutes: 20 - - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for docker - agents: - queue: n2-4-virt - timeout_in_minutes: 20 + # - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh + # label: Package testing for docker + # agents: + # queue: n2-4-virt + # timeout_in_minutes: 20 - - command: .buildkite/scripts/steps/artifacts/docker_context.sh - label: 'Docker Build Context' - agents: - queue: n2-2 - timeout_in_minutes: 20 + # - command: .buildkite/scripts/steps/artifacts/docker_context.sh + # label: 'Docker Build Context' + # agents: + # queue: n2-2 + # timeout_in_minutes: 20 - command: .buildkite/scripts/steps/artifacts/cloud.sh label: 'FTR on Cloud' From ba3bdaca8976a2935512193b872a74538fa0a062 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 6 Apr 2022 14:54:46 -0500 Subject: [PATCH 05/48] build cloud artifact --- .buildkite/scripts/steps/artifacts/build.sh | 31 ++++----------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/build.sh b/.buildkite/scripts/steps/artifacts/build.sh index 9d40a713d4b0a..494405cc331ff 100644 --- a/.buildkite/scripts/steps/artifacts/build.sh +++ b/.buildkite/scripts/steps/artifacts/build.sh @@ -7,41 +7,22 @@ set -euo pipefail if [[ "${RELEASE_BUILD:-}" == "true" ]]; then VERSION="$(jq -r '.version' package.json)" RELEASE_ARG="--release" + + # This doesn't meet the requirements for a release image, implementation TBD + # Beats artifacts will need to match a specific commit sha that matches other stack iamges + BUILD_CLOUD_ARG="--skip-docker-cloud" else VERSION="$(jq -r '.version' package.json)-SNAPSHOT" RELEASE_ARG="" + BUILD_CLOUD_ARG="" fi echo "--- Build Kibana Distribution" -node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile --skip-docker-cloud +node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile "$BUILD_CLOUD_ARG" echo "--- Build dependencies report" node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv" -# Release verification -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - echo "--- Build and push Kibana Cloud Distribution" - # This doesn't meet the requirements for a release image, implementation TBD - # Beats artifacts will need to match a specific commit sha that matches other stack iamges - # For now this is a placeholder step that will allow us to run automated Cloud tests - # against a best guess approximation of a release image - echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co - trap 'docker logout docker.elastic.co' EXIT - - node scripts/build \ - "$RELEASE_ARG" \ - --skip-initialize \ - --skip-generic-folders \ - --skip-platform-folders \ - --skip-archives \ - --docker-images \ - --docker-tag-qualifier="$GIT_COMMIT" \ - --docker-push \ - --skip-docker-ubi \ - --skip-docker-ubuntu \ - --skip-docker-contexts -fi - cd target buildkite-agent artifact upload "*" cd - \ No newline at end of file From 5a1de47b994ceb66e2f660540b6cb38bf088c5de Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 6 Apr 2022 16:26:19 -0500 Subject: [PATCH 06/48] fix tag --- .buildkite/scripts/steps/artifacts/cloud.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index 8d4c6bd8b587b..a66da1824d869 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -23,7 +23,7 @@ TAG="$VERSION-$GIT_COMMIT" KIBANA_BASE_IMAGE="docker.elastic.co/kibana/kibana-cloud:$VERSION" KIBANA_TEST_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$TAG" -docker tag "$KIBANA_BASE_IMAGE" "KIBANA_TEST_IMAGE" +docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE" echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT From e812256c69d2070482bc9662698200fca92763f4 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 6 Apr 2022 16:52:10 -0500 Subject: [PATCH 07/48] add missing artifacts --- .buildkite/scripts/steps/artifacts/publish.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index d881d4afd6f6b..a7478aee5f6c4 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -31,11 +31,13 @@ download "kibana-$VERSION-aarch64.rpm" download "kibana-$VERSION-docker-build-context.tar.gz" download "kibana-ironbank-$VERSION-docker-build-context.tar.gz" +download "kibana-ubi8-$VERSION-docker-build-context.tar.gz" download "kibana-$VERSION-linux-aarch64.tar.gz" download "kibana-$VERSION-linux-x86_64.tar.gz" download "kibana-$VERSION-darwin-x86_64.tar.gz" +download "kibana-$VERSION-darwin-aarch64.gz" download "kibana-$VERSION-windows-x86_64.zip" From 63bf201a2771e545af39c58af1f946eac55e3d23 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 6 Apr 2022 16:54:48 -0500 Subject: [PATCH 08/48] enable rm --- .buildkite/scripts/steps/artifacts/publish.sh | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index a7478aee5f6c4..bc579f3e7b470 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -45,28 +45,28 @@ download "dependencies-$VERSION.csv" cd - -echo "--- Pull latest release manager CLI" +echo "--- Pull latest release manager" echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT docker pull docker.elastic.co/infra/release-manager:latest -# echo "--- Publish artifacts" -# VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" -# VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" -# VAULT_ADDR="https://secrets.elastic.co:8200" -# QUALIFIER="" -# docker run --rm \ -# --name release-manager \ -# -e VAULT_ADDR \ -# -e VAULT_ROLE_ID \ -# -e VAULT_SECRET_ID \ -# --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts \ -# docker.elastic.co/infra/release-manager:latest \ -# cli collect \ -# --project kibana \ -# --branch "$KIBANA_BASE_BRANCH" \ -# --commit "$GIT_COMMIT" \ -# --workflow "$WORKFLOW" \ -# --version "$VERSION" -# --qualifier "$QUALIFIER" -# --artifact-set main +echo "--- Publish artifacts" +VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" +VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" +VAULT_ADDR="https://secrets.elastic.co:8200" +QUALIFIER="" +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project kibana \ + --branch "$KIBANA_BASE_BRANCH" \ + --commit "$GIT_COMMIT" \ + --workflow "$WORKFLOW" \ + --version "$VERSION" + --qualifier "$QUALIFIER" + --artifact-set main From 8e9c6700c749b05de4732876fbc8dc8a4c0e9ad6 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 6 Apr 2022 16:58:49 -0500 Subject: [PATCH 09/48] fix syntax --- .buildkite/scripts/steps/artifacts/publish.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index bc579f3e7b470..210ee7cf515c9 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -67,6 +67,6 @@ docker run --rm \ --branch "$KIBANA_BASE_BRANCH" \ --commit "$GIT_COMMIT" \ --workflow "$WORKFLOW" \ - --version "$VERSION" - --qualifier "$QUALIFIER" + --version "$VERSION" \ + --qualifier "$QUALIFIER" \ --artifact-set main From 4baf37ad842ab6e2d1a64b6fe0fcb23282c38b03 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 6 Apr 2022 17:20:32 -0500 Subject: [PATCH 10/48] fix tag --- .buildkite/scripts/steps/artifacts/cloud.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index a66da1824d869..d73e036acb976 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -20,7 +20,7 @@ buildkite-agent artifact download "kibana-cloud-$VERSION-docker-image.tar.gz" . docker load --input kibana-cloud-$VERSION-docker-image.tar.gz TAG="$VERSION-$GIT_COMMIT" -KIBANA_BASE_IMAGE="docker.elastic.co/kibana/kibana-cloud:$VERSION" +KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$VERSION" KIBANA_TEST_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$TAG" docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE" From 148c9ded260df9e9b5939b6f525f6d502b4f8d0e Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 6 Apr 2022 17:58:10 -0500 Subject: [PATCH 11/48] fix artifact --- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 210ee7cf515c9..7fab4f4f7f2af 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -37,7 +37,7 @@ download "kibana-$VERSION-linux-aarch64.tar.gz" download "kibana-$VERSION-linux-x86_64.tar.gz" download "kibana-$VERSION-darwin-x86_64.tar.gz" -download "kibana-$VERSION-darwin-aarch64.gz" +download "kibana-$VERSION-darwin-aarch64.tar.gz" download "kibana-$VERSION-windows-x86_64.zip" From 7729686187374321a2e02a1fec8806cd6e6ec2e2 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 08:30:11 -0500 Subject: [PATCH 12/48] fix variable --- .buildkite/scripts/steps/artifacts/cloud.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index d73e036acb976..7eb77ab13f058 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -28,7 +28,7 @@ docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE" echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT -docker push "KIBANA_TEST_IMAGE" +docker push "$KIBANA_TEST_IMAGE" docker logout docker.elastic.co cd - @@ -50,7 +50,6 @@ jq ' ' .buildkite/scripts/steps/cloud/deploy.json > "$DEPLOYMENT_SPEC" ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" &> "$LOGS" - CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS") CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS") CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS") @@ -59,6 +58,8 @@ CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url') CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url') +trap 'echo "--- Shutdown deployment" && ecctl deployment shutdown --force --track --output json &> "$LOGS"' EXIT + echo "--- Run functional tests" # TODO # TEST_KIBANA_PROTOCOL=https @@ -72,6 +73,3 @@ echo "--- Run functional tests" # TEST_ES_PORT=9200 # TEST_ES_USER="$CLOUD_DEPLOYMENT_USERNAME" # TEST_ES_PASS="$CLOUD_DEPLOYMENT_PASSWORD" - -echo "--- Shutdown deployment" -ecctl deployment shutdown --force --track --output json &> "$LOGS" From f19df7b484ac1ca58f8fd364ed6b0720c031fff0 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 09:25:58 -0500 Subject: [PATCH 13/48] source utils --- .buildkite/scripts/steps/artifacts/publish.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 7fab4f4f7f2af..5c107a8a91a40 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -2,6 +2,8 @@ set -euo pipefail +source .buildkite/scripts/common/util.sh + if [[ "${RELEASE_BUILD:-}" == "true" ]]; then VERSION="$(jq -r '.version' package.json)" WORKFLOW="staging" From 04f99e93fb02a950c6f0ee99064a381333642a9f Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 10:30:31 -0500 Subject: [PATCH 14/48] fix path --- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 5c107a8a91a40..d9d9c3011b501 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -62,7 +62,7 @@ docker run --rm \ -e VAULT_ADDR \ -e VAULT_ROLE_ID \ -e VAULT_SECRET_ID \ - --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ docker.elastic.co/infra/release-manager:latest \ cli collect \ --project kibana \ From 24e45c65ae51ef5da0e36b911236e9907551e03d Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 12:42:47 -0500 Subject: [PATCH 15/48] fix version --- .buildkite/scripts/steps/artifacts/publish.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index d9d9c3011b501..1412e93a1f9f4 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -17,6 +17,7 @@ function download { buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" sha512sum -c "$1.sha512.txt" + rm "$1.sha512.txt" } mkdir -p target @@ -57,6 +58,7 @@ VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kiban VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" VAULT_ADDR="https://secrets.elastic.co:8200" QUALIFIER="" +BASE_VERSION="$(jq -r '.version' package.json)" docker run --rm \ --name release-manager \ -e VAULT_ADDR \ @@ -69,6 +71,6 @@ docker run --rm \ --branch "$KIBANA_BASE_BRANCH" \ --commit "$GIT_COMMIT" \ --workflow "$WORKFLOW" \ - --version "$VERSION" \ + --version "$BASE_VERSION" \ --qualifier "$QUALIFIER" \ --artifact-set main From 57031a130f1906dfa58d17d508735a4dafb21a77 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 12:43:47 -0500 Subject: [PATCH 16/48] skip steps --- .buildkite/pipelines/artifacts.yml | 42 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index f472a7b77943b..5165f0f55ea76 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -7,31 +7,31 @@ steps: - wait - - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for deb - agents: - queue: n2-4-virt - timeout_in_minutes: 30 + # - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh + # label: Package testing for deb + # agents: + # queue: n2-4-virt + # timeout_in_minutes: 30 - - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for rpm - agents: - queue: n2-4-virt - timeout_in_minutes: 30 + # - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh + # label: Package testing for rpm + # agents: + # queue: n2-4-virt + # timeout_in_minutes: 30 - - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for docker - agents: - queue: n2-4-virt - timeout_in_minutes: 30 + # - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh + # label: Package testing for docker + # agents: + # queue: n2-4-virt + # timeout_in_minutes: 30 - - command: .buildkite/scripts/steps/artifacts/docker_context.sh - label: 'Docker Build Context' - agents: - queue: n2-2 - timeout_in_minutes: 30 + # - command: .buildkite/scripts/steps/artifacts/docker_context.sh + # label: 'Docker Build Context' + # agents: + # queue: n2-2 + # timeout_in_minutes: 30 - - wait + # - wait - command: .buildkite/scripts/steps/artifacts/publish.sh label: 'Publish artifacts' From 56b6fdd9f1fdf14af05e6ff1721f186c84303249 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 12:45:16 -0500 Subject: [PATCH 17/48] re-add pipeline steps --- .buildkite/pipelines/artifacts.yml | 42 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index 5165f0f55ea76..f472a7b77943b 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -7,31 +7,31 @@ steps: - wait - # - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh - # label: Package testing for deb - # agents: - # queue: n2-4-virt - # timeout_in_minutes: 30 + - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh + label: Package testing for deb + agents: + queue: n2-4-virt + timeout_in_minutes: 30 - # - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh - # label: Package testing for rpm - # agents: - # queue: n2-4-virt - # timeout_in_minutes: 30 + - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh + label: Package testing for rpm + agents: + queue: n2-4-virt + timeout_in_minutes: 30 - # - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh - # label: Package testing for docker - # agents: - # queue: n2-4-virt - # timeout_in_minutes: 30 + - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh + label: Package testing for docker + agents: + queue: n2-4-virt + timeout_in_minutes: 30 - # - command: .buildkite/scripts/steps/artifacts/docker_context.sh - # label: 'Docker Build Context' - # agents: - # queue: n2-2 - # timeout_in_minutes: 30 + - command: .buildkite/scripts/steps/artifacts/docker_context.sh + label: 'Docker Build Context' + agents: + queue: n2-2 + timeout_in_minutes: 30 - # - wait + - wait - command: .buildkite/scripts/steps/artifacts/publish.sh label: 'Publish artifacts' From 8df0b235af82e777b48353622846ecdf51bb9429 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 13:44:50 -0500 Subject: [PATCH 18/48] set permissions --- .buildkite/scripts/steps/artifacts/publish.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 1412e93a1f9f4..5a1d324890722 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -48,6 +48,10 @@ download "dependencies-$VERSION.csv" cd - +echo "--- Set artifact permissions" +chmod -R a+r target/* +chmod -R a+w target + echo "--- Pull latest release manager" echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT From a6228f52cd75297f67a6d5505129c5257934f8ad Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 14:49:40 -0500 Subject: [PATCH 19/48] Update .buildkite/scripts/steps/artifacts/publish.sh Co-authored-by: Chris --- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 5a1d324890722..c945e77384155 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -68,7 +68,7 @@ docker run --rm \ -e VAULT_ADDR \ -e VAULT_ROLE_ID \ -e VAULT_SECRET_ID \ - --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts \ docker.elastic.co/infra/release-manager:latest \ cli collect \ --project kibana \ From eb271e4e6214e5e56da257cf269a30cf51a19672 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 14:51:41 -0500 Subject: [PATCH 20/48] Revert "Update .buildkite/scripts/steps/artifacts/publish.sh" This reverts commit a6228f52cd75297f67a6d5505129c5257934f8ad. --- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index c945e77384155..5a1d324890722 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -68,7 +68,7 @@ docker run --rm \ -e VAULT_ADDR \ -e VAULT_ROLE_ID \ -e VAULT_SECRET_ID \ - --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts \ + --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \ docker.elastic.co/infra/release-manager:latest \ cli collect \ --project kibana \ From 9b39eac4510b8bae27919f733b275707729baee0 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 7 Apr 2022 14:52:36 -0500 Subject: [PATCH 21/48] export --- .buildkite/scripts/steps/artifacts/publish.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 5a1d324890722..30758511a0b04 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -58,9 +58,9 @@ trap 'docker logout docker.elastic.co' EXIT docker pull docker.elastic.co/infra/release-manager:latest echo "--- Publish artifacts" -VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" -VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" -VAULT_ADDR="https://secrets.elastic.co:8200" +export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" +export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" +export VAULT_ADDR="https://secrets.elastic.co:8200" QUALIFIER="" BASE_VERSION="$(jq -r '.version' package.json)" docker run --rm \ From 482625d41261c17e2f149c23c534adf5e853b745 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:01:06 -0500 Subject: [PATCH 22/48] support version qualifier --- .buildkite/scripts/steps/artifacts/build.sh | 42 ++--------------- .../scripts/steps/artifacts/docker_context.sh | 13 ++--- .buildkite/scripts/steps/artifacts/publish.sh | 47 ++++++++----------- 3 files changed, 27 insertions(+), 75 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/build.sh b/.buildkite/scripts/steps/artifacts/build.sh index 83527f227cc3b..3b7e7eacf35c2 100644 --- a/.buildkite/scripts/steps/artifacts/build.sh +++ b/.buildkite/scripts/steps/artifacts/build.sh @@ -3,46 +3,14 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh - -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - VERSION="$(jq -r '.version' package.json)" - RELEASE_ARG="--release" -else - VERSION="$(jq -r '.version' package.json)-SNAPSHOT" - RELEASE_ARG="" -fi +.buildkite/scripts/steps/artifacts/env.sh echo "--- Build Kibana Distribution" -node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile --skip-docker-cloud - -echo "--- Build dependencies report" -node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv" - -# Release verification -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - echo "--- Build and push Kibana Cloud Distribution" - # This doesn't meet the requirements for a release image, implementation TBD - # Beats artifacts will need to match a specific commit sha that matches other stack iamges - # For now this is a placeholder step that will allow us to run automated Cloud tests - # against a best guess approximation of a release image - echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co - trap 'docker logout docker.elastic.co' EXIT - - node scripts/build \ - "$RELEASE_ARG" \ - --skip-initialize \ - --skip-generic-folders \ - --skip-platform-folders \ - --skip-archives \ - --docker-images \ - --docker-tag-qualifier="$GIT_COMMIT" \ - --docker-push \ - --skip-docker-ubi \ - --skip-docker-ubuntu \ - --skip-docker-contexts -fi +node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS") +echo "--- Build and upload dependencies report" +node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv" cd target -sha512sum "dependencies-$VERSION.csv" > "dependencies-$VERSION.csv.sha512.txt" +sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt" buildkite-agent artifact upload "*" cd - diff --git a/.buildkite/scripts/steps/artifacts/docker_context.sh b/.buildkite/scripts/steps/artifacts/docker_context.sh index 787f5d278febc..029b5174f7805 100644 --- a/.buildkite/scripts/steps/artifacts/docker_context.sh +++ b/.buildkite/scripts/steps/artifacts/docker_context.sh @@ -3,18 +3,11 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh - -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - VERSION="$(jq -r '.version' package.json)" - RELEASE_ARG="--release" -else - VERSION="$(jq -r '.version' package.json)-SNAPSHOT" - RELEASE_ARG="" -fi +.buildkite/scripts/steps/artifacts/env.sh echo "--- Create contexts" mkdir -p target -node scripts/build "$RELEASE_ARG" --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact +node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact $(echo "$BUILD_ARGS") echo "--- Setup default context" DOCKER_BUILD_FOLDER=$(mktemp -d) @@ -22,7 +15,7 @@ DOCKER_BUILD_FOLDER=$(mktemp -d) tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER" cd $DOCKER_BUILD_FOLDER -buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" +buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" echo "--- Build context" docker build . diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 30758511a0b04..4bc7c04282209 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -2,15 +2,8 @@ set -euo pipefail -source .buildkite/scripts/common/util.sh - -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then - VERSION="$(jq -r '.version' package.json)" - WORKFLOW="staging" -else - VERSION="$(jq -r '.version' package.json)-SNAPSHOT" - WORKFLOW="snapshot" -fi +.buildkite/scripts/common/util.sh +.buildkite/scripts/steps/artifacts/env.sh echo "--- Download and verify artifacts" function download { @@ -23,28 +16,28 @@ function download { mkdir -p target cd target -download "kibana-$VERSION-docker-image.tar.gz" -download "kibana-$VERSION-docker-image-aarch64.tar.gz" -download "kibana-ubi8-$VERSION-docker-image.tar.gz" +download "kibana-$FULL_VERSION-docker-image.tar.gz" +download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz" +download "kibana-ubi8-$FULL_VERSION-docker-image.tar.gz" -download "kibana-$VERSION-arm64.deb" -download "kibana-$VERSION-amd64.deb" -download "kibana-$VERSION-x86_64.rpm" -download "kibana-$VERSION-aarch64.rpm" +download "kibana-$FULL_VERSION-arm64.deb" +download "kibana-$FULL_VERSION-amd64.deb" +download "kibana-$FULL_VERSION-x86_64.rpm" +download "kibana-$FULL_VERSION-aarch64.rpm" -download "kibana-$VERSION-docker-build-context.tar.gz" -download "kibana-ironbank-$VERSION-docker-build-context.tar.gz" -download "kibana-ubi8-$VERSION-docker-build-context.tar.gz" +download "kibana-$FULL_VERSION-docker-build-context.tar.gz" +download "kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz" +download "kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz" -download "kibana-$VERSION-linux-aarch64.tar.gz" -download "kibana-$VERSION-linux-x86_64.tar.gz" +download "kibana-$FULL_VERSION-linux-aarch64.tar.gz" +download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" -download "kibana-$VERSION-darwin-x86_64.tar.gz" -download "kibana-$VERSION-darwin-aarch64.tar.gz" +download "kibana-$FULL_VERSION-darwin-x86_64.tar.gz" +download "kibana-$FULL_VERSION-darwin-aarch64.tar.gz" -download "kibana-$VERSION-windows-x86_64.zip" +download "kibana-$FULL_VERSION-windows-x86_64.zip" -download "dependencies-$VERSION.csv" +download "dependencies-$FULL_VERSION.csv" cd - @@ -61,8 +54,6 @@ echo "--- Publish artifacts" export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" export VAULT_ADDR="https://secrets.elastic.co:8200" -QUALIFIER="" -BASE_VERSION="$(jq -r '.version' package.json)" docker run --rm \ --name release-manager \ -e VAULT_ADDR \ @@ -76,5 +67,5 @@ docker run --rm \ --commit "$GIT_COMMIT" \ --workflow "$WORKFLOW" \ --version "$BASE_VERSION" \ - --qualifier "$QUALIFIER" \ + --qualifier "$VERSION_QUALIFIER" \ --artifact-set main From 8fe30999c1be25164cd8314604170c0a7164f664 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:03:09 -0500 Subject: [PATCH 23/48] setup env --- .buildkite/scripts/steps/artifacts/env.sh | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .buildkite/scripts/steps/artifacts/env.sh diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh new file mode 100644 index 0000000000000..5f08dec611682 --- /dev/null +++ b/.buildkite/scripts/steps/artifacts/env.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo pipefail + +BASE_VERSION="$(jq -r '.version' package.json)" +VERSION_QUALIFIER="${VERSION_QUALIFIER:=}" + +if [[ "${VERSION_QUALIFIER:-}" == "" ]]; then + QUALIFIER_VERSION="$BASE_VERSION" +else + QUALIFIER_VERSION="$BASE_VERSION-$VERSION_QUALIFIER" +fi + +if [[ "${RELEASE_BUILD:-}" == "true" ]]; then + FULL_VERSION="$QUALIFIER_VERSION" + + # Beats artifacts will need to match a specific commit sha that matches other stack images + # for release builds. + # For now we are skipping staging Cloud builds until there's a pointer. + BUILD_ARGS="--release --skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER" + WORKFLOW="staging" +else + FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT" + BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER" + WORKFLOW="snapshot" +fi + +export VERSION_QUALIFIER +export BASE_VERSION +export QUALIFIER_VERSION +export FULL_VERSION +export BUILD_ARGS +export WORKFLOW \ No newline at end of file From 9cde7a431c66c1b62d310d1f0c1ec00a6c8f0aa3 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:07:07 -0500 Subject: [PATCH 24/48] cleanup --- .buildkite/pipelines/artifacts.yml | 10 +++++----- .buildkite/scripts/steps/artifacts/build.sh | 2 +- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index f472a7b77943b..1f60f5ae60520 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -8,25 +8,25 @@ steps: - wait - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for deb + label: Artifact Testing agents: queue: n2-4-virt timeout_in_minutes: 30 - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for rpm + label: Artifact Testing agents: queue: n2-4-virt timeout_in_minutes: 30 - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh - label: Package testing for docker + label: Artifact Testing agents: queue: n2-4-virt timeout_in_minutes: 30 - command: .buildkite/scripts/steps/artifacts/docker_context.sh - label: 'Docker Build Context' + label: 'Docker Context Verification' agents: queue: n2-2 timeout_in_minutes: 30 @@ -34,7 +34,7 @@ steps: - wait - command: .buildkite/scripts/steps/artifacts/publish.sh - label: 'Publish artifacts' + label: 'Publish Artifacts' agents: queue: n2-2 timeout_in_minutes: 30 diff --git a/.buildkite/scripts/steps/artifacts/build.sh b/.buildkite/scripts/steps/artifacts/build.sh index 3b7e7eacf35c2..bbec37ccc3efc 100644 --- a/.buildkite/scripts/steps/artifacts/build.sh +++ b/.buildkite/scripts/steps/artifacts/build.sh @@ -5,7 +5,7 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh .buildkite/scripts/steps/artifacts/env.sh -echo "--- Build Kibana Distribution" +echo "--- Build Kibana artifacts" node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS") echo "--- Build and upload dependencies report" diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 4bc7c04282209..1b8f4e5006934 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -45,7 +45,7 @@ echo "--- Set artifact permissions" chmod -R a+r target/* chmod -R a+w target -echo "--- Pull latest release manager" +echo "--- Pull latest Release Manager CLI" echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co trap 'docker logout docker.elastic.co' EXIT docker pull docker.elastic.co/infra/release-manager:latest From 25165df24c7b026ef0bf672d0540843fe2728889 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:09:03 -0500 Subject: [PATCH 25/48] consistency --- .buildkite/pipelines/artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index 1f60f5ae60520..647f2c317569c 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -34,7 +34,7 @@ steps: - wait - command: .buildkite/scripts/steps/artifacts/publish.sh - label: 'Publish Artifacts' + label: 'Publish Kibana Artifacts' agents: queue: n2-2 timeout_in_minutes: 30 From b70d19d5bfdf8fd236022d89308381a613d6c67d Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:11:13 -0500 Subject: [PATCH 26/48] \n --- .buildkite/scripts/steps/artifacts/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh index 5f08dec611682..ab8906fa9a840 100644 --- a/.buildkite/scripts/steps/artifacts/env.sh +++ b/.buildkite/scripts/steps/artifacts/env.sh @@ -30,4 +30,4 @@ export BASE_VERSION export QUALIFIER_VERSION export FULL_VERSION export BUILD_ARGS -export WORKFLOW \ No newline at end of file +export WORKFLOW From 87035b4d530fbe092cc2ebf4d18b718ff11c6dc3 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:12:06 -0500 Subject: [PATCH 27/48] comment --- .buildkite/scripts/steps/artifacts/env.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh index ab8906fa9a840..91c4da42a0495 100644 --- a/.buildkite/scripts/steps/artifacts/env.sh +++ b/.buildkite/scripts/steps/artifacts/env.sh @@ -15,8 +15,7 @@ if [[ "${RELEASE_BUILD:-}" == "true" ]]; then FULL_VERSION="$QUALIFIER_VERSION" # Beats artifacts will need to match a specific commit sha that matches other stack images - # for release builds. - # For now we are skipping staging Cloud builds until there's a pointer. + # for release builds. For now we are skipping Cloud builds until there's a pointer. BUILD_ARGS="--release --skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER" WORKFLOW="staging" else From 6183eef67e5ebd1ac8d0da474a43bd39187eb3be Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:14:48 -0500 Subject: [PATCH 28/48] unneccessary fallback --- .buildkite/scripts/steps/artifacts/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh index 91c4da42a0495..609a3d2bf515d 100644 --- a/.buildkite/scripts/steps/artifacts/env.sh +++ b/.buildkite/scripts/steps/artifacts/env.sh @@ -5,7 +5,7 @@ set -euo pipefail BASE_VERSION="$(jq -r '.version' package.json)" VERSION_QUALIFIER="${VERSION_QUALIFIER:=}" -if [[ "${VERSION_QUALIFIER:-}" == "" ]]; then +if [[ "$VERSION_QUALIFIER" == "" ]]; then QUALIFIER_VERSION="$BASE_VERSION" else QUALIFIER_VERSION="$BASE_VERSION-$VERSION_QUALIFIER" From 0b9c997cc10cd84c899ffa72a000c6464fd7d40b Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:18:14 -0500 Subject: [PATCH 29/48] +x --- .buildkite/scripts/steps/artifacts/env.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .buildkite/scripts/steps/artifacts/env.sh diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh old mode 100644 new mode 100755 From 2466e7df65223cbcbee0bc399eaef02155774db1 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:23:34 -0500 Subject: [PATCH 30/48] hoist env args --- .buildkite/scripts/steps/artifacts/env.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh index 609a3d2bf515d..ea2850ff9a38a 100755 --- a/.buildkite/scripts/steps/artifacts/env.sh +++ b/.buildkite/scripts/steps/artifacts/env.sh @@ -2,8 +2,10 @@ set -euo pipefail +RELEASE_BUILD="${RELEASE_BUILD:="false"}" +VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}" + BASE_VERSION="$(jq -r '.version' package.json)" -VERSION_QUALIFIER="${VERSION_QUALIFIER:=}" if [[ "$VERSION_QUALIFIER" == "" ]]; then QUALIFIER_VERSION="$BASE_VERSION" @@ -11,7 +13,7 @@ else QUALIFIER_VERSION="$BASE_VERSION-$VERSION_QUALIFIER" fi -if [[ "${RELEASE_BUILD:-}" == "true" ]]; then +if [[ "$RELEASE_BUILD" == "true" ]]; then FULL_VERSION="$QUALIFIER_VERSION" # Beats artifacts will need to match a specific commit sha that matches other stack images From cf99ad722c86c9f917df5682b6b0fca9b9a3f2ed Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 11:25:54 -0500 Subject: [PATCH 31/48] source --- .buildkite/scripts/steps/artifacts/build.sh | 3 ++- .buildkite/scripts/steps/artifacts/docker_context.sh | 3 ++- .buildkite/scripts/steps/artifacts/publish.sh | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/build.sh b/.buildkite/scripts/steps/artifacts/build.sh index bbec37ccc3efc..152b3ea9bf72b 100644 --- a/.buildkite/scripts/steps/artifacts/build.sh +++ b/.buildkite/scripts/steps/artifacts/build.sh @@ -3,7 +3,8 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh -.buildkite/scripts/steps/artifacts/env.sh + +source .buildkite/scripts/steps/artifacts/env.sh echo "--- Build Kibana artifacts" node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS") diff --git a/.buildkite/scripts/steps/artifacts/docker_context.sh b/.buildkite/scripts/steps/artifacts/docker_context.sh index 029b5174f7805..a20544de18fd9 100644 --- a/.buildkite/scripts/steps/artifacts/docker_context.sh +++ b/.buildkite/scripts/steps/artifacts/docker_context.sh @@ -3,7 +3,8 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh -.buildkite/scripts/steps/artifacts/env.sh + +source .buildkite/scripts/steps/artifacts/env.sh echo "--- Create contexts" mkdir -p target diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 1b8f4e5006934..9d04ce6cb81a2 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -2,8 +2,8 @@ set -euo pipefail -.buildkite/scripts/common/util.sh -.buildkite/scripts/steps/artifacts/env.sh +source .buildkite/scripts/common/util.sh +source .buildkite/scripts/steps/artifacts/env.sh echo "--- Download and verify artifacts" function download { From 37448ca747eb5c8567bf53c78e3c9fb0ce632588 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 12:04:38 -0500 Subject: [PATCH 32/48] link to artifacts --- .buildkite/scripts/steps/artifacts/publish.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 9d04ce6cb81a2..e93cedd7e2388 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -69,3 +69,13 @@ docker run --rm \ --version "$BASE_VERSION" \ --qualifier "$VERSION_QUALIFIER" \ --artifact-set main + +BUILD_SUBDOMAIN="artifacts-$WORKFLOW" +BUILD_ID="$BASE_VERSION-${GIT_COMMIT:0:8}" +BUILD_REPORT="summary-$FULL_VERSION.html" + +cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary + ### Artifacts Summary + + https://$BUILD_SUBDOMAIN.elastic.co/kibana/$BUILD_ID/$BUILD_REPORT +EOF \ No newline at end of file From 82cc0f94f1db05ad9aee70259016047dc0394a71 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 13:13:51 -0500 Subject: [PATCH 33/48] fix artifacts summary --- .buildkite/scripts/steps/artifacts/publish.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index e93cedd7e2388..38c13c8b1798d 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -70,12 +70,11 @@ docker run --rm \ --qualifier "$VERSION_QUALIFIER" \ --artifact-set main -BUILD_SUBDOMAIN="artifacts-$WORKFLOW" -BUILD_ID="$BASE_VERSION-${GIT_COMMIT:0:8}" -BUILD_REPORT="summary-$FULL_VERSION.html" +ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW" +ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url' cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary ### Artifacts Summary - https://$BUILD_SUBDOMAIN.elastic.co/kibana/$BUILD_ID/$BUILD_REPORT + $ARTIFACTS_SUMMARY EOF \ No newline at end of file From 338de07694dfeb2958a2c8c6cbc918aeca93904d Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 13:15:18 -0500 Subject: [PATCH 34/48] relax docker glob --- .buildkite/scripts/steps/package_testing/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/package_testing/test.sh b/.buildkite/scripts/steps/package_testing/test.sh index a9a46502d5b3b..4c8bac026af84 100755 --- a/.buildkite/scripts/steps/package_testing/test.sh +++ b/.buildkite/scripts/steps/package_testing/test.sh @@ -15,7 +15,7 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" KIBANA_IP_ADDRESS="192.168.56.6" elif [[ "$TEST_PACKAGE" == "docker" ]]; then - buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" + buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" KIBANA_IP_ADDRESS="192.168.56.7" fi cd .. From c988f4d42f25c25163cc289306d5d4467e72c16e Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 13:34:39 -0500 Subject: [PATCH 35/48] \n --- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 38c13c8b1798d..ed96f3907027f 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -77,4 +77,4 @@ cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary ### Artifacts Summary $ARTIFACTS_SUMMARY -EOF \ No newline at end of file +EOF From 33250f83c3f7baa57367bdb181d12a6ef626224e Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 14:44:59 -0500 Subject: [PATCH 36/48] fix summary --- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index ed96f3907027f..d36191c5cf60b 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -71,7 +71,7 @@ docker run --rm \ --artifact-set main ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW" -ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url' +ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$BASE_VERSION.json" | jq -re '.summary_url') cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary ### Artifacts Summary From cfb6442a3e0a82f3d93c26af7bc4b3baa1f26bee Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 11 Apr 2022 14:47:07 -0500 Subject: [PATCH 37/48] fix variable --- .buildkite/scripts/steps/artifacts/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index d36191c5cf60b..edb82d6f83ca0 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -71,7 +71,7 @@ docker run --rm \ --artifact-set main ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW" -ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$BASE_VERSION.json" | jq -re '.summary_url') +ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url') cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary ### Artifacts Summary From 9fd823339307ce3ce7d6daf86faef1c83c50dad4 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 09:01:16 -0500 Subject: [PATCH 38/48] rm es docker image --- .buildkite/scripts/steps/artifacts/cloud.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index 7eb77ab13f058..7496f5feccc81 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -42,7 +42,6 @@ DEPLOYMENT_SPEC=$(mktemp --suffix ".json") jq ' .name = "'$CLOUD_DEPLOYMENT_NAME'" | .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_TEST_IMAGE'" | - .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$VERSION'" | .resources.kibana[0].plan.kibana.version = "'$VERSION'" | .resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'" | .resources.enterprise_search[0].plan.enterprise_search.version = "'$VERSION'" | From a634d44e63cb6dfd1bc0a7a5fdf82863df8a79a8 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 11:26:51 -0500 Subject: [PATCH 39/48] setup cloud env --- .buildkite/scripts/steps/artifacts/cloud.sh | 41 +++++++++++++-------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index 7496f5feccc81..11e9fdcaf8eae 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -57,18 +57,29 @@ CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url') CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url') -trap 'echo "--- Shutdown deployment" && ecctl deployment shutdown --force --track --output json &> "$LOGS"' EXIT - -echo "--- Run functional tests" -# TODO -# TEST_KIBANA_PROTOCOL=https -# TEST_KIBANA_HOSTNAME=my-kibana-instance.internal.net -# TEST_KIBANA_PORT=443 -# TEST_KIBANA_USER=kibana -# TEST_KIBANA_PASS= - -# TEST_ES_PROTOCOL=https -# TEST_ES_HOSTNAME=my-es-cluster.internal.net -# TEST_ES_PORT=9200 -# TEST_ES_USER="$CLOUD_DEPLOYMENT_USERNAME" -# TEST_ES_PASS="$CLOUD_DEPLOYMENT_PASSWORD" +# NOTE: disabled pending log sanitization +# echo "--- Setup FTR" +# export TEST_KIBANA_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)") +# export TEST_KIBANA_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)") +# export TEST_KIBANA_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)") +# export TEST_KIBANA_USERNAME=$CLOUD_DEPLOYMENT_USERNAME" +# export TEST_KIBANA_PASS=$CLOUD_DEPLOYMENT_PASSWORD" + +# export TEST_ES_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)") +# export TEST_ES_HOSTNAME==$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)") +# export TEST_ES_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)") +# export TEST_ES_USER="$CLOUD_DEPLOYMENT_USERNAME" +# export TEST_ES_PASS="$CLOUD_DEPLOYMENT_PASSWORD" + +# Error: attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ConnectionError: self signed certificate in certificate chain +# export NODE_TLS_REJECT_UNAUTHORIZED=0 + +# echo "--- Run default functional tests" +# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud + +# echo "--- Run x-pack functional tests" +# cd x-pack +# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud + +echo "--- Shutdown deployment" +ecctl deployment shutdown --force --track --output json &> "$LOGS"' From 8ff4fe87883f3519d96bc23f82fbaa04526a6bbf Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 11:28:04 -0500 Subject: [PATCH 40/48] rename deployment --- .buildkite/scripts/steps/artifacts/cloud.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index 11e9fdcaf8eae..6fd926e03f5ce 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -34,7 +34,7 @@ docker logout docker.elastic.co cd - echo "--- Create deployment" -CLOUD_DEPLOYMENT_NAME="kibana-release-$TAG" +CLOUD_DEPLOYMENT_NAME="kibana-artifacts-$TAG" LOGS=$(mktemp --suffix ".json") DEPLOYMENT_SPEC=$(mktemp --suffix ".json") From 3cf9db6a2b852ce18c249bb04c0767a2bc030c25 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 11:29:19 -0500 Subject: [PATCH 41/48] exclude skipCloud --- .buildkite/scripts/steps/artifacts/cloud.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index 6fd926e03f5ce..5956cec9f3807 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -75,11 +75,11 @@ CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID # export NODE_TLS_REJECT_UNAUTHORIZED=0 # echo "--- Run default functional tests" -# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud +# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud # echo "--- Run x-pack functional tests" # cd x-pack -# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud +# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud echo "--- Shutdown deployment" ecctl deployment shutdown --force --track --output json &> "$LOGS"' From 4c5996ea7a36bcc7caec1df619264b4260c632b4 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 11:40:36 -0500 Subject: [PATCH 42/48] only run on snapshot builds --- .buildkite/pipelines/artifacts.yml | 41 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index f0d381c9b9166..b87fb342b206b 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -7,32 +7,33 @@ steps: - wait - # - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh - # label: Package testing for deb - # agents: - # queue: n2-4-virt - # timeout_in_minutes: 20 + - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh + label: Package testing for deb + agents: + queue: n2-4-virt + timeout_in_minutes: 20 - # - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh - # label: Package testing for rpm - # agents: - # queue: n2-4-virt - # timeout_in_minutes: 20 + - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh + label: Package testing for rpm + agents: + queue: n2-4-virt + timeout_in_minutes: 20 - # - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh - # label: Package testing for docker - # agents: - # queue: n2-4-virt - # timeout_in_minutes: 20 + - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh + label: Package testing for docker + agents: + queue: n2-4-virt + timeout_in_minutes: 20 - # - command: .buildkite/scripts/steps/artifacts/docker_context.sh - # label: 'Docker Build Context' - # agents: - # queue: n2-2 - # timeout_in_minutes: 20 + - command: .buildkite/scripts/steps/artifacts/docker_context.sh + label: 'Docker Build Context' + agents: + queue: n2-2 + timeout_in_minutes: 20 - command: .buildkite/scripts/steps/artifacts/cloud.sh label: 'FTR on Cloud' agents: queue: n2-2 timeout_in_minutes: 60 + if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'" \ No newline at end of file From 477f6dfdbcfa993251d84889e17d422d3732cd8a Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 11:46:21 -0500 Subject: [PATCH 43/48] update step name --- .buildkite/pipelines/artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index b87fb342b206b..43b30e86878a3 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -32,7 +32,7 @@ steps: timeout_in_minutes: 20 - command: .buildkite/scripts/steps/artifacts/cloud.sh - label: 'FTR on Cloud' + label: 'Cloud Deployment' agents: queue: n2-2 timeout_in_minutes: 60 From fcfa52ced7020be69ba76d367dfc356299cf77c7 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 11:53:05 -0500 Subject: [PATCH 44/48] TEST_BROWSER_HEADLESS=1 --- .buildkite/scripts/steps/artifacts/cloud.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index 5956cec9f3807..32b226e282b5c 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -71,6 +71,8 @@ CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID # export TEST_ES_USER="$CLOUD_DEPLOYMENT_USERNAME" # export TEST_ES_PASS="$CLOUD_DEPLOYMENT_PASSWORD" +# export TEST_BROWSER_HEADLESS=1 + # Error: attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ConnectionError: self signed certificate in certificate chain # export NODE_TLS_REJECT_UNAUTHORIZED=0 From 7cedcce0eeb143d46681a89ab06887b314cd5bfc Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 12:22:15 -0500 Subject: [PATCH 45/48] cleanup --- .buildkite/scripts/steps/artifacts/cloud.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index 32b226e282b5c..ea38e26b2c761 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -84,4 +84,4 @@ CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID # node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud echo "--- Shutdown deployment" -ecctl deployment shutdown --force --track --output json &> "$LOGS"' +ecctl deployment shutdown --force --track --output json &> "$LOGS" From 62deac4cbdae6d422268c74285b8088355503528 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 12:40:34 -0500 Subject: [PATCH 46/48] newline --- .buildkite/pipelines/artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index 43b30e86878a3..64c9ce1070bb6 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -36,4 +36,4 @@ steps: agents: queue: n2-2 timeout_in_minutes: 60 - if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'" \ No newline at end of file + if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'" From f47e6b4cc7046f4f4d3bf46549c49b8c19b407ef Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 16:53:03 -0500 Subject: [PATCH 47/48] consistency --- .buildkite/pipelines/artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml index a2ba41f29ba4b..ab3201472b5c2 100644 --- a/.buildkite/pipelines/artifacts.yml +++ b/.buildkite/pipelines/artifacts.yml @@ -35,7 +35,7 @@ steps: label: 'Cloud Deployment' agents: queue: n2-2 - timeout_in_minutes: 60 + timeout_in_minutes: 30 if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'" - wait From 5a3a3c63bba89adb3e65e3dd116c769595204dd8 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 12 Apr 2022 16:58:35 -0500 Subject: [PATCH 48/48] pass deployment id --- .buildkite/scripts/steps/artifacts/cloud.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index 952aafb453a42..8fa04a5d176b0 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -78,4 +78,4 @@ CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID # node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud echo "--- Shutdown deployment" -ecctl deployment shutdown --force --track --output json &> "$LOGS" +ecctl deployment shutdown "$CLOUD_DEPLOYMENT_ID" --force --track --output json &> "$LOGS"