[artifacts] Deploy Cloud image

.buildkite/pipelines/artifacts.yml

@@ -8,25 +8,40 @@ steps:
   - wait
 
   - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for deb
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
 
   - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for rpm
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
 
   - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for docker
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
 
   - command: .buildkite/scripts/steps/artifacts/docker_context.sh
-    label: 'Docker Build Context'
+    label: 'Docker Context Verification'
     agents:
       queue: n2-2
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+
+  - command: .buildkite/scripts/steps/artifacts/cloud.sh
+    label: 'Cloud Deployment'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
+
+  - wait
+
+  - command: .buildkite/scripts/steps/artifacts/publish.sh
+    label: 'Publish Kibana Artifacts'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30

.buildkite/scripts/steps/artifacts/build.sh

@@ -4,44 +4,14 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  VERSION="$(jq -r '.version' package.json)"
-  RELEASE_ARG="--release"
-else
-  VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
-  RELEASE_ARG=""
-fi
+source .buildkite/scripts/steps/artifacts/env.sh
 
-echo "--- Build Kibana Distribution"
-node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile --skip-docker-cloud
-
-echo "--- Build dependencies report"
-node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv"
-
-# Release verification
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  echo "--- Build and push Kibana Cloud Distribution"
-  # This doesn't meet the requirements for a release image, implementation TBD
-  # Beats artifacts will need to match a specific commit sha that matches other stack iamges
-  # For now this is a placeholder step that will allow us to run automated Cloud tests
-  # against a best guess approximation of a release image
-  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-  trap 'docker logout docker.elastic.co' EXIT
-
-  node scripts/build \
-    "$RELEASE_ARG" \
-    --skip-initialize \
-    --skip-generic-folders \
-    --skip-platform-folders \
-    --skip-archives \
-    --docker-images \
-    --docker-tag-qualifier="$GIT_COMMIT" \
-    --docker-push \
-    --skip-docker-ubi \
-    --skip-docker-ubuntu \
-    --skip-docker-contexts
-fi
+echo "--- Build Kibana artifacts"
+node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS")
 
+echo "--- Build and upload dependencies report"
+node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv"
 cd target
+sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt"
 buildkite-agent artifact upload "*"
-cd -
+cd -

.buildkite/scripts/steps/artifacts/cloud.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+source .buildkite/scripts/steps/artifacts/env.sh
+
+echo "--- Publish Cloud image"
+mkdir -p target
+cd target
+
+buildkite-agent artifact download "kibana-cloud-$FULL_VERSION-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+docker load --input kibana-cloud-$FULL_VERSION-docker-image.tar.gz
+
+TAG="$FULL_VERSION-$GIT_COMMIT"
+KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$FULL_VERSION"
+KIBANA_TEST_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$TAG"
+
+docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE"
+
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+trap 'docker logout docker.elastic.co' EXIT
+
+docker push "$KIBANA_TEST_IMAGE"
+docker logout docker.elastic.co
+
+cd -
+
+echo "--- Create deployment"
+CLOUD_DEPLOYMENT_NAME="kibana-artifacts-$TAG"
+
+LOGS=$(mktemp --suffix ".json")
+DEPLOYMENT_SPEC=$(mktemp --suffix ".json")
+
+jq '
+  .name = "'$CLOUD_DEPLOYMENT_NAME'" |
+  .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_TEST_IMAGE'" |
+  .resources.kibana[0].plan.kibana.version = "'$FULL_VERSION'" |
+  .resources.elasticsearch[0].plan.elasticsearch.version = "'$FULL_VERSION'" |
+  .resources.enterprise_search[0].plan.enterprise_search.version = "'$FULL_VERSION'" |
+  .resources.integrations_server[0].plan.integrations_server.version = "'$FULL_VERSION'"
+  ' .buildkite/scripts/steps/cloud/deploy.json > "$DEPLOYMENT_SPEC"
+
+ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" &> "$LOGS"
+CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
+CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
+CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS")
+CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$LOGS")
+
+CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
+CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
+
+# NOTE: disabled pending log sanitization
+# echo "--- Setup FTR"
+# export TEST_KIBANA_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)")
+# export TEST_KIBANA_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
+# export TEST_KIBANA_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
+# export TEST_KIBANA_USERNAME=$CLOUD_DEPLOYMENT_USERNAME"
+# export TEST_KIBANA_PASS=$CLOUD_DEPLOYMENT_PASSWORD"
+
+# export TEST_ES_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)")
+# export TEST_ES_HOSTNAME==$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
+# export TEST_ES_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
+# export TEST_ES_USER="$CLOUD_DEPLOYMENT_USERNAME"
+# export TEST_ES_PASS="$CLOUD_DEPLOYMENT_PASSWORD"
+
+# export TEST_BROWSER_HEADLESS=1
+
+# Error: attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ConnectionError: self signed certificate in certificate chain
+# export NODE_TLS_REJECT_UNAUTHORIZED=0
+
+# echo "--- Run default functional tests"
+# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud
+
+# echo "--- Run x-pack functional tests"
+# cd x-pack
+# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud 
+
+echo "--- Shutdown deployment"
+ecctl deployment shutdown "$CLOUD_DEPLOYMENT_ID" --force --track --output json &> "$LOGS"

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -4,25 +4,19 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  VERSION="$(jq -r '.version' package.json)"
-  RELEASE_ARG="--release"
-else
-  VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
-  RELEASE_ARG=""
-fi
+source .buildkite/scripts/steps/artifacts/env.sh
 
 echo "--- Create contexts"
 mkdir -p target
-node scripts/build "$RELEASE_ARG" --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact
+node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact $(echo "$BUILD_ARGS")
 
 echo "--- Setup default context"
 DOCKER_BUILD_FOLDER=$(mktemp -d)
 
 tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER"
 cd $DOCKER_BUILD_FOLDER
 
-buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 echo "--- Build context"
 docker build .

.buildkite/scripts/steps/artifacts/env.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+RELEASE_BUILD="${RELEASE_BUILD:="false"}"
+VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
+
+BASE_VERSION="$(jq -r '.version' package.json)"
+
+if [[ "$VERSION_QUALIFIER" == "" ]]; then
+  QUALIFIER_VERSION="$BASE_VERSION"
+else
+  QUALIFIER_VERSION="$BASE_VERSION-$VERSION_QUALIFIER"
+fi
+
+if [[ "$RELEASE_BUILD" == "true" ]]; then
+  FULL_VERSION="$QUALIFIER_VERSION"
+
+  # Beats artifacts will need to match a specific commit sha that matches other stack images
+  # for release builds.  For now we are skipping Cloud builds until there's a pointer.
+  BUILD_ARGS="--release --skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER"
+  WORKFLOW="staging"
+else
+  FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT"
+  BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER"
+  WORKFLOW="snapshot"
+fi
+
+export VERSION_QUALIFIER
+export BASE_VERSION
+export QUALIFIER_VERSION
+export FULL_VERSION
+export BUILD_ARGS
+export WORKFLOW

.buildkite/scripts/steps/artifacts/publish.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/artifacts/env.sh
+
+echo "--- Download and verify artifacts"
+function download {
+  buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  sha512sum -c "$1.sha512.txt"
+  rm "$1.sha512.txt"
+}
+
+mkdir -p target
+cd target
+
+download "kibana-$FULL_VERSION-docker-image.tar.gz"
+download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz"
+download "kibana-ubi8-$FULL_VERSION-docker-image.tar.gz"
+
+download "kibana-$FULL_VERSION-arm64.deb"
+download "kibana-$FULL_VERSION-amd64.deb"
+download "kibana-$FULL_VERSION-x86_64.rpm"
+download "kibana-$FULL_VERSION-aarch64.rpm"
+
+download "kibana-$FULL_VERSION-docker-build-context.tar.gz"
+download "kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz"
+download "kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
+
+download "kibana-$FULL_VERSION-linux-aarch64.tar.gz"
+download "kibana-$FULL_VERSION-linux-x86_64.tar.gz"
+
+download "kibana-$FULL_VERSION-darwin-x86_64.tar.gz"
+download "kibana-$FULL_VERSION-darwin-aarch64.tar.gz"
+
+download "kibana-$FULL_VERSION-windows-x86_64.zip"
+
+download "dependencies-$FULL_VERSION.csv"
+
+cd - 
+
+echo "--- Set artifact permissions"
+chmod -R a+r target/*
+chmod -R a+w target
+
+echo "--- Pull latest Release Manager CLI"
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+trap 'docker logout docker.elastic.co' EXIT
+docker pull docker.elastic.co/infra/release-manager:latest
+
+echo "--- Publish artifacts"
+export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
+export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+export VAULT_ADDR="https://secrets.elastic.co:8200"
+docker run --rm \
+  --name release-manager \
+  -e VAULT_ADDR \
+  -e VAULT_ROLE_ID \
+  -e VAULT_SECRET_ID \
+  --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
+  docker.elastic.co/infra/release-manager:latest \
+    cli collect \
+      --project kibana \
+      --branch "$KIBANA_BASE_BRANCH" \
+      --commit "$GIT_COMMIT" \
+      --workflow "$WORKFLOW" \
+      --version "$BASE_VERSION" \
+      --qualifier "$VERSION_QUALIFIER" \
+      --artifact-set main
+
+ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
+ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url')
+
+cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
+  ### Artifacts Summary
+
+  $ARTIFACTS_SUMMARY
+EOF

.buildkite/scripts/steps/package_testing/test.sh

@@ -15,7 +15,7 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
   buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.6"
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
-  buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.7"
 fi
 cd ..