From db3b0d2a6d345129bfe64da08cb14101aebcea18 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 29 Jun 2022 15:13:35 -0500 Subject: [PATCH] [cft] Improve error handling (#134519) * [cft] Write to vault as soon as deployment is ready * cleanup * add status logging * quicker rebuilds * log stderr * login * fix conditional * disable self logging * cleanup logging * cleanup --- .../scripts/steps/cloud/build_and_deploy.sh | 80 +++++++++++-------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/.buildkite/scripts/steps/cloud/build_and_deploy.sh b/.buildkite/scripts/steps/cloud/build_and_deploy.sh index 91d207f8fcb31..9c45e9fa4847e 100755 --- a/.buildkite/scripts/steps/cloud/build_and_deploy.sh +++ b/.buildkite/scripts/steps/cloud/build_and_deploy.sh @@ -10,40 +10,48 @@ export KBN_NP_PLUGINS_BUILT=true VERSION="$(jq -r '.version' package.json)-SNAPSHOT" -echo "--- Download kibana distribution" +echo "--- Download Kibana Distribution" mkdir -p ./target buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" -echo "--- Build and push Kibana Cloud Distribution" - -echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co -trap 'docker logout docker.elastic.co' EXIT - -node scripts/build \ - --skip-initialize \ - --skip-generic-folders \ - --skip-platform-folders \ - --skip-archives \ - --docker-images \ - --docker-tag-qualifier="$GIT_COMMIT" \ - --docker-push \ - --skip-docker-ubi \ - --skip-docker-ubuntu \ - --skip-docker-contexts - +echo "--- Build Cloud Distribution" ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json" ELASTICSEARCH_SHA=$(curl -s $ELASTICSEARCH_MANIFEST_URL | jq -r '.sha') ELASTICSEARCH_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$VERSION-$ELASTICSEARCH_SHA" -CLOUD_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-cloud) +KIBANA_CLOUD_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$VERSION-$GIT_COMMIT" CLOUD_DEPLOYMENT_NAME="kibana-pr-$BUILDKITE_PULL_REQUEST" +echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co +set +e +DISTRIBUTION_EXISTS=$(docker manifest inspect $KIBANA_CLOUD_IMAGE &> /dev/null; echo $?) +set -e + +if [ $DISTRIBUTION_EXISTS -eq 0 ]; then + echo "Distribution already exists, skipping build" +else + node scripts/build \ + --skip-initialize \ + --skip-generic-folders \ + --skip-platform-folders \ + --skip-archives \ + --docker-images \ + --docker-tag-qualifier="$GIT_COMMIT" \ + --docker-push \ + --skip-docker-ubi \ + --skip-docker-ubuntu \ + --skip-docker-contexts +fi + +docker logout docker.elastic.co + +echo "--- Create Deployment" CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id') JSON_FILE=$(mktemp --suffix ".json") if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then jq ' - .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" | + .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" | .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$ELASTICSEARCH_CLOUD_IMAGE'" | .name = "'$CLOUD_DEPLOYMENT_NAME'" | .resources.kibana[0].plan.kibana.version = "'$VERSION'" | @@ -52,32 +60,38 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then .resources.integrations_server[0].plan.integrations_server.version = "'$VERSION'" ' .buildkite/scripts/steps/cloud/deploy.json > /tmp/deploy.json - ecctl deployment create --track --output json --file /tmp/deploy.json &> "$JSON_FILE" + echo -n "Creating deployment..." + ecctl deployment create --track --output json --file /tmp/deploy.json > "$JSON_FILE" + echo "done" + CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$JSON_FILE") CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$JSON_FILE") CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$JSON_FILE") CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$JSON_FILE") - # Enable stack monitoring - jq ' - .settings.observability.metrics.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'" | - .settings.observability.logging.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'" - ' .buildkite/scripts/steps/cloud/stack_monitoring.json > /tmp/stack_monitoring.json - ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/stack_monitoring.json &> "$JSON_FILE" - - # Refresh vault token + echo -n "Writing to vault..." VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)" VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)" VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") retry 5 30 vault login -no-print "$VAULT_TOKEN" - retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD" + + echo -n "Enabling Stack Monitoring..." + jq ' + .settings.observability.metrics.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'" | + .settings.observability.logging.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'" + ' .buildkite/scripts/steps/cloud/stack_monitoring.json > /tmp/stack_monitoring.json + ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/stack_monitoring.json > "$JSON_FILE" + echo "done" else ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq ' - .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" | + .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" | (.. | select(.version? != null).version) = "'$VERSION'" ' > /tmp/deploy.json - ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json &> "$JSON_FILE" + + echo -n "Updating deployment..." + ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$JSON_FILE" + echo "done" fi CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url') @@ -92,7 +106,7 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\` - Kibana image: \`$CLOUD_IMAGE\` + Kibana image: \`$KIBANA_CLOUD_IMAGE\` Elasticsearch image: \`$ELASTICSEARCH_CLOUD_IMAGE\` EOF