From 81910b2ebe92387cc736e51e98e84f25d71fbb70 Mon Sep 17 00:00:00 2001
From: Alex Szabo <alex.szabo@elastic.co>
Date: Wed, 15 May 2024 09:42:42 +0200
Subject: [PATCH] [CI] Fix issues related to publish (#183393)

## Summary
On the new infra, the publish step will still require legacy vault
credentials and login.
(https://buildkite.com/elastic/kibana-artifacts-staging/builds/3513#018f7691-73c8-4e6f-862b-328b05d9de3b)

As a fix: this PR digs up the credentials from the vault instead of
gcloud secrets on the new infra.

Also, other usages of role-id/secret-id is used are moved in the
legacy-vault usages, plus minor code re-org, to reduce branching, and
future cleanup.

(cherry picked from commit 05fce3b4badb0ed5abc7cfab0e529dd8a39e2827)

# Conflicts:
#	.buildkite/scripts/steps/serverless/build_and_deploy.sh
---
 .buildkite/scripts/common/vault_fns.sh        | 20 +++++++++++++++++++
 .buildkite/scripts/steps/artifacts/publish.sh |  4 ++--
 .../scripts/steps/cloud/build_and_deploy.sh   | 14 ++++++-------
 .../steps/serverless/build_and_deploy.sh      |  8 ++++----
 4 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/.buildkite/scripts/common/vault_fns.sh b/.buildkite/scripts/common/vault_fns.sh
index a7b92a4b05d6d..022a22541d6bf 100644
--- a/.buildkite/scripts/common/vault_fns.sh
+++ b/.buildkite/scripts/common/vault_fns.sh
@@ -65,3 +65,23 @@ vault_kv_set() {
 
   vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}"
 }
+
+function get_vault_role_id() {
+  if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+    VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
+  else
+    VAULT_ROLE_ID="$(vault_get kibana-buildkite-vault-credentials role-id)"
+  fi
+
+  echo "$VAULT_ROLE_ID"
+}
+
+function get_vault_secret_id() {
+    if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+      VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+    else
+      VAULT_SECRET_ID="$(vault_get kibana-buildkite-vault-credentials secret-id)"
+    fi
+
+    echo "$VAULT_SECRET_ID"
+}
diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh
index 2621242fe0aa7..949fe89ebdb01 100644
--- a/.buildkite/scripts/steps/artifacts/publish.sh
+++ b/.buildkite/scripts/steps/artifacts/publish.sh
@@ -55,8 +55,8 @@ docker pull docker.elastic.co/infra/release-manager:latest
 
 echo "--- Publish artifacts"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
-  export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
-  export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+  export VAULT_ROLE_ID="$(get_vault_role_id)"
+  export VAULT_SECRET_ID="$(get_vault_secret_id)"
   export VAULT_ADDR="https://secrets.elastic.co:8200"
 
   download_artifact beats_manifest.json /tmp --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
diff --git a/.buildkite/scripts/steps/cloud/build_and_deploy.sh b/.buildkite/scripts/steps/cloud/build_and_deploy.sh
index 6a7a95f8eaf10..270aad9c6327b 100755
--- a/.buildkite/scripts/steps/cloud/build_and_deploy.sh
+++ b/.buildkite/scripts/steps/cloud/build_and_deploy.sh
@@ -83,13 +83,13 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
   CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$ECCTL_LOGS")
 
   echo "Writing to vault..."
-  VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
-  VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
-  VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
-  retry 5 30 vault login -no-print "$VAULT_TOKEN"
 
   # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
   if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+    VAULT_ROLE_ID="$(get_vault_role_id)"
+    VAULT_SECRET_ID="$(get_vault_secret_id)"
+    VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+    retry 5 30 vault login -no-print "$VAULT_TOKEN"
     vault_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
   else
     vault_kv_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
@@ -125,9 +125,6 @@ else
   ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
 fi
 
-CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
-CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
-
 # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
 if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
   VAULT_READ_COMMAND="vault read $VAULT_PATH_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME"
@@ -135,6 +132,9 @@ else
   VAULT_READ_COMMAND="vault kv get $VAULT_KV_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME"
 fi
 
+CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
+CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
+
 cat << EOF | buildkite-agent annotate --style "info" --context cloud
   ### Cloud Deployment
 
diff --git a/.buildkite/scripts/steps/serverless/build_and_deploy.sh b/.buildkite/scripts/steps/serverless/build_and_deploy.sh
index 44529c6dba5f5..5f00c19972792 100644
--- a/.buildkite/scripts/steps/serverless/build_and_deploy.sh
+++ b/.buildkite/scripts/steps/serverless/build_and_deploy.sh
@@ -74,13 +74,13 @@ deploy() {
     PROJECT_PASSWORD=$(jq -r --slurp '.[2].password' $DEPLOY_LOGS)
 
     echo "Write to vault..."
-    VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
-    VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
-    VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
-    retry 5 30 vault login -no-print "$VAULT_TOKEN"
 
     # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
     if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+      VAULT_ROLE_ID="$(get_vault_role_id)"
+      VAULT_SECRET_ID="$(get_vault_secret_id)"
+      VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+      retry 5 30 vault login -no-print "$VAULT_TOKEN"
       vault_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
     else
       vault_kv_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"