Skip to content

Commit

Permalink
[Ops] Switch to using the migrated secret paths for vault secrets (#1…
Browse files Browse the repository at this point in the history
…72175)

## Summary
We're moving to a different vault address/instance when we're on the
elastic-wide buildkite infra. While the migration is in progress, we can
bridge between using this solution.

✅ Tested the status quo by running the PR pipeline (tests all the loads
from `pre-command`) and by using `ci:cloud-deploy` (tests vault
writing).

🟠 Tested the new vault provider on this PR:
#171317
The secrets can be accessed, *but they can't be written* (neither by me
nor) by the PR pipeline. Change requested here:
https://elasticco.atlassian.net/browse/ENGPRD-414

However, this PR can be merged without figuring out write access to
secrets, this will work as long as we're on the `kibana-buildkite`
infra.

---
Closes: elastic/kibana-operations#28

Based on: #157220

---------

Co-authored-by: Jon <[email protected]>
  • Loading branch information
delanni and jbudz authored Dec 6, 2023
1 parent b45c189 commit 1022ccd
Show file tree
Hide file tree
Showing 12 changed files with 93 additions and 44 deletions.
31 changes: 31 additions & 0 deletions .buildkite/scripts/common/util.sh
Original file line number Diff line number Diff line change
Expand Up @@ -170,3 +170,34 @@ npm_install_global() {
download_artifact() {
retry 3 1 timeout 3m buildkite-agent artifact download "$@"
}


vault_get() {
path=$1
field=$2

fullPath="secret/ci/elastic-kibana/$path"
if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
fullPath="secret/kibana-issues/dev/$path"
fi

if [[ -z "${2:-}" ]]; then
retry 5 5 vault read "$fullPath"
else
retry 5 5 vault read -field="$field" "$fullPath"
fi
}

vault_set() {
path=$1
shift
fields=("$@")

fullPath="secret/ci/elastic-kibana/$path"
if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
fullPath="secret/kibana-issues/dev/$path"
fi

# shellcheck disable=SC2068
retry 5 5 vault write "$fullPath" ${fields[@]}
}
48 changes: 24 additions & 24 deletions .buildkite/scripts/lifecycle/pre_command.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ echo '--- Setup environment vars'
source .buildkite/scripts/common/env.sh
source .buildkite/scripts/common/setup_node.sh

BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)"
export BUILDKITE_TOKEN

echo '--- Install/build buildkite dependencies'
Expand Down Expand Up @@ -77,11 +77,11 @@ EOF
{
CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"
export CI_STATS_BUILD_ID
CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-issues/dev/kibana_ci_stats)"

CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)"
export CI_STATS_TOKEN
CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"

CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)"
export CI_STATS_HOST

if [[ "$CI_STATS_BUILD_ID" ]]; then
Expand All @@ -97,66 +97,66 @@ EOF
fi
}

GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)
GITHUB_TOKEN=$(vault_get kibanamachine github_token)
export GITHUB_TOKEN

KIBANA_CI_GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibana-ci-github)
KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token)
export KIBANA_CI_GITHUB_TOKEN

KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value)
export KIBANA_CI_REPORTER_KEY

KIBANA_DOCKER_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/container-registry)"
KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
export KIBANA_DOCKER_USERNAME

KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
export KIBANA_DOCKER_PASSWORD

EC_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-cloud-deploy)"
EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)"
export EC_API_KEY

PROJECT_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-project-deploy)"
PROJECT_API_KEY="$(vault_get kibana-ci-project-deploy pr_deploy_api_key)"
export PROJECT_API_KEY

PROJECT_API_DOMAIN="$(retry 5 5 vault read -field=pr_deploy_domain secret/kibana-issues/dev/kibana-ci-project-deploy)"
PROJECT_API_DOMAIN="$(vault_get kibana-ci-project-deploy pr_deploy_domain)"
export PROJECT_API_DOMAIN

SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)"
export SYNTHETICS_SERVICE_USERNAME

SYNTHETICS_SERVICE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)"
export SYNTHETICS_SERVICE_PASSWORD

SYNTHETICS_SERVICE_MANIFEST="$(retry 5 5 vault read -field=manifest secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)"
export SYNTHETICS_SERVICE_MANIFEST

SYNTHETICS_REMOTE_KIBANA_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)"
export SYNTHETICS_REMOTE_KIBANA_USERNAME

SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)"
export SYNTHETICS_REMOTE_KIBANA_PASSWORD

SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(retry 5 5 vault read -field=url secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"}
SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"}
export SYNTHETICS_REMOTE_KIBANA_URL

DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(retry 5 5 vault read -field=DEPLOY_TAGGER_SLACK_WEBHOOK_URL secret/kibana-issues/dev/kibana-serverless-release-tools)"}
DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"}
export DEPLOY_TAGGER_SLACK_WEBHOOK_URL

# Setup Failed Test Reporter Elasticsearch credentials
{
TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)
TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id)
export TEST_FAILURES_ES_CLOUD_ID

TEST_FAILURES_ES_USERNAME=$(retry 5 5 vault read -field=username secret/kibana-issues/dev/failed_tests_reporter_es)
TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username)
export TEST_FAILURES_ES_USERNAME

TEST_FAILURES_ES_PASSWORD=$(retry 5 5 vault read -field=password secret/kibana-issues/dev/failed_tests_reporter_es)
TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password)
export TEST_FAILURES_ES_PASSWORD
}

BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"

PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ echo "--- Serverless Security Second Quality Gate"
cd x-pack/test/security_solution_api_integration
set +e

QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
QA_API_KEY=$(vault_get security-solution-qg-enc-key qa_api_key)

# Generate a random 5-digit number
random_number=$((10000 + $RANDOM % 90000))
Expand Down Expand Up @@ -47,7 +47,7 @@ while : ; do
echo "Sleeping for 40s to wait for ES status to be green..."
sleep 40
else
echo "Elasticsearch has status green."
echo "Elasticsearch has status green."
break
fi
done
Expand All @@ -59,17 +59,17 @@ while : ; do
echo "Sleeping for 15s to wait for Kibana to be available..."
sleep 15
else
echo "Kibana is available."
echo "Kibana is available."
break
fi
done

# Removing the https:// part of the url provided in order to use it in the command below.
FORMATTED_ES_URL="${ES_URL/https:\/\//}"
FORMATTED_ES_URL="${ES_URL/https:\/\//}"
FORMATTED_KB_URL="${KB_URL/https:\/\//}"

# Find a way to remove this in the future
# This is used in order to wait for the environment to be ready.
# This is used in order to wait for the environment to be ready.
sleep 150

TEST_CLOUD=1 TEST_ES_URL="https://elastic:$PASSWORD@$FORMATTED_ES_URL:443" TEST_KIBANA_URL="https://elastic:$PASSWORD@$FORMATTED_KB_URL:443" yarn run $1
Expand All @@ -79,4 +79,4 @@ echo "Exit code with status: $cmd_status"
curl --location --request DELETE "https://global.qa.cld.elstc.co/api/v1/serverless/projects/security/$ID" \
--header "Authorization: ApiKey $QA_API_KEY"

exit $cmd_status
exit $cmd_status
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@ buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true
cd x-pack/test/security_solution_cypress
set +e

QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
QA_API_KEY=$(vault_get security-solution-qg-enc-key qa_api_key)

CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status
CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status
10 changes: 8 additions & 2 deletions .buildkite/scripts/steps/cloud/build_and_deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
retry 5 30 vault login -no-print "$VAULT_TOKEN"
retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
vault_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"

echo "Enabling Stack Monitoring..."
jq '
Expand Down Expand Up @@ -121,14 +121,20 @@ fi
CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')

if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
VAULT_PATH_PREFIX="secret/kibana-issues/dev"
else
VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
fi

cat << EOF | buildkite-agent annotate --style "info" --context cloud
### Cloud Deployment
Kibana: $CLOUD_DEPLOYMENT_KIBANA_URL
Elasticsearch: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL
Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
Credentials: \`vault read $VAULT_PATH_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
Kibana image: \`$KIBANA_CLOUD_IMAGE\`
Expand Down
5 changes: 4 additions & 1 deletion .buildkite/scripts/steps/cloud/purge_deployment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ import { execSync } from 'child_process';

const deploymentsListJson = execSync('ecctl deployment list --output json').toString();
const { deployments } = JSON.parse(deploymentsListJson);
const secretBasePath = process.env.VAULT_ADDR?.match(/secrets\.elastic\.co/g)
? 'secret/kibana-issues/dev'
: 'secret/ci/elastic-kibana';

const prNumber = parseInt(
process.env.KIBANA_PULL_REQUEST ||
Expand All @@ -27,7 +30,7 @@ if (!prNumber || !deployment) {
console.log(`Scheduling deployment for deletion: ${deployment.name} / ${deployment.id}`);
try {
execSync(`ecctl deployment shutdown --force '${deployment.id}'`, { stdio: 'inherit' });
execSync(`vault delete secret/kibana-issues/dev/cloud-deploy/${deployment.name}`, {
execSync(`vault delete ${secretBasePath}/cloud-deploy/${deployment.name}`, {
stdio: 'inherit',
});
} catch (ex) {
Expand Down
5 changes: 4 additions & 1 deletion .buildkite/scripts/steps/cloud/purge_deployments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ import { execSync } from 'child_process';

const deploymentsListJson = execSync('ecctl deployment list --output json').toString();
const { deployments } = JSON.parse(deploymentsListJson);
const secretBasePath = process.env.VAULT_ADDR?.match(/secrets\.elastic\.co/g)
? 'secret/kibana-issues/dev'
: 'secret/ci/elastic-kibana';

const prDeployments = deployments.filter((deployment: any) =>
deployment.name.startsWith('kibana-pr-')
Expand Down Expand Up @@ -67,7 +70,7 @@ for (const deployment of deploymentsToPurge) {
console.log(`Scheduling deployment for deletion: ${deployment.name} / ${deployment.id}`);
try {
execSync(`ecctl deployment shutdown --force '${deployment.id}'`, { stdio: 'inherit' });
execSync(`vault delete secret/kibana-issues/dev/cloud-deploy/${deployment.name}`, {
execSync(`vault delete ${secretBasePath}/cloud-deploy/${deployment.name}`, {
stdio: 'inherit',
});
} catch (ex) {
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/scripts/steps/functional/apm_cypress.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ set -euo pipefail

source .buildkite/scripts/common/util.sh

APM_CYPRESS_RECORD_KEY="$(retry 5 5 vault read -field=CYPRESS_RECORD_KEY secret/kibana-issues/dev/apm-cypress-dashboard-record-key)"
APM_CYPRESS_RECORD_KEY="$(vault_get apm-cypress-dashboard-record-key CYPRESS_RECORD_KEY)"

.buildkite/scripts/bootstrap.sh
.buildkite/scripts/download_build_artifacts.sh
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ set -euo pipefail
source .buildkite/scripts/common/util.sh

# TODO: Add new user and change lines accordingly
USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/ci_stats_performance_metrics)"
PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/ci_stats_performance_metrics)"
USER_FROM_VAULT="$(vault_get ci_stats_performance_metrics username)"
PASS_FROM_VAULT="$(vault_get ci_stats_performance_metrics password)"
APM_SERVER_URL="https://kibana-ops-e2e-perf.kb.us-central1.gcp.cloud.es.io:9243/internal/apm"
BUILD_ID=${BUILDKITE_BUILD_ID}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ set -euo pipefail

source .buildkite/scripts/common/util.sh

USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/apm_parser_performance)"
PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/apm_parser_performance)"
USER_FROM_VAULT="$(vault_get apm_parser_performance username)"
PASS_FROM_VAULT="$(vault_get apm_parser_performance password)"
ES_SERVER_URL="https://kibana-ops-e2e-perf.es.us-central1.gcp.cloud.es.io:9243"
BUILD_ID="${BUILDKITE_BUILD_ID}"
GCS_BUCKET="gs://kibana-performance/scalability-tests"
Expand Down
10 changes: 8 additions & 2 deletions .buildkite/scripts/steps/serverless/build_and_deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ deploy() {
VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
retry 5 30 vault login -no-print "$VAULT_TOKEN"
retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
vault_set "cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
else
echo "Updating project..."
curl -s \
Expand All @@ -86,14 +86,20 @@ deploy() {
PROJECT_KIBANA_LOGIN_URL="${PROJECT_KIBANA_URL}/login"
PROJECT_ELASTICSEARCH_URL=$(jq -r --slurp '.[1].endpoints.elasticsearch' $DEPLOY_LOGS)

if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
VAULT_PATH_PREFIX="secret/kibana-issues/dev"
else
VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
fi

cat << EOF | buildkite-agent annotate --style "info" --context "project-$PROJECT_TYPE"
### $PROJECT_TYPE_LABEL Deployment
Kibana: $PROJECT_KIBANA_LOGIN_URL
Elasticsearch: $PROJECT_ELASTICSEARCH_URL
Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME\`
Credentials: \`vault read $VAULT_PATH_PREFIX/cloud-deploy/$PROJECT_NAME\`
Kibana image: \`$KIBANA_IMAGE\`
EOF
Expand Down
2 changes: 1 addition & 1 deletion .ci/Jenkinsfile_baseline_capture
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ kibanaPipeline(timeoutMinutes: 210) {
runErrorReporter: false,
bootstrapped: false
) {
withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') {
withGcpServiceAccount.fromVaultSecret('secret/ci/elastic-kibana/ci-artifacts-key', 'value') {
withEnv([
'DISABLE_BOOTSTRAP_VALIDATION=true',
]) {
Expand Down

0 comments on commit 1022ccd

Please sign in to comment.