From bbf68840fb94236fcb1f3aafcd87d1410400033a Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 11 May 2022 14:21:23 -0400 Subject: [PATCH] [bazel] Use a GCS bucket for local dev remote bazel cache (#131866) --- .buildkite/pipelines/on_merge.yml | 10 +++ .buildkite/scripts/common/setup_bazel.sh | 13 ++- .buildkite/scripts/lifecycle/pre_command.sh | 4 + .../steps/bazel_cache/bootstrap_linux.sh | 13 +++ .../steps/bazel_cache/bootstrap_mac.sh | 8 +- packages/kbn-pm/dist/index.js | 70 +++------------- .../src/utils/bazel/setup_remote_cache.ts | 81 +++---------------- 7 files changed, 65 insertions(+), 134 deletions(-) create mode 100755 .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 586199f082925..512f0a2c279a3 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -107,6 +107,16 @@ steps: - exit_status: '-1' limit: 3 + - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh + label: 'Populate local dev bazel cache (Linux)' + agents: + queue: n2-4-spot + timeout_in_minutes: 15 + retry: + automatic: + - exit_status: '-1' + limit: 3 + - wait: ~ continue_on_failure: true diff --git a/.buildkite/scripts/common/setup_bazel.sh b/.buildkite/scripts/common/setup_bazel.sh index e3791dfa393c7..40159ba9eaf69 100755 --- a/.buildkite/scripts/common/setup_bazel.sh +++ b/.buildkite/scripts/common/setup_bazel.sh @@ -32,6 +32,15 @@ cat <> $KIBANA_DIR/.bazelrc EOF fi +if [[ "$BAZEL_CACHE_MODE" == "populate-local-gcs" ]]; then + echo "[bazel] enabling caching with GCS buckets for local dev" + +cat <> $KIBANA_DIR/.bazelrc + build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache + build --google_credentials=$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE +EOF +fi + if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then echo "[bazel] enabling caching with Buildbuddy" cat <> $KIBANA_DIR/.bazelrc @@ -43,7 +52,7 @@ cat <> $KIBANA_DIR/.bazelrc EOF fi -if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then - echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]" +if [[ "$BAZEL_CACHE_MODE" != @(gcs|populate-local-gcs|buildbuddy|none|) ]]; then + echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,populate-local-gcs|buildbuddy,none]" exit 1 fi diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh index 8f3776db3ca6b..11806ebf10e73 100755 --- a/.buildkite/scripts/lifecycle/pre_command.sh +++ b/.buildkite/scripts/lifecycle/pre_command.sh @@ -132,6 +132,10 @@ export SYNTHETICS_REMOTE_KIBANA_URL KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key) export KIBANA_BUILDBUDDY_CI_API_KEY +BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json" +export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE +retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE" + # By default, all steps should set up these things to get a full environment before running # It can be skipped for pipeline upload steps though, to make job start time a little faster if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then diff --git a/.buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh b/.buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh new file mode 100755 index 0000000000000..7798370bfbd35 --- /dev/null +++ b/.buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +export BAZEL_CACHE_MODE=populate-local-gcs +export DISABLE_BOOTSTRAP_VALIDATION=true + +# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded +rm -rf ~/.bazel-cache + +.buildkite/scripts/bootstrap.sh diff --git a/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh b/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh index 62aabf496fd8a..ab642ec431486 100755 --- a/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh +++ b/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh @@ -2,13 +2,19 @@ set -euo pipefail -export BAZEL_CACHE_MODE=buildbuddy +source .buildkite/scripts/common/util.sh + +export BAZEL_CACHE_MODE=populate-local-gcs export DISABLE_BOOTSTRAP_VALIDATION=true +# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded +rm -rf ~/.bazel-cache + # Since our Mac agents are currently static, # use a temporary HOME directory that gets cleaned out between builds TMP_HOME="$WORKSPACE/tmp_home" rm -rf "$TMP_HOME" +mkdir -p "$TMP_HOME" export HOME="$TMP_HOME" .buildkite/scripts/bootstrap.sh diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index f2d5a60cd325e..1043c357da8d1 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -60672,17 +60672,6 @@ async function installBazelTools(repoRootPath) { -async function isVaultAvailable() { - try { - await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__[/* spawn */ "a"])('vault', ['--version'], { - stdio: 'pipe' - }); - return true; - } catch { - return false; - } -} - async function isElasticCommitter() { try { const { @@ -60696,21 +60685,13 @@ async function isElasticCommitter() { } } -async function migrateToNewServersIfNeeded(settingsPath) { +async function upToDate(settingsPath) { if (!(await Object(_fs__WEBPACK_IMPORTED_MODULE_5__[/* isFile */ "d"])(settingsPath))) { return false; } const readSettingsFile = await Object(_fs__WEBPACK_IMPORTED_MODULE_5__[/* readFile */ "f"])(settingsPath, 'utf8'); - const newReadSettingsFile = readSettingsFile.replace(/cloud\.buildbuddy\.io/g, 'remote.buildbuddy.io'); - - if (newReadSettingsFile === readSettingsFile) { - return false; - } - - Object(_fs__WEBPACK_IMPORTED_MODULE_5__[/* writeFile */ "i"])(settingsPath, newReadSettingsFile); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info(`[bazel_tools] upgrade remote cache settings to use new server address`); - return true; + return readSettingsFile.startsWith('# V2 '); } async function setupRemoteCache(repoRootPath) { @@ -60720,52 +60701,19 @@ async function setupRemoteCache(repoRootPath) { } _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].debug(`[bazel_tools] setting up remote cache settings if necessary`); - const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade the servers used on .bazelrc.cache - // - // NOTE: this can be removed in the future once everyone is migrated into the new servers + const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade or install the config file - if (await migrateToNewServersIfNeeded(settingsPath)) { - return; - } - - if (Object(fs__WEBPACK_IMPORTED_MODULE_1__["existsSync"])(settingsPath)) { - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].debug(`[bazel_tools] remote cache settings already exist, skipping`); - return; - } - - if (!(await isVaultAvailable())) { - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] vault is not available, unable to setup remote cache settings.'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] building packages will work, but will be slower in many cases.'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] use the following guide or reach out to Operations for assistance'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] https://github.com/elastic/infra/tree/master/docs/vault'); - return; - } - - let apiKey = ''; - - try { - const { - stdout - } = await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__[/* spawn */ "a"])('vault', ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], { - stdio: 'pipe' - }); - apiKey = stdout.trim(); - } catch (ex) { - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] building packages will work, but will be slower in many cases.'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] reach out to Operations if you need assistance with this.'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info(`[bazel_tools] ${ex}`); + if (await upToDate(settingsPath)) { + _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].debug(`[bazel_tools] remote cache config already exists and is up-to-date, skipping`); return; } const contents = dedent__WEBPACK_IMPORTED_MODULE_0___default.a` - # V1 - This file is automatically generated by 'yarn kbn bootstrap' + # V2 - This file is automatically generated by 'yarn kbn bootstrap' # To regenerate this file, delete it and run 'yarn kbn bootstrap' again. - build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://remote.buildbuddy.io - build --remote_cache=grpcs://remote.buildbuddy.io - build --remote_timeout=3600 - build --remote_header=${apiKey} + build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache + build --noremote_upload_local_results + build --incompatible_remote_results_ignore_disk `; Object(fs__WEBPACK_IMPORTED_MODULE_1__["writeFileSync"])(settingsPath, contents); _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info(`[bazel_tools] remote cache settings written to ${settingsPath}`); diff --git a/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts b/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts index 0c5213e2dacc6..5c28d6550d0f7 100644 --- a/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts +++ b/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts @@ -6,21 +6,11 @@ * Side Public License, v 1. */ import dedent from 'dedent'; -import { existsSync, writeFileSync } from 'fs'; +import { writeFileSync } from 'fs'; import { resolve } from 'path'; import { spawn } from '../child_process'; import { log } from '../log'; -import { isFile, readFile, writeFile } from '../fs'; - -async function isVaultAvailable() { - try { - await spawn('vault', ['--version'], { stdio: 'pipe' }); - - return true; - } catch { - return false; - } -} +import { isFile, readFile } from '../fs'; async function isElasticCommitter() { try { @@ -34,24 +24,13 @@ async function isElasticCommitter() { } } -async function migrateToNewServersIfNeeded(settingsPath: string) { +async function upToDate(settingsPath: string) { if (!(await isFile(settingsPath))) { return false; } const readSettingsFile = await readFile(settingsPath, 'utf8'); - const newReadSettingsFile = readSettingsFile.replace( - /cloud\.buildbuddy\.io/g, - 'remote.buildbuddy.io' - ); - - if (newReadSettingsFile === readSettingsFile) { - return false; - } - - writeFile(settingsPath, newReadSettingsFile); - log.info(`[bazel_tools] upgrade remote cache settings to use new server address`); - return true; + return readSettingsFile.startsWith('# V2 '); } export async function setupRemoteCache(repoRootPath: string) { @@ -64,56 +43,18 @@ export async function setupRemoteCache(repoRootPath: string) { const settingsPath = resolve(repoRootPath, '.bazelrc.cache'); - // Checks if we should upgrade the servers used on .bazelrc.cache - // - // NOTE: this can be removed in the future once everyone is migrated into the new servers - if (await migrateToNewServersIfNeeded(settingsPath)) { - return; - } - - if (existsSync(settingsPath)) { - log.debug(`[bazel_tools] remote cache settings already exist, skipping`); - return; - } - - if (!(await isVaultAvailable())) { - log.info('[bazel_tools] vault is not available, unable to setup remote cache settings.'); - log.info('[bazel_tools] building packages will work, but will be slower in many cases.'); - log.info('[bazel_tools] use the following guide or reach out to Operations for assistance'); - log.info('[bazel_tools] https://github.com/elastic/infra/tree/master/docs/vault'); - return; - } - - let apiKey = ''; - - try { - const { stdout } = await spawn( - 'vault', - ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], - { - stdio: 'pipe', - } - ); - apiKey = stdout.trim(); - } catch (ex: unknown) { - log.info( - '[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?' - ); - log.info('[bazel_tools] building packages will work, but will be slower in many cases.'); - log.info('[bazel_tools] reach out to Operations if you need assistance with this.'); - log.info(`[bazel_tools] ${ex}`); - + // Checks if we should upgrade or install the config file + if (await upToDate(settingsPath)) { + log.debug(`[bazel_tools] remote cache config already exists and is up-to-date, skipping`); return; } const contents = dedent` - # V1 - This file is automatically generated by 'yarn kbn bootstrap' + # V2 - This file is automatically generated by 'yarn kbn bootstrap' # To regenerate this file, delete it and run 'yarn kbn bootstrap' again. - build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://remote.buildbuddy.io - build --remote_cache=grpcs://remote.buildbuddy.io - build --remote_timeout=3600 - build --remote_header=${apiKey} + build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache + build --noremote_upload_local_results + build --incompatible_remote_results_ignore_disk `; writeFileSync(settingsPath, contents);