From d50b5591503a4d96e8f495a356b9cc4fc78c1b98 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 9 May 2022 16:17:07 -0400 Subject: [PATCH] [bazel] Use a GCS bucket for local dev remote bazel cache --- .buildkite/scripts/common/setup_bazel.sh | 13 ++- .../steps/bazel_cache/bootstrap_mac.sh | 5 +- packages/kbn-pm/dist/index.js | 69 ++-------------- .../src/utils/bazel/setup_remote_cache.ts | 80 +++---------------- 4 files changed, 33 insertions(+), 134 deletions(-) diff --git a/.buildkite/scripts/common/setup_bazel.sh b/.buildkite/scripts/common/setup_bazel.sh index e3791dfa393c7..5f81a86fdce4d 100755 --- a/.buildkite/scripts/common/setup_bazel.sh +++ b/.buildkite/scripts/common/setup_bazel.sh @@ -32,6 +32,15 @@ cat <> $KIBANA_DIR/.bazelrc EOF fi +if [[ "$BAZEL_CACHE_MODE" == "populate-local-gcs" ]]; then + echo "[bazel] enabling caching with GCS buckets for local dev" + +cat <> $KIBANA_DIR/.bazelrc + build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache + build --google_default_credentials +EOF +fi + if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then echo "[bazel] enabling caching with Buildbuddy" cat <> $KIBANA_DIR/.bazelrc @@ -43,7 +52,7 @@ cat <> $KIBANA_DIR/.bazelrc EOF fi -if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then - echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]" +if [[ "$BAZEL_CACHE_MODE" != @(gcs|populate-local-gcs|buildbuddy|none|) ]]; then + echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,populate-local-gcs|buildbuddy,none]" exit 1 fi diff --git a/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh b/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh index 62aabf496fd8a..359ac661b4007 100755 --- a/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh +++ b/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh @@ -2,9 +2,12 @@ set -euo pipefail -export BAZEL_CACHE_MODE=buildbuddy +export BAZEL_CACHE_MODE=populate-local-gcs export DISABLE_BOOTSTRAP_VALIDATION=true +# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded +rm -rf ~/.bazel-cache + # Since our Mac agents are currently static, # use a temporary HOME directory that gets cleaned out between builds TMP_HOME="$WORKSPACE/tmp_home" diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index f2d5a60cd325e..20bb93d69d99a 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -60672,17 +60672,6 @@ async function installBazelTools(repoRootPath) { -async function isVaultAvailable() { - try { - await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__[/* spawn */ "a"])('vault', ['--version'], { - stdio: 'pipe' - }); - return true; - } catch { - return false; - } -} - async function isElasticCommitter() { try { const { @@ -60696,21 +60685,13 @@ async function isElasticCommitter() { } } -async function migrateToNewServersIfNeeded(settingsPath) { +async function upToDate(settingsPath) { if (!(await Object(_fs__WEBPACK_IMPORTED_MODULE_5__[/* isFile */ "d"])(settingsPath))) { return false; } const readSettingsFile = await Object(_fs__WEBPACK_IMPORTED_MODULE_5__[/* readFile */ "f"])(settingsPath, 'utf8'); - const newReadSettingsFile = readSettingsFile.replace(/cloud\.buildbuddy\.io/g, 'remote.buildbuddy.io'); - - if (newReadSettingsFile === readSettingsFile) { - return false; - } - - Object(_fs__WEBPACK_IMPORTED_MODULE_5__[/* writeFile */ "i"])(settingsPath, newReadSettingsFile); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info(`[bazel_tools] upgrade remote cache settings to use new server address`); - return true; + return readSettingsFile.startsWith('# V2 '); } async function setupRemoteCache(repoRootPath) { @@ -60720,52 +60701,18 @@ async function setupRemoteCache(repoRootPath) { } _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].debug(`[bazel_tools] setting up remote cache settings if necessary`); - const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade the servers used on .bazelrc.cache - // - // NOTE: this can be removed in the future once everyone is migrated into the new servers + const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade or install the config file - if (await migrateToNewServersIfNeeded(settingsPath)) { - return; - } - - if (Object(fs__WEBPACK_IMPORTED_MODULE_1__["existsSync"])(settingsPath)) { - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].debug(`[bazel_tools] remote cache settings already exist, skipping`); - return; - } - - if (!(await isVaultAvailable())) { - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] vault is not available, unable to setup remote cache settings.'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] building packages will work, but will be slower in many cases.'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] use the following guide or reach out to Operations for assistance'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] https://github.com/elastic/infra/tree/master/docs/vault'); - return; - } - - let apiKey = ''; - - try { - const { - stdout - } = await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__[/* spawn */ "a"])('vault', ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], { - stdio: 'pipe' - }); - apiKey = stdout.trim(); - } catch (ex) { - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] building packages will work, but will be slower in many cases.'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] reach out to Operations if you need assistance with this.'); - _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info(`[bazel_tools] ${ex}`); + if (await upToDate(settingsPath)) { + _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].debug(`[bazel_tools] remote cache config already exists and is up-to-date, skipping`); return; } const contents = dedent__WEBPACK_IMPORTED_MODULE_0___default.a` - # V1 - This file is automatically generated by 'yarn kbn bootstrap' + # V2 - This file is automatically generated by 'yarn kbn bootstrap' # To regenerate this file, delete it and run 'yarn kbn bootstrap' again. - build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://remote.buildbuddy.io - build --remote_cache=grpcs://remote.buildbuddy.io - build --remote_timeout=3600 - build --remote_header=${apiKey} + build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache + build --noremote_upload_local_results `; Object(fs__WEBPACK_IMPORTED_MODULE_1__["writeFileSync"])(settingsPath, contents); _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info(`[bazel_tools] remote cache settings written to ${settingsPath}`); diff --git a/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts b/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts index 0c5213e2dacc6..16ec5b361ba9b 100644 --- a/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts +++ b/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts @@ -6,21 +6,11 @@ * Side Public License, v 1. */ import dedent from 'dedent'; -import { existsSync, writeFileSync } from 'fs'; +import { writeFileSync } from 'fs'; import { resolve } from 'path'; import { spawn } from '../child_process'; import { log } from '../log'; -import { isFile, readFile, writeFile } from '../fs'; - -async function isVaultAvailable() { - try { - await spawn('vault', ['--version'], { stdio: 'pipe' }); - - return true; - } catch { - return false; - } -} +import { isFile, readFile } from '../fs'; async function isElasticCommitter() { try { @@ -34,24 +24,13 @@ async function isElasticCommitter() { } } -async function migrateToNewServersIfNeeded(settingsPath: string) { +async function upToDate(settingsPath: string) { if (!(await isFile(settingsPath))) { return false; } const readSettingsFile = await readFile(settingsPath, 'utf8'); - const newReadSettingsFile = readSettingsFile.replace( - /cloud\.buildbuddy\.io/g, - 'remote.buildbuddy.io' - ); - - if (newReadSettingsFile === readSettingsFile) { - return false; - } - - writeFile(settingsPath, newReadSettingsFile); - log.info(`[bazel_tools] upgrade remote cache settings to use new server address`); - return true; + return readSettingsFile.startsWith('# V2 '); } export async function setupRemoteCache(repoRootPath: string) { @@ -64,56 +43,17 @@ export async function setupRemoteCache(repoRootPath: string) { const settingsPath = resolve(repoRootPath, '.bazelrc.cache'); - // Checks if we should upgrade the servers used on .bazelrc.cache - // - // NOTE: this can be removed in the future once everyone is migrated into the new servers - if (await migrateToNewServersIfNeeded(settingsPath)) { - return; - } - - if (existsSync(settingsPath)) { - log.debug(`[bazel_tools] remote cache settings already exist, skipping`); - return; - } - - if (!(await isVaultAvailable())) { - log.info('[bazel_tools] vault is not available, unable to setup remote cache settings.'); - log.info('[bazel_tools] building packages will work, but will be slower in many cases.'); - log.info('[bazel_tools] use the following guide or reach out to Operations for assistance'); - log.info('[bazel_tools] https://github.com/elastic/infra/tree/master/docs/vault'); - return; - } - - let apiKey = ''; - - try { - const { stdout } = await spawn( - 'vault', - ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], - { - stdio: 'pipe', - } - ); - apiKey = stdout.trim(); - } catch (ex: unknown) { - log.info( - '[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?' - ); - log.info('[bazel_tools] building packages will work, but will be slower in many cases.'); - log.info('[bazel_tools] reach out to Operations if you need assistance with this.'); - log.info(`[bazel_tools] ${ex}`); - + // Checks if we should upgrade or install the config file + if (await upToDate(settingsPath)) { + log.debug(`[bazel_tools] remote cache config already exists and is up-to-date, skipping`); return; } const contents = dedent` - # V1 - This file is automatically generated by 'yarn kbn bootstrap' + # V2 - This file is automatically generated by 'yarn kbn bootstrap' # To regenerate this file, delete it and run 'yarn kbn bootstrap' again. - build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://remote.buildbuddy.io - build --remote_cache=grpcs://remote.buildbuddy.io - build --remote_timeout=3600 - build --remote_header=${apiKey} + build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache + build --noremote_upload_local_results `; writeFileSync(settingsPath, contents);