[7.17] [bazel] Use a GCS bucket for local dev remote bazel cache (#14…

…8393) 08a8c64 removes the legacy remote caching setup we had with buildbuddy 7982f60 backports the `populate-local-gcs` option from #131866. It adds support for keeping the local development cache in gcs up to date. It's an old backport and I don't think the complexity/maintenance is worth the development performance on a feature frozen branch, but I'll leave it up for discussion. --------- Co-authored-by: Tiago Costa <[email protected]>
elastic · Mar 20, 2023 · 13916b3 · 13916b3
1 parent 652eea2
commit 13916b3
Show file tree

Hide file tree

Showing 9 changed files with 72 additions and 153 deletions.
diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
@@ -213,6 +213,16 @@ steps:
     key: storybooks
     timeout_in_minutes: 60
 
+  - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
+    label: 'Populate local dev bazel cache (Linux)'
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 15
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - wait: ~
     continue_on_failure: true
 

diff --git a/.buildkite/scripts/common/setup_bazel.sh b/.buildkite/scripts/common/setup_bazel.sh
@@ -33,18 +33,16 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
 EOF
 fi
 
-if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then
-  echo "[bazel] enabling caching with Buildbuddy"
+if [[ "$BAZEL_CACHE_MODE" == "populate-local-gcs" ]]; then
+  echo "[bazel] enabling caching with GCS buckets for local dev"
+
 cat <<EOF >> $KIBANA_DIR/.bazelrc
-  build --bes_results_url=https://app.buildbuddy.io/invocation/
-  build --bes_backend=grpcs://remote.buildbuddy.io
-  build --remote_cache=grpcs://remote.buildbuddy.io
-  build --remote_timeout=3600
-  build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY
+  build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
+  build --google_credentials=$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
 EOF
 fi
 
-if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then
-  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]"
+if [[ "$BAZEL_CACHE_MODE" != @(gcs|populate-local-gcs|none|) ]]; then
+  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,populate-local-gcs,none]"
   exit 1
 fi
diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh
@@ -96,6 +96,11 @@ export KIBANA_DOCKER_PASSWORD
 KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
 export KIBANA_CI_REPORTER_KEY
 
+
+BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
+export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
+retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
+
 # Setup Failed Test Reporter Elasticsearch credentials
 {
   TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)
@@ -108,9 +113,6 @@ export KIBANA_CI_REPORTER_KEY
   export TEST_FAILURES_ES_PASSWORD
 }
 
-KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
-export KIBANA_BUILDBUDDY_CI_API_KEY
-
 # By default, all steps should set up these things to get a full environment before running
 # It can be skipped for pipeline upload steps though, to make job start time a little faster
 if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then

diff --git a/.buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh b/.buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+export BAZEL_CACHE_MODE=populate-local-gcs
+export DISABLE_BOOTSTRAP_VALIDATION=true
+
+# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
+rm -rf ~/.bazel-cache
+
+.buildkite/scripts/bootstrap.sh
diff --git a/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh b/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh
@@ -2,13 +2,19 @@
 
 set -euo pipefail
 
-export BAZEL_CACHE_MODE=buildbuddy
+source .buildkite/scripts/common/util.sh
+
+export BAZEL_CACHE_MODE=populate-local-gcs
 export DISABLE_BOOTSTRAP_VALIDATION=true
 
+# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
+rm -rf ~/.bazel-cache
+
 # Since our Mac agents are currently static,
 # use a temporary HOME directory that gets cleaned out between builds
 TMP_HOME="$WORKSPACE/tmp_home"
 rm -rf "$TMP_HOME"
+mkdir -p "$TMP_HOME"
 export HOME="$TMP_HOME"
 
 .buildkite/scripts/bootstrap.sh
diff --git a/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh b/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh
@@ -2,7 +2,6 @@
 
 set -euo pipefail
 
-export BAZEL_CACHE_MODE=buildbuddy # Populate Buildbuddy bazel remote cache for linux
 export BUILD_TS_REFS_CACHE_ENABLE=true
 export BUILD_TS_REFS_CACHE_CAPTURE=true
 export DISABLE_BOOTSTRAP_VALIDATION=true

diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js
@@ -61599,7 +61599,6 @@ __webpack_require__.r(__webpack_exports__);
 /* harmony import */ var path__WEBPACK_IMPORTED_MODULE_2___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_2__);
 /* harmony import */ var _child_process__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(242);
 /* harmony import */ var _log__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(241);
-/* harmony import */ var _fs__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(252);
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
@@ -61613,18 +61612,6 @@ __webpack_require__.r(__webpack_exports__);
 
 
 
-
-async function isVaultAvailable() {
-  try {
-    await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__["spawn"])('vault', ['--version'], {
-      stdio: 'pipe'
-    });
-    return true;
-  } catch {
-    return false;
-  }
-}
-
 async function isElasticCommitter() {
   try {
     const {
@@ -61638,21 +61625,17 @@ async function isElasticCommitter() {
   }
 }
 
-async function migrateToNewServersIfNeeded(settingsPath) {
-  if (!(await Object(_fs__WEBPACK_IMPORTED_MODULE_5__["isFile"])(settingsPath))) {
-    return false;
-  }
-
-  const readSettingsFile = await Object(_fs__WEBPACK_IMPORTED_MODULE_5__["readFile"])(settingsPath, 'utf8');
-  const newReadSettingsFile = readSettingsFile.replace(/cloud\.buildbuddy\.io/g, 'remote.buildbuddy.io');
+function upToDate(settingsPath) {
+  try {
+    const stat = Object(fs__WEBPACK_IMPORTED_MODULE_1__["statSync"])(settingsPath);
 
-  if (newReadSettingsFile === readSettingsFile) {
+    if (stat.isFile()) {
+      const readSettingsFile = Object(fs__WEBPACK_IMPORTED_MODULE_1__["readFileSync"])(settingsPath, 'utf8');
+      return readSettingsFile.startsWith('# V2 ');
+    }
+  } catch (error) {
     return false;
   }
-
-  Object(_fs__WEBPACK_IMPORTED_MODULE_5__["writeFile"])(settingsPath, newReadSettingsFile);
-  _log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] upgrade remote cache settings to use new server address`);
-  return true;
 }
 
 async function setupRemoteCache(repoRootPath) {
@@ -61662,51 +61645,19 @@ async function setupRemoteCache(repoRootPath) {
   }
 
   _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] setting up remote cache settings if necessary`);
-  const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade the servers used on .bazelrc.cache
-  //
-  // NOTE: this can be removed in the future once everyone is migrated into the new servers
-
-  if (await migrateToNewServersIfNeeded(settingsPath)) {
-    return;
-  }
+  const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache');
 
-  if (Object(fs__WEBPACK_IMPORTED_MODULE_1__["existsSync"])(settingsPath)) {
-    _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] remote cache settings already exist, skipping`);
-    return;
-  }
-
-  if (!(await isVaultAvailable())) {
-    _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] vault is not available, unable to setup remote cache settings.');
-    _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] building packages will work, but will be slower in many cases.');
-    _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] reach out to Operations if you need assistance with this.');
-    return;
-  }
-
-  let apiKey = '';
-
-  try {
-    const {
-      stdout
-    } = await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__["spawn"])('vault', ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], {
-      stdio: 'pipe'
-    });
-    apiKey = stdout.trim();
-  } catch (ex) {
-    _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?');
-    _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] building packages will work, but will be slower in many cases.');
-    _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] reach out to Operations if you need assistance with this.');
-    _log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] ${ex}`);
+  if (upToDate(settingsPath)) {
+    _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] remote cache settings already up to date, skipping`);
     return;
   }
 
   const contents = dedent__WEBPACK_IMPORTED_MODULE_0___default.a`
-    # V1 - This file is automatically generated by 'yarn kbn bootstrap'
+    # V2 - This file is automatically generated by 'yarn kbn bootstrap'
     # To regenerate this file, delete it and run 'yarn kbn bootstrap' again.
-    build --bes_results_url=https://app.buildbuddy.io/invocation/
-    build --bes_backend=grpcs://remote.buildbuddy.io
-    build --remote_cache=grpcs://remote.buildbuddy.io
-    build --remote_timeout=3600
-    build --remote_header=${apiKey}
+    build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
+    build --noremote_upload_local_results
+    build --incompatible_remote_results_ignore_disk
   `;
   Object(fs__WEBPACK_IMPORTED_MODULE_1__["writeFileSync"])(settingsPath, contents);
   _log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] remote cache settings written to ${settingsPath}`);

diff --git a/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts b/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts
@@ -6,21 +6,10 @@
  * Side Public License, v 1.
  */
 import dedent from 'dedent';
-import { existsSync, writeFileSync } from 'fs';
+import { writeFileSync, readFileSync, statSync } from 'fs';
 import { resolve } from 'path';
 import { spawn } from '../child_process';
 import { log } from '../log';
-import { isFile, readFile, writeFile } from '../fs';
-
-async function isVaultAvailable() {
-  try {
-    await spawn('vault', ['--version'], { stdio: 'pipe' });
-
-    return true;
-  } catch {
-    return false;
-  }
-}
 
 async function isElasticCommitter() {
   try {
@@ -34,24 +23,16 @@ async function isElasticCommitter() {
   }
 }
 
-async function migrateToNewServersIfNeeded(settingsPath: string) {
-  if (!(await isFile(settingsPath))) {
-    return false;
-  }
-
-  const readSettingsFile = await readFile(settingsPath, 'utf8');
-  const newReadSettingsFile = readSettingsFile.replace(
-    /cloud\.buildbuddy\.io/g,
-    'remote.buildbuddy.io'
-  );
-
-  if (newReadSettingsFile === readSettingsFile) {
+function upToDate(settingsPath: string) {
+  try {
+    const stat = statSync(settingsPath);
+    if (stat.isFile()) {
+      const readSettingsFile = readFileSync(settingsPath, 'utf8');
+      return readSettingsFile.startsWith('# V2 ');
+    }
+  } catch (error) {
     return false;
   }
-
-  writeFile(settingsPath, newReadSettingsFile);
-  log.info(`[bazel_tools] upgrade remote cache settings to use new server address`);
-  return true;
 }
 
 export async function setupRemoteCache(repoRootPath: string) {
@@ -64,55 +45,17 @@ export async function setupRemoteCache(repoRootPath: string) {
 
   const settingsPath = resolve(repoRootPath, '.bazelrc.cache');
 
-  // Checks if we should upgrade the servers used on .bazelrc.cache
-  //
-  // NOTE: this can be removed in the future once everyone is migrated into the new servers
-  if (await migrateToNewServersIfNeeded(settingsPath)) {
-    return;
-  }
-
-  if (existsSync(settingsPath)) {
-    log.debug(`[bazel_tools] remote cache settings already exist, skipping`);
-    return;
-  }
-
-  if (!(await isVaultAvailable())) {
-    log.info('[bazel_tools] vault is not available, unable to setup remote cache settings.');
-    log.info('[bazel_tools] building packages will work, but will be slower in many cases.');
-    log.info('[bazel_tools] reach out to Operations if you need assistance with this.');
-    return;
-  }
-
-  let apiKey = '';
-
-  try {
-    const { stdout } = await spawn(
-      'vault',
-      ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'],
-      {
-        stdio: 'pipe',
-      }
-    );
-    apiKey = stdout.trim();
-  } catch (ex: unknown) {
-    log.info(
-      '[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?'
-    );
-    log.info('[bazel_tools] building packages will work, but will be slower in many cases.');
-    log.info('[bazel_tools] reach out to Operations if you need assistance with this.');
-    log.info(`[bazel_tools] ${ex}`);
-
+  if (upToDate(settingsPath)) {
+    log.debug(`[bazel_tools] remote cache settings already up to date, skipping`);
     return;
   }
 
   const contents = dedent`
-    # V1 - This file is automatically generated by 'yarn kbn bootstrap'
+    # V2 - This file is automatically generated by 'yarn kbn bootstrap'
     # To regenerate this file, delete it and run 'yarn kbn bootstrap' again.
-    build --bes_results_url=https://app.buildbuddy.io/invocation/
-    build --bes_backend=grpcs://remote.buildbuddy.io
-    build --remote_cache=grpcs://remote.buildbuddy.io
-    build --remote_timeout=3600
-    build --remote_header=${apiKey}
+    build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
+    build --noremote_upload_local_results
+    build --incompatible_remote_results_ignore_disk
   `;
 
   writeFileSync(settingsPath, contents);

diff --git a/src/dev/ci_setup/load_env_keys.sh b/src/dev/ci_setup/load_env_keys.sh
@@ -34,9 +34,6 @@ else
   PERCY_TOKEN=$(retry 5 vault read -field=value secret/kibana-issues/dev/percy)
   export PERCY_TOKEN
 
-  KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
-  export KIBANA_BUILDBUDDY_CI_API_KEY
-
   # remove vault related secrets
   unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN VAULT_ADDR
 fi