[bazel] Use a GCS bucket for local dev remote bazel cache (elastic#13…

…1866)

.buildkite/pipelines/on_merge.yml

@@ -107,6 +107,16 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
+    label: 'Populate local dev bazel cache (Linux)'
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 15
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - wait: ~
     continue_on_failure: true
 

.buildkite/scripts/common/setup_bazel.sh

@@ -32,6 +32,15 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
 EOF
 fi
 
+if [[ "$BAZEL_CACHE_MODE" == "populate-local-gcs" ]]; then
+  echo "[bazel] enabling caching with GCS buckets for local dev"
+
+cat <<EOF >> $KIBANA_DIR/.bazelrc
+  build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
+  build --google_credentials=$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
+EOF
+fi
+
 if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then
   echo "[bazel] enabling caching with Buildbuddy"
 cat <<EOF >> $KIBANA_DIR/.bazelrc
@@ -43,7 +52,7 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
 EOF
 fi
 
-if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then
-  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]"
+if [[ "$BAZEL_CACHE_MODE" != @(gcs|populate-local-gcs|buildbuddy|none|) ]]; then
+  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,populate-local-gcs|buildbuddy,none]"
   exit 1
 fi

.buildkite/scripts/lifecycle/pre_command.sh

@@ -132,6 +132,10 @@ export SYNTHETICS_REMOTE_KIBANA_URL
 KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
 export KIBANA_BUILDBUDDY_CI_API_KEY
 
+BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
+export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
+retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
+
 # By default, all steps should set up these things to get a full environment before running
 # It can be skipped for pipeline upload steps though, to make job start time a little faster
 if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then

.buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+export BAZEL_CACHE_MODE=populate-local-gcs
+export DISABLE_BOOTSTRAP_VALIDATION=true
+
+# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
+rm -rf ~/.bazel-cache
+
+.buildkite/scripts/bootstrap.sh

.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh

@@ -2,13 +2,19 @@
 
 set -euo pipefail
 
-export BAZEL_CACHE_MODE=buildbuddy
+source .buildkite/scripts/common/util.sh
+
+export BAZEL_CACHE_MODE=populate-local-gcs
 export DISABLE_BOOTSTRAP_VALIDATION=true
 
+# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
+rm -rf ~/.bazel-cache
+
 # Since our Mac agents are currently static,
 # use a temporary HOME directory that gets cleaned out between builds
 TMP_HOME="$WORKSPACE/tmp_home"
 rm -rf "$TMP_HOME"
+mkdir -p "$TMP_HOME"
 export HOME="$TMP_HOME"
 
 .buildkite/scripts/bootstrap.sh

packages/kbn-pm/dist/index.js

@@ -60672,17 +60672,6 @@ async function installBazelTools(repoRootPath) {
 
 
 
-async function isVaultAvailable() {
-  try {
-    await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__[/* spawn */ "a"])('vault', ['--version'], {
-      stdio: 'pipe'
-    });
-    return true;
-  } catch {
-    return false;
-  }
-}
-
 async function isElasticCommitter() {
   try {
     const {
@@ -60696,21 +60685,13 @@ async function isElasticCommitter() {
   }
 }
 
-async function migrateToNewServersIfNeeded(settingsPath) {
+async function upToDate(settingsPath) {
   if (!(await Object(_fs__WEBPACK_IMPORTED_MODULE_5__[/* isFile */ "d"])(settingsPath))) {
     return false;
   }
 
   const readSettingsFile = await Object(_fs__WEBPACK_IMPORTED_MODULE_5__[/* readFile */ "f"])(settingsPath, 'utf8');
-  const newReadSettingsFile = readSettingsFile.replace(/cloud\.buildbuddy\.io/g, 'remote.buildbuddy.io');
-
-  if (newReadSettingsFile === readSettingsFile) {
-    return false;
-  }
-
-  Object(_fs__WEBPACK_IMPORTED_MODULE_5__[/* writeFile */ "i"])(settingsPath, newReadSettingsFile);
-  _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info(`[bazel_tools] upgrade remote cache settings to use new server address`);
-  return true;
+  return readSettingsFile.startsWith('# V2 ');
 }
 
 async function setupRemoteCache(repoRootPath) {
@@ -60720,52 +60701,19 @@ async function setupRemoteCache(repoRootPath) {
   }
 
   _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].debug(`[bazel_tools] setting up remote cache settings if necessary`);
-  const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade the servers used on .bazelrc.cache
-  //
-  // NOTE: this can be removed in the future once everyone is migrated into the new servers
+  const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade or install the config file
 
-  if (await migrateToNewServersIfNeeded(settingsPath)) {
-    return;
-  }
-
-  if (Object(fs__WEBPACK_IMPORTED_MODULE_1__["existsSync"])(settingsPath)) {
-    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].debug(`[bazel_tools] remote cache settings already exist, skipping`);
-    return;
-  }
-
-  if (!(await isVaultAvailable())) {
-    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] vault is not available, unable to setup remote cache settings.');
-    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] building packages will work, but will be slower in many cases.');
-    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] use the following guide or reach out to Operations for assistance');
-    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] https://github.com/elastic/infra/tree/master/docs/vault');
-    return;
-  }
-
-  let apiKey = '';
-
-  try {
-    const {
-      stdout
-    } = await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__[/* spawn */ "a"])('vault', ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], {
-      stdio: 'pipe'
-    });
-    apiKey = stdout.trim();
-  } catch (ex) {
-    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?');
-    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] building packages will work, but will be slower in many cases.');
-    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info('[bazel_tools] reach out to Operations if you need assistance with this.');
-    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info(`[bazel_tools] ${ex}`);
+  if (await upToDate(settingsPath)) {
+    _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].debug(`[bazel_tools] remote cache config already exists and is up-to-date, skipping`);
     return;
   }
 
   const contents = dedent__WEBPACK_IMPORTED_MODULE_0___default.a`
-    # V1 - This file is automatically generated by 'yarn kbn bootstrap'
+    # V2 - This file is automatically generated by 'yarn kbn bootstrap'
     # To regenerate this file, delete it and run 'yarn kbn bootstrap' again.
-    build --bes_results_url=https://app.buildbuddy.io/invocation/
-    build --bes_backend=grpcs://remote.buildbuddy.io
-    build --remote_cache=grpcs://remote.buildbuddy.io
-    build --remote_timeout=3600
-    build --remote_header=${apiKey}
+    build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
+    build --noremote_upload_local_results
+    build --incompatible_remote_results_ignore_disk
   `;
   Object(fs__WEBPACK_IMPORTED_MODULE_1__["writeFileSync"])(settingsPath, contents);
   _log__WEBPACK_IMPORTED_MODULE_4__[/* log */ "a"].info(`[bazel_tools] remote cache settings written to ${settingsPath}`);

packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts

@@ -6,21 +6,11 @@
  * Side Public License, v 1.
  */
 import dedent from 'dedent';
-import { existsSync, writeFileSync } from 'fs';
+import { writeFileSync } from 'fs';
 import { resolve } from 'path';
 import { spawn } from '../child_process';
 import { log } from '../log';
-import { isFile, readFile, writeFile } from '../fs';
-
-async function isVaultAvailable() {
-  try {
-    await spawn('vault', ['--version'], { stdio: 'pipe' });
-
-    return true;
-  } catch {
-    return false;
-  }
-}
+import { isFile, readFile } from '../fs';
 
 async function isElasticCommitter() {
   try {
@@ -34,24 +24,13 @@ async function isElasticCommitter() {
   }
 }
 
-async function migrateToNewServersIfNeeded(settingsPath: string) {
+async function upToDate(settingsPath: string) {
   if (!(await isFile(settingsPath))) {
     return false;
   }
 
   const readSettingsFile = await readFile(settingsPath, 'utf8');
-  const newReadSettingsFile = readSettingsFile.replace(
-    /cloud\.buildbuddy\.io/g,
-    'remote.buildbuddy.io'
-  );
-
-  if (newReadSettingsFile === readSettingsFile) {
-    return false;
-  }
-
-  writeFile(settingsPath, newReadSettingsFile);
-  log.info(`[bazel_tools] upgrade remote cache settings to use new server address`);
-  return true;
+  return readSettingsFile.startsWith('# V2 ');
 }
 
 export async function setupRemoteCache(repoRootPath: string) {
@@ -64,56 +43,18 @@ export async function setupRemoteCache(repoRootPath: string) {
 
   const settingsPath = resolve(repoRootPath, '.bazelrc.cache');
 
-  // Checks if we should upgrade the servers used on .bazelrc.cache
-  //
-  // NOTE: this can be removed in the future once everyone is migrated into the new servers
-  if (await migrateToNewServersIfNeeded(settingsPath)) {
-    return;
-  }
-
-  if (existsSync(settingsPath)) {
-    log.debug(`[bazel_tools] remote cache settings already exist, skipping`);
-    return;
-  }
-
-  if (!(await isVaultAvailable())) {
-    log.info('[bazel_tools] vault is not available, unable to setup remote cache settings.');
-    log.info('[bazel_tools] building packages will work, but will be slower in many cases.');
-    log.info('[bazel_tools] use the following guide or reach out to Operations for assistance');
-    log.info('[bazel_tools] https://github.com/elastic/infra/tree/master/docs/vault');
-    return;
-  }
-
-  let apiKey = '';
-
-  try {
-    const { stdout } = await spawn(
-      'vault',
-      ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'],
-      {
-        stdio: 'pipe',
-      }
-    );
-    apiKey = stdout.trim();
-  } catch (ex: unknown) {
-    log.info(
-      '[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?'
-    );
-    log.info('[bazel_tools] building packages will work, but will be slower in many cases.');
-    log.info('[bazel_tools] reach out to Operations if you need assistance with this.');
-    log.info(`[bazel_tools] ${ex}`);
-
+  // Checks if we should upgrade or install the config file
+  if (await upToDate(settingsPath)) {
+    log.debug(`[bazel_tools] remote cache config already exists and is up-to-date, skipping`);
     return;
   }
 
   const contents = dedent`
-    # V1 - This file is automatically generated by 'yarn kbn bootstrap'
+    # V2 - This file is automatically generated by 'yarn kbn bootstrap'
     # To regenerate this file, delete it and run 'yarn kbn bootstrap' again.
-    build --bes_results_url=https://app.buildbuddy.io/invocation/
-    build --bes_backend=grpcs://remote.buildbuddy.io
-    build --remote_cache=grpcs://remote.buildbuddy.io
-    build --remote_timeout=3600
-    build --remote_header=${apiKey}
+    build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
+    build --noremote_upload_local_results
+    build --incompatible_remote_results_ignore_disk
   `;
 
   writeFileSync(settingsPath, contents);