diff --git a/.buildkite/scripts/common/setup_bazel.sh b/.buildkite/scripts/common/setup_bazel.sh index ae38a61e7c5e9..37f800efe6e2a 100755 --- a/.buildkite/scripts/common/setup_bazel.sh +++ b/.buildkite/scripts/common/setup_bazel.sh @@ -33,18 +33,7 @@ cat <> $KIBANA_DIR/.bazelrc EOF fi -if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then - echo "[bazel] enabling caching with Buildbuddy" -cat <> $KIBANA_DIR/.bazelrc - build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://remote.buildbuddy.io - build --remote_cache=grpcs://remote.buildbuddy.io - build --remote_timeout=3600 - build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY -EOF -fi - -if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then - echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]" +if [[ "$BAZEL_CACHE_MODE" != @(gcs|none|) ]]; then + echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,none]" exit 1 fi diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh index 53e5591efe0fe..e08adc3bdbd98 100755 --- a/.buildkite/scripts/lifecycle/pre_command.sh +++ b/.buildkite/scripts/lifecycle/pre_command.sh @@ -108,9 +108,6 @@ export KIBANA_CI_REPORTER_KEY export TEST_FAILURES_ES_PASSWORD } -KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key) -export KIBANA_BUILDBUDDY_CI_API_KEY - # By default, all steps should set up these things to get a full environment before running # It can be skipped for pipeline upload steps though, to make job start time a little faster if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then diff --git a/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh b/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh index 5659db50e1404..d1542a884cf42 100755 --- a/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh +++ b/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh @@ -2,7 +2,6 @@ set -euo pipefail -export BAZEL_CACHE_MODE=buildbuddy # Populate Buildbuddy bazel remote cache for linux export BUILD_TS_REFS_CACHE_ENABLE=true export BUILD_TS_REFS_CACHE_CAPTURE=true export DISABLE_BOOTSTRAP_VALIDATION=true diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index 5c9f53622a324..247d91541ddda 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -61595,7 +61595,6 @@ __webpack_require__.r(__webpack_exports__); /* harmony import */ var path__WEBPACK_IMPORTED_MODULE_2___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_2__); /* harmony import */ var _child_process__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(242); /* harmony import */ var _log__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(241); -/* harmony import */ var _fs__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(252); /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -61609,18 +61608,6 @@ __webpack_require__.r(__webpack_exports__); - -async function isVaultAvailable() { - try { - await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__["spawn"])('vault', ['--version'], { - stdio: 'pipe' - }); - return true; - } catch { - return false; - } -} - async function isElasticCommitter() { try { const { @@ -61634,21 +61621,17 @@ async function isElasticCommitter() { } } -async function migrateToNewServersIfNeeded(settingsPath) { - if (!(await Object(_fs__WEBPACK_IMPORTED_MODULE_5__["isFile"])(settingsPath))) { - return false; - } - - const readSettingsFile = await Object(_fs__WEBPACK_IMPORTED_MODULE_5__["readFile"])(settingsPath, 'utf8'); - const newReadSettingsFile = readSettingsFile.replace(/cloud\.buildbuddy\.io/g, 'remote.buildbuddy.io'); +function upToDate(settingsPath) { + try { + const stat = Object(fs__WEBPACK_IMPORTED_MODULE_1__["statSync"])(settingsPath); - if (newReadSettingsFile === readSettingsFile) { + if (stat.isFile()) { + const readSettingsFile = Object(fs__WEBPACK_IMPORTED_MODULE_1__["readFileSync"])(settingsPath, 'utf8'); + return readSettingsFile.startsWith('# V2 '); + } + } catch (error) { return false; } - - Object(_fs__WEBPACK_IMPORTED_MODULE_5__["writeFile"])(settingsPath, newReadSettingsFile); - _log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] upgrade remote cache settings to use new server address`); - return true; } async function setupRemoteCache(repoRootPath) { @@ -61658,51 +61641,19 @@ async function setupRemoteCache(repoRootPath) { } _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] setting up remote cache settings if necessary`); - const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade the servers used on .bazelrc.cache - // - // NOTE: this can be removed in the future once everyone is migrated into the new servers - - if (await migrateToNewServersIfNeeded(settingsPath)) { - return; - } + const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); - if (Object(fs__WEBPACK_IMPORTED_MODULE_1__["existsSync"])(settingsPath)) { - _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] remote cache settings already exist, skipping`); - return; - } - - if (!(await isVaultAvailable())) { - _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] vault is not available, unable to setup remote cache settings.'); - _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] building packages will work, but will be slower in many cases.'); - _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] reach out to Operations if you need assistance with this.'); - return; - } - - let apiKey = ''; - - try { - const { - stdout - } = await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__["spawn"])('vault', ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], { - stdio: 'pipe' - }); - apiKey = stdout.trim(); - } catch (ex) { - _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?'); - _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] building packages will work, but will be slower in many cases.'); - _log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] reach out to Operations if you need assistance with this.'); - _log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] ${ex}`); + if (upToDate(settingsPath)) { + _log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] remote cache settings already up to date, skipping`); return; } const contents = dedent__WEBPACK_IMPORTED_MODULE_0___default.a` - # V1 - This file is automatically generated by 'yarn kbn bootstrap' + # V2 - This file is automatically generated by 'yarn kbn bootstrap' # To regenerate this file, delete it and run 'yarn kbn bootstrap' again. - build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://remote.buildbuddy.io - build --remote_cache=grpcs://remote.buildbuddy.io - build --remote_timeout=3600 - build --remote_header=${apiKey} + build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache + build --noremote_upload_local_results + build --incompatible_remote_results_ignore_disk `; Object(fs__WEBPACK_IMPORTED_MODULE_1__["writeFileSync"])(settingsPath, contents); _log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] remote cache settings written to ${settingsPath}`); diff --git a/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts b/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts index 19997799a1234..184ab891f202f 100644 --- a/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts +++ b/packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts @@ -6,21 +6,10 @@ * Side Public License, v 1. */ import dedent from 'dedent'; -import { existsSync, writeFileSync } from 'fs'; +import { writeFileSync, readFileSync, statSync } from 'fs'; import { resolve } from 'path'; import { spawn } from '../child_process'; import { log } from '../log'; -import { isFile, readFile, writeFile } from '../fs'; - -async function isVaultAvailable() { - try { - await spawn('vault', ['--version'], { stdio: 'pipe' }); - - return true; - } catch { - return false; - } -} async function isElasticCommitter() { try { @@ -34,24 +23,16 @@ async function isElasticCommitter() { } } -async function migrateToNewServersIfNeeded(settingsPath: string) { - if (!(await isFile(settingsPath))) { - return false; - } - - const readSettingsFile = await readFile(settingsPath, 'utf8'); - const newReadSettingsFile = readSettingsFile.replace( - /cloud\.buildbuddy\.io/g, - 'remote.buildbuddy.io' - ); - - if (newReadSettingsFile === readSettingsFile) { +function upToDate(settingsPath: string) { + try { + const stat = statSync(settingsPath); + if (stat.isFile()) { + const readSettingsFile = readFileSync(settingsPath, 'utf8'); + return readSettingsFile.startsWith('# V2 '); + } + } catch (error) { return false; } - - writeFile(settingsPath, newReadSettingsFile); - log.info(`[bazel_tools] upgrade remote cache settings to use new server address`); - return true; } export async function setupRemoteCache(repoRootPath: string) { @@ -64,55 +45,17 @@ export async function setupRemoteCache(repoRootPath: string) { const settingsPath = resolve(repoRootPath, '.bazelrc.cache'); - // Checks if we should upgrade the servers used on .bazelrc.cache - // - // NOTE: this can be removed in the future once everyone is migrated into the new servers - if (await migrateToNewServersIfNeeded(settingsPath)) { - return; - } - - if (existsSync(settingsPath)) { - log.debug(`[bazel_tools] remote cache settings already exist, skipping`); - return; - } - - if (!(await isVaultAvailable())) { - log.info('[bazel_tools] vault is not available, unable to setup remote cache settings.'); - log.info('[bazel_tools] building packages will work, but will be slower in many cases.'); - log.info('[bazel_tools] reach out to Operations if you need assistance with this.'); - return; - } - - let apiKey = ''; - - try { - const { stdout } = await spawn( - 'vault', - ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], - { - stdio: 'pipe', - } - ); - apiKey = stdout.trim(); - } catch (ex: unknown) { - log.info( - '[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?' - ); - log.info('[bazel_tools] building packages will work, but will be slower in many cases.'); - log.info('[bazel_tools] reach out to Operations if you need assistance with this.'); - log.info(`[bazel_tools] ${ex}`); - + if (upToDate(settingsPath)) { + log.debug(`[bazel_tools] remote cache settings already up to date, skipping`); return; } const contents = dedent` - # V1 - This file is automatically generated by 'yarn kbn bootstrap' + # V2 - This file is automatically generated by 'yarn kbn bootstrap' # To regenerate this file, delete it and run 'yarn kbn bootstrap' again. - build --bes_results_url=https://app.buildbuddy.io/invocation/ - build --bes_backend=grpcs://remote.buildbuddy.io - build --remote_cache=grpcs://remote.buildbuddy.io - build --remote_timeout=3600 - build --remote_header=${apiKey} + build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache + build --noremote_upload_local_results + build --incompatible_remote_results_ignore_disk `; writeFileSync(settingsPath, contents); diff --git a/src/dev/ci_setup/load_env_keys.sh b/src/dev/ci_setup/load_env_keys.sh index 5f7a6c26bab21..62d29db232eae 100644 --- a/src/dev/ci_setup/load_env_keys.sh +++ b/src/dev/ci_setup/load_env_keys.sh @@ -34,9 +34,6 @@ else PERCY_TOKEN=$(retry 5 vault read -field=value secret/kibana-issues/dev/percy) export PERCY_TOKEN - KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key) - export KIBANA_BUILDBUDDY_CI_API_KEY - # remove vault related secrets unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN VAULT_ADDR fi