Skip to content

Commit

Permalink
[7.17] Remove buildbuddy bazel cache (#145038)
Browse files Browse the repository at this point in the history 
This is no longer used.
  • Loading branch information
@jbudz
jbudz committed Jan 4, 2023
1 parent f9eb6b9 commit 08a8c64
Show file tree
Hide file tree
Showing 6 changed files with 32 additions and 156 deletions.
15 changes: 2 additions & 13 deletions .buildkite/scripts/common/setup_bazel.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,7 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
EOF
fi

if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then
echo "[bazel] enabling caching with Buildbuddy"
cat <<EOF >> $KIBANA_DIR/.bazelrc
build --bes_results_url=https://app.buildbuddy.io/invocation/
build --bes_backend=grpcs://remote.buildbuddy.io
build --remote_cache=grpcs://remote.buildbuddy.io
build --remote_timeout=3600
build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY
EOF
fi

if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then
echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]"
if [[ "$BAZEL_CACHE_MODE" != @(gcs|none|) ]]; then
echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,none]"
exit 1
fi
3 changes: 0 additions & 3 deletions .buildkite/scripts/lifecycle/pre_command.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,6 @@ export KIBANA_CI_REPORTER_KEY
export TEST_FAILURES_ES_PASSWORD
}

KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
export KIBANA_BUILDBUDDY_CI_API_KEY

# By default, all steps should set up these things to get a full environment before running
# It can be skipped for pipeline upload steps though, to make job start time a little faster
if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then
Expand Down
1 change: 0 additions & 1 deletion .buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

set -euo pipefail

export BAZEL_CACHE_MODE=buildbuddy # Populate Buildbuddy bazel remote cache for linux
export BUILD_TS_REFS_CACHE_ENABLE=true
export BUILD_TS_REFS_CACHE_CAPTURE=true
export DISABLE_BOOTSTRAP_VALIDATION=true
Expand Down
79 changes: 15 additions & 64 deletions packages/kbn-pm/dist/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61595,7 +61595,6 @@ __webpack_require__.r(__webpack_exports__);
/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_2___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_2__);
/* harmony import */ var _child_process__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(242);
/* harmony import */ var _log__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(241);
/* harmony import */ var _fs__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(252);
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
Expand All @@ -61609,18 +61608,6 @@ __webpack_require__.r(__webpack_exports__);




async function isVaultAvailable() {
try {
await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__["spawn"])('vault', ['--version'], {
stdio: 'pipe'
});
return true;
} catch {
return false;
}
}

async function isElasticCommitter() {
try {
const {
Expand All @@ -61634,21 +61621,17 @@ async function isElasticCommitter() {
}
}

async function migrateToNewServersIfNeeded(settingsPath) {
if (!(await Object(_fs__WEBPACK_IMPORTED_MODULE_5__["isFile"])(settingsPath))) {
return false;
}

const readSettingsFile = await Object(_fs__WEBPACK_IMPORTED_MODULE_5__["readFile"])(settingsPath, 'utf8');
const newReadSettingsFile = readSettingsFile.replace(/cloud\.buildbuddy\.io/g, 'remote.buildbuddy.io');
function upToDate(settingsPath) {
try {
const stat = Object(fs__WEBPACK_IMPORTED_MODULE_1__["statSync"])(settingsPath);

if (newReadSettingsFile === readSettingsFile) {
if (stat.isFile()) {
const readSettingsFile = Object(fs__WEBPACK_IMPORTED_MODULE_1__["readFileSync"])(settingsPath, 'utf8');
return readSettingsFile.startsWith('# V2 ');
}
} catch (error) {
return false;
}

Object(_fs__WEBPACK_IMPORTED_MODULE_5__["writeFile"])(settingsPath, newReadSettingsFile);
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] upgrade remote cache settings to use new server address`);
return true;
}

async function setupRemoteCache(repoRootPath) {
Expand All @@ -61658,51 +61641,19 @@ async function setupRemoteCache(repoRootPath) {
}

_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] setting up remote cache settings if necessary`);
const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache'); // Checks if we should upgrade the servers used on .bazelrc.cache
//
// NOTE: this can be removed in the future once everyone is migrated into the new servers

if (await migrateToNewServersIfNeeded(settingsPath)) {
return;
}
const settingsPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(repoRootPath, '.bazelrc.cache');

if (Object(fs__WEBPACK_IMPORTED_MODULE_1__["existsSync"])(settingsPath)) {
_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] remote cache settings already exist, skipping`);
return;
}

if (!(await isVaultAvailable())) {
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] vault is not available, unable to setup remote cache settings.');
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] building packages will work, but will be slower in many cases.');
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] reach out to Operations if you need assistance with this.');
return;
}

let apiKey = '';

try {
const {
stdout
} = await Object(_child_process__WEBPACK_IMPORTED_MODULE_3__["spawn"])('vault', ['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'], {
stdio: 'pipe'
});
apiKey = stdout.trim();
} catch (ex) {
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?');
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] building packages will work, but will be slower in many cases.');
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info('[bazel_tools] reach out to Operations if you need assistance with this.');
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] ${ex}`);
if (upToDate(settingsPath)) {
_log__WEBPACK_IMPORTED_MODULE_4__["log"].debug(`[bazel_tools] remote cache settings already up to date, skipping`);
return;
}

const contents = dedent__WEBPACK_IMPORTED_MODULE_0___default.a`
# V1 - This file is automatically generated by 'yarn kbn bootstrap'
# V2 - This file is automatically generated by 'yarn kbn bootstrap'
# To regenerate this file, delete it and run 'yarn kbn bootstrap' again.
build --bes_results_url=https://app.buildbuddy.io/invocation/
build --bes_backend=grpcs://remote.buildbuddy.io
build --remote_cache=grpcs://remote.buildbuddy.io
build --remote_timeout=3600
build --remote_header=${apiKey}
build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
build --noremote_upload_local_results
build --incompatible_remote_results_ignore_disk
`;
Object(fs__WEBPACK_IMPORTED_MODULE_1__["writeFileSync"])(settingsPath, contents);
_log__WEBPACK_IMPORTED_MODULE_4__["log"].info(`[bazel_tools] remote cache settings written to ${settingsPath}`);
Expand Down
87 changes: 15 additions & 72 deletions packages/kbn-pm/src/utils/bazel/setup_remote_cache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,10 @@
* Side Public License, v 1.
*/
import dedent from 'dedent';
import { existsSync, writeFileSync } from 'fs';
import { writeFileSync, readFileSync, statSync } from 'fs';
import { resolve } from 'path';
import { spawn } from '../child_process';
import { log } from '../log';
import { isFile, readFile, writeFile } from '../fs';

async function isVaultAvailable() {
try {
await spawn('vault', ['--version'], { stdio: 'pipe' });

return true;
} catch {
return false;
}
}

async function isElasticCommitter() {
try {
Expand All @@ -34,24 +23,16 @@ async function isElasticCommitter() {
}
}

async function migrateToNewServersIfNeeded(settingsPath: string) {
if (!(await isFile(settingsPath))) {
return false;
}

const readSettingsFile = await readFile(settingsPath, 'utf8');
const newReadSettingsFile = readSettingsFile.replace(
/cloud\.buildbuddy\.io/g,
'remote.buildbuddy.io'
);

if (newReadSettingsFile === readSettingsFile) {
function upToDate(settingsPath: string) {
try {
const stat = statSync(settingsPath);
if (stat.isFile()) {
const readSettingsFile = readFileSync(settingsPath, 'utf8');
return readSettingsFile.startsWith('# V2 ');
}
} catch (error) {
return false;
}

writeFile(settingsPath, newReadSettingsFile);
log.info(`[bazel_tools] upgrade remote cache settings to use new server address`);
return true;
}

export async function setupRemoteCache(repoRootPath: string) {
Expand All @@ -64,55 +45,17 @@ export async function setupRemoteCache(repoRootPath: string) {

const settingsPath = resolve(repoRootPath, '.bazelrc.cache');

// Checks if we should upgrade the servers used on .bazelrc.cache
//
// NOTE: this can be removed in the future once everyone is migrated into the new servers
if (await migrateToNewServersIfNeeded(settingsPath)) {
return;
}

if (existsSync(settingsPath)) {
log.debug(`[bazel_tools] remote cache settings already exist, skipping`);
return;
}

if (!(await isVaultAvailable())) {
log.info('[bazel_tools] vault is not available, unable to setup remote cache settings.');
log.info('[bazel_tools] building packages will work, but will be slower in many cases.');
log.info('[bazel_tools] reach out to Operations if you need assistance with this.');
return;
}

let apiKey = '';

try {
const { stdout } = await spawn(
'vault',
['read', '-field=readonly-key', 'secret/ui-team/kibana-bazel-remote-cache'],
{
stdio: 'pipe',
}
);
apiKey = stdout.trim();
} catch (ex: unknown) {
log.info(
'[bazel_tools] unable to read bazel remote cache key from vault, are you authenticated?'
);
log.info('[bazel_tools] building packages will work, but will be slower in many cases.');
log.info('[bazel_tools] reach out to Operations if you need assistance with this.');
log.info(`[bazel_tools] ${ex}`);

if (upToDate(settingsPath)) {
log.debug(`[bazel_tools] remote cache settings already up to date, skipping`);
return;
}

const contents = dedent`
# V1 - This file is automatically generated by 'yarn kbn bootstrap'
# V2 - This file is automatically generated by 'yarn kbn bootstrap'
# To regenerate this file, delete it and run 'yarn kbn bootstrap' again.
build --bes_results_url=https://app.buildbuddy.io/invocation/
build --bes_backend=grpcs://remote.buildbuddy.io
build --remote_cache=grpcs://remote.buildbuddy.io
build --remote_timeout=3600
build --remote_header=${apiKey}
build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
build --noremote_upload_local_results
build --incompatible_remote_results_ignore_disk
`;

writeFileSync(settingsPath, contents);
Expand Down
3 changes: 0 additions & 3 deletions src/dev/ci_setup/load_env_keys.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,6 @@ else
PERCY_TOKEN=$(retry 5 vault read -field=value secret/kibana-issues/dev/percy)
export PERCY_TOKEN

KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
export KIBANA_BUILDBUDDY_CI_API_KEY

# remove vault related secrets
unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN VAULT_ADDR
fi

0 comments on commit 08a8c64

Please sign in to comment.