Skip to content

Commit

Permalink
backport populate-gcs from #131866
Browse files Browse the repository at this point in the history
  • Loading branch information
jbudz committed Jan 4, 2023
1 parent 08a8c64 commit 7982f60
Show file tree
Hide file tree
Showing 5 changed files with 46 additions and 3 deletions.
10 changes: 10 additions & 0 deletions .buildkite/pipelines/on_merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,16 @@ steps:
key: storybooks
timeout_in_minutes: 60

- command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
label: 'Populate local dev bazel cache (Linux)'
agents:
queue: n2-4-spot
timeout_in_minutes: 15
retry:
automatic:
- exit_status: '-1'
limit: 3

- wait: ~
continue_on_failure: true

Expand Down
13 changes: 11 additions & 2 deletions .buildkite/scripts/common/setup_bazel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,16 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
EOF
fi

if [[ "$BAZEL_CACHE_MODE" != @(gcs|none|) ]]; then
echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,none]"
if [[ "$BAZEL_CACHE_MODE" == "populate-local-gcs" ]]; then
echo "[bazel] enabling caching with GCS buckets for local dev"

cat <<EOF >> $KIBANA_DIR/.bazelrc
build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
build --google_credentials=$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
EOF
fi

if [[ "$BAZEL_CACHE_MODE" != @(gcs|populate-local-gcs|none|) ]]; then
echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,populate-local-gcs,none]"
exit 1
fi
5 changes: 5 additions & 0 deletions .buildkite/scripts/lifecycle/pre_command.sh
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,11 @@ export KIBANA_DOCKER_PASSWORD
KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
export KIBANA_CI_REPORTER_KEY


BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"

# Setup Failed Test Reporter Elasticsearch credentials
{
TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)
Expand Down
13 changes: 13 additions & 0 deletions .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/bin/bash

set -euo pipefail

source .buildkite/scripts/common/util.sh

export BAZEL_CACHE_MODE=populate-local-gcs
export DISABLE_BOOTSTRAP_VALIDATION=true

# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
rm -rf ~/.bazel-cache

.buildkite/scripts/bootstrap.sh
8 changes: 7 additions & 1 deletion .buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,19 @@

set -euo pipefail

export BAZEL_CACHE_MODE=buildbuddy
source .buildkite/scripts/common/util.sh

export BAZEL_CACHE_MODE=populate-local-gcs
export DISABLE_BOOTSTRAP_VALIDATION=true

# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
rm -rf ~/.bazel-cache

# Since our Mac agents are currently static,
# use a temporary HOME directory that gets cleaned out between builds
TMP_HOME="$WORKSPACE/tmp_home"
rm -rf "$TMP_HOME"
mkdir -p "$TMP_HOME"
export HOME="$TMP_HOME"

.buildkite/scripts/bootstrap.sh

0 comments on commit 7982f60

Please sign in to comment.