From 3a81e78e3654aef827962c8c4cb587a23eaa3eaf Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 2 May 2022 14:29:05 -0400 Subject: [PATCH 01/13] Trying our own bazel remote cache --- .buildkite/pipelines/pull_request/base.yml | 191 +----------------- .../pipelines/pull_request/post_build.yml | 8 - .buildkite/scripts/steps/bazel_cache/test.sh | 12 ++ .../scripts/steps/bazel_cache/test_clean.sh | 14 ++ 4 files changed, 34 insertions(+), 191 deletions(-) create mode 100755 .buildkite/scripts/steps/bazel_cache/test.sh create mode 100755 .buildkite/scripts/steps/bazel_cache/test_clean.sh diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 4b4104f18c627..4577440253248 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -1,190 +1,15 @@ steps: - - command: .buildkite/scripts/lifecycle/pre_build.sh - label: Pre-Build - timeout_in_minutes: 10 - agents: - queue: kibana-default - - - wait - - - command: .buildkite/scripts/steps/build_kibana.sh - label: Build Kibana Distribution and Plugins - agents: - queue: c2-16 - key: build - if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" - timeout_in_minutes: 60 - - - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh - label: 'Default CI Group' - parallelism: 31 - agents: - queue: n2-4-spot-2 - depends_on: build - timeout_in_minutes: 150 - key: default-cigroup - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/oss_cigroup.sh - label: 'OSS CI Group' - parallelism: 12 - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 120 - key: oss-cigroup - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/oss_accessibility.sh - label: 'OSS Accessibility Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 120 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh - label: 'Default Accessibility Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 120 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/oss_firefox.sh - label: 'OSS Firefox Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 120 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/xpack_firefox.sh - label: 'Default Firefox Tests' + - command: .buildkite/scripts/steps/bazel_cache/test.sh + label: Bootstrap + timeout_in_minutes: 20 agents: queue: n2-4-spot - depends_on: build - timeout_in_minutes: 120 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - command: .buildkite/scripts/steps/functional/oss_misc.sh - label: 'OSS Misc Functional Tests' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 120 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh - label: 'Saved Object Field Metrics' - agents: - queue: n2-4-spot - depends_on: build - timeout_in_minutes: 120 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh - label: 'Pick Jest Config Run Order' - agents: - queue: kibana-default - retry: - automatic: - - exit_status: '*' - limit: 1 - - - command: .buildkite/scripts/steps/test/api_integration.sh - label: 'API Integration Tests' - agents: - queue: n2-2-spot - timeout_in_minutes: 120 - key: api-integration - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - command: .buildkite/scripts/steps/lint.sh - label: 'Linting' - agents: - queue: n2-2 - key: linting - timeout_in_minutes: 90 - - - command: .buildkite/scripts/steps/lint_with_types.sh - label: 'Linting (with types)' - agents: - queue: c2-16 - key: linting_with_types - timeout_in_minutes: 90 - - - command: .buildkite/scripts/steps/checks.sh - label: 'Checks' - agents: - queue: c2-8 - key: checks - timeout_in_minutes: 120 - - - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh - label: 'Build Storybooks' - agents: - queue: c2-4 - key: storybooks - timeout_in_minutes: 60 + - wait - - command: .buildkite/scripts/steps/build_api_docs.sh - label: 'Build API Docs' + - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh + label: Bootstrap with no local cache + timeout_in_minutes: 20 + parallelism: 10 agents: queue: n2-4-spot - key: build_api_docs - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - command: .buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh - label: 'Build Webpack Bundle Analyzer reports' - agents: - queue: c2-4 - key: webpack_bundle_analyzer - timeout_in_minutes: 60 diff --git a/.buildkite/pipelines/pull_request/post_build.yml b/.buildkite/pipelines/pull_request/post_build.yml index 63f7169334584..e69de29bb2d1d 100644 --- a/.buildkite/pipelines/pull_request/post_build.yml +++ b/.buildkite/pipelines/pull_request/post_build.yml @@ -1,8 +0,0 @@ -steps: - - wait: ~ - continue_on_failure: true - - - command: .buildkite/scripts/lifecycle/post_build.sh - label: Post-Build - agents: - queue: kibana-default diff --git a/.buildkite/scripts/steps/bazel_cache/test.sh b/.buildkite/scripts/steps/bazel_cache/test.sh new file mode 100755 index 0000000000000..bcbf977b665d8 --- /dev/null +++ b/.buildkite/scripts/steps/bazel_cache/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" + +cat << EOF >> .bazelrc +build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 +EOF + +.buildkite/scripts/bootstrap.sh + diff --git a/.buildkite/scripts/steps/bazel_cache/test_clean.sh b/.buildkite/scripts/steps/bazel_cache/test_clean.sh new file mode 100755 index 0000000000000..1d389adf8f361 --- /dev/null +++ b/.buildkite/scripts/steps/bazel_cache/test_clean.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" + +cat << EOF >> .bazelrc +build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 +EOF + +rm -rf ~/.bazel-cache + +.buildkite/scripts/bootstrap.sh + From a4cd372290729f534c38bafa3d1cd576f73dcbb8 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 2 May 2022 14:36:56 -0400 Subject: [PATCH 02/13] Add util --- .buildkite/scripts/steps/bazel_cache/test.sh | 2 ++ .buildkite/scripts/steps/bazel_cache/test_clean.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.buildkite/scripts/steps/bazel_cache/test.sh b/.buildkite/scripts/steps/bazel_cache/test.sh index bcbf977b665d8..18e6fb74aa8ed 100755 --- a/.buildkite/scripts/steps/bazel_cache/test.sh +++ b/.buildkite/scripts/steps/bazel_cache/test.sh @@ -2,6 +2,8 @@ set -euo pipefail +source .buildkite/scripts/common/util.sh + REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" cat << EOF >> .bazelrc diff --git a/.buildkite/scripts/steps/bazel_cache/test_clean.sh b/.buildkite/scripts/steps/bazel_cache/test_clean.sh index 1d389adf8f361..d28b06c83cd13 100755 --- a/.buildkite/scripts/steps/bazel_cache/test_clean.sh +++ b/.buildkite/scripts/steps/bazel_cache/test_clean.sh @@ -2,6 +2,8 @@ set -euo pipefail +source .buildkite/scripts/common/util.sh + REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" cat << EOF >> .bazelrc From 63f926529cb9f9da5c5fedb3abcd2e62361b216e Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 2 May 2022 14:46:09 -0400 Subject: [PATCH 03/13] Initial bootstrap should be clean also --- .buildkite/pipelines/pull_request/base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 4577440253248..c4acccfd12588 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -1,6 +1,6 @@ steps: - - command: .buildkite/scripts/steps/bazel_cache/test.sh - label: Bootstrap + - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh + label: Initial bootstrap with no local cache timeout_in_minutes: 20 agents: queue: n2-4-spot From 39975027afda5b5cb085b1259bc0d05feb1a73f5 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 2 May 2022 14:47:40 -0400 Subject: [PATCH 04/13] Overwrite bazelrc altogether --- .buildkite/scripts/steps/bazel_cache/test.sh | 3 ++- .buildkite/scripts/steps/bazel_cache/test_clean.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/steps/bazel_cache/test.sh b/.buildkite/scripts/steps/bazel_cache/test.sh index 18e6fb74aa8ed..28fcfc7c68e7b 100755 --- a/.buildkite/scripts/steps/bazel_cache/test.sh +++ b/.buildkite/scripts/steps/bazel_cache/test.sh @@ -6,7 +6,8 @@ source .buildkite/scripts/common/util.sh REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" -cat << EOF >> .bazelrc +cat << EOF > .bazelrc +import %workspace%/.bazelrc.common build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 EOF diff --git a/.buildkite/scripts/steps/bazel_cache/test_clean.sh b/.buildkite/scripts/steps/bazel_cache/test_clean.sh index d28b06c83cd13..ee8d065f830b8 100755 --- a/.buildkite/scripts/steps/bazel_cache/test_clean.sh +++ b/.buildkite/scripts/steps/bazel_cache/test_clean.sh @@ -6,7 +6,8 @@ source .buildkite/scripts/common/util.sh REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" -cat << EOF >> .bazelrc +cat << EOF > .bazelrc +import %workspace%/.bazelrc.common build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 EOF From 1e3c05682db77e9d16cda047561b13aa01259d69 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 2 May 2022 14:54:24 -0400 Subject: [PATCH 05/13] Overwrite in correct spot --- .buildkite/scripts/bootstrap.sh | 7 +++++++ .buildkite/scripts/steps/bazel_cache/test.sh | 7 ------- .buildkite/scripts/steps/bazel_cache/test_clean.sh | 7 ------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh index 34461ca6db194..65644a701ab30 100755 --- a/.buildkite/scripts/bootstrap.sh +++ b/.buildkite/scripts/bootstrap.sh @@ -5,6 +5,13 @@ set -euo pipefail source .buildkite/scripts/common/util.sh source .buildkite/scripts/common/setup_bazel.sh +REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" + +cat << EOF > .bazelrc +import %workspace%/.bazelrc.common +build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 +EOF + echo "--- yarn install and bootstrap" if ! yarn kbn bootstrap; then echo "bootstrap failed, trying again in 15 seconds" diff --git a/.buildkite/scripts/steps/bazel_cache/test.sh b/.buildkite/scripts/steps/bazel_cache/test.sh index 28fcfc7c68e7b..08be73c1cb59b 100755 --- a/.buildkite/scripts/steps/bazel_cache/test.sh +++ b/.buildkite/scripts/steps/bazel_cache/test.sh @@ -4,12 +4,5 @@ set -euo pipefail source .buildkite/scripts/common/util.sh -REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" - -cat << EOF > .bazelrc -import %workspace%/.bazelrc.common -build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 -EOF - .buildkite/scripts/bootstrap.sh diff --git a/.buildkite/scripts/steps/bazel_cache/test_clean.sh b/.buildkite/scripts/steps/bazel_cache/test_clean.sh index ee8d065f830b8..faca4101515a3 100755 --- a/.buildkite/scripts/steps/bazel_cache/test_clean.sh +++ b/.buildkite/scripts/steps/bazel_cache/test_clean.sh @@ -4,13 +4,6 @@ set -euo pipefail source .buildkite/scripts/common/util.sh -REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" - -cat << EOF > .bazelrc -import %workspace%/.bazelrc.common -build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 -EOF - rm -rf ~/.bazel-cache .buildkite/scripts/bootstrap.sh From 85664c7e98b5376e4a10248ed5b72750d93d7bd9 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 2 May 2022 15:14:44 -0400 Subject: [PATCH 06/13] Try partial cache x100 --- .buildkite/pipelines/pull_request/base.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index c4acccfd12588..ea9cfe834bf7d 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -1,15 +1,15 @@ steps: - - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh - label: Initial bootstrap with no local cache - timeout_in_minutes: 20 - agents: - queue: n2-4-spot + # - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh + # label: Initial bootstrap with no local cache + # timeout_in_minutes: 20 + # agents: + # queue: n2-4-spot - - wait + # - wait - - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh - label: Bootstrap with no local cache + - command: .buildkite/scripts/steps/bazel_cache/test.sh + label: Bootstrap with partial cache timeout_in_minutes: 20 - parallelism: 10 + parallelism: 100 agents: queue: n2-4-spot From abfb676f73e706552420f167234ef54cc71e9dab Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 2 May 2022 15:28:31 -0400 Subject: [PATCH 07/13] 100x full pull --- .buildkite/pipelines/pull_request/base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index ea9cfe834bf7d..3d9711df93677 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -7,8 +7,8 @@ steps: # - wait - - command: .buildkite/scripts/steps/bazel_cache/test.sh - label: Bootstrap with partial cache + - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh + label: Bootstrap with no disk cache timeout_in_minutes: 20 parallelism: 100 agents: From 8ff132bc0f8a252f7438c38dd8392b7f89ecc0f4 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 2 May 2022 16:04:24 -0400 Subject: [PATCH 08/13] Try experimental_remote_cache_compression --- .buildkite/scripts/bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh index 65644a701ab30..24d62c2ed0143 100755 --- a/.buildkite/scripts/bootstrap.sh +++ b/.buildkite/scripts/bootstrap.sh @@ -10,6 +10,7 @@ REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issu cat << EOF > .bazelrc import %workspace%/.bazelrc.common build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 +build --experimental_remote_cache_compression EOF echo "--- yarn install and bootstrap" From d3e1a889a15b8c17f89b91fce1867642e95386f9 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 3 May 2022 12:44:37 -0400 Subject: [PATCH 09/13] Try gcs bucket --- .buildkite/pipelines/pull_request/base.yml | 19 +++++++++++++------ .buildkite/scripts/bootstrap.sh | 6 ++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 3d9711df93677..e623fae6d7cec 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -1,11 +1,18 @@ steps: - # - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh - # label: Initial bootstrap with no local cache - # timeout_in_minutes: 20 - # agents: - # queue: n2-4-spot + - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh + label: Initial bootstrap with no local cache + timeout_in_minutes: 20 + agents: + queue: n2-4-spot + + - wait - # - wait + - command: .buildkite/scripts/steps/bazel_cache/test.sh + label: Bootstrap with disk cache + timeout_in_minutes: 20 + parallelism: 100 + agents: + queue: n2-4-spot - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh label: Bootstrap with no disk cache diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh index 24d62c2ed0143..ceb7e623749cd 100755 --- a/.buildkite/scripts/bootstrap.sh +++ b/.buildkite/scripts/bootstrap.sh @@ -9,8 +9,10 @@ REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issu cat << EOF > .bazelrc import %workspace%/.bazelrc.common -build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 -build --experimental_remote_cache_compression +#build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 +#build --experimental_remote_cache_compression +build --remote_cache=https://storage.googleapis.com/kibana_ci_bazel_remote_cache +build --google_default_credentials EOF echo "--- yarn install and bootstrap" From d4899ec5a224ee4ddd361b92d09a417d3e3c6085 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 3 May 2022 13:58:42 -0400 Subject: [PATCH 10/13] Try regional gcs buckets --- .buildkite/pipelines/pull_request/base.yml | 24 +++++++++++----------- .buildkite/scripts/bootstrap.sh | 11 ++++++---- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index e623fae6d7cec..be243e1672205 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -1,18 +1,18 @@ steps: - - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh - label: Initial bootstrap with no local cache - timeout_in_minutes: 20 - agents: - queue: n2-4-spot + # - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh + # label: Initial bootstrap with no local cache + # timeout_in_minutes: 20 + # agents: + # queue: n2-4-spot - - wait + # - wait - - command: .buildkite/scripts/steps/bazel_cache/test.sh - label: Bootstrap with disk cache - timeout_in_minutes: 20 - parallelism: 100 - agents: - queue: n2-4-spot + # - command: .buildkite/scripts/steps/bazel_cache/test.sh + # label: Bootstrap with disk cache + # timeout_in_minutes: 20 + # parallelism: 100 + # agents: + # queue: n2-4-spot - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh label: Bootstrap with no disk cache diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh index ceb7e623749cd..d2143cc87ee7e 100755 --- a/.buildkite/scripts/bootstrap.sh +++ b/.buildkite/scripts/bootstrap.sh @@ -5,13 +5,16 @@ set -euo pipefail source .buildkite/scripts/common/util.sh source .buildkite/scripts/common/setup_bazel.sh -REMOTE_CACHE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/bazel-remote-cache-test)" +BAZEL_REGION="us-central1" +if [[ "$(curl -is metadata.google.internal || true)" ]]; then + BAZEL_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -c3- | rev) +fi + +BAZEL_BUCKET="kibana-ci-bazel_$BAZEL_REGION" cat << EOF > .bazelrc import %workspace%/.bazelrc.common -#build --remote_cache=grpc://test:${REMOTE_CACHE_PASSWORD}@34.121.74.141:9092 -#build --experimental_remote_cache_compression -build --remote_cache=https://storage.googleapis.com/kibana_ci_bazel_remote_cache +build --remote_cache=https://storage.googleapis.com/$BAZEL_BUCKET build --google_default_credentials EOF From a63ba243ea026f36ccfe0d2a03de2ad17186055c Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 3 May 2022 14:09:12 -0400 Subject: [PATCH 11/13] Fix region parsing --- .buildkite/scripts/bootstrap.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh index d2143cc87ee7e..2db7315e69356 100755 --- a/.buildkite/scripts/bootstrap.sh +++ b/.buildkite/scripts/bootstrap.sh @@ -7,7 +7,8 @@ source .buildkite/scripts/common/setup_bazel.sh BAZEL_REGION="us-central1" if [[ "$(curl -is metadata.google.internal || true)" ]]; then - BAZEL_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -c3- | rev) + # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1 + BAZEL_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev) fi BAZEL_BUCKET="kibana-ci-bazel_$BAZEL_REGION" @@ -19,6 +20,8 @@ build --google_default_credentials EOF echo "--- yarn install and bootstrap" +echo "Using Bazel remote cache bucket: $BAZEL_BUCKET" + if ! yarn kbn bootstrap; then echo "bootstrap failed, trying again in 15 seconds" sleep 15 From 3d8d11230ee7a90569d0cf8bfc4baf749898953f Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 3 May 2022 15:30:01 -0400 Subject: [PATCH 12/13] Prepare final changes for bazel remote cache in CI --- .buildkite/pipelines/pull_request/base.yml | 18 +++++----- .buildkite/scripts/bootstrap.sh | 15 -------- .buildkite/scripts/common/setup_bazel.sh | 36 +++++++++++++------ .../steps/bazel_cache/bootstrap_mac.sh | 2 +- .../steps/on_merge_ts_refs_api_docs.sh | 2 +- 5 files changed, 36 insertions(+), 37 deletions(-) diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index be243e1672205..2896fc1359351 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -7,16 +7,16 @@ steps: # - wait - # - command: .buildkite/scripts/steps/bazel_cache/test.sh - # label: Bootstrap with disk cache - # timeout_in_minutes: 20 - # parallelism: 100 - # agents: - # queue: n2-4-spot - - - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh - label: Bootstrap with no disk cache + - command: .buildkite/scripts/steps/bazel_cache/test.sh + label: Bootstrap with disk cache timeout_in_minutes: 20 parallelism: 100 agents: queue: n2-4-spot + + # - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh + # label: Bootstrap with no disk cache + # timeout_in_minutes: 20 + # parallelism: 100 + # agents: + # queue: n2-4-spot diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh index 2db7315e69356..62d13b02f69f8 100755 --- a/.buildkite/scripts/bootstrap.sh +++ b/.buildkite/scripts/bootstrap.sh @@ -5,22 +5,7 @@ set -euo pipefail source .buildkite/scripts/common/util.sh source .buildkite/scripts/common/setup_bazel.sh -BAZEL_REGION="us-central1" -if [[ "$(curl -is metadata.google.internal || true)" ]]; then - # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1 - BAZEL_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev) -fi - -BAZEL_BUCKET="kibana-ci-bazel_$BAZEL_REGION" - -cat << EOF > .bazelrc -import %workspace%/.bazelrc.common -build --remote_cache=https://storage.googleapis.com/$BAZEL_BUCKET -build --google_default_credentials -EOF - echo "--- yarn install and bootstrap" -echo "Using Bazel remote cache bucket: $BAZEL_BUCKET" if ! yarn kbn bootstrap; then echo "bootstrap failed, trying again in 15 seconds" diff --git a/.buildkite/scripts/common/setup_bazel.sh b/.buildkite/scripts/common/setup_bazel.sh index 96cd04fa612fd..e3791dfa393c7 100755 --- a/.buildkite/scripts/common/setup_bazel.sh +++ b/.buildkite/scripts/common/setup_bazel.sh @@ -11,8 +11,29 @@ cat < $KIBANA_DIR/.bazelrc build --build_metadata=ROLE=CI EOF -if [[ "${BAZEL_CACHE_MODE:-none}" == read* ]]; then - echo "[bazel] enabling caching" +BAZEL_CACHE_MODE=${BAZEL_CACHE_MODE:-gcs} + +if [[ "$BAZEL_CACHE_MODE" == "gcs" ]]; then + echo "[bazel] enabling caching with GCS buckets" + + BAZEL_REGION="us-central1" + if [[ "$(curl -is metadata.google.internal || true)" ]]; then + # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1 + BAZEL_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev) + fi + + BAZEL_BUCKET="kibana-ci-bazel_$BAZEL_REGION" + + echo "[bazel] using GCS bucket: $BAZEL_BUCKET" + +cat <> $KIBANA_DIR/.bazelrc + build --remote_cache=https://storage.googleapis.com/$BAZEL_BUCKET + build --google_default_credentials +EOF +fi + +if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then + echo "[bazel] enabling caching with Buildbuddy" cat <> $KIBANA_DIR/.bazelrc build --bes_results_url=https://app.buildbuddy.io/invocation/ build --bes_backend=grpcs://remote.buildbuddy.io @@ -22,14 +43,7 @@ cat <> $KIBANA_DIR/.bazelrc EOF fi -if [[ "${BAZEL_CACHE_MODE:-none}" == "read" ]]; then - echo "[bazel] cache set to read-only" -cat <> $KIBANA_DIR/.bazelrc - build --noremote_upload_local_results -EOF -fi - -if [[ "${BAZEL_CACHE_MODE:-none}" != @(read|read-write|none|) ]]; then - echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [read,read-write,none]" +if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then + echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]" exit 1 fi diff --git a/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh b/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh index 1417137f0b6f0..62aabf496fd8a 100755 --- a/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh +++ b/.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh @@ -2,7 +2,7 @@ set -euo pipefail -export BAZEL_CACHE_MODE=read-write +export BAZEL_CACHE_MODE=buildbuddy export DISABLE_BOOTSTRAP_VALIDATION=true # Since our Mac agents are currently static, diff --git a/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh b/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh index 6d08fbb2c6835..5659db50e1404 100755 --- a/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh +++ b/.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh @@ -2,7 +2,7 @@ set -euo pipefail -export BAZEL_CACHE_MODE=read-write # Populate bazel remote cache for linux +export BAZEL_CACHE_MODE=buildbuddy # Populate Buildbuddy bazel remote cache for linux export BUILD_TS_REFS_CACHE_ENABLE=true export BUILD_TS_REFS_CACHE_CAPTURE=true export DISABLE_BOOTSTRAP_VALIDATION=true From ced09f0ff62943cbb8fae9909943e2ea90c56d83 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 3 May 2022 15:37:39 -0400 Subject: [PATCH 13/13] Remove test files and revert test changes --- .buildkite/pipelines/pull_request/base.yml | 200 ++++++++++++++++-- .../pipelines/pull_request/post_build.yml | 8 + .buildkite/scripts/bootstrap.sh | 1 - .buildkite/scripts/steps/bazel_cache/test.sh | 8 - .../scripts/steps/bazel_cache/test_clean.sh | 10 - 5 files changed, 192 insertions(+), 35 deletions(-) delete mode 100755 .buildkite/scripts/steps/bazel_cache/test.sh delete mode 100755 .buildkite/scripts/steps/bazel_cache/test_clean.sh diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 2896fc1359351..4b4104f18c627 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -1,22 +1,190 @@ steps: - # - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh - # label: Initial bootstrap with no local cache - # timeout_in_minutes: 20 - # agents: - # queue: n2-4-spot + - command: .buildkite/scripts/lifecycle/pre_build.sh + label: Pre-Build + timeout_in_minutes: 10 + agents: + queue: kibana-default + + - wait + + - command: .buildkite/scripts/steps/build_kibana.sh + label: Build Kibana Distribution and Plugins + agents: + queue: c2-16 + key: build + if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" + timeout_in_minutes: 60 + + - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh + label: 'Default CI Group' + parallelism: 31 + agents: + queue: n2-4-spot-2 + depends_on: build + timeout_in_minutes: 150 + key: default-cigroup + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/oss_cigroup.sh + label: 'OSS CI Group' + parallelism: 12 + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 120 + key: oss-cigroup + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/oss_accessibility.sh + label: 'OSS Accessibility Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh + label: 'Default Accessibility Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 - # - wait + - command: .buildkite/scripts/steps/functional/oss_firefox.sh + label: 'OSS Firefox Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/functional/xpack_firefox.sh + label: 'Default Firefox Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 - - command: .buildkite/scripts/steps/bazel_cache/test.sh - label: Bootstrap with disk cache - timeout_in_minutes: 20 - parallelism: 100 + - command: .buildkite/scripts/steps/functional/oss_misc.sh + label: 'OSS Misc Functional Tests' agents: queue: n2-4-spot + depends_on: build + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 - # - command: .buildkite/scripts/steps/bazel_cache/test_clean.sh - # label: Bootstrap with no disk cache - # timeout_in_minutes: 20 - # parallelism: 100 - # agents: - # queue: n2-4-spot + - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh + label: 'Saved Object Field Metrics' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh + label: 'Pick Jest Config Run Order' + agents: + queue: kibana-default + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: .buildkite/scripts/steps/test/api_integration.sh + label: 'API Integration Tests' + agents: + queue: n2-2-spot + timeout_in_minutes: 120 + key: api-integration + retry: + automatic: + - exit_status: '-1' + limit: 3 + + - command: .buildkite/scripts/steps/lint.sh + label: 'Linting' + agents: + queue: n2-2 + key: linting + timeout_in_minutes: 90 + + - command: .buildkite/scripts/steps/lint_with_types.sh + label: 'Linting (with types)' + agents: + queue: c2-16 + key: linting_with_types + timeout_in_minutes: 90 + + - command: .buildkite/scripts/steps/checks.sh + label: 'Checks' + agents: + queue: c2-8 + key: checks + timeout_in_minutes: 120 + + - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh + label: 'Build Storybooks' + agents: + queue: c2-4 + key: storybooks + timeout_in_minutes: 60 + + - command: .buildkite/scripts/steps/build_api_docs.sh + label: 'Build API Docs' + agents: + queue: n2-4-spot + key: build_api_docs + timeout_in_minutes: 60 + retry: + automatic: + - exit_status: '-1' + limit: 3 + + - command: .buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh + label: 'Build Webpack Bundle Analyzer reports' + agents: + queue: c2-4 + key: webpack_bundle_analyzer + timeout_in_minutes: 60 diff --git a/.buildkite/pipelines/pull_request/post_build.yml b/.buildkite/pipelines/pull_request/post_build.yml index e69de29bb2d1d..63f7169334584 100644 --- a/.buildkite/pipelines/pull_request/post_build.yml +++ b/.buildkite/pipelines/pull_request/post_build.yml @@ -0,0 +1,8 @@ +steps: + - wait: ~ + continue_on_failure: true + + - command: .buildkite/scripts/lifecycle/post_build.sh + label: Post-Build + agents: + queue: kibana-default diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh index 62d13b02f69f8..34461ca6db194 100755 --- a/.buildkite/scripts/bootstrap.sh +++ b/.buildkite/scripts/bootstrap.sh @@ -6,7 +6,6 @@ source .buildkite/scripts/common/util.sh source .buildkite/scripts/common/setup_bazel.sh echo "--- yarn install and bootstrap" - if ! yarn kbn bootstrap; then echo "bootstrap failed, trying again in 15 seconds" sleep 15 diff --git a/.buildkite/scripts/steps/bazel_cache/test.sh b/.buildkite/scripts/steps/bazel_cache/test.sh deleted file mode 100755 index 08be73c1cb59b..0000000000000 --- a/.buildkite/scripts/steps/bazel_cache/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -source .buildkite/scripts/common/util.sh - -.buildkite/scripts/bootstrap.sh - diff --git a/.buildkite/scripts/steps/bazel_cache/test_clean.sh b/.buildkite/scripts/steps/bazel_cache/test_clean.sh deleted file mode 100755 index faca4101515a3..0000000000000 --- a/.buildkite/scripts/steps/bazel_cache/test_clean.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -source .buildkite/scripts/common/util.sh - -rm -rf ~/.bazel-cache - -.buildkite/scripts/bootstrap.sh -