chore(NA): merge and solve conflicts with main

.bazelrc.common

@@ -14,9 +14,18 @@ query --experimental_guard_against_concurrent_changes
 
 ## Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches)
 build --disk_cache=~/.bazel-cache/disk-cache
+fetch --disk_cache=~/.bazel-cache/disk-cache
+query --disk_cache=~/.bazel-cache/disk-cache
+sync --disk_cache=~/.bazel-cache/disk-cache
+test --disk_cache=~/.bazel-cache/disk-cache
 
 ## Bazel repo cache settings
 build --repository_cache=~/.bazel-cache/repository-cache
+fetch --repository_cache=~/.bazel-cache/repository-cache
+query --repository_cache=~/.bazel-cache/repository-cache
+run --repository_cache=~/.bazel-cache/repository-cache
+sync --repository_cache=~/.bazel-cache/repository-cache
+test --repository_cache=~/.bazel-cache/repository-cache
 
 # Bazel will create symlinks from the workspace directory to output artifacts.
 # Build results will be placed in a directory called "bazel-bin"

.buildkite/pipelines/es_snapshots/verify.yml

@@ -27,9 +27,9 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 13
+    parallelism: 27
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
@@ -41,7 +41,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -77,7 +77,7 @@ steps:
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
-      queue: jest
+      queue: n2-2
     timeout_in_minutes: 120
     key: api-integration
 

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -8,7 +8,7 @@ const stepInput = (key, nameOfSuite) => {
 };
 
 const OSS_CI_GROUPS = 12;
-const XPACK_CI_GROUPS = 13;
+const XPACK_CI_GROUPS = 27;
 
 const inputs = [
   {
@@ -23,11 +23,16 @@ for (let i = 1; i <= OSS_CI_GROUPS; i++) {
   inputs.push(stepInput(`oss/cigroup/${i}`, `OSS CI Group ${i}`));
 }
 
+inputs.push(stepInput(`oss/firefox`, 'OSS Firefox'));
+inputs.push(stepInput(`oss/accessibility`, 'OSS Accessibility'));
+
 for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
   inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
 }
 
 inputs.push(stepInput(`xpack/cigroup/Docker`, 'Default CI Group Docker'));
+inputs.push(stepInput(`xpack/firefox`, 'Default Firefox'));
+inputs.push(stepInput(`xpack/accessibility`, 'Default Accessibility'));
 
 const pipeline = {
   steps: [

.buildkite/pipelines/flaky_tests/runner.js

@@ -65,34 +65,67 @@ for (const testSuite of testSuites) {
 
   const JOB_PARTS = TEST_SUITE.split('/');
   const IS_XPACK = JOB_PARTS[0] === 'xpack';
+  const TASK = JOB_PARTS[1];
   const CI_GROUP = JOB_PARTS.length > 2 ? JOB_PARTS[2] : '';
 
   if (RUN_COUNT < 1) {
     continue;
   }
 
-  if (IS_XPACK) {
-    steps.push({
-      command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
-      label: `Default CI Group ${CI_GROUP}`,
-      agents: { queue: 'ci-group-6' },
-      depends_on: 'build',
-      parallelism: RUN_COUNT,
-      concurrency: concurrency,
-      concurrency_group: UUID,
-      concurrency_method: 'eager',
-    });
-  } else {
-    steps.push({
-      command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/oss_cigroup.sh`,
-      label: `OSS CI Group ${CI_GROUP}`,
-      agents: { queue: 'ci-group-4d' },
-      depends_on: 'build',
-      parallelism: RUN_COUNT,
-      concurrency: concurrency,
-      concurrency_group: UUID,
-      concurrency_method: 'eager',
-    });
+  switch (TASK) {
+    case 'cigroup':
+      if (IS_XPACK) {
+        steps.push({
+          command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
+          label: `Default CI Group ${CI_GROUP}`,
+          agents: { queue: 'n2-4' },
+          depends_on: 'build',
+          parallelism: RUN_COUNT,
+          concurrency: concurrency,
+          concurrency_group: UUID,
+          concurrency_method: 'eager',
+        });
+      } else {
+        steps.push({
+          command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/oss_cigroup.sh`,
+          label: `OSS CI Group ${CI_GROUP}`,
+          agents: { queue: 'ci-group-4d' },
+          depends_on: 'build',
+          parallelism: RUN_COUNT,
+          concurrency: concurrency,
+          concurrency_group: UUID,
+          concurrency_method: 'eager',
+        });
+      }
+      break;
+
+    case 'firefox':
+      steps.push({
+        command: `.buildkite/scripts/steps/functional/${IS_XPACK ? 'xpack' : 'oss'}_firefox.sh`,
+        label: `${IS_XPACK ? 'Default' : 'OSS'} Firefox`,
+        agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
+        depends_on: 'build',
+        parallelism: RUN_COUNT,
+        concurrency: concurrency,
+        concurrency_group: UUID,
+        concurrency_method: 'eager',
+      });
+      break;
+
+    case 'accessibility':
+      steps.push({
+        command: `.buildkite/scripts/steps/functional/${
+          IS_XPACK ? 'xpack' : 'oss'
+        }_accessibility.sh`,
+        label: `${IS_XPACK ? 'Default' : 'OSS'} Accessibility`,
+        agents: { queue: IS_XPACK ? 'n2-4' : 'ci-group-4d' },
+        depends_on: 'build',
+        parallelism: RUN_COUNT,
+        concurrency: concurrency,
+        concurrency_group: UUID,
+        concurrency_method: 'eager',
+      });
+      break;
   }
 }
 

.buildkite/pipelines/hourly.yml

@@ -17,9 +17,9 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 13
+    parallelism: 27
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 250
     key: default-cigroup
@@ -31,7 +31,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -67,7 +67,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -89,7 +89,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -100,7 +100,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -111,14 +111,22 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/test/jest.sh
+    label: 'Jest Tests'
+    parallelism: 8
+    agents:
+      queue: n2-4
+    timeout_in_minutes: 90
+    key: jest
+
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
     agents:
@@ -133,13 +141,6 @@ steps:
     timeout_in_minutes: 120
     key: api-integration
 
-  - command: .buildkite/scripts/steps/test/jest.sh
-    label: 'Jest Tests'
-    agents:
-      queue: c2-16
-    timeout_in_minutes: 120
-    key: jest
-
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:

.buildkite/pipelines/pull_request/base.yml

@@ -15,9 +15,9 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 13
+    parallelism: 27
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
@@ -29,7 +29,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -65,7 +65,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -87,7 +87,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -98,7 +98,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -109,14 +109,22 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: ci-group-6
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/test/jest.sh
+    label: 'Jest Tests'
+    parallelism: 8
+    agents:
+      queue: n2-4
+    timeout_in_minutes: 90
+    key: jest
+
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
     agents:
@@ -131,13 +139,6 @@ steps:
     timeout_in_minutes: 120
     key: api-integration
 
-  - command: .buildkite/scripts/steps/test/jest.sh
-    label: 'Jest Tests'
-    agents:
-      queue: c2-16
-    timeout_in_minutes: 120
-    key: jest
-
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:
@@ -155,7 +156,7 @@ steps:
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
     agents:
-      queue: c2-4
+      queue: c2-8
     key: checks
     timeout_in_minutes: 120
 

.buildkite/scripts/bootstrap.sh

@@ -18,6 +18,10 @@ if ! yarn kbn bootstrap; then
   yarn kbn bootstrap
 fi
 
+if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then
+  verify_no_git_changes 'yarn kbn bootstrap'
+fi
+
 ###
 ### upload ts-refs-cache artifacts as quickly as possible so they are available for download
 ###
@@ -27,7 +31,3 @@ if [[ "${BUILD_TS_REFS_CACHE_CAPTURE:-}" == "true" ]]; then
   gsutil cp "*.zip" 'gs://kibana-ci-ts-refs-cache/'
   cd "$KIBANA_DIR"
 fi
-
-if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then
-  verify_no_git_changes 'yarn kbn bootstrap'
-fi

.buildkite/scripts/common/env.sh

@@ -38,8 +38,10 @@ export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
 if is_pr; then
   if [[ "${GITHUB_PR_LABELS:-}" == *"ci:collect-apm"* ]]; then
     export ELASTIC_APM_ACTIVE=true
+    export ELASTIC_APM_CONTEXT_PROPAGATION_ONLY=false
   else
-    export ELASTIC_APM_ACTIVE=false
+    export ELASTIC_APM_ACTIVE=true
+    export ELASTIC_APM_CONTEXT_PROPAGATION_ONLY=true
   fi
 
   if [[ "${GITHUB_STEP_COMMIT_STATUS_ENABLED:-}" != "true" ]]; then
@@ -61,6 +63,7 @@ if is_pr; then
   export PR_TARGET_BRANCH="$GITHUB_PR_TARGET_BRANCH"
 else
   export ELASTIC_APM_ACTIVE=true
+  export ELASTIC_APM_CONTEXT_PROPAGATION_ONLY=false
   export CHECKS_REPORTER_ACTIVE=false
 fi
 

.buildkite/scripts/common/util.sh

@@ -16,12 +16,14 @@ is_pr() {
 
 verify_no_git_changes() {
   RED='\033[0;31m'
+  YELLOW='\033[0;33m'
   C_RESET='\033[0m' # Reset color
 
   GIT_CHANGES="$(git ls-files --modified -- . ':!:.bazelrc')"
   if [ "$GIT_CHANGES" ]; then
     echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
     echo -e "$GIT_CHANGES\n"
+    echo -e "\n${YELLOW}TO FIX: Run '$1' locally, commit the changes and push to your branch${C_RESET}\n"
     exit 1
   fi
 }