Merge remote-tracking branch 'upstream/master' into fix/105410

.bazelrc

@@ -2,12 +2,13 @@
 # Import shared settings first so we can override below
 import %workspace%/.bazelrc.common
 
-## Disabled for now
 # Remote cache settings for local env
-# build --remote_cache=https://storage.googleapis.com/kibana-bazel-cache
-# build --incompatible_remote_results_ignore_disk=true
-# build --remote_accept_cached=true
-# build --remote_upload_local_results=false
+build --remote_cache=grpcs://cloud.buildbuddy.io
+build --incompatible_remote_results_ignore_disk=true
+build --noremote_upload_local_results
+build --remote_timeout=30
+build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze
+build --remote_accept_cached=true
 
 # BuildBuddy
 ## Metadata settings

.buildkite/hooks/post-command

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+.buildkite/scripts/lifecycle/post_command.sh

.buildkite/pipelines/es_snapshots/verify.yml

@@ -19,7 +19,7 @@ steps:
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: c2-8
+      queue: c2-16
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
 
@@ -28,8 +28,8 @@ steps:
     parallelism: 13
     agents:
       queue: ci-group-6
-    artifact_paths: target/junit/**/*.xml
     depends_on: build
+    timeout_in_minutes: 150
     key: default-cigroup
     retry:
       automatic:
@@ -40,8 +40,8 @@ steps:
     label: 'Docker CI Group'
     agents:
       queue: ci-group-6
-    artifact_paths: target/junit/**/*.xml
     depends_on: build
+    timeout_in_minutes: 120
     key: default-cigroup-docker
     retry:
       automatic:
@@ -50,11 +50,11 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 12
+    parallelism: 11
     agents:
       queue: ci-group-4d
-    artifact_paths: target/junit/**/*.xml
     depends_on: build
+    timeout_in_minutes: 120
     key: oss-cigroup
     retry:
       automatic:
@@ -64,8 +64,8 @@ steps:
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
     agents:
-      queue: jest
-    artifact_paths: target/junit/**/*.xml
+      queue: n2-4
+    timeout_in_minutes: 120
     key: jest-integration
     retry:
       automatic:
@@ -76,7 +76,7 @@ steps:
     label: 'API Integration Tests'
     agents:
       queue: jest
-    artifact_paths: target/junit/**/*.xml
+    timeout_in_minutes: 120
     key: api-integration
 
   - command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
@@ -91,12 +91,5 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - plugins:
-      - junit-annotate#v1.9.0:
-          artifacts: target/junit/**/*.xml
-
-  - wait: ~
-    continue_on_failure: true
-
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -0,0 +1,52 @@
+const stepInput = (key, nameOfSuite) => {
+  return {
+    key: `ftsr-suite/${key}`,
+    text: nameOfSuite,
+    required: false,
+    default: '0',
+  };
+};
+
+const OSS_CI_GROUPS = 12;
+const XPACK_CI_GROUPS = 13;
+
+const inputs = [
+  {
+    key: 'ftsr-override-count',
+    text: 'Override for all suites',
+    default: 0,
+    required: true,
+  },
+  {
+    key: 'ftsr-concurrency',
+    text: 'Max concurrency per step',
+    default: 20,
+    required: true,
+  },
+];
+
+for (let i = 1; i <= OSS_CI_GROUPS; i++) {
+  inputs.push(stepInput(`oss/cigroup/${i}`, `OSS CI Group ${i}`));
+}
+
+for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
+  inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
+}
+
+const pipeline = {
+  steps: [
+    {
+      input: 'Number of Runs',
+      fields: inputs,
+    },
+    {
+      wait: '~',
+    },
+    {
+      command: '.buildkite/pipelines/flaky_tests/runner.sh',
+      label: 'Create pipeline',
+    },
+  ],
+};
+
+console.log(JSON.stringify(pipeline, null, 2));

.buildkite/pipelines/flaky_tests/pipeline.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+node .buildkite/pipelines/flaky_tests/pipeline.js | buildkite-agent pipeline upload

.buildkite/pipelines/flaky_tests/runner.js

@@ -0,0 +1,82 @@
+const { execSync } = require('child_process');
+
+const keys = execSync('buildkite-agent meta-data keys')
+  .toString()
+  .split('\n')
+  .filter((k) => k.startsWith('ftsr-suite/'));
+
+const overrideCount = parseInt(
+  execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
+);
+
+const concurrency =
+  parseInt(execSync(`buildkite-agent meta-data get 'ftsr-concurrency'`).toString().trim()) || 20;
+
+const testSuites = [];
+for (const key of keys) {
+  if (!key) {
+    continue;
+  }
+
+  const value =
+    overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+
+  testSuites.push({
+    key: key.replace('ftsr-suite/', ''),
+    count: value === '' ? defaultCount : parseInt(value),
+  });
+}
+
+const steps = [];
+const pipeline = {
+  env: {
+    IGNORE_SHIP_CI_STATS_ERROR: 'true',
+  },
+  steps: steps,
+};
+
+steps.push({
+  command: '.buildkite/scripts/steps/build_kibana.sh',
+  label: 'Build Kibana Distribution and Plugins',
+  agents: { queue: 'c2-8' },
+  key: 'build',
+  if: "build.env('BUILD_ID_FOR_ARTIFACTS') == null || build.env('BUILD_ID_FOR_ARTIFACTS') == ''",
+});
+
+for (const testSuite of testSuites) {
+  const TEST_SUITE = testSuite.key;
+  const RUN_COUNT = testSuite.count;
+  const UUID = TEST_SUITE + process.env.UUID;
+
+  const JOB_PARTS = TEST_SUITE.split('/');
+  const IS_XPACK = JOB_PARTS[0] === 'xpack';
+  const CI_GROUP = JOB_PARTS.length > 2 ? JOB_PARTS[2] : '';
+
+  if (RUN_COUNT < 1) {
+    continue;
+  }
+
+  if (IS_XPACK) {
+    steps.push({
+      command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
+      label: `Default CI Group ${CI_GROUP}`,
+      agents: { queue: 'ci-group-6' },
+      depends_on: 'build',
+      parallelism: RUN_COUNT,
+      concurrency: concurrency,
+      concurrency_group: UUID,
+    });
+  } else {
+    steps.push({
+      command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/oss_cigroup.sh`,
+      label: `OSS CI Group ${CI_GROUP}`,
+      agents: { queue: 'ci-group-4d' },
+      depends_on: 'build',
+      parallelism: RUN_COUNT,
+      concurrency: concurrency,
+      concurrency_group: UUID,
+    });
+  }
+}
+
+console.log(JSON.stringify(pipeline, null, 2));

.buildkite/pipelines/flaky_tests/runner.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+UUID="$(cat /proc/sys/kernel/random/uuid)"
+export UUID
+
+node .buildkite/pipelines/flaky_tests/runner.js | buildkite-agent pipeline upload