Merge branch '7.16' into ua/show_on_prem_step_when_in_cloud_and_missi…

…ng_foundsnapshots

.buildkite/pipelines/bazel_cache.yml

@@ -0,0 +1,33 @@
+steps:
+  - label: ':pipeline: Create pipeline with priority'
+    concurrency_group: bazel_macos
+    concurrency: 1
+    concurrency_method: eager
+    # We have to use a dynamic pipeline to set PRIORITY at runtime based on the branch
+    # We want the main branch to be prioritized higher than other branches
+    # The other option would be to have a slightly different version of this yaml live on different branches
+    # But this makes backports easier / everything cleaner
+    command: |
+      if [[ "$${BUILDKITE_BRANCH}" == "$${BUILDKITE_PIPELINE_DEFAULT_BRANCH}" ]]; then
+        export PRIORITY=1
+      else
+        export PRIORITY=0
+      fi
+      buildkite-agent pipeline upload <<YAML
+      steps:
+      - command: .buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh
+        label: Bootstrap (MacOS x86)
+        priority: $${PRIORITY}
+        agents:
+          queue: macos-x86
+        timeout_in_minutes: 60
+        concurrency_group: bazel_macos
+        concurrency: 1
+        concurrency_method: eager
+      - command: .buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh
+        label: Bootstrap (MacOS ARM)
+        priority: $${PRIORITY}
+        agents:
+          queue: macos-arm
+        timeout_in_minutes: 60
+      YAML

.buildkite/pipelines/flaky_tests/groups.json

@@ -0,0 +1,34 @@
+{
+  "groups": [
+    {
+      "key": "oss/cigroup",
+      "name": "OSS CI Group",
+      "ciGroups": 12
+    },
+    {
+      "key": "oss/firefox",
+      "name": "OSS Firefox"
+    },
+    {
+      "key": "oss/accessibility",
+      "name": "OSS Accessibility"
+    },
+    {
+      "key": "xpack/cigroup",
+      "name": "Default CI Group",
+      "ciGroups": 27
+    },
+    {
+      "key": "xpack/cigroup/Docker",
+      "name": "Default CI Group Docker"
+    },
+    {
+      "key": "xpack/firefox",
+      "name": "Default Firefox"
+    },
+    {
+      "key": "xpack/accessibility",
+      "name": "Default Accessibility"
+    }
+  ]
+}

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -1,3 +1,7 @@
+const groups = /** @type {Array<{key: string, name: string, ciGroups: number }>} */(
+  require('./groups.json').groups
+)
+
 const stepInput = (key, nameOfSuite) => {
   return {
     key: `ftsr-suite/${key}`,
@@ -7,38 +11,31 @@ const stepInput = (key, nameOfSuite) => {
   };
 };
 
-const OSS_CI_GROUPS = 12;
-const XPACK_CI_GROUPS = 27;
-
 const inputs = [
   {
     key: 'ftsr-override-count',
     text: 'Override for all suites',
-    default: 0,
+    default: '0',
     required: true,
   },
 ];
 
-for (let i = 1; i <= OSS_CI_GROUPS; i++) {
-  inputs.push(stepInput(`oss/cigroup/${i}`, `OSS CI Group ${i}`));
+for (const group of groups) {
+  if (!group.ciGroups) {
+    inputs.push(stepInput(group.key, group.name))
+  } else {
+    for (let i = 1; i <= group.ciGroups; i++) {
+      inputs.push(stepInput(`${group.key}/${i}`, `${group.name} ${i}`))
+    }
+  }
 }
 
-inputs.push(stepInput(`oss/firefox`, 'OSS Firefox'));
-inputs.push(stepInput(`oss/accessibility`, 'OSS Accessibility'));
-
-for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
-  inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
-}
-
-inputs.push(stepInput(`xpack/cigroup/Docker`, 'Default CI Group Docker'));
-inputs.push(stepInput(`xpack/firefox`, 'Default Firefox'));
-inputs.push(stepInput(`xpack/accessibility`, 'Default Accessibility'));
-
 const pipeline = {
   steps: [
     {
       input: 'Number of Runs - Click Me',
       fields: inputs,
+      if: `build.env('KIBANA_FLAKY_TEST_RUNNER_CONFIG') == null`
     },
     {
       wait: '~',

.buildkite/pipelines/flaky_tests/runner.js

@@ -1,37 +1,85 @@
 const { execSync } = require('child_process');
 
-const keys = execSync('buildkite-agent meta-data keys')
-  .toString()
-  .split('\n')
-  .filter((k) => k.startsWith('ftsr-suite/'));
-
-const overrideCount = parseInt(
-  execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
-);
-
 const concurrency = 25;
+const defaultCount = concurrency * 2;
 const initialJobs = 3;
 
-let totalJobs = initialJobs;
+function getTestSuitesFromMetadata() {
+  const keys = execSync('buildkite-agent meta-data keys')
+    .toString()
+    .split('\n')
+    .filter((k) => k.startsWith('ftsr-suite/'));
 
-const testSuites = [];
-for (const key of keys) {
-  if (!key) {
-    continue;
+  const overrideCount = execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim();
+
+  const testSuites = [];
+  for (const key of keys) {
+    if (!key) {
+      continue;
+    }
+
+    const value =
+      overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+
+    const count = value === '' ? defaultCount : parseInt(value);
+    totalJobs += count;
+
+    testSuites.push({
+      key: key.replace('ftsr-suite/', ''),
+      count: count,
+    });
   }
 
-  const value =
-    overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+  return testSuites
+}
 
-  const count = value === '' ? defaultCount : parseInt(value);
-  totalJobs += count;
+function getTestSuitesFromJson(json) {
+  const fail = (errorMsg) => {
+    console.error('+++ Invalid test config provided')
+    console.error(`${errorMsg}: ${json}`);
+    process.exit(1);
+  }
 
-  testSuites.push({
-    key: key.replace('ftsr-suite/', ''),
-    count: count,
-  });
+  let parsed;
+  try {
+    parsed = JSON.parse(json)
+  } catch (error) {
+    fail(`JSON test config did not parse correctly`)
+  }
+
+  if (!Array.isArray(parsed)) {
+    fail(`JSON test config must be an array`)
+  }
+
+  /** @type {Array<{ key: string, count: number }>} */
+  const testSuites = []
+  for (const item of parsed) {
+    if (typeof item !== 'object' || item === null) {
+      fail(`testSuites must be objects`)
+    }
+    const key = item.key
+    if (typeof key !== 'string') {
+      fail(`testSuite.key must be a string`)
+    }
+    const count = item.count;
+    if (typeof count !== 'number') {
+      fail(`testSuite.count must be a number`)
+    }
+    testSuites.push({
+      key,
+      count,
+    })
+  }
+
+  return testSuites
 }
 
+const testSuites = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG
+  ? getTestSuitesFromJson(process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG)
+  : getTestSuitesFromMetadata();
+
+let totalJobs = testSuites.reduce((acc, t) => acc + t.count, initialJobs);
+
 if (totalJobs > 500) {
   console.error('+++ Too many tests');
   console.error(

.buildkite/scripts/build_kibana.sh

@@ -7,6 +7,8 @@ export KBN_NP_PLUGINS_BUILT=true
 echo "--- Build Kibana Distribution"
 if [[ "${GITHUB_PR_LABELS:-}" == *"ci:build-all-platforms"* ]]; then
   node scripts/build --all-platforms --skip-os-packages
+elif [[ "${GITHUB_PR_LABELS:-}" == *"ci:build-os-packages"* ]]; then
+  node scripts/build --all-platforms
 else
   node scripts/build
 fi

.buildkite/scripts/common/env.sh

@@ -73,8 +73,14 @@ export GIT_BRANCH="${BUILDKITE_BRANCH:-}"
 export FLEET_PACKAGE_REGISTRY_PORT=6104
 export TEST_CORS_SERVER_PORT=6105
 
-export DETECT_CHROMEDRIVER_VERSION=true
-export CHROMEDRIVER_FORCE_DOWNLOAD=true
+# Mac agents currently don't have Chrome
+if [[ "$(which google-chrome-stable)" || "$(which google-chrome)" ]]; then
+  echo "Chrome detected, setting DETECT_CHROMEDRIVER_VERSION=true"
+  export DETECT_CHROMEDRIVER_VERSION=true
+  export CHROMEDRIVER_FORCE_DOWNLOAD=true
+else
+  echo "Chrome not detected, installing default chromedriver binary for the package version"
+fi
 
 export GCS_UPLOAD_PREFIX=FAKE_UPLOAD_PREFIX # TODO remove the need for this
 

.buildkite/scripts/common/setup_bazel.sh

@@ -2,9 +2,6 @@
 
 source .buildkite/scripts/common/util.sh
 
-KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
-export KIBANA_BUILDBUDDY_CI_API_KEY
-
 echo "[bazel] writing .bazelrc"
 cat <<EOF > $KIBANA_DIR/.bazelrc
   # Generated by .buildkite/scripts/common/setup_bazel.sh

.buildkite/scripts/common/setup_node.sh

@@ -8,28 +8,53 @@ export NODE_DIR="$CACHE_DIR/node/$NODE_VERSION"
 export NODE_BIN_DIR="$NODE_DIR/bin"
 export YARN_OFFLINE_CACHE="$CACHE_DIR/yarn-offline-cache"
 
-if [[ ! -d "$NODE_DIR" ]]; then
-  hostArch="$(command uname -m)"
-  case "${hostArch}" in
-    x86_64 | amd64) nodeArch="x64" ;;
-    aarch64) nodeArch="arm64" ;;
-    *) nodeArch="${hostArch}" ;;
-  esac
+## Install node for whatever the current os/arch are
+hostArch="$(command uname -m)"
+case "${hostArch}" in
+  x86_64 | amd64) nodeArch="x64" ;;
+  aarch64) nodeArch="arm64" ;;
+  *) nodeArch="${hostArch}" ;;
+esac
+classifier="$nodeArch.tar.gz"
 
-  nodeUrl="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$nodeArch.tar.gz"
+UNAME=$(uname)
+OS="linux"
+if [[ "$UNAME" = *"MINGW64_NT"* ]]; then
+  OS="win"
+  NODE_BIN_DIR="$HOME/node"
+  classifier="x64.zip"
+elif [[ "$UNAME" == "Darwin" ]]; then
+  OS="darwin"
+fi
+echo " -- Running on OS: $OS"
 
-  echo "node.js v$NODE_VERSION not found at $NODE_DIR, downloading from $nodeUrl"
+nodeUrl="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v$NODE_VERSION/node-v$NODE_VERSION-${OS}-${classifier}"
 
-  mkdir -p "$NODE_DIR"
-  curl --silent -L "$nodeUrl" | tar -xz -C "$NODE_DIR" --strip-components=1
+echo " -- node: version=v${NODE_VERSION} dir=$NODE_DIR"
+
+echo " -- setting up node.js"
+if [ -x "$NODE_BIN_DIR/node" ] && [ "$("$NODE_BIN_DIR/node" --version)" == "v$NODE_VERSION" ]; then
+  echo " -- reusing node.js install"
 else
-  echo "node.js v$NODE_VERSION already installed to $NODE_DIR, re-using"
-  ls -alh "$NODE_BIN_DIR"
+  if [ -d "$NODE_DIR" ]; then
+    echo " -- clearing previous node.js install"
+    rm -rf "$NODE_DIR"
+  fi
+
+  echo " -- downloading node.js from $nodeUrl"
+  mkdir -p "$NODE_DIR"
+  if [[ "$OS" == "win" ]]; then
+    nodePkg="$NODE_DIR/${nodeUrl##*/}"
+    curl --silent -L -o "$nodePkg" "$nodeUrl"
+    unzip -qo "$nodePkg" -d "$NODE_DIR"
+    mv "${nodePkg%.*}" "$NODE_BIN_DIR"
+  else
+    curl --silent -L "$nodeUrl" | tar -xz -C "$NODE_DIR" --strip-components=1
+  fi
 fi
 
 export PATH="$NODE_BIN_DIR:$PATH"
 
-
 echo "--- Setup Yarn"
 
 YARN_VERSION=$(node -e "console.log(String(require('./package.json').engines.yarn || '').replace(/^[^\d]+/,''))")

.buildkite/scripts/lifecycle/pre_command.sh

@@ -84,6 +84,9 @@ export KIBANA_CI_REPORTER_KEY
   export TEST_FAILURES_ES_PASSWORD
 }
 
+KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
+export KIBANA_BUILDBUDDY_CI_API_KEY
+
 # By default, all steps should set up these things to get a full environment before running
 # It can be skipped for pipeline upload steps though, to make job start time a little faster
 if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then

.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euo pipefail
+
+export BAZEL_CACHE_MODE=read-write
+export DISABLE_BOOTSTRAP_VALIDATION=true
+
+# Since our Mac agents are currently static,
+# use a temporary HOME directory that gets cleaned out between builds
+TMP_HOME="$WORKSPACE/tmp_home"
+rm -rf "$TMP_HOME"
+export HOME="$TMP_HOME"
+
+.buildkite/scripts/bootstrap.sh

.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh

@@ -2,6 +2,7 @@
 
 set -euo pipefail
 
+export BAZEL_CACHE_MODE=read-write # Populate bazel remote cache for linux
 export BUILD_TS_REFS_CACHE_ENABLE=true
 export BUILD_TS_REFS_CACHE_CAPTURE=true
 export DISABLE_BOOTSTRAP_VALIDATION=true