merge conflicts

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "elastic/kibana-buildkite-library"
+    "kibana-buildkite-library": "git+https://[email protected]/elastic/kibana-buildkite-library#0f95579ace8100de9f1ad4cc16976b9ec6d5841e"
   }
 }

.buildkite/pipelines/es_snapshots/verify.yml

@@ -53,13 +53,12 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/jest_integration.sh
-    label: 'Jest Integration Tests'
-    parallelism: 3
+  - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
+    label: 'Pick Jest Config Run Order'
     agents:
-      queue: n2-4
-    timeout_in_minutes: 120
-    key: jest-integration
+      queue: kibana-default
+    env:
+      FILTER_JEST_CONFIG_TYPE: integration
     retry:
       automatic:
         - exit_status: '*'

.buildkite/pipelines/on_merge.yml

@@ -157,29 +157,14 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/jest.sh
-    label: 'Jest Tests'
-    parallelism: 8
+  - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
+    label: 'Pick Jest Config Run Order'
     agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 90
-    key: jest
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/test/jest_integration.sh
-    label: 'Jest Integration Tests'
-    parallelism: 3
-    agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 120
-    key: jest-integration
+      queue: kibana-default
     retry:
       automatic:
-        - exit_status: '-1'
-          limit: 3
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'

.buildkite/pipelines/pull_request/base.yml

@@ -123,29 +123,14 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/jest.sh
-    label: 'Jest Tests'
-    parallelism: 8
+  - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
+    label: 'Pick Jest Config Run Order'
     agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 90
-    key: jest
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/test/jest_integration.sh
-    label: 'Jest Integration Tests'
-    parallelism: 3
-    agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 120
-    key: jest-integration
+      queue: kibana-default
     retry:
       automatic:
-        - exit_status: '-1'
-          limit: 3
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'

.buildkite/pipelines/pull_request/response_ops.yml

@@ -1,6 +1,6 @@
 steps:
-  - command: .buildkite/scripts/steps/functional/response_ops_cases.sh
-    label: 'Cases Cypress Tests on Security Solution'
+  - command: .buildkite/scripts/steps/functional/response_ops.sh
+    label: 'Rules, Alerts and Exceptions ResponseOps Cypress Tests on Security Solution'
     agents:
       queue: ci-group-6
     depends_on: build

.buildkite/pipelines/pull_request/response_ops_cases.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/response_ops_cases.sh
+    label: 'Cases Cypress Tests on Security Solution'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/lifecycle/pre_command.sh

@@ -9,21 +9,7 @@ export BUILDKITE_TOKEN
 
 echo '--- Install buildkite dependencies'
 cd '.buildkite'
-
-# If this yarn install is terminated early, e.g. if the build is cancelled in buildkite,
-# A node module could end up in a bad state that can cause all future builds to fail
-# So, let's cache clean and try again to make sure that's not what caused the error
-install_deps() {
-  yarn install --production --pure-lockfile
-  EXIT=$?
-  if [[ "$EXIT" != "0" ]]; then
-    yarn cache clean
-  fi
-  return $EXIT
-}
-
-retry 5 15 install_deps
-
+retry 5 15 npm ci
 cd ..
 
 echo '--- Agent Debug/SSH Info'

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -64,25 +64,39 @@ const uploadPipeline = (pipelineContent) => {
 
     if (
       (await doAnyChangesMatch([
-        /^x-pack\/plugins\/security_solution/,
         /^x-pack\/plugins\/lists/,
+        /^x-pack\/plugins\/security_solution/,
         /^x-pack\/plugins\/timelines/,
-        /^x-pack\/test\/security_solution_cypress/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
+        /^x-pack\/test\/security_solution_cypress/,
       ])) ||
       process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
     }
 
     if (
-      (await doAnyChangesMatch([/^x-pack\/plugins\/cases/])) ||
+      (await doAnyChangesMatch([
+        /^src\/plugins\/data/,
+        /^x-pack\/plugins\/actions/,
+        /^x-pack\/plugins\/alerting/,
+        /^x-pack\/plugins\/event_log/,
+        /^x-pack\/plugins\/rule_registry/,
+        /^x-pack\/plugins\/task_manager/,
+      ])) ||
       process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([/^x-pack\/plugins\/cases/])) ||
+      process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops_cases.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
       process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')

.buildkite/scripts/steps/checks.sh

@@ -8,6 +8,7 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/commit/commit.sh
 .buildkite/scripts/steps/checks/bazel_packages.sh
 .buildkite/scripts/steps/checks/telemetry.sh
+.buildkite/scripts/steps/checks/validate_ci_groups.sh
 .buildkite/scripts/steps/checks/ts_projects.sh
 .buildkite/scripts/steps/checks/jest_configs.sh
 .buildkite/scripts/steps/checks/doc_api_changes.sh

.buildkite/scripts/steps/checks/validate_ci_groups.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Ensure that all tests are in a CI Group
+checks-reporter-with-killswitch "Ensure that all tests are in a CI Group" \
+  node scripts/ensure_all_tests_in_ci_group

.buildkite/scripts/steps/functional/response_ops.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+
+export JOB=kibana-security-solution-chrome
+
+echo "--- Response Ops Cypress Tests on Security Solution"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Response Ops Cypress Tests on Security Solution" \
+ node scripts/functional_tests \
+   --debug --bail \
+   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+   --config test/security_solution_cypress/response_ops_cli_config.ts

.buildkite/scripts/steps/test/jest_env.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# keys used to associate test group data in ci-stats with Jest execution order
+export TEST_GROUP_TYPE_UNIT="Jest Unit Tests"
+export TEST_GROUP_TYPE_INTEGRATION="Jest Integration Tests"

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -2,36 +2,38 @@
 
 set -uo pipefail
 
-JOB=$BUILDKITE_PARALLEL_JOB
-JOB_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT
+source .buildkite/scripts/steps/test/jest_env.sh
 
-# a jest failure will result in the script returning an exit code of 10
+export JOB=$BUILDKITE_PARALLEL_JOB
 
-i=0
+# a jest failure will result in the script returning an exit code of 10
 exitCode=0
 
-# run unit tests in parallel
 if [[ "$1" == 'jest.config.js' ]]; then
+  # run unit tests in parallel
   parallelism="-w2"
+  TEST_TYPE="unit"
 else
+  # run integration tests in-band
   parallelism="--runInBand"
+  TEST_TYPE="integration"
 fi
 
+export TEST_TYPE
+echo "--- downloading integration test run order"
+buildkite-agent artifact download jest_run_order.json .
+configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
+
 while read -r config; do
-  if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then
-    echo "--- $ node scripts/jest --config $config"
-    node --max-old-space-size=14336 ./scripts/jest --config="$config" "$parallelism" --coverage=false --passWithNoTests
-    lastCode=$?
-
-    if [ $lastCode -ne 0 ]; then
-      exitCode=10
-      echo "Jest exited with code $lastCode"
-      echo "^^^ +++"
-    fi
+  echo "--- $ node scripts/jest --config $config"
+  NODE_OPTIONS="--max-old-space-size=14336" node ./scripts/jest --config="$config" "$parallelism" --coverage=false --passWithNoTests
+  lastCode=$?
+
+  if [ $lastCode -ne 0 ]; then
+    exitCode=10
+    echo "Jest exited with code $lastCode"
+    echo "^^^ +++"
   fi
-
-  ((i=i+1))
-# uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode
-done <<< "$(find src x-pack packages -name "$1" -not -path "*/__fixtures__/*" | sort)"
+done <<< "$configs"
 
 exit $exitCode

.buildkite/scripts/steps/test/pick_jest_config_run_order.js

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const { CiStats } = require('kibana-buildkite-library');
+
+(async () => {
+  try {
+    await CiStats.pickTestGroupRunOrder();
+  } catch (ex) {
+    console.error('CI Stats Error', ex.message);
+    if (ex.response) {
+      console.error('HTTP Error Response Status', ex.response.status);
+      console.error('HTTP Error Response Body', ex.response.data);
+    }
+    process.exit(1);
+  }
+})();

.buildkite/scripts/steps/test/pick_jest_config_run_order.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/test/jest_env.sh
+
+echo '--- Pick Jest Config Run Order'
+node "$(dirname "${0}")/pick_jest_config_run_order.js"