Merge remote-tracking branch 'upstream/main' into kbn-146881-capabili…

…ties-second-attempt

.backportrc.json

@@ -3,6 +3,9 @@
   "repoName": "kibana",
   "targetBranchChoices": [
     "main",
+    "8.10",
+    "8.9",
+    "8.8",
     "8.7",
     "8.6",
     "8.5",
@@ -44,7 +47,7 @@
     "backport"
   ],
   "branchLabelMapping": {
-    "^v8.8.0$": "main",
+    "^v8.11.0$": "main",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/disabled_jest_configs.json

@@ -1,4 +1 @@
-[
-  "x-pack/plugins/watcher/jest.config.js",
-  "src/core/server/integration_tests/ui_settings/jest.integration.config.js"
-]
+[]

.buildkite/hooks/post-command

@@ -1,3 +1,7 @@
 #!/usr/bin/env bash
 
-.buildkite/scripts/lifecycle/post_command.sh
+if [[ "$BUILDKITE_AGENT_NAME" =~ ^bk-agent ]]; then
+  echo "Pipeline file triggered from outside the kibana executors, skipping post_command"
+else
+  .buildkite/scripts/lifecycle/post_command.sh
+fi

.buildkite/hooks/pre-command

@@ -1,3 +1,7 @@
 #!/usr/bin/env bash
 
-source .buildkite/scripts/lifecycle/pre_command.sh
+if [[ "$BUILDKITE_AGENT_NAME" =~ ^bk-agent ]]; then
+  echo "Pipeline file triggered from outside the kibana executors, skipping pre_command"
+else
+  source .buildkite/scripts/lifecycle/pre_command.sh
+fi

.buildkite/pipelines/artifacts_container_image.yml

@@ -1,6 +1,6 @@
 steps:
   - command: .buildkite/scripts/steps/artifacts/docker_image.sh
-    label: Build default container images
+    label: Build serverless container images
     agents:
       queue: n2-16-spot
     timeout_in_minutes: 60

.buildkite/pipelines/artifacts_trigger.yml

@@ -0,0 +1,6 @@
+steps:
+  - command: .buildkite/scripts/steps/artifacts/trigger.sh
+    label: Trigger artifacts build
+    agents:
+      queue: kibana-default
+    timeout_in_minutes: 10

.buildkite/pipelines/flaky_tests/groups.json

@@ -1,9 +1,17 @@
 {
   "groups": [
+    {
+      "key": "cypress/security_solution_investigations",
+      "name": "Security Solution Investigations - Cypress"
+    },
     {
       "key": "cypress/security_solution",
       "name": "Security Solution - Cypress"
     },
+    {
+      "key": "cypress/security_solution_explore",
+      "name": "Security Solution Explore - Cypress"
+    },
     {
       "key": "cypress/osquery_cypress",
       "name": "Osquery - Cypress"

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -153,18 +153,17 @@ for (const testSuite of testSuites) {
     continue;
   }
 
-  const keyParts = testSuite.key.split('/');
-  switch (keyParts[0]) {
+  const [category, suiteName] = testSuite.key.split('/');
+  switch (category) {
     case 'cypress':
-      const CYPRESS_SUITE = keyParts[1];
-      const group = groups.find((g) => g.key.includes(CYPRESS_SUITE));
+      const group = groups.find((g) => g.key === testSuite.key);
       if (!group) {
         throw new Error(
-          `Group configuration was not found in groups.json for the following cypress suite: {${CYPRESS_SUITE}}.`
+          `Group configuration was not found in groups.json for the following cypress suite: {${suiteName}}.`
         );
       }
       steps.push({
-        command: `.buildkite/scripts/steps/functional/${CYPRESS_SUITE}.sh`,
+        command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
         agents: { queue: 'n2-4-spot' },
         depends_on: 'build',
@@ -177,6 +176,9 @@ for (const testSuite of testSuites) {
           // by setting chunks vars to value 1, which means all test will run in one job
           CLI_NUMBER: 1,
           CLI_COUNT: 1,
+          // The security solution cypress tests don't recognize CLI_NUMBER and CLI_COUNT, they use `BUILDKITE_PARALLEL_JOB_COUNT` and `BUILDKITE_PARALLEL_JOB`, which cannot be overridden here.
+          // Use `RUN_ALL_TESTS` to make Security Solution Cypress tests run all tests instead of a subset.
+          RUN_ALL_TESTS: 'true',
         },
       });
       break;

.buildkite/pipelines/on_merge.yml

@@ -78,6 +78,50 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless.sh
+    label: 'Serverless Security Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
+    label: 'Serverless Explore - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
+    label: 'Serverless Investigations - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: '.buildkite/scripts/steps/functional/on_merge_unsupported_ftrs.sh'
+    label: Trigger unsupported ftr tests
+    timeout_in_minutes: 10
+    depends_on:
+      - build
+    agents:
+      queue: 'kibana-default'
+
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:
@@ -123,7 +167,7 @@ steps:
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'
     agents:
-      queue: n2-4-spot
+      queue: n2-8-spot
     key: storybooks
     timeout_in_minutes: 60
     retry:

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -0,0 +1,125 @@
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      queue: n2-16-spot
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/functional/apm_cypress.sh
+    label: 'APM Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
+    label: 'Profling Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution.sh
+    label: 'Security Solution Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 14
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
+    label: 'Investigations - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_explore.sh
+    label: 'Explore - Security Solution Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows.sh
+    label: 'Defend Workflows Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/osquery_cypress.sh
+    label: 'Osquery Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 50
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/synthetics_plugin.sh
+    label: 'Synthetics @elastic/synthetics Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    artifact_paths:
+      - 'x-pack/plugins/synthetics/e2e/.journeys/**/*'
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pipeline.kibana-serverless-release.yaml

@@ -0,0 +1,15 @@
+steps:
+  - label: ":releasethekraken: Release kibana"
+    # https://regex101.com/r/tY52jo/1
+    if: build.tag =~ /^deploy@\d+\$/
+    trigger: gpctl-promote-with-e2e-tests
+    build:
+      env:
+        SERVICE_COMMIT_HASH: "${BUILDKITE_COMMIT:0:12}"
+        REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/config.yaml
+        SERVICE: kibana-controller
+        NAMESPACE: kibana-ci
+        IMAGE_NAME: kibana-serverless
+
+notify:
+  - slack: "#kibana-mission-control"

.buildkite/pipelines/pull_request/base.yml

@@ -56,6 +56,54 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless.sh
+    label: 'Serverless Security Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  # status_exception: Native role management is not enabled in this Elasticsearch instance
+  # - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh
+  #   label: 'Serverless Security Defend Workflows Cypress Tests'
+  #   agents:
+  #     queue: n2-4-spot
+  #   depends_on: build
+  #   timeout_in_minutes: 40
+  #   retry:
+  #     automatic:
+  #       - exit_status: '*'
+  #         limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
+    label: 'Serverless Security Investigations Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
+    label: 'Serverless Security Explore Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:

.buildkite/pipelines/pull_request/defend_workflows.yml

@@ -4,12 +4,21 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows_vagrant.sh
+    label: 'Defend Workflows Endpoint Cypress Tests'
+    agents:
+      queue: n2-4-virt
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
     retry:
       automatic:
-        - exit_status: '-1'
-          limit: 3
         - exit_status: '*'
           limit: 1
-    artifact_paths:
-      - "target/kibana-security-solution/**/*"