Merge branch 'main' into cypress_spike

.buildkite/ftr_configs.yml

@@ -232,8 +232,6 @@ enabled:
   - x-pack/test/cloud_security_posture_api/config.ts
   - x-pack/test/dataset_quality_api_integration/basic/config.ts
   - x-pack/test/detection_engine_api_integration/basic/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group1/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts
   - x-pack/test/disable_ems/config.ts
   - x-pack/test/encrypted_saved_objects_api_integration/config.ts
   - x-pack/test/examples/config.ts
@@ -499,5 +497,17 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/ess.config.ts
-
-
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_delete/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_delete/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_update/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_update/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_patch/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_patch/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_import_export/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_import_export/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_read/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_read/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/configs/serverless.config.ts

.buildkite/pipelines/on_merge.yml

@@ -271,30 +271,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/defend_workflows.sh
-    label: 'Defend Workflows Cypress Tests'
-    agents:
-      queue: n2-4-virt
-    depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 10
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
-    label: 'Defend Workflows Cypress Tests on Serverless'
-    agents:
-      queue: n2-4-virt
-    depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 6
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
     label: 'Threat Intelligence Cypress Tests'
     agents:

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -61,3 +61,27 @@ steps:
           limit: 3
         - exit_status: '*'
           limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows.sh
+    label: 'Defend Workflows Cypress Tests'
+    agents:
+      queue: n2-4-virt
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 10
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
+    label: 'Defend Workflows Cypress Tests on Serverless'
+    agents:
+      queue: n2-4-virt
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/base.yml

@@ -249,30 +249,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/defend_workflows.sh
-    label: 'Defend Workflows Cypress Tests'
-    agents:
-      queue: n2-4-virt
-    depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 16
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
-    label: 'Defend Workflows Cypress Tests on Serverless'
-    agents:
-      queue: n2-4-virt
-    depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 6
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
     label: 'Threat Intelligence Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/defend_workflows.yml

@@ -0,0 +1,24 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/defend_workflows.sh
+    label: 'Defend Workflows Cypress Tests'
+    agents:
+      queue: n2-4-virt
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 16
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
+    label: 'Defend Workflows Cypress Tests on Serverless'
+    agents:
+      queue: n2-4-virt
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml

@@ -10,6 +10,11 @@
 #
 # Docs: https://docs.elastic.dev/serverless/qualitygates
 
+agents:
+  cpu: 2
+  ephemeralStorage: "20G"
+  memory: "8G"
+
 env:
   TEAM_CHANNEL: "#kibana-mission-control"
   ENVIRONMENT: ${ENVIRONMENT?}

.buildkite/pipelines/security_solution/api_integration.yml

@@ -15,7 +15,7 @@ steps:
     key: exception_operators_date_numeric_types:qa:serverless
     agents:
       queue: n2-4-spot
-    timeout_in_minutes: 120  
+    timeout_in_minutes: 120
     retry:
       automatic:
         - exit_status: '*'
@@ -31,7 +31,7 @@ steps:
       automatic:
         - exit_status: '*'
           limit: 2
-  
+
   - label: Running exception_operators_ips:qa:serverless
     command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_operators_ips:qa:serverless
     key: exception_operators_ips:qa:serverless
@@ -53,7 +53,6 @@ steps:
       automatic:
         - exit_status: '1'
           limit: 2
-
 
   - label: Running exception_operators_text:qa:serverless
     command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_operators_text:qa:serverless
@@ -76,7 +75,7 @@ steps:
       automatic:
         - exit_status: '1'
           limit: 2
-          
+
   - label: Running actions:qa:serverless
     command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh actions:qa:serverless
     key: actions:qa:serverless
@@ -97,7 +96,7 @@ steps:
     retry:
       automatic:
         - exit_status: '1'
-          limit: 2 
+          limit: 2
 
   - label: Running prebuilt_rules_management:qa:serverless
     command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh prebuilt_rules_management:qa:serverless
@@ -119,7 +118,7 @@ steps:
     retry:
       automatic:
         - exit_status: '1'
-          limit: 2 
+          limit: 2
 
   - label: Running prebuilt_rules_large_prebuilt_rules_package:qa:serverless
     command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh prebuilt_rules_large_prebuilt_rules_package:qa:serverless
@@ -130,7 +129,7 @@ steps:
     retry:
       automatic:
         - exit_status: '1'
-          limit: 2  
+          limit: 2
 
   - label: Running prebuilt_rules_update_prebuilt_rules_package:qa:serverless
     command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh prebuilt_rules_update_prebuilt_rules_package:qa:serverless
@@ -152,8 +151,8 @@ steps:
     retry:
       automatic:
         - exit_status: '1'
-          limit: 2 
-        
+          limit: 2
+
   - label: Running user_roles:qa:serverless
     command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh user_roles:qa:serverless
     key: user_roles:qa:serverless
@@ -163,7 +162,7 @@ steps:
     retry:
       automatic:
         - exit_status: '1'
-          limit: 2 
+          limit: 2
 
   - label: Running telemetry:qa:serverless
     command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh telemetry:qa:serverless
@@ -175,4 +174,80 @@ steps:
       automatic:
         - exit_status: '1'
           limit: 2
-
+
+  - label: Running rule_delete:qa:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh rule_delete:qa:serverless
+    key: rule_delete:qa:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running rule_update:qa:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh rule_update:qa:serverless
+    key: rule_update:qa:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running rule_patch:qa:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh rule_patch:qa:serverless
+    key: rule_patch:qa:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running rule_import_export:qa:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh rule_import_export:qa:serverless
+    key: rule_import_export:qa:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running rule_management:qa:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh rule_management:qa:serverless
+    key: rule_management:qa:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running rule_bulk_actions:qa:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh rule_bulk_actions:qa:serverless
+    key: rule_bulk_actions:qa:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running rule_read:qa:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh rule_read:qa:serverless
+    key: rule_read:qa:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -179,6 +179,19 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/cypress_burn.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^packages\/kbn-securitysolution-.*/,
+        /^x-pack\/plugins\/security_solution/,
+        /^x-pack\/test\/defend_workflows_cypress/,
+        /^x-pack\/test\/security_solution_cypress/,
+        /^fleet_packages\.json/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/defend_workflows.yml'));
+    }
+
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
     // remove duplicated steps

.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh

@@ -16,6 +16,9 @@ export JOB=kibana-security-solution-chrome
 
 buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
 
+mkdir .ftr
+retry 5 5 vault kv get -format=json -field=data secret/kibana-issues/dev/security-quality-gate/role-users > .ftr/role_users.json
+
 cd x-pack/test/security_solution_cypress
 set +e
 

.buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.ts

@@ -13,6 +13,7 @@ import {
   getSelectedCommitHash,
   getCommitByHash,
   makeCommitInfoHtml,
+  getRecentCommits,
 } from './info_sections/commit_info';
 import {
   getArtifactBuild,
@@ -45,7 +46,8 @@ async function main() {
   const buildkiteBuild = await getOnMergePRBuild(selectedSha);
   const nextBuildContainingCommit = await getQAFBuildContainingCommit(
     selectedSha,
-    selectedCommitInfo.date!
+    selectedCommitInfo.date!,
+    await getRecentCommits(50)
   );
   const artifactBuild = await getArtifactBuild(selectedSha);
   addBuildkiteInfoSection(

.buildkite/scripts/serverless/create_deploy_tag/create_deploy_tag.sh

@@ -11,9 +11,12 @@ fi
 
 echo "--- Creating deploy tag $DEPLOY_TAG at $KIBANA_COMMIT_SHA"
 
+echo "Fetching user identity from GitHub..."
+IDENTITY_JSON=$(ts-node .buildkite/scripts/serverless/create_deploy_tag/get_github_identity.ts)
+
 # Set git identity to whomever triggered the buildkite job
-git config user.email "$BUILDKITE_BUILD_CREATOR_EMAIL"
-git config user.name "$BUILDKITE_BUILD_CREATOR"
+git config user.email "${BUILDKITE_BUILD_CREATOR_EMAIL:-$(echo ${IDENTITY_JSON} | jq .email)}"
+git config user.name "${BUILDKITE_BUILD_CREATOR:-$(echo ${IDENTITY_JSON} | jq .name)}"
 
 # Create a tag for the deploy
 git tag -a "$DEPLOY_TAG" "$KIBANA_COMMIT_SHA" \