[Detection Engine][FTR] Audit detection engine rule execution FTRs

.buildkite/ftr_configs.yml

@@ -489,6 +489,8 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/basic_license_essentials_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/trial_license_complete_tier/configs/ess.config.ts

.buildkite/pipelines/security_solution/api_integration.yml

@@ -133,7 +133,7 @@ steps:
         timeout_in_minutes: 120
         retry:
           automatic:
-            - exit_status: "1"
+            - exit_status: '1'
               limit: 2
 
       - label: Running rule_execution_logic:qa:serverless
@@ -147,6 +147,17 @@ steps:
             - exit_status: '1'
               limit: 2
 
+      - label: Running rule_execution_logic:essentials:qa:serverless
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_execution_logic:essentials:qa:serverless
+        key: rule_execution_logic:essentials:qa:serverless
+        agents:
+          queue: n2-4-spot
+        timeout_in_minutes: 120
+        retry:
+          automatic:
+            - exit_status: '1'
+              limit: 2
+
       - label: Running rule_patch:qa:serverless
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_patch:qa:serverless
         key: rule_patch:qa:serverless
@@ -290,7 +301,6 @@ steps:
             - exit_status: '1'
               limit: 2
 
-
       - label: Running rule_management:qa:serverless
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh rule_management:qa:serverless
         key: rule_management:qa:serverless

x-pack/test/security_solution_api_integration/package.json

@@ -134,6 +134,12 @@
     "rule_execution_logic:server:ess": "npm run initialize-server:de rule_execution_logic ess",
     "rule_execution_logic:runner:ess": "npm run run-tests:de rule_execution_logic ess essEnv",
 
+    "rule_execution_logic:essentials:server:serverless": "npm run initialize-server:de:basic_essentials rule_execution_logic serverless",
+    "rule_execution_logic:essentials:runner:serverless": "npm run run-tests:de:basic_essentials rule_execution_logic serverless serverlessEnv",
+    "rule_execution_logic:essentials:qa:serverless": "npm run run-tests:de:basic_essentials rule_execution_logic serverless qaEnv",
+    "rule_execution_logic:basic:server:ess": "npm run initialize-server:de:basic_essentials rule_execution_logic ess",
+    "rule_execution_logic:basic:runner:ess": "npm run run-tests:de:basic_essentials rule_execution_logic ess essEnv",
+
     "rule_creation:server:serverless": "npm run initialize-server:rm rule_creation serverless",
     "rule_creation:runner:serverless": "npm run run-tests:rm rule_creation serverless serverlessEnv",
     "rule_creation:qa:serverless": "npm run run-tests:rm rule_creation serverless qaEnv",

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/alert_status/alert_status.ts

@@ -43,11 +43,11 @@ export default ({ getService }: FtrProviderContext) => {
   const dataPathBuilder = new EsArchivePathBuilder(isServerless);
   const path = dataPathBuilder.getPath('auditbeat/hosts');
 
-  // Failing: See https://github.com/elastic/kibana/issues/179704
-  describe.skip('@ess @serverless change alert status endpoints', () => {
+  describe('@ess @serverless change alert status endpoints', () => {
     describe('validation checks', () => {
       describe('update by ids', () => {
-        it('should not give errors when querying and the alerts index does not exist yet', async () => {
+        // Failing: See https://github.com/elastic/kibana/issues/179693
+        it.skip('should not give errors when querying and the alerts index does not exist yet', async () => {
           const { body } = await supertest
             .post(DETECTION_ENGINE_SIGNALS_STATUS_URL)
             .set('kbn-xsrf', 'true')
@@ -60,7 +60,8 @@ export default ({ getService }: FtrProviderContext) => {
           expect(body).to.eql(getAlertUpdateByQueryEmptyResponse());
         });
 
-        it('should not give errors when querying and the alerts index does exist and is empty', async () => {
+        // Failing: See https://github.com/elastic/kibana/issues/179691
+        it.skip('should not give errors when querying and the alerts index does exist and is empty', async () => {
           await createAlertsIndex(supertest, log);
           const { body } = await supertest
             .post(DETECTION_ENGINE_SIGNALS_STATUS_URL)
@@ -78,7 +79,8 @@ export default ({ getService }: FtrProviderContext) => {
       });
 
       describe('update by query', () => {
-        it('should not give errors when querying and the alerts index does not exist yet', async () => {
+        // Failing: See https://github.com/elastic/kibana/issues/179681
+        it.skip('should not give errors when querying and the alerts index does not exist yet', async () => {
           const { body } = await supertest
             .post(DETECTION_ENGINE_SIGNALS_STATUS_URL)
             .set('kbn-xsrf', 'true')
@@ -91,7 +93,8 @@ export default ({ getService }: FtrProviderContext) => {
           expect(body).to.eql(getAlertUpdateByQueryEmptyResponse());
         });
 
-        it('should not give errors when querying and the alerts index does exist and is empty', async () => {
+        // Failing: See https://github.com/elastic/kibana/issues/179704
+        it.skip('should not give errors when querying and the alerts index does exist and is empty', async () => {
           await createAlertsIndex(supertest, log);
           const { body } = await supertest
             .post(DETECTION_ENGINE_SIGNALS_STATUS_URL)

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/configs/ess.config.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(
+    require.resolve('../../../../../../config/ess/config.base.basic')
+  );
+
+  return {
+    ...functionalConfig.getAll(),
+    testFiles: [require.resolve('..')],
+    junit: {
+      reportName:
+        'Detection Engine - Rule Execution Logic Integration Tests - ESS Env - Basic License',
+    },
+  };
+}

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/configs/serverless.config.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createTestConfig } from '../../../../../../config/serverless/config.base.essentials';
+
+export default createTestConfig({
+  testFiles: [require.resolve('..')],
+  junit: {
+    reportName:
+      'Detection Engine - Rule Execution Logic Integration Tests - Serverless Env - Essentials Tier ',
+  },
+});

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/ignore_fields.ts → x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/ignore_fields.ts

@@ -46,7 +46,9 @@ export default ({ getService }: FtrProviderContext): void => {
    * server/lib/detection_engine/signals/source_fields_merging/utils/is_ignored.ts
    * server/lib/detection_engine/signals/source_fields_merging/utils/is_eql_bug_77152.ts
    */
-  describe('@ess @serverless ignore_fields', () => {
+
+  // TODO: Fix for serverless - https://github.com/elastic/kibana/issues/179767
+  describe('@ess @serverless @brokenInServerless ignore_fields', () => {
     const supertest = getService('supertest');
     const esArchiver = getService('esArchiver');
     const log = getService('log');
@@ -69,7 +71,7 @@ export default ({ getService }: FtrProviderContext): void => {
       await deleteAllRules(supertest, log);
     });
 
-    it('@skipInQA should ignore the field of "testing_ignored"', async () => {
+    it('should ignore the field of "testing_ignored"', async () => {
       const rule = getEqlRuleForAlertTesting(['ignore_fields']);
 
       const { id } = await createRule(supertest, log, rule);
@@ -84,7 +86,7 @@ export default ({ getService }: FtrProviderContext): void => {
       expect(hits).to.eql([undefined, undefined, undefined, undefined]);
     });
 
-    it('@skipInQA should ignore the field of "testing_regex"', async () => {
+    it('should ignore the field of "testing_regex"', async () => {
       const rule = getEqlRuleForAlertTesting(['ignore_fields']);
 
       const { id } = await createRule(supertest, log, rule);
@@ -111,24 +113,5 @@ export default ({ getService }: FtrProviderContext): void => {
       // Value should be "constant_value for all records"
       expect(hits).to.eql(['constant_value', 'constant_value', 'constant_value', 'constant_value']);
     });
-
-    // TODO: Remove this test once https://github.com/elastic/elasticsearch/issues/77152 is fixed
-    it('should ignore the field of "_ignored" when using EQL and index the data', async () => {
-      const rule = getEqlRuleForAlertTesting(['ignore_fields']);
-
-      const { id } = await createRule(supertest, log, rule);
-      await waitForRuleSuccess({ supertest, log, id });
-      await waitForAlertsToBePresent(supertest, log, 4, [id]);
-      const alertsOpen = await getAlertsById(supertest, log, id);
-      const hits = alertsOpen.hits.hits.map((hit) => (hit._source as Ignore).small_field).sort();
-
-      // We just test a constant value to ensure this did not blow up on us and did index data.
-      expect(hits).to.eql([
-        '1 indexed',
-        '2 large not indexed',
-        '3 large not indexed',
-        '4 large not indexed',
-      ]);
-    });
   });
 };

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Rule execution logic', function () {
+    loadTestFile(require.resolve('./ignore_fields'));
+    loadTestFile(require.resolve('./runtime_fields'));
+    loadTestFile(require.resolve('./timestamps'));
+    loadTestFile(require.resolve('./non_ecs_fields'));
+    loadTestFile(require.resolve('./rules'));
+    loadTestFile(require.resolve('./keyword_family'));
+  });
+}

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword.ts → x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/keyword_family/keyword.ts

@@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => {
   const log = getService('log');
   const es = getService('es');
 
-  describe('Rule detects against a keyword of event.dataset', () => {
+  describe('@ess @serverless Rule detects against a keyword of event.dataset', () => {
     before(async () => {
       await esArchiver.load('x-pack/test/functional/es_archives/rule_keyword_family/keyword');
     });

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/non_ecs_fields.ts → x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/non_ecs_fields.ts

@@ -11,13 +11,13 @@ import {
   previewRule,
   dataGeneratorFactory,
   enhanceDocument,
-} from '../../../../utils';
+} from '../../../utils';
 import {
   deleteAllRules,
   deleteAllAlerts,
   getRuleForAlertTesting,
-} from '../../../../../../../common/utils/security_solution';
-import { FtrProviderContext } from '../../../../../../ftr_provider_context';
+} from '../../../../../../common/utils/security_solution';
+import { FtrProviderContext } from '../../../../../ftr_provider_context';
 
 const getQueryRule = (docIdToQuery: string) => ({
   ...getRuleForAlertTesting(['ecs_non_compliant']),

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts → x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/rules/index.ts

@@ -8,18 +8,14 @@
 import { FtrProviderContext } from '../../../../../../ftr_provider_context';
 
 export default ({ loadTestFile }: FtrProviderContext): void => {
-  describe('Detection Engine - Execution logic', function () {
+  describe('Detection Engine - rule execution', function () {
+    loadTestFile(require.resolve('./new_terms'));
     loadTestFile(require.resolve('./eql'));
     loadTestFile(require.resolve('./esql'));
-    loadTestFile(require.resolve('./machine_learning'));
-    loadTestFile(require.resolve('./new_terms'));
-    loadTestFile(require.resolve('./new_terms_alert_suppression'));
+    loadTestFile(require.resolve('./query'));
+    loadTestFile(require.resolve('./query_ess'));
     loadTestFile(require.resolve('./saved_query'));
     loadTestFile(require.resolve('./threat_match'));
-    loadTestFile(require.resolve('./threat_match_alert_suppression'));
     loadTestFile(require.resolve('./threshold'));
-    loadTestFile(require.resolve('./threshold_alert_suppression'));
-    loadTestFile(require.resolve('./non_ecs_fields'));
-    loadTestFile(require.resolve('./query'));
   });
 };

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts → x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/basic_license_essentials_tier/rules/new_terms.ts

@@ -51,6 +51,7 @@ export default ({ getService }: FtrProviderContext) => {
   const isServerless = config.get('serverless');
   const dataPathBuilder = new EsArchivePathBuilder(isServerless);
   const path = dataPathBuilder.getPath('auditbeat/hosts');
+
   /**
    * indexes 2 sets of documents:
    * - documents in historical window
@@ -80,8 +81,7 @@ export default ({ getService }: FtrProviderContext) => {
     return testId;
   };
 
-  // Failing: See https://github.com/elastic/kibana/issues/180236
-  describe.skip('@ess @serverless New terms type rules', () => {
+  describe('@ess @serverless New terms type rules', () => {
     before(async () => {
       await esArchiver.load(path);
       await esArchiver.load('x-pack/test/functional/es_archives/security_solution/new_terms');
@@ -94,12 +94,13 @@ export default ({ getService }: FtrProviderContext) => {
       await deleteAllRules(supertest, log);
     });
 
+    // Failing: See https://github.com/elastic/kibana/issues/180236
     // First test creates a real rule - remaining tests use preview API
 
     // This test also tests that alerts are NOT created for terms that are not new: the host name
     // suricata-sensor-san-francisco appears in a document at 2019-02-19T20:42:08.230Z, but also appears
     // in earlier documents so is not new. An alert should not be generated for that term.
-    it('should generate 1 alert with 1 selected field', async () => {
+    it.skip('should generate 1 alert with 1 selected field', async () => {
       const rule: NewTermsRuleCreateProps = {
         ...getCreateNewTermsRulesSchemaMock('rule-1', true),
         new_terms_fields: ['host.name'],