[Security Solution] [Platform] Adds support for data views and runtime field mappings in rule creation, exceptions, and during execution

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -111,6 +111,12 @@ export type IdOrUndefined = t.TypeOf<typeof idOrUndefined>;
 export const index = t.array(t.string);
 export type Index = t.TypeOf<typeof index>;
 
+export const data_view_id = t.string;
+export type DataViewId = t.TypeOf<typeof data_view_id>;
+
+export const dataViewIdOrUndefined = t.union([data_view_id, t.undefined]);
+export type DataViewIdOrUndefined = t.TypeOf<typeof dataViewIdOrUndefined>;
+
 export const indexOrUndefined = t.union([index, t.undefined]);
 export type IndexOrUndefined = t.TypeOf<typeof indexOrUndefined>;
 

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts

@@ -53,6 +53,7 @@ import {
   filters,
   RuleId,
   index,
+  data_view_id,
   output_index,
   saved_id,
   timeline_id,
@@ -116,6 +117,7 @@ export const importRulesSchema = t.intersection([
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode
       index, // defaults to undefined if not set during decode
+      data_view_id, // defaults to undefined if not set during decode
       immutable: OnlyFalseAllowed, // defaults to "false" if not set during decode
       interval: DefaultIntervalString, // defaults to "5m" if not set during decode
       query, // defaults to undefined if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts

@@ -56,7 +56,7 @@ export const getCreateThreatMatchRulesSchemaMock = (
   language: 'kuery',
   rule_id: ruleId,
   threat_query: '*:*',
-  threat_index: ['list-index'],
+  threat_index: ['auditbeat-*'],
   threat_indicator_path: DEFAULT_INDICATOR_SOURCE_PATH,
   interval: '5m',
   from: 'now-6m',

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts

@@ -32,6 +32,7 @@ import { version } from '@kbn/securitysolution-io-ts-types';
 import {
   id,
   index,
+  data_view_id,
   filters,
   event_category_override,
   building_block_type,
@@ -146,6 +147,7 @@ const baseParams = {
   },
   optional: {
     building_block_type,
+    data_view_id,
     note,
     license,
     outcome,
@@ -209,6 +211,7 @@ const eqlRuleParams = {
   },
   optional: {
     index,
+    data_view_id,
     filters,
     event_category_override,
   },
@@ -231,6 +234,7 @@ const threatMatchRuleParams = {
   },
   optional: {
     index,
+    data_view_id,
     filters,
     saved_id,
     threat_filters,
@@ -256,6 +260,7 @@ const queryRuleParams = {
   },
   optional: {
     index,
+    data_view_id,
     filters,
     saved_id,
   },
@@ -281,6 +286,7 @@ const savedQueryRuleParams = {
     // Having language, query, and filters possibly defined adds more code confusion and probably user confusion
     // if the saved object gets deleted for some reason
     index,
+    data_view_id,
     query,
     filters,
   },
@@ -304,6 +310,7 @@ const thresholdRuleParams = {
   },
   optional: {
     index,
+    data_view_id,
     filters,
     saved_id,
   },

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts

@@ -146,7 +146,7 @@ export const getThreatMatchingSchemaPartialMock = (enabled = false): Partial<Rul
     query: 'user.name: root or user.name: admin',
     language: 'kuery',
     threat_query: '*:*',
-    threat_index: ['list-index'],
+    threat_index: ['auditbeat-*'],
     threat_indicator_path: DEFAULT_INDICATOR_SOURCE_PATH,
     threat_mapping: [
       {

x-pack/plugins/security_solution/common/detection_engine/utils.ts

@@ -37,12 +37,14 @@ export const hasEqlSequenceQuery = (ruleQuery: string | undefined): boolean => {
   return false;
 };
 
+// these functions should be typeguards and accept an entire rule.
 export const isEqlRule = (ruleType: Type | undefined): boolean => ruleType === 'eql';
 export const isThresholdRule = (ruleType: Type | undefined): boolean => ruleType === 'threshold';
 export const isQueryRule = (ruleType: Type | undefined): boolean =>
   ruleType === 'query' || ruleType === 'saved_query';
 export const isThreatMatchRule = (ruleType: Type | undefined): boolean =>
   ruleType === 'threat_match';
+export const isMlRule = (ruleType: Type | undefined): boolean => ruleType === 'machine_learning';
 
 export const normalizeThresholdField = (
   thresholdField: string | string[] | null | undefined

x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts

@@ -124,8 +124,9 @@ export async function indexEndpointHostDocs({
     generator.updateHostData();
     generator.updateHostPolicyData();
 
+    // timestamp - timeBetweenDocs * (numDocs - j - 1),
     hostMetadata = generator.generateHostMetadata(
-      timestamp - timeBetweenDocs * (numDocs - j - 1),
+      timestamp,
       EndpointDocGenerator.createDataStreamFromIndex(metadataIndex)
     );
 
@@ -198,7 +199,8 @@ export async function indexEndpointHostDocs({
       // there is an extra delay and fleet-agents gets populated much sooner.
       // we manually add a delay to the time sync field so that the united transform
       // will pick up the latest metadata doc.
-      '@timestamp': hostMetadata['@timestamp'] + 60000,
+      // hostMetadata['@timestamp'] + 60000,
+      '@timestamp': hostMetadata['@timestamp'],
     };
     await client
       .index({

x-pack/plugins/security_solution/cypress/integration/detection_rules/bulk_edit_rules.spec.ts

@@ -45,7 +45,7 @@ import {
   typeTags,
 } from '../../tasks/rules_bulk_edit';
 
-import { hasIndexPatterns } from '../../tasks/rule_details';
+import { hasIndexPatterns, doesNotHaveDataView } from '../../tasks/rule_details';
 import { login, visitWithoutDateRange } from '../../tasks/login';
 
 import { SECURITY_DETECTIONS_RULES_URL } from '../../urls/navigation';
@@ -61,6 +61,7 @@ import {
 import { esArchiverResetKibana } from '../../tasks/es_archiver';
 
 const RULE_NAME = 'Custom rule for bulk actions';
+const DATA_VIEW_RULE_NAME = 'Custom rule for bulk actions + data view';
 
 const CUSTOM_INDEX_PATTERN_1 = 'custom-cypress-test-*';
 const DEFAULT_INDEX_PATTERNS = ['index-1-*', 'index-2-*'];
@@ -73,6 +74,13 @@ const customRule = {
   name: RULE_NAME,
 };
 
+const customRuleWithDataViewId = {
+  ...getNewRule(),
+  index: DEFAULT_INDEX_PATTERNS,
+  dataViewId: 'myfakedataview',
+  name: DATA_VIEW_RULE_NAME,
+};
+
 describe('Detection rules, bulk edit', () => {
   before(() => {
     cleanKibana();
@@ -123,6 +131,61 @@ describe('Detection rules, bulk edit', () => {
     hasIndexPatterns([...DEFAULT_INDEX_PATTERNS, CUSTOM_INDEX_PATTERN_1].join(''));
   });
 
+  it('should warn before add/delete/overwrite index patterns in rules when rule with dataview id is selected', () => {
+    cy.log('Creates custom rule with dataViewId');
+    createCustomRule(customRuleWithDataViewId, 'rule with dataview id');
+
+    cy.log('Adds index patterns');
+    // Switch to 5 rules per page, so we can edit all existing rules, not only ones on a page
+    // this way we will use underlying bulk edit API with query parameter, which update all rules based on query search results
+    changeRowsPerPageTo(5);
+    selectAllRules();
+
+    openBulkEditAddIndexPatternsForm();
+    typeIndexPatterns([CUSTOM_INDEX_PATTERN_1]);
+    confirmBulkEditForm();
+    waitForBulkEditActionToFinish({ rulesCount: 7 });
+
+    // check if rule has been updated
+    changeRowsPerPageTo(20);
+    goToTheRuleDetailsOf(DATA_VIEW_RULE_NAME);
+    hasIndexPatterns([...DEFAULT_INDEX_PATTERNS, CUSTOM_INDEX_PATTERN_1].join(''));
+    cy.go('back');
+
+    cy.log('Deletes index patterns');
+    // select all rules on page (as page displays all existing rules).
+    // this way we will use underlying bulk edit API with ids parameter, which updates rules based their ids
+    cy.get(SELECT_ALL_RULES_ON_PAGE_CHECKBOX).click();
+    openBulkEditDeleteIndexPatternsForm();
+    typeIndexPatterns([CUSTOM_INDEX_PATTERN_1]);
+    confirmBulkEditForm();
+    waitForBulkEditActionToFinish({ rulesCount: 7 });
+
+    // check if rule has been updated
+    goToTheRuleDetailsOf(DATA_VIEW_RULE_NAME);
+    hasIndexPatterns(DEFAULT_INDEX_PATTERNS.join(''));
+    cy.go('back');
+
+    cy.log('Overwrites index patterns');
+    cy.get(SELECT_ALL_RULES_ON_PAGE_CHECKBOX).click();
+    openBulkEditAddIndexPatternsForm();
+    cy.get(RULES_BULK_EDIT_OVERWRITE_INDEX_PATTERNS_CHECKBOX)
+      .should('have.text', 'Overwrite all selected rules index patterns')
+      .click();
+    cy.get(RULES_BULK_EDIT_INDEX_PATTERNS_WARNING).should(
+      'have.text',
+      'You’re about to overwrite index patterns for 7 selected rules, press Save to apply changes.'
+    );
+    typeIndexPatterns(OVERWRITE_INDEX_PATTERNS);
+    confirmBulkEditForm();
+    waitForBulkEditActionToFinish({ rulesCount: 7 });
+
+    // check if rule has been updated
+    goToTheRuleDetailsOf(DATA_VIEW_RULE_NAME);
+    hasIndexPatterns(OVERWRITE_INDEX_PATTERNS.join(''));
+    doesNotHaveDataView();
+  });
+
   it('should add/delete/overwrite index patterns in rules', () => {
     cy.log('Adds index patterns');
     // Switch to 5 rules per page, so we can edit all existing rules, not only ones on a page

x-pack/plugins/security_solution/cypress/integration/detection_rules/custom_query_rule.spec.ts

@@ -331,6 +331,10 @@ describe('Custom query rules', () => {
         // expect define step to populate
         cy.get(CUSTOM_QUERY_INPUT).should('have.value', getExistingRule().customQuery);
         if (getExistingRule().index && getExistingRule().index.length > 0) {
+          // cy.get('[id="indexPatterns"]').click({ force: true });
+          // cy.get('[data-test-subj="detectionEngineStepDefineRuleIndexPatternsAccordion"]').click({
+          //   force: true,
+          // });
           cy.get(DEFINE_INDEX_INPUT).should('have.text', getExistingRule().index.join(''));
         }
 

x-pack/plugins/security_solution/cypress/integration/detection_rules/override.spec.ts

@@ -169,8 +169,8 @@ describe('Detection rules, override', () => {
     cy.get(NUMBER_OF_ALERTS)
       .invoke('text')
       .should('match', /^[1-9].+$/); // Any number of alerts
-    cy.get(ALERT_GRID_CELL).contains('auditbeat');
-    cy.get(ALERT_GRID_CELL).contains('critical');
-    cy.get(ALERT_GRID_CELL).contains('80');
+    cy.get(ALERT_GRID_CELL).contains('siem-kibana');
+    cy.get(ALERT_GRID_CELL).contains('test');
+    cy.get(ALERT_GRID_CELL).contains('zsh');
   });
 });

x-pack/plugins/security_solution/cypress/integration/exceptions/exceptions_flyout.spec.ts

@@ -130,12 +130,12 @@ describe('Exceptions flyout', () => {
     cy.get(ADD_AND_BTN).click();
     addExceptionEntryFieldValue('@timestamp', 1);
     cy.get(ADD_AND_BTN).click();
-    addExceptionEntryFieldValue('c', 2);
+    addExceptionEntryFieldValue('agent.hostname', 2);
 
     // delete second item, invalid values 'a' and 'c' should remain
     cy.get(ENTRY_DELETE_BTN).eq(1).click();
     cy.get(FIELD_INPUT).eq(0).should('have.text', 'agent.name');
-    cy.get(FIELD_INPUT).eq(1).should('have.text', 'c');
+    cy.get(FIELD_INPUT).eq(1).should('have.text', 'agent.hostname');
 
     closeExceptionBuilderFlyout();
   });
@@ -158,7 +158,7 @@ describe('Exceptions flyout', () => {
     cy.get(ADD_AND_BTN).click();
     addExceptionEntryFieldValueOfItemX('user.last', 1, 1);
     cy.get(ADD_AND_BTN).click();
-    addExceptionEntryFieldValueOfItemX('e', 1, 2);
+    addExceptionEntryFieldValueOfItemX('host.architecture', 1, 2);
 
     // delete single entry from exception item 2
     cy.get(ENTRY_DELETE_BTN).eq(3).click();
@@ -177,7 +177,11 @@ describe('Exceptions flyout', () => {
       .find(FIELD_INPUT)
       .eq(0)
       .should('have.text', 'user.first');
-    cy.get(EXCEPTION_ITEM_CONTAINER).eq(1).find(FIELD_INPUT).eq(1).should('have.text', 'e');
+    cy.get(EXCEPTION_ITEM_CONTAINER)
+      .eq(1)
+      .find(FIELD_INPUT)
+      .eq(1)
+      .should('have.text', 'host.architecture');
 
     // delete remaining entries in exception item 2
     cy.get(ENTRY_DELETE_BTN).eq(2).click();
@@ -210,9 +214,9 @@ describe('Exceptions flyout', () => {
     cy.get(ADD_OR_BTN).click();
     addExceptionEntryFieldValueOfItemX('agent.name', 1, 0);
     cy.get(ADD_NESTED_BTN).click();
-    addExceptionEntryFieldValueOfItemX('user.id{downarrow}{enter}', 1, 1);
+    addExceptionEntryFieldValueOfItemX('file.elf.sections{downArrow}{enter}', 1, 1);
     cy.get(ADD_AND_BTN).click();
-    addExceptionEntryFieldValueOfItemX('last{downarrow}{enter}', 1, 3);
+    addExceptionEntryFieldValueOfItemX('chi2{downArrow}{enter}', 1, 3);
     // This button will now read `Add non-nested button`
     cy.get(ADD_NESTED_BTN).scrollIntoView();
     cy.get(ADD_NESTED_BTN).focus().click();
@@ -225,14 +229,22 @@ describe('Exceptions flyout', () => {
       .find(FIELD_INPUT)
       .eq(0)
       .should('have.text', 'agent.name');
-    cy.get(EXCEPTION_ITEM_CONTAINER).eq(0).find(FIELD_INPUT).eq(1).should('have.text', 'b');
+    cy.get(EXCEPTION_ITEM_CONTAINER)
+      .eq(0)
+      .find(FIELD_INPUT)
+      .eq(1)
+      .should('have.text', 'agent.build.original');
     cy.get(EXCEPTION_ITEM_CONTAINER)
       .eq(1)
       .find(FIELD_INPUT)
       .eq(0)
       .should('have.text', 'agent.name');
-    cy.get(EXCEPTION_ITEM_CONTAINER).eq(1).find(FIELD_INPUT).eq(1).should('have.text', 'user');
-    cy.get(EXCEPTION_ITEM_CONTAINER).eq(1).find(FIELD_INPUT).eq(2).should('have.text', 'last');
+    cy.get(EXCEPTION_ITEM_CONTAINER)
+      .eq(1)
+      .find(FIELD_INPUT)
+      .eq(1)
+      .should('have.text', 'file.elf.sections');
+    cy.get(EXCEPTION_ITEM_CONTAINER).eq(1).find(FIELD_INPUT).eq(2).should('have.text', 'chi2');
     cy.get(EXCEPTION_ITEM_CONTAINER)
       .eq(1)
       .find(FIELD_INPUT)
@@ -246,7 +258,11 @@ describe('Exceptions flyout', () => {
       .find(FIELD_INPUT)
       .eq(0)
       .should('have.text', 'agent.name');
-    cy.get(EXCEPTION_ITEM_CONTAINER).eq(0).find(FIELD_INPUT).eq(1).should('have.text', 'b');
+    cy.get(EXCEPTION_ITEM_CONTAINER)
+      .eq(0)
+      .find(FIELD_INPUT)
+      .eq(1)
+      .should('have.text', 'agent.build.original');
     cy.get(EXCEPTION_ITEM_CONTAINER)
       .eq(1)
       .find(FIELD_INPUT)
@@ -268,6 +284,7 @@ describe('Exceptions flyout', () => {
         return $el.find(ADD_AND_BTN);
       })
       .should('be.visible');
+    addExceptionEntryFieldValueOfItemX('unique_value.test', 0, 0);
     cy.get(FIELD_INPUT).eq(0).click({ force: true });
     cy.get(EXCEPTION_FIELD_LIST).contains('unique_value.test');
 

x-pack/plugins/security_solution/cypress/screens/rule_details.ts

@@ -39,6 +39,8 @@ export const FALSE_POSITIVES_DETAILS = 'False positive examples';
 
 export const INDEX_PATTERNS_DETAILS = 'Index patterns';
 
+export const DATA_VIEW_DETAILS = 'Data View';
+
 export const INDICATOR_INDEX_PATTERNS = 'Indicator index patterns';
 
 export const INDICATOR_INDEX_QUERY = 'Indicator index query';