[Security Solution] [Platform] Adds support for data views and runtime field mappings in rule creation, exceptions, and during execution

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -111,6 +111,12 @@ export type IdOrUndefined = t.TypeOf<typeof idOrUndefined>;
 export const index = t.array(t.string);
 export type Index = t.TypeOf<typeof index>;
 
+export const data_view_id = t.string;
+export type DataViewId = t.TypeOf<typeof data_view_id>;
+
+export const dataViewIdOrUndefined = t.union([data_view_id, t.undefined]);
+export type DataViewIdOrUndefined = t.TypeOf<typeof dataViewIdOrUndefined>;
+
 export const indexOrUndefined = t.union([index, t.undefined]);
 export type IndexOrUndefined = t.TypeOf<typeof indexOrUndefined>;
 
@@ -450,14 +456,17 @@ const bulkActionEditPayloadTags = t.type({
 
 export type BulkActionEditPayloadTags = t.TypeOf<typeof bulkActionEditPayloadTags>;
 
-const bulkActionEditPayloadIndexPatterns = t.type({
-  type: t.union([
-    t.literal(BulkActionEditType.add_index_patterns),
-    t.literal(BulkActionEditType.delete_index_patterns),
-    t.literal(BulkActionEditType.set_index_patterns),
-  ]),
-  value: index,
-});
+const bulkActionEditPayloadIndexPatterns = t.intersection([
+  t.type({
+    type: t.union([
+      t.literal(BulkActionEditType.add_index_patterns),
+      t.literal(BulkActionEditType.delete_index_patterns),
+      t.literal(BulkActionEditType.set_index_patterns),
+    ]),
+    value: index,
+  }),
+  t.exact(t.partial({ overwriteDataViews: t.boolean })),
+]);
 
 export type BulkActionEditPayloadIndexPatterns = t.TypeOf<
   typeof bulkActionEditPayloadIndexPatterns

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -50,6 +50,7 @@ import {
   anomaly_threshold,
   filters,
   index,
+  data_view_id,
   saved_id,
   timeline_id,
   timeline_title,
@@ -107,6 +108,7 @@ export const addPrepackagedRulesSchema = t.intersection([
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode
       index, // defaults to undefined if not set during decode
+      data_view_id, // defaults to undefined if not set during decode
       interval: DefaultIntervalString, // defaults to "5m" if not set during decode
       query, // defaults to undefined if not set during decode
       language, // defaults to undefined if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts

@@ -1773,4 +1773,124 @@ describe('import rules schema', () => {
       expect(message.schema).toEqual(expected);
     });
   });
+
+  describe('data_view_id', () => {
+    test('Defined data_view_id and empty index does validate', () => {
+      const payload: ImportRulesSchema = {
+        rule_id: 'rule-1',
+        risk_score: 50,
+        description: 'some description',
+        from: 'now-5m',
+        to: 'now',
+        name: 'some-name',
+        severity: 'low',
+        type: 'query',
+        query: 'some query',
+        data_view_id: 'logs-*',
+        index: [],
+        interval: '5m',
+      };
+
+      const decoded = importRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      const expected: ImportRulesSchemaDecoded = {
+        author: [],
+        severity_mapping: [],
+        risk_score_mapping: [],
+        rule_id: 'rule-1',
+        risk_score: 50,
+        description: 'some description',
+        from: 'now-5m',
+        to: 'now',
+        name: 'some-name',
+        severity: 'low',
+        type: 'query',
+        query: 'some query',
+        index: [],
+        data_view_id: 'logs-*',
+        interval: '5m',
+        references: [],
+        actions: [],
+        enabled: true,
+        false_positives: [],
+        max_signals: DEFAULT_MAX_SIGNALS,
+        tags: [],
+        threat: [],
+        throttle: null,
+        version: 1,
+        exceptions_list: [],
+        immutable: false,
+      };
+      expect(message.schema).toEqual(expected);
+    });
+
+    // Both can be defined, but if a data_view_id is defined, rule will use that one
+    test('Defined data_view_id and index does validate', () => {
+      const payload: ImportRulesSchema = {
+        rule_id: 'rule-1',
+        risk_score: 50,
+        description: 'some description',
+        from: 'now-5m',
+        to: 'now',
+        name: 'some-name',
+        severity: 'low',
+        type: 'query',
+        query: 'some query',
+        data_view_id: 'logs-*',
+        index: ['auditbeat-*'],
+        interval: '5m',
+      };
+
+      const decoded = importRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      const expected: ImportRulesSchemaDecoded = {
+        author: [],
+        severity_mapping: [],
+        risk_score_mapping: [],
+        rule_id: 'rule-1',
+        risk_score: 50,
+        description: 'some description',
+        from: 'now-5m',
+        to: 'now',
+        name: 'some-name',
+        severity: 'low',
+        type: 'query',
+        query: 'some query',
+        index: ['auditbeat-*'],
+        data_view_id: 'logs-*',
+        interval: '5m',
+        references: [],
+        actions: [],
+        enabled: true,
+        false_positives: [],
+        max_signals: DEFAULT_MAX_SIGNALS,
+        tags: [],
+        threat: [],
+        throttle: null,
+        version: 1,
+        exceptions_list: [],
+        immutable: false,
+      };
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('data_view_id cannot be a number', () => {
+      const payload: Omit<ImportRulesSchema, 'data_view_id'> & { data_view_id: number } = {
+        ...getImportRulesSchemaMock(),
+        data_view_id: 5,
+      };
+
+      const decoded = importRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "5" supplied to "data_view_id"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+  });
 });

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts

@@ -53,6 +53,7 @@ import {
   filters,
   RuleId,
   index,
+  data_view_id,
   output_index,
   saved_id,
   timeline_id,
@@ -116,6 +117,7 @@ export const importRulesSchema = t.intersection([
       filters, // defaults to undefined if not set during decode
       from: DefaultFromString, // defaults to "now-6m" if not set during decode
       index, // defaults to undefined if not set during decode
+      data_view_id, // defaults to undefined if not set during decode
       immutable: OnlyFalseAllowed, // defaults to "false" if not set during decode
       interval: DefaultIntervalString, // defaults to "5m" if not set during decode
       query, // defaults to undefined if not set during decode

x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts

@@ -38,6 +38,7 @@ import {
   anomaly_threshold,
   filters,
   index,
+  data_view_id,
   output_index,
   saved_id,
   timeline_id,
@@ -85,6 +86,7 @@ export const patchRulesSchema = t.exact(
     from,
     rule_id,
     index,
+    data_view_id,
     interval,
     query,
     language,

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts

@@ -29,6 +29,18 @@ export const getCreateRulesSchemaMock = (ruleId = 'rule-1'): QueryCreateSchema =
   rule_id: ruleId,
 });
 
+export const getCreateRulesSchemaMockWithDataView = (ruleId = 'rule-1'): QueryCreateSchema => ({
+  data_view_id: 'logs-*',
+  description: 'Detecting root and admin users',
+  name: 'Query with a rule id',
+  query: 'user.name: root or user.name: admin',
+  severity: 'high',
+  type: 'query',
+  risk_score: 55,
+  language: 'kuery',
+  rule_id: ruleId,
+});
+
 export const getCreateSavedQueryRulesSchemaMock = (ruleId = 'rule-1'): SavedQueryCreateSchema => ({
   description: 'Detecting root and admin users',
   name: 'Query with a rule id',
@@ -56,7 +68,7 @@ export const getCreateThreatMatchRulesSchemaMock = (
   language: 'kuery',
   rule_id: ruleId,
   threat_query: '*:*',
-  threat_index: ['list-index'],
+  threat_index: ['auditbeat-*'],
   threat_indicator_path: DEFAULT_INDICATOR_SOURCE_PATH,
   interval: '5m',
   from: 'now-6m',

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts

@@ -14,6 +14,8 @@ import {
   getCreateThreatMatchRulesSchemaMock,
   getCreateRulesSchemaMock,
   getCreateThresholdRulesSchemaMock,
+  getCreateRulesSchemaMockWithDataView,
+  getCreateMachineLearningRulesSchemaMock,
 } from './rule_schemas.mock';
 import { getListArrayMock } from '../types/lists.mock';
 
@@ -1199,4 +1201,89 @@ describe('create rules schema', () => {
       expect(message.schema).toEqual({});
     });
   });
+
+  describe('data_view_id', () => {
+    test('validates when "data_view_id" and index are defined', () => {
+      const payload = { ...getCreateRulesSchemaMockWithDataView(), index: ['auditbeat-*'] };
+      const decoded = createRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(payload);
+    });
+
+    test('"data_view_id" cannot be a number', () => {
+      const payload: Omit<CreateRulesSchema, 'data_view_id'> & { data_view_id: number } = {
+        ...getCreateRulesSchemaMockWithDataView(),
+        data_view_id: 5,
+      };
+
+      const decoded = createRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "5" supplied to "data_view_id"',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+
+    test('it should validate a type of "query" with "data_view_id" defined', () => {
+      const payload = getCreateRulesSchemaMockWithDataView();
+
+      const decoded = createRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = getCreateRulesSchemaMockWithDataView();
+
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('it should validate a type of "saved_query" with "data_view_id" defined', () => {
+      const payload = { ...getCreateSavedQueryRulesSchemaMock(), data_view_id: 'logs-*' };
+
+      const decoded = createRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = { ...getCreateSavedQueryRulesSchemaMock(), data_view_id: 'logs-*' };
+
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('it should validate a type of "threat_match" with "data_view_id" defined', () => {
+      const payload = { ...getCreateThreatMatchRulesSchemaMock(), data_view_id: 'logs-*' };
+
+      const decoded = createRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = { ...getCreateThreatMatchRulesSchemaMock(), data_view_id: 'logs-*' };
+
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('it should validate a type of "threshold" with "data_view_id" defined', () => {
+      const payload = { ...getCreateThresholdRulesSchemaMock(), data_view_id: 'logs-*' };
+
+      const decoded = createRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      const expected = { ...getCreateThresholdRulesSchemaMock(), data_view_id: 'logs-*' };
+
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(expected);
+    });
+
+    test('it should NOT validate a type of "machine_learning" with "data_view_id" defined', () => {
+      const payload = { ...getCreateMachineLearningRulesSchemaMock(), data_view_id: 'logs-*' };
+
+      const decoded = createRulesSchema.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+
+      expect(message.schema).toEqual({});
+      expect(getPaths(left(message.errors))).toEqual(['invalid keys "data_view_id"']);
+    });
+  });
 });

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts

@@ -32,6 +32,7 @@ import { version } from '@kbn/securitysolution-io-ts-types';
 import {
   id,
   index,
+  data_view_id,
   filters,
   event_category_override,
   building_block_type,
@@ -209,6 +210,7 @@ const eqlRuleParams = {
   },
   optional: {
     index,
+    data_view_id,
     filters,
     event_category_override,
   },
@@ -231,6 +233,7 @@ const threatMatchRuleParams = {
   },
   optional: {
     index,
+    data_view_id,
     filters,
     saved_id,
     threat_filters,
@@ -256,6 +259,7 @@ const queryRuleParams = {
   },
   optional: {
     index,
+    data_view_id,
     filters,
     saved_id,
   },
@@ -281,6 +285,7 @@ const savedQueryRuleParams = {
     // Having language, query, and filters possibly defined adds more code confusion and probably user confusion
     // if the saved object gets deleted for some reason
     index,
+    data_view_id,
     query,
     filters,
   },
@@ -304,6 +309,7 @@ const thresholdRuleParams = {
   },
   optional: {
     index,
+    data_view_id,
     filters,
     saved_id,
   },

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts

@@ -146,7 +146,7 @@ export const getThreatMatchingSchemaPartialMock = (enabled = false): Partial<Rul
     query: 'user.name: root or user.name: admin',
     language: 'kuery',
     threat_query: '*:*',
-    threat_index: ['list-index'],
+    threat_index: ['auditbeat-*'],
     threat_indicator_path: DEFAULT_INDICATOR_SOURCE_PATH,
     threat_mapping: [
       {