[Security Solution][Detections][Threshold Rules] Threshold multiple aggregations with cardinality

x-pack/plugins/osquery/common/ecs/rule/index.ts

@@ -28,7 +28,7 @@ export interface RuleEcs {
   tags?: string[];
   threat?: unknown;
   threshold?: {
-    field: string;
+    field: string | string[];
     value: number;
   };
   type?: string[];

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -459,12 +459,21 @@ export type Threats = t.TypeOf<typeof threats>;
 export const threatsOrUndefined = t.union([threats, t.undefined]);
 export type ThreatsOrUndefined = t.TypeOf<typeof threatsOrUndefined>;
 
-export const threshold = t.exact(
-  t.type({
-    field: t.string,
-    value: PositiveIntegerGreaterThanZero,
-  })
-);
+export const threshold = t.intersection([
+  t.exact(
+    t.type({
+      field: t.union([t.string, t.array(t.string)]),
+      value: PositiveIntegerGreaterThanZero,
+    })
+  ),
+  t.exact(
+    t.partial({
+      cardinality_field: t.union([t.string, t.array(t.string), t.undefined]),
+      cardinality_value: t.union([PositiveInteger, t.undefined]), // TODO: cardinality_value should be set if cardinality_field is set
+    })
+  ),
+]);
+// TODO: codec to transform threshold field string to string[] ?
 export type Threshold = t.TypeOf<typeof threshold>;
 
 export const thresholdOrUndefined = t.union([threshold, t.undefined]);

x-pack/plugins/security_solution/common/ecs/rule/index.ts

@@ -27,10 +27,7 @@ export interface RuleEcs {
   severity?: string[];
   tags?: string[];
   threat?: unknown;
-  threshold?: {
-    field: string;
-    value: number;
-  };
+  threshold?: unknown;
   type?: string[];
   size?: string[];
   to?: string[];

x-pack/plugins/security_solution/common/ecs/signal/index.ts

@@ -14,4 +14,5 @@ export interface SignalEcs {
   group?: {
     id?: string[];
   };
+  threshold_result?: unknown;
 }

x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts

@@ -36,7 +36,14 @@ export interface MatrixHistogramRequestOptions extends RequestBasicOptions {
   timerange: TimerangeInput;
   histogramType: MatrixHistogramType;
   stackByField: string;
-  threshold?: { field: string | undefined; value: number } | undefined;
+  threshold?:
+    | {
+        field: string | string[] | undefined;
+        value: number;
+        cardinality_field?: string;
+        cardinality_value?: number;
+      }
+    | undefined;
   inspect?: Maybe<Inspect>;
   isPtrIncluded?: boolean;
 }

x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts

@@ -74,7 +74,14 @@ export interface MatrixHistogramQueryProps {
   stackByField: string;
   startDate: string;
   histogramType: MatrixHistogramType;
-  threshold?: { field: string | undefined; value: number } | undefined;
+  threshold?:
+    | {
+        field: string | string[] | undefined;
+        value: number;
+        cardinality_field?: string | undefined;
+        cardinality_value?: number;
+      }
+    | undefined;
   skip?: boolean;
   isPtrIncluded?: boolean;
 }

x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx

@@ -8,7 +8,7 @@
 /* eslint-disable complexity */
 
 import dateMath from '@elastic/datemath';
-import { get, getOr, isEmpty, find } from 'lodash/fp';
+import { getOr, isEmpty } from 'lodash/fp';
 import moment from 'moment';
 import { i18n } from '@kbn/i18n';
 
@@ -38,15 +38,18 @@ import {
   replaceTemplateFieldFromDataProviders,
 } from './helpers';
 import { KueryFilterQueryKind } from '../../../common/store';
-import { DataProvider } from '../../../timelines/components/timeline/data_providers/data_provider';
+import {
+  DataProvider,
+  QueryOperator,
+} from '../../../timelines/components/timeline/data_providers/data_provider';
 
 export const getUpdateAlertsQuery = (eventIds: Readonly<string[]>) => {
   return {
     query: {
       bool: {
         filter: {
           terms: {
-            _id: [...eventIds],
+            _id: eventIds,
           },
         },
       },
@@ -131,22 +134,48 @@ export const getThresholdAggregationDataProvider = (
   ecsData: Ecs,
   nonEcsData: TimelineNonEcsData[]
 ): DataProvider[] => {
-  const aggregationField = ecsData.signal?.rule?.threshold?.field!;
-  const aggregationValue =
-    get(aggregationField, ecsData) ?? find(['field', aggregationField], nonEcsData)?.value;
-  const dataProviderValue = Array.isArray(aggregationValue)
-    ? aggregationValue[0]
-    : aggregationValue;
+  const threshold = ecsData.signal?.rule?.threshold as string[];
+
+  let aggField: string[] = [];
+  let thresholdResult: {
+    terms?: Array<{
+      field?: string;
+      value: string;
+    }>;
+    count: number;
+  };
 
-  if (!dataProviderValue) {
-    return [];
+  try {
+    thresholdResult = JSON.parse((ecsData.signal?.threshold_result as string[])[0]);
+    aggField = JSON.parse(threshold[0]).field;
+  } catch (err) {
+    thresholdResult = {
+      terms: [
+        {
+          field: (ecsData.rule?.threshold as { field: string }).field,
+          value: (ecsData.signal?.threshold_result as { value: string }).value,
+        },
+      ],
+      count: (ecsData.signal?.threshold_result as { count: number }).count,
+    };
   }
 
-  const aggregationFieldId = aggregationField.replace('.', '-');
+  const aggregationFields = Array.isArray(aggField) ? aggField : [aggField];
+
+  return aggregationFields.reduce<DataProvider[]>((acc, aggregationField, i) => {
+    const aggregationValue = (thresholdResult.terms ?? []).filter(
+      (term: { field?: string | undefined; value: string }) => term.field === aggregationField
+    )[0].value;
+    const dataProviderValue = Array.isArray(aggregationValue)
+      ? aggregationValue[0]
+      : aggregationValue;
 
-  return [
-    {
-      and: [],
+    if (!dataProviderValue) {
+      return acc;
+    }
+
+    const aggregationFieldId = aggregationField.replace('.', '-');
+    const dataProviderPartial = {
       id: `send-alert-to-timeline-action-default-draggable-event-details-value-formatted-field-value-${TimelineId.active}-${aggregationFieldId}-${dataProviderValue}`,
       name: aggregationField,
       enabled: true,
@@ -155,10 +184,23 @@ export const getThresholdAggregationDataProvider = (
       queryMatch: {
         field: aggregationField,
         value: dataProviderValue,
-        operator: ':',
+        operator: ':' as QueryOperator,
       },
-    },
-  ];
+    };
+
+    if (i === 0) {
+      return [
+        ...acc,
+        {
+          ...dataProviderPartial,
+          and: [],
+        },
+      ];
+    } else {
+      acc[0].and.push(dataProviderPartial);
+      return acc;
+    }
+  }, []);
 };
 
 export const isEqlRuleWithGroupId = (ecsData: Ecs) =>
@@ -275,7 +317,7 @@ export const sendAlertToTimelineAction = async ({
         ...timelineDefaults,
         description: `_id: ${ecsData._id}`,
         filters: getFiltersFromRule(ecsData.signal?.rule?.filters as string[]),
-        dataProviders: [...getThresholdAggregationDataProvider(ecsData, nonEcsData)],
+        dataProviders: getThresholdAggregationDataProvider(ecsData, nonEcsData),
         id: TimelineId.active,
         indexNames: [],
         dateRange: {

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx

@@ -288,7 +288,12 @@ describe('PreviewQuery', () => {
             idAria="queryPreview"
             query={{ query: { query: 'file where true', language: 'kuery' }, filters: [] }}
             index={['foo-*']}
-            threshold={{ field: 'agent.hostname', value: 200 }}
+            threshold={{
+              field: 'agent.hostname',
+              value: 200,
+              cardinality_field: 'user.name',
+              cardinality_value: 2,
+            }}
             isDisabled={false}
           />
         </TestProviders>
@@ -330,7 +335,12 @@ describe('PreviewQuery', () => {
             idAria="queryPreview"
             query={{ query: { query: 'file where true', language: 'kuery' }, filters: [] }}
             index={['foo-*']}
-            threshold={{ field: 'agent.hostname', value: 200 }}
+            threshold={{
+              field: 'agent.hostname',
+              value: 200,
+              cardinality_field: 'user.name',
+              cardinality_value: 2,
+            }}
             isDisabled={false}
           />
         </TestProviders>
@@ -369,7 +379,12 @@ describe('PreviewQuery', () => {
             idAria="queryPreview"
             query={{ query: { query: 'file where true', language: 'kuery' }, filters: [] }}
             index={['foo-*']}
-            threshold={{ field: undefined, value: 200 }}
+            threshold={{
+              field: undefined,
+              value: 200,
+              cardinality_field: 'user.name',
+              cardinality_value: 2,
+            }}
             isDisabled={false}
           />
         </TestProviders>
@@ -396,7 +411,12 @@ describe('PreviewQuery', () => {
             idAria="queryPreview"
             query={{ query: { query: 'file where true', language: 'kuery' }, filters: [] }}
             index={['foo-*']}
-            threshold={{ field: '   ', value: 200 }}
+            threshold={{
+              field: '   ',
+              value: 200,
+              cardinality_field: 'user.name',
+              cardinality_value: 2,
+            }}
             isDisabled={false}
           />
         </TestProviders>

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx

@@ -56,7 +56,14 @@ export const initialState: State = {
   showNonEqlHistogram: false,
 };
 
-export type Threshold = { field: string | undefined; value: number } | undefined;
+export type Threshold =
+  | {
+      field: string | string[] | undefined;
+      value: number;
+      cardinality_field: string | undefined;
+      cardinality_value: number;
+    }
+  | undefined;
 
 interface PreviewQueryProps {
   dataTestSubj: string;

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts

@@ -334,7 +334,12 @@ describe('queryPreviewReducer', () => {
     test('should set thresholdFieldExists to true if threshold field is defined and not empty string', () => {
       const update = reducer(initialState, {
         type: 'setThresholdQueryVals',
-        threshold: { field: 'agent.hostname', value: 200 },
+        threshold: {
+          field: 'agent.hostname',
+          value: 200,
+          cardinality_field: 'user.name',
+          cardinality_value: 2,
+        },
         ruleType: 'threshold',
       });
 
@@ -347,7 +352,12 @@ describe('queryPreviewReducer', () => {
     test('should set thresholdFieldExists to false if threshold field is not defined', () => {
       const update = reducer(initialState, {
         type: 'setThresholdQueryVals',
-        threshold: { field: undefined, value: 200 },
+        threshold: {
+          field: undefined,
+          value: 200,
+          cardinality_field: 'user.name',
+          cardinality_value: 2,
+        },
         ruleType: 'threshold',
       });
 
@@ -360,7 +370,12 @@ describe('queryPreviewReducer', () => {
     test('should set thresholdFieldExists to false if threshold field is empty string', () => {
       const update = reducer(initialState, {
         type: 'setThresholdQueryVals',
-        threshold: { field: '    ', value: 200 },
+        threshold: {
+          field: '    ',
+          value: 200,
+          cardinality_field: 'user.name',
+          cardinality_value: 2,
+        },
         ruleType: 'threshold',
       });
 
@@ -373,7 +388,12 @@ describe('queryPreviewReducer', () => {
     test('should set showNonEqlHistogram to false if ruleType is eql', () => {
       const update = reducer(initialState, {
         type: 'setThresholdQueryVals',
-        threshold: { field: 'agent.hostname', value: 200 },
+        threshold: {
+          field: 'agent.hostname',
+          value: 200,
+          cardinality_field: 'user.name',
+          cardinality_value: 2,
+        },
         ruleType: 'eql',
       });
 
@@ -385,7 +405,12 @@ describe('queryPreviewReducer', () => {
     test('should set showNonEqlHistogram to true if ruleType is query', () => {
       const update = reducer(initialState, {
         type: 'setThresholdQueryVals',
-        threshold: { field: 'agent.hostname', value: 200 },
+        threshold: {
+          field: 'agent.hostname',
+          value: 200,
+          cardinality_field: 'user.name',
+          cardinality_value: 2,
+        },
         ruleType: 'query',
       });
 
@@ -397,7 +422,12 @@ describe('queryPreviewReducer', () => {
     test('should set showNonEqlHistogram to true if ruleType is saved_query', () => {
       const update = reducer(initialState, {
         type: 'setThresholdQueryVals',
-        threshold: { field: 'agent.hostname', value: 200 },
+        threshold: {
+          field: 'agent.hostname',
+          value: 200,
+          cardinality_field: 'user.name',
+          cardinality_value: 2,
+        },
         ruleType: 'saved_query',
       });
 

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.ts

@@ -67,6 +67,7 @@ export type Action =
       type: 'setToFrom';
     };
 
+/* eslint-disable-next-line complexity */
 export const queryPreviewReducer = () => (state: State, action: Action): State => {
   switch (action.type) {
     case 'setQueryInfo': {
@@ -131,7 +132,9 @@ export const queryPreviewReducer = () => (state: State, action: Action): State =
       const thresholdField =
         action.threshold != null &&
         action.threshold.field != null &&
-        action.threshold.field.trim() !== '';
+        ((typeof action.threshold.field === 'string' && action.threshold.field.trim() !== '') ||
+          (Array.isArray(action.threshold.field) &&
+            action.threshold.field.every((field) => field.trim() !== '')));
       const showNonEqlHist =
         action.ruleType === 'query' ||
         action.ruleType === 'saved_query' ||