[Security Solution][Detections][Threshold Rules] Threshold multiple aggregations with cardinality

x-pack/plugins/osquery/common/ecs/rule/index.ts

@@ -28,7 +28,7 @@ export interface RuleEcs {
   tags?: string[];
   threat?: unknown;
   threshold?: {
-    field: string;
+    field: string | string[];
     value: number;
   };
   type?: string[];

x-pack/plugins/security_solution/common/ecs/rule/index.ts

@@ -27,10 +27,7 @@ export interface RuleEcs {
   severity?: string[];
   tags?: string[];
   threat?: unknown;
-  threshold?: {
-    field: string;
-    value: number;
-  };
+  threshold?: unknown;
   type?: string[];
   size?: string[];
   to?: string[];

x-pack/plugins/security_solution/common/ecs/signal/index.ts

@@ -14,4 +14,5 @@ export interface SignalEcs {
   group?: {
     id?: string[];
   };
+  threshold_result?: unknown;
 }

x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx

@@ -8,7 +8,7 @@
 /* eslint-disable complexity */
 
 import dateMath from '@elastic/datemath';
-import { get, getOr, isEmpty, find } from 'lodash/fp';
+import { getOr, isEmpty } from 'lodash/fp';
 import moment from 'moment';
 import { i18n } from '@kbn/i18n';
 
@@ -131,34 +131,51 @@ export const getThresholdAggregationDataProvider = (
   ecsData: Ecs,
   nonEcsData: TimelineNonEcsData[]
 ): DataProvider[] => {
-  const aggregationField = ecsData.signal?.rule?.threshold?.field!;
-  const aggregationValue =
-    get(aggregationField, ecsData) ?? find(['field', aggregationField], nonEcsData)?.value;
-  const dataProviderValue = Array.isArray(aggregationValue)
-    ? aggregationValue[0]
-    : aggregationValue;
+  const threshold = ecsData.signal?.rule?.threshold as string[];
+  const thresholdResult = JSON.parse((ecsData.signal?.threshold_result as string[])[0]);
 
-  if (!dataProviderValue) {
-    return [];
-  }
+  const aggField = JSON.parse(threshold[0]).field;
+  const aggregationFields = Array.isArray(aggField) ? aggField : [aggField];
+
+  return aggregationFields.reduce<DataProvider[]>((acc, aggregationField, i) => {
+    const aggregationValue = thresholdResult.terms.filter(
+      (term) => term.field === aggregationField
+    )[0].value;
+    const dataProviderValue = Array.isArray(aggregationValue)
+      ? aggregationValue[0]
+      : aggregationValue;
 
-  const aggregationFieldId = aggregationField.replace('.', '-');
+    if (!dataProviderValue) {
+      return acc;
+    }
 
-  return [
-    {
-      and: [],
+    const aggregationFieldId = aggregationField.replace('.', '-');
+    const dataProviderPartial = {
       id: `send-alert-to-timeline-action-default-draggable-event-details-value-formatted-field-value-${TimelineId.active}-${aggregationFieldId}-${dataProviderValue}`,
       name: aggregationField,
       enabled: true,
       excluded: false,
       kqlQuery: '',
       queryMatch: {
-        field: aggregationField,
-        value: dataProviderValue,
+        field: aggregationField as string,
+        value: dataProviderValue as string,
         operator: ':',
       },
-    },
-  ];
+    };
+
+    if (i === 0) {
+      return [
+        ...acc,
+        {
+          ...dataProviderPartial,
+          and: [],
+        },
+      ];
+    } else {
+      acc[0].and.push(dataProviderPartial);
+      return acc;
+    }
+  }, []);
 };
 
 export const isEqlRuleWithGroupId = (ecsData: Ecs) =>

x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx

@@ -20,7 +20,7 @@ export interface FieldValueThreshold {
   field: string[];
   value: string;
   cardinality_field: string[];
-  cardinality_value: number;
+  cardinality_value: string;
 }
 
 interface ThresholdInputProps {

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts

@@ -222,7 +222,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep
             field: ruleFields.threshold?.field ?? [],
             value: parseInt(ruleFields.threshold?.value, 10) ?? 0,
             cardinality_field: ruleFields.threshold.cardinality_field[0] ?? '',
-            cardinality_value: ruleFields.threshold.cardinality_value,
+            cardinality_value: parseInt(ruleFields.threshold?.cardinality_value, 10) ?? 0,
           },
         }),
       }

x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts

@@ -36,7 +36,13 @@ describe('buildBulkBody', () => {
     delete doc._source.source;
     const fakeSignalSourceHit = buildBulkBody({
       doc,
-      ruleParams: sampleParams,
+      ruleParams: {
+        ...sampleParams,
+        threshold: {
+          field: ['host.name'],
+          value: 100,
+        },
+      },
       id: sampleRuleGuid,
       name: 'rule-name',
       actions: [],
@@ -111,7 +117,7 @@ describe('buildBulkBody', () => {
           tags: ['some fake tag 1', 'some fake tag 2'],
           threat: [],
           threshold: {
-            field: 'host.name',
+            field: ['host.name'],
             value: 100,
           },
           throttle: 'no_actions',
@@ -142,7 +148,6 @@ describe('buildBulkBody', () => {
         threshold_result: {
           terms: [
             {
-              field: '',
               value: 'abcd',
             },
           ],
@@ -153,7 +158,13 @@ describe('buildBulkBody', () => {
     delete doc._source.source;
     const fakeSignalSourceHit = buildBulkBody({
       doc,
-      ruleParams: sampleParams,
+      ruleParams: {
+        ...sampleParams,
+        threshold: {
+          field: [],
+          value: 4,
+        },
+      },
       id: sampleRuleGuid,
       name: 'rule-name',
       actions: [],
@@ -227,6 +238,10 @@ describe('buildBulkBody', () => {
           severity_mapping: [],
           tags: ['some fake tag 1', 'some fake tag 2'],
           threat: [],
+          threshold: {
+            field: [],
+            value: 4,
+          },
           throttle: 'no_actions',
           type: 'query',
           to: 'now',
@@ -240,10 +255,11 @@ describe('buildBulkBody', () => {
           exceptions_list: getListArrayMock(),
         },
         threshold_result: {
-          terms: {
-            field: 'host.name',
-            value: 'abcd',
-          },
+          terms: [
+            {
+              value: 'abcd',
+            },
+          ],
           count: 5,
         },
         depth: 1,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts

@@ -106,10 +106,12 @@ describe('transformThresholdResultsToEcs', () => {
                     value: 'garden-gnomes',
                   },
                 ],
-                cardinality: {
-                  field: 'destination.ip',
-                  value: 7,
-                },
+                cardinality: [
+                  {
+                    field: 'destination.ip',
+                    value: 7,
+                  },
+                ],
                 count: 12,
               },
             },

x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts

@@ -91,7 +91,6 @@ const getTransformedHits = (
             value: ruleId,
           },
         ],
-        // TODO: cardinality?
         count: totalResults,
       },
     };
@@ -106,7 +105,7 @@ const getTransformedHits = (
   }
 
   const aggParts = results.aggregations && getThresholdAggregationParts(results.aggregations);
-  if (!aggParts.key) {
+  if (!aggParts) {
     return [];
   }
 
@@ -118,7 +117,7 @@ const getTransformedHits = (
         if (nextLevelAggParts == null) {
           throw new Error('Something went horribly wrong');
         }
-        const nextLevelPath = `${nextLevelAggParts.name}.buckets`;
+        const nextLevelPath = `['${nextLevelAggParts.name}']['buckets']`;
         const nextBuckets = get(nextLevelPath, bucket);
         const combinations = getCombinations(nextBuckets, nextLevelIdx, nextLevelAggParts.field);
         combinations.forEach((val) => {
@@ -205,7 +204,7 @@ const getTransformedHits = (
           ruleId,
           startedAt,
           threshold.field as string[],
-          bucket.terms.join(',')
+          bucket.terms.map((term) => term.value).join(',')
         ),
         _source: source,
       });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_find_previous_signals.ts

@@ -44,22 +44,6 @@ export const findPreviousThresholdSignals = async ({
   searchDuration: string;
   searchErrors: string[];
 }> => {
-  const aggregations = {
-    threshold: {
-      terms: {
-        field: 'signal.threshold_result.value',
-        size: 10000,
-      },
-      aggs: {
-        lastSignalTimestamp: {
-          max: {
-            field: 'signal.original_time', // timestamp of last event captured by bucket
-          },
-        },
-      },
-    },
-  };
-
   const filter = {
     bool: {
       must: [
@@ -80,7 +64,6 @@ export const findPreviousThresholdSignals = async ({
   };
 
   return singleSearchAfter({
-    aggregations,
     searchAfterSortId: undefined,
     timestampOverride,
     index: indexPattern,
@@ -89,7 +72,7 @@ export const findPreviousThresholdSignals = async ({
     services,
     logger,
     filter,
-    pageSize: 0,
+    pageSize: 10000, // TODO: multiple pages?
     buildRuleMessage,
     excludeDocsWithTimestampOverride: false,
   });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+import { loggingSystemMock } from '../../../../../../../src/core/server/mocks';
+import { sampleDocNoSortId, sampleDocSearchResultsNoSortId } from './__mocks__/es_results';
+import { transformThresholdResultsToEcs } from './bulk_create_threshold_signals';
+import { calculateThresholdSignalUuid } from './utils';
+import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas';
+
+describe('getCombinations', () => {
+  it('should get all combinations for multiple aggregations without cardinality', () => {});
+
+  it('should get all combinations for multiple aggregations with cardinality', () => {});
+
+  it('should exclude empty buckets', () => {});
+});
+*/