From 82e35be89c4b1e4e455290b07459ab2bff110f23 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Sep 2021 13:05:14 -0400 Subject: [PATCH 01/15] Initial commit --- .../query/create_query_alert_type.test.ts | 2 +- .../scripts/create_rule_threshold.sh | 63 +++++++++++++ .../create_threshold_alert_type.test.ts | 10 ++ .../threshold/create_threshold_alert_type.ts | 92 +++++++++++++++++++ .../signals/executors/threshold.test.ts | 1 + .../signals/executors/threshold.ts | 45 +++++---- .../signals/signal_rule_alert_type.ts | 4 +- .../threshold/build_signal_history.test.ts | 10 ++ .../signals/threshold/build_signal_history.ts | 64 +++++++++++++ .../lib/detection_engine/signals/types.ts | 4 + 10 files changed, 274 insertions(+), 21 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts index ed791af08890..e45d8440386f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts @@ -24,7 +24,7 @@ jest.mock('../utils/get_list_client', () => ({ jest.mock('../../rule_execution_log/rule_execution_log_client'); -describe('Custom query alerts', () => { +describe('Custom Query Alerts', () => { it('does not send an alert when no events found', async () => { const { services, dependencies, executor } = createRuleTypeMocks(); const queryAlertType = createQueryAlertType({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh new file mode 100644 index 000000000000..4b8371c3d07a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +curl -X POST ${KIBANA_URL}${SPACE_URL}/api/alerts/alert \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -H 'kbn-xsrf: true' \ + -H 'Content-Type: application/json' \ + --verbose \ + -d ' +{ + "params":{ + "author": [], + "description": "Basic threshold rule", + "exceptionsList": [], + "falsePositives": [], + "from": "now-300s", + "query": "*:*", + "immutable": false, + "index": ["*"], + "language": "kuery", + "maxSignals": 10, + "outputIndex": "", + "references": [], + "riskScore": 21, + "riskScoreMapping": [], + "ruleId": "52dec1ba-b779-469c-9667-6b0e865fb89a", + "severity": "low", + "severityMapping": [], + "threat": [], + "threshold": { + "field": ["host.id", "host.ip"], + "value": 5, + "cardinality": [ + { + "field": "source.ip", + "value": 11, + } + ] + }, + "to": "now", + "type": "threshold", + "version": 1 + }, + "consumer":"alerts", + "alertTypeId":"siem.thresholdRule", + "schedule":{ + "interval":"1m" + }, + "actions":[], + "tags":[ + "custom", + "persistence" + ], + "notifyWhen":"onActionGroupChange", + "name":"Basic threshold rule" +}' + + diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts new file mode 100644 index 000000000000..f3eb3154dfe3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +describe('Threshold Alerts', () => { + it('TODO', async () => {}); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts new file mode 100644 index 000000000000..f0663d9d15f8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { validateNonExact } from '@kbn/securitysolution-io-ts-utils'; + +import { PersistenceServices } from '../../../../../../rule_registry/server'; +import { THRESHOLD_ALERT_TYPE_ID } from '../../../../../common/constants'; +import { thresholdRuleParams, ThresholdRuleParams } from '../../schemas/rule_schemas'; +import { thresholdExecutor } from '../../signals/executors/threshold'; +import { ThresholdAlertState } from '../../signals/types'; +import { createSecurityRuleTypeFactory } from '../create_security_rule_type_factory'; +import { CreateRuleOptions } from '../types'; + +export const createThresholdAlertType = (createOptions: CreateRuleOptions) => { + const { + experimentalFeatures, + lists, + logger, + mergeStrategy, + ignoreFields, + ruleDataClient, + version, + ruleDataService, + } = createOptions; + const createSecurityRuleType = createSecurityRuleTypeFactory({ + lists, + logger, + mergeStrategy, + ignoreFields, + ruleDataClient, + ruleDataService, + }); + return createSecurityRuleType({ + id: THRESHOLD_ALERT_TYPE_ID, + name: 'Threshold Rule', + validate: { + params: { + validate: (object: unknown): ThresholdRuleParams => { + const [validated, errors] = validateNonExact(object, thresholdRuleParams); + if (errors != null) { + throw new Error(errors); + } + if (validated == null) { + throw new Error('Validation of rule params failed'); + } + return validated; + }, + }, + }, + actionGroups: [ + { + id: 'default', + name: 'Default', + }, + ], + defaultActionGroupId: 'default', + actionVariables: { + context: [{ name: 'server', description: 'the server' }], + }, + minimumLicenseRequired: 'basic', + isExportable: false, + producer: 'security-solution', + async executor(execOptions) { + const { + runOpts: { buildRuleMessage, bulkCreate, exceptionItems, rule, tuple, wrapHits }, + services, + startedAt, + state, + } = execOptions; + + const result = await thresholdExecutor({ + buildRuleMessage, + bulkCreate, + exceptionItems, + experimentalFeatures, + logger, + rule, + services, + startedAt, + state, + tuple, + version, + wrapHits, + }); + return { ...result, state }; + }, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index afcb3707591f..6d60608bf97c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -73,6 +73,7 @@ describe('threshold_executor', () => { exceptionItems, experimentalFeatures: allowedExperimentalValues, services: alertServices, + state: { signalHistory: {} }, // TODO version, logger, buildRuleMessage, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index ffd90f3b90b9..e4aedae66d08 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -7,7 +7,10 @@ import { Logger } from 'src/core/server'; import { SavedObject } from 'src/core/types'; + +import { SearchHit } from '@elastic/elasticsearch/api/types'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; + import { AlertInstanceContext, AlertInstanceState, @@ -21,13 +24,14 @@ import { bulkCreateThresholdSignals, findThresholdSignals, getThresholdBucketFilters, - getThresholdSignalHistory, } from '../threshold'; import { AlertAttributes, BulkCreate, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, + SignalSource, + ThresholdAlertState, WrapHits, } from '../types'; import { @@ -37,6 +41,7 @@ import { } from '../utils'; import { BuildRuleMessage } from '../rule_messages'; import { ExperimentalFeatures } from '../../../../../common/experimental_features'; +import { buildThresholdSignalHistory } from '../threshold/build_signal_history'; export const thresholdExecutor = async ({ rule, @@ -48,6 +53,7 @@ export const thresholdExecutor = async ({ logger, buildRuleMessage, startedAt, + state, bulkCreate, wrapHits, }: { @@ -60,11 +66,17 @@ export const thresholdExecutor = async ({ logger: Logger; buildRuleMessage: BuildRuleMessage; startedAt: Date; + state: ThresholdAlertState; bulkCreate: BulkCreate; wrapHits: WrapHits; -}): Promise => { +}): Promise => { let result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; + + const { signalHistory: thresholdSignalHistory } = state; + // TODO: clean up any signal history that has fallen outside the window + // TODO: handle case where we have no signal history? (upgrades) + if (hasLargeValueItem(exceptionItems)) { result.warningMessages.push( 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' @@ -78,21 +90,6 @@ export const thresholdExecutor = async ({ index: ruleParams.index, }); - const { - thresholdSignalHistory, - searchErrors: previousSearchErrors, - } = await getThresholdSignalHistory({ - indexPattern: [ruleParams.outputIndex], - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - ruleId: ruleParams.ruleId, - bucketByFields: ruleParams.threshold.field, - timestampOverride: ruleParams.timestampOverride, - buildRuleMessage, - }); - const bucketFilters = await getThresholdBucketFilters({ thresholdSignalHistory, timestampOverride: ruleParams.timestampOverride, @@ -154,12 +151,22 @@ export const thresholdExecutor = async ({ }), createSearchAfterReturnType({ success, - errors: [...errors, ...previousSearchErrors, ...searchErrors], + errors: [...errors, ...searchErrors], createdSignalsCount: createdItemsCount, createdSignals: createdItems, bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], searchAfterTimes: [thresholdSearchDuration], }), ]); - return result; + + return { + ...result, + state: { + ...state, + signalHistory: await buildThresholdSignalHistory({ + alerts: result.createdSignals as Array>, + bucketByFields: ruleParams.threshold.field, + }), + }, + }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 68d60f7757e4..9a6c099ed176 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -29,7 +29,7 @@ import { } from '../../../../common/detection_engine/utils'; import { SetupPlugins } from '../../../plugin'; import { getInputIndex } from './get_input_output_index'; -import { AlertAttributes, SignalRuleAlertTypeDefinition } from './types'; +import { AlertAttributes, SignalRuleAlertTypeDefinition, ThresholdAlertState } from './types'; import { getListsClient, getExceptions, @@ -125,6 +125,7 @@ export const signalRulesAlertType = ({ async executor({ previousStartedAt, startedAt, + state, alertId, services, params, @@ -316,6 +317,7 @@ export const signalRulesAlertType = ({ logger, buildRuleMessage, startedAt, + state: state as ThresholdAlertState, bulkCreate, wrapHits, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts new file mode 100644 index 000000000000..b15e025391e2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +describe('buildSignalHistory', () => { + it('TODO', () => {}); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts new file mode 100644 index 000000000000..845ce29015b5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchHit } from '@elastic/elasticsearch/api/types'; + +import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema'; +import { SignalSource, ThresholdSignalHistory } from '../types'; +import { getThresholdTermsHash } from '../utils'; + +interface GetThresholdSignalHistoryParams { + alerts: Array>; + bucketByFields: string[]; +} + +export const buildThresholdSignalHistory = async ({ + alerts, + bucketByFields, +}: GetThresholdSignalHistoryParams): Promise => { + const thresholdSignalHistory = alerts.reduce((acc, hit) => { + if (!hit._source) { + return acc; + } + + // TODO: get all values from `bucketByFields` on source. Store in terms. + const terms = + hit._source.signal?.threshold_result?.terms != null + ? hit._source.signal.threshold_result.terms + : [ + // Pre-7.12 signals + { + field: + (((hit._source.signal?.rule as RulesSchema).threshold as unknown) as { + field: string; + }).field ?? '', + value: ((hit._source.signal?.threshold_result as unknown) as { value: string }).value, + }, + ]; + + const hash = getThresholdTermsHash(terms); + const existing = acc[hash]; + const originalTime = + hit._source.signal?.original_time != null + ? new Date(hit._source.signal?.original_time).getTime() + : undefined; + + if (existing != null) { + if (originalTime && originalTime > existing.lastSignalTimestamp) { + acc[hash].lastSignalTimestamp = originalTime; + } + } else if (originalTime) { + acc[hash] = { + terms, + lastSignalTimestamp: originalTime, + }; + } + return acc; + }, {}); + + return thresholdSignalHistory; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 3dfe3fb650ec..2bb9b3579389 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -362,3 +362,7 @@ export interface ThresholdQueryBucket extends TermAggregationBucket { value_as_string: string; }; } + +export interface ThresholdAlertState extends AlertTypeState { + signalHistory: ThresholdSignalHistory; +} From 3ae81a7dbcd451dbdfdbf3d9fdc54ace08c8799f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Sep 2021 16:51:06 -0400 Subject: [PATCH 02/15] Properly handle signal history --- .../signals/executors/threshold.test.ts | 2 +- .../signals/executors/threshold.ts | 52 +++++++++++---- .../signals/threshold/build_signal_history.ts | 63 ++++++++++--------- .../bulk_create_threshold_signals.ts | 12 ++-- .../get_threshold_bucket_filters.test.ts | 2 +- .../threshold/get_threshold_bucket_filters.ts | 12 +++- .../threshold/get_threshold_signal_history.ts | 52 ++------------- .../lib/detection_engine/signals/types.ts | 1 + 8 files changed, 99 insertions(+), 97 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index 6d60608bf97c..5766390099e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -73,7 +73,7 @@ describe('threshold_executor', () => { exceptionItems, experimentalFeatures: allowedExperimentalValues, services: alertServices, - state: { signalHistory: {} }, // TODO + state: { initialized: true, signalHistory: {} }, version, logger, buildRuleMessage, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index e4aedae66d08..301c0df760b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -24,13 +24,13 @@ import { bulkCreateThresholdSignals, findThresholdSignals, getThresholdBucketFilters, + getThresholdSignalHistory, } from '../threshold'; import { AlertAttributes, BulkCreate, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, - SignalSource, ThresholdAlertState, WrapHits, } from '../types'; @@ -73,9 +73,33 @@ export const thresholdExecutor = async ({ let result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; - const { signalHistory: thresholdSignalHistory } = state; - // TODO: clean up any signal history that has fallen outside the window - // TODO: handle case where we have no signal history? (upgrades) + // Get state or build initial state (on upgrade) + const { signalHistory, searchErrors: previousSearchErrors } = state.initialized + ? { signalHistory: state.signalHistory, searchErrors: [] } + : await getThresholdSignalHistory({ + indexPattern: ['*'], // TODO: get outputIndex? + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + ruleId: ruleParams.ruleId, + bucketByFields: ruleParams.threshold.field, + timestampOverride: ruleParams.timestampOverride, + buildRuleMessage, + }); + + if (!state.initialized) { + // Clean up any signal history that has fallen outside the window + const toDelete: string[] = []; + for (const [hash, entry] of Object.entries(signalHistory)) { + if (entry.lastSignalTimestamp < tuple.from.valueOf()) { + toDelete.push(hash); + } + } + for (const hash of toDelete) { + delete signalHistory[hash]; + } + } if (hasLargeValueItem(exceptionItems)) { result.warningMessages.push( @@ -83,6 +107,7 @@ export const thresholdExecutor = async ({ ); result.warning = true; } + const inputIndex = await getInputIndex({ experimentalFeatures, services, @@ -91,7 +116,7 @@ export const thresholdExecutor = async ({ }); const bucketFilters = await getThresholdBucketFilters({ - thresholdSignalHistory, + signalHistory, timestampOverride: ruleParams.timestampOverride, }); @@ -138,7 +163,7 @@ export const thresholdExecutor = async ({ signalsIndex: ruleParams.outputIndex, startedAt, from: tuple.from.toDate(), - thresholdSignalHistory, + signalHistory, bulkCreate, wrapHits, }); @@ -151,7 +176,7 @@ export const thresholdExecutor = async ({ }), createSearchAfterReturnType({ success, - errors: [...errors, ...searchErrors], + errors: [...errors, ...previousSearchErrors, ...searchErrors], createdSignalsCount: createdItemsCount, createdSignals: createdItems, bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], @@ -159,14 +184,19 @@ export const thresholdExecutor = async ({ }), ]); + const newSignalHistory = await buildThresholdSignalHistory({ + alerts: result.createdSignals as Array>, + }); + return { ...result, state: { ...state, - signalHistory: await buildThresholdSignalHistory({ - alerts: result.createdSignals as Array>, - bucketByFields: ruleParams.threshold.field, - }), + initialized: true, + signalHistory: { + ...signalHistory, + ...newSignalHistory, + }, }, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts index 845ce29015b5..1a057aabd08b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts @@ -7,45 +7,52 @@ import { SearchHit } from '@elastic/elasticsearch/api/types'; -import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema'; -import { SignalSource, ThresholdSignalHistory } from '../types'; -import { getThresholdTermsHash } from '../utils'; +import { SimpleHit, ThresholdSignalHistory } from '../types'; +import { getThresholdTermsHash, isWrappedRACAlert, isWrappedSignalHit } from '../utils'; interface GetThresholdSignalHistoryParams { - alerts: Array>; - bucketByFields: string[]; + alerts: Array>; } +const getTerms = (alert: SimpleHit) => { + if (isWrappedRACAlert(alert)) { + return (alert._source['kibana.alert.threshold.field'] as string[]).map((field) => ({ + field, + value: alert._source[field] as string, + })); + } else if (isWrappedSignalHit(alert)) { + return alert._source.signal?.threshold_result?.terms ?? []; + } else { + // We shouldn't be here + return []; + } +}; + +const getOriginalTime = (alert: SimpleHit) => { + if (isWrappedRACAlert(alert)) { + const originalTime = alert._source['kibana.alert.original_time']; + return originalTime != null ? new Date(originalTime as string).getTime() : undefined; + } else if (isWrappedSignalHit(alert)) { + const originalTime = alert._source.signal?.original_time; + return originalTime != null ? new Date(originalTime).getTime() : undefined; + } else { + // We shouldn't be here + return undefined; + } +}; + export const buildThresholdSignalHistory = async ({ alerts, - bucketByFields, }: GetThresholdSignalHistoryParams): Promise => { - const thresholdSignalHistory = alerts.reduce((acc, hit) => { - if (!hit._source) { + const signalHistory = alerts.reduce((acc, alert) => { + if (!alert._source) { return acc; } - // TODO: get all values from `bucketByFields` on source. Store in terms. - const terms = - hit._source.signal?.threshold_result?.terms != null - ? hit._source.signal.threshold_result.terms - : [ - // Pre-7.12 signals - { - field: - (((hit._source.signal?.rule as RulesSchema).threshold as unknown) as { - field: string; - }).field ?? '', - value: ((hit._source.signal?.threshold_result as unknown) as { value: string }).value, - }, - ]; - + const terms = getTerms(alert as SimpleHit); const hash = getThresholdTermsHash(terms); const existing = acc[hash]; - const originalTime = - hit._source.signal?.original_time != null - ? new Date(hit._source.signal?.original_time).getTime() - : undefined; + const originalTime = getOriginalTime(alert as SimpleHit); if (existing != null) { if (originalTime && originalTime > existing.lastSignalTimestamp) { @@ -60,5 +67,5 @@ export const buildThresholdSignalHistory = async ({ return acc; }, {}); - return thresholdSignalHistory; + return signalHistory; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts index afb0353c4ba0..ce8ee4542d60 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts @@ -46,7 +46,7 @@ interface BulkCreateThresholdSignalsParams { signalsIndex: string; startedAt: Date; from: Date; - thresholdSignalHistory: ThresholdSignalHistory; + signalHistory: ThresholdSignalHistory; bulkCreate: BulkCreate; wrapHits: WrapHits; } @@ -61,7 +61,7 @@ const getTransformedHits = ( ruleId: string, filter: unknown, timestampOverride: TimestampOverrideOrUndefined, - thresholdSignalHistory: ThresholdSignalHistory + signalHistory: ThresholdSignalHistory ) => { const aggParts = threshold.field.length ? results.aggregations && getThresholdAggregationParts(results.aggregations) @@ -148,7 +148,7 @@ const getTransformedHits = ( } const termsHash = getThresholdTermsHash(bucket.terms); - const signalHit = thresholdSignalHistory[termsHash]; + const signalHit = signalHistory[termsHash]; const source = { '@timestamp': timestamp, @@ -202,7 +202,7 @@ export const transformThresholdResultsToEcs = ( threshold: ThresholdNormalized, ruleId: string, timestampOverride: TimestampOverrideOrUndefined, - thresholdSignalHistory: ThresholdSignalHistory + signalHistory: ThresholdSignalHistory ): SignalSearchResponse => { const transformedHits = getTransformedHits( results, @@ -214,7 +214,7 @@ export const transformThresholdResultsToEcs = ( ruleId, filter, timestampOverride, - thresholdSignalHistory + signalHistory ); const thresholdResults = { ...results, @@ -246,7 +246,7 @@ export const bulkCreateThresholdSignals = async ( ruleParams.threshold, ruleParams.ruleId, ruleParams.timestampOverride, - params.thresholdSignalHistory + params.signalHistory ); return params.bulkCreate( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.test.ts index d621868a0956..e67a6fa3dfa9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.test.ts @@ -11,7 +11,7 @@ import { getThresholdBucketFilters } from './get_threshold_bucket_filters'; describe('getThresholdBucketFilters', () => { it('should generate filters for threshold signal detection with dupe mitigation', async () => { const result = await getThresholdBucketFilters({ - thresholdSignalHistory: sampleThresholdSignalHistory(), + signalHistory: sampleThresholdSignalHistory(), timestampOverride: undefined, }); expect(result).toEqual([ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.ts index e6a188a20b5d..c4569ef9818d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.ts @@ -9,14 +9,18 @@ import { Filter } from 'src/plugins/data/common'; import { ESFilter } from '../../../../../../../../src/core/types/elasticsearch'; import { ThresholdSignalHistory, ThresholdSignalHistoryRecord } from '../types'; +/* + * Returns a filter to exclude events that have already been included in a + * previous threshold signal. Uses the threshold signal history to achieve this. + */ export const getThresholdBucketFilters = async ({ - thresholdSignalHistory, + signalHistory, timestampOverride, }: { - thresholdSignalHistory: ThresholdSignalHistory; + signalHistory: ThresholdSignalHistory; timestampOverride: string | undefined; }): Promise => { - const filters = Object.values(thresholdSignalHistory).reduce( + const filters = Object.values(signalHistory).reduce( (acc: ESFilter[], bucket: ThresholdSignalHistoryRecord): ESFilter[] => { const filter = { bool: { @@ -24,6 +28,7 @@ export const getThresholdBucketFilters = async ({ { range: { [timestampOverride ?? '@timestamp']: { + // Timestamp of last event signaled on for this set of terms. lte: new Date(bucket.lastSignalTimestamp).toISOString(), }, }, @@ -32,6 +37,7 @@ export const getThresholdBucketFilters = async ({ }, } as ESFilter; + // Terms to filter events older than `lastSignalTimestamp`. bucket.terms.forEach((term) => { if (term.field != null) { (filter.bool!.filter as ESFilter[]).push({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts index b93d8423c025..9ef782e3c0bf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema'; import { TimestampOverrideOrUndefined } from '../../../../../common/detection_engine/schemas/common/schemas'; import { AlertInstanceContext, @@ -16,7 +15,7 @@ import { Logger } from '../../../../../../../../src/core/server'; import { ThresholdSignalHistory } from '../types'; import { BuildRuleMessage } from '../rule_messages'; import { findPreviousThresholdSignals } from './find_previous_threshold_signals'; -import { getThresholdTermsHash } from '../utils'; +import { buildThresholdSignalHistory } from './build_signal_history'; interface GetThresholdSignalHistoryParams { from: string; @@ -41,7 +40,7 @@ export const getThresholdSignalHistory = async ({ timestampOverride, buildRuleMessage, }: GetThresholdSignalHistoryParams): Promise<{ - thresholdSignalHistory: ThresholdSignalHistory; + signalHistory: ThresholdSignalHistory; searchErrors: string[]; }> => { const { searchResult, searchErrors } = await findPreviousThresholdSignals({ @@ -56,51 +55,10 @@ export const getThresholdSignalHistory = async ({ buildRuleMessage, }); - const thresholdSignalHistory = searchResult.hits.hits.reduce( - (acc, hit) => { - if (!hit._source) { - return acc; - } - - const terms = - hit._source.signal?.threshold_result?.terms != null - ? hit._source.signal.threshold_result.terms - : [ - // Pre-7.12 signals - { - field: - (((hit._source.signal?.rule as RulesSchema).threshold as unknown) as { - field: string; - }).field ?? '', - value: ((hit._source.signal?.threshold_result as unknown) as { value: string }) - .value, - }, - ]; - - const hash = getThresholdTermsHash(terms); - const existing = acc[hash]; - const originalTime = - hit._source.signal?.original_time != null - ? new Date(hit._source.signal?.original_time).getTime() - : undefined; - - if (existing != null) { - if (originalTime && originalTime > existing.lastSignalTimestamp) { - acc[hash].lastSignalTimestamp = originalTime; - } - } else if (originalTime) { - acc[hash] = { - terms, - lastSignalTimestamp: originalTime, - }; - } - return acc; - }, - {} - ); - return { - thresholdSignalHistory, + signalHistory: await buildThresholdSignalHistory({ + alerts: searchResult.hits.hits, + }), searchErrors, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 2bb9b3579389..d237fca3541a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -364,5 +364,6 @@ export interface ThresholdQueryBucket extends TermAggregationBucket { } export interface ThresholdAlertState extends AlertTypeState { + initialized: boolean; signalHistory: ThresholdSignalHistory; } From ee7ce2abba306f0a2753adbda9e94112f63d0b21 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 8 Sep 2021 09:14:59 -0400 Subject: [PATCH 03/15] Fix #95258 - cardinality sort bug --- .../signals/threshold/find_threshold_signals.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts index ca7f22e4a757..927a8c1f0005 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts @@ -89,6 +89,11 @@ export const findThresholdSignals = async ({ const thresholdFields = threshold.field; + // order buckets by cardinality (https://github.com/elastic/kibana/issues/95258) + const orderByCardinality = threshold.cardinality?.length + ? { order: { cardinality_count: 'desc' } } + : {}; + // Generate a nested terms aggregation for each threshold grouping field provided, appending leaf // aggregations to 1) filter out buckets that don't meet the cardinality threshold, if provided, and // 2) return the latest hit for each bucket so that we can persist the timestamp of the event in the @@ -104,6 +109,7 @@ export const findThresholdSignals = async ({ set(acc, aggPath, { terms: { field, + ...orderByCardinality, min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set size: 10000, // max 10k buckets }, @@ -121,6 +127,7 @@ export const findThresholdSignals = async ({ source: '""', // Group everything in the same bucket lang: 'painless', }, + ...orderByCardinality, min_doc_count: threshold.value, }, aggs: leafAggs, From e28c671db8a3cd3eddbb04e94be764132051ee64 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 8 Sep 2021 13:37:46 -0400 Subject: [PATCH 04/15] Init threshold rule --- .../server/lib/detection_engine/rule_types/index.ts | 1 + x-pack/plugins/security_solution/server/plugin.ts | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/index.ts index 0fde90c991e4..661737b263c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/index.ts @@ -8,3 +8,4 @@ export { createQueryAlertType } from './query/create_query_alert_type'; export { createIndicatorMatchAlertType } from './indicator_match/create_indicator_match_alert_type'; export { createMlAlertType } from './ml/create_ml_alert_type'; +export { createThresholdAlertType } from './threshold/create_threshold_alert_type'; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 4e4d0be5a741..2c3201c1db5a 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -48,8 +48,12 @@ import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { ILicense, LicensingPluginStart } from '../../licensing/server'; import { FleetStartContract } from '../../fleet/server'; import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; -import { createQueryAlertType } from './lib/detection_engine/rule_types'; -import { createMlAlertType } from './lib/detection_engine/rule_types/ml/create_ml_alert_type'; +import { + createIndicatorMatchAlertType, + createMlAlertType, + createQueryAlertType, + createThresholdAlertType, +} from './lib/detection_engine/rule_types'; import { initRoutes } from './routes'; import { isAlertExecutor } from './lib/detection_engine/signals/types'; import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule_alert_type'; @@ -96,7 +100,6 @@ import { rulesFieldMap } from './lib/detection_engine/rule_types/field_maps/rule import { RuleExecutionLogClient } from './lib/detection_engine/rule_execution_log/rule_execution_log_client'; import { getKibanaPrivilegesFeaturePrivileges } from './features'; import { EndpointMetadataService } from './endpoint/services/metadata'; -import { createIndicatorMatchAlertType } from './lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type'; import { CreateRuleOptions } from './lib/detection_engine/rule_types/types'; import { ctiFieldMap } from './lib/detection_engine/rule_types/field_maps/cti'; @@ -262,6 +265,7 @@ export class Plugin implements IPlugin Date: Thu, 9 Sep 2021 11:47:44 -0400 Subject: [PATCH 05/15] Create working threshold rule --- .../field_maps/technical_rule_field_map.ts | 2 +- .../schemas/response/rules_schema.ts | 1 + .../rule_types/factories/utils/build_alert.ts | 5 +- .../rule_types/field_maps/alerts.ts | 70 ------------------- .../rule_types/field_maps/rules.ts | 70 +++++++++++++++++++ .../scripts/create_rule_threshold.sh | 6 +- .../signals/single_search_after.ts | 2 + .../threshold/find_threshold_signals.ts | 12 ++-- 8 files changed, 88 insertions(+), 80 deletions(-) diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts index 3c8b96976792..54a4b80a35bb 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts @@ -115,7 +115,7 @@ export const technicalRuleFieldMap = { required: false, }, [Fields.ALERT_RULE_FROM]: { - type: 'date', + type: 'keyword', array: false, required: false, }, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 0efd6dc5067c..5e5322aac031 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -175,6 +175,7 @@ export const partialRulesSchema = t.partial({ meta, index, note, + uuid: id, // Move to 'required' post-migration }); /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index 1377e9ef9207..9b4db33cc763 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -105,6 +105,9 @@ export const buildAlert = ( [] ); + const { actions, id, output_index: outputIndex, throttle, ...mappedRule } = rule; + mappedRule.uuid = id; + return ({ '@timestamp': new Date().toISOString(), [ALERT_RULE_CONSUMER]: SERVER_APP_ID, @@ -114,7 +117,7 @@ export const buildAlert = ( [ALERT_WORKFLOW_STATUS]: 'open', [ALERT_DEPTH]: depth, [ALERT_REASON]: reason, - ...flattenWithPrefix(ALERT_RULE_NAMESPACE, rule), + ...flattenWithPrefix(ALERT_RULE_NAMESPACE, mappedRule as RulesSchema), } as unknown) as RACAlert; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts index 1c4b7f03fd73..c3d6a79a59e4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts @@ -198,76 +198,6 @@ export const alertsFieldMap: FieldMap = { array: false, required: false, }, - 'kibana.alert.threat': { - type: 'object', - array: false, - required: false, - }, - 'kibana.alert.threat.framework': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.tactic': { - type: 'object', - array: false, - required: true, - }, - 'kibana.alert.threat.tactic.id': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.tactic.name': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.tactic.reference': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique': { - type: 'object', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.id': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.name': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.reference': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.subtechnique': { - type: 'object', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.subtechnique.id': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.subtechnique.name': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.subtechnique.reference': { - type: 'keyword', - array: false, - required: true, - }, 'kibana.alert.threshold_result': { type: 'object', array: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts index 21405672fdf7..00b1db13829d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts @@ -11,6 +11,11 @@ export const rulesFieldMap = { array: false, required: false, }, + 'kibana.alert.rule.exceptions_list': { + type: 'object', + array: true, + required: false, + }, 'kibana.alert.rule.false_positives': { type: 'keyword', array: true, @@ -46,6 +51,71 @@ export const rulesFieldMap = { array: true, required: true, }, + 'kibana.alert.rule.threat.framework': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.tactic': { + type: 'object', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.tactic.id': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.tactic.name': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.tactic.reference': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique': { + type: 'object', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.id': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.name': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.reference': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.subtechnique': { + type: 'object', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.subtechnique.id': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.subtechnique.name': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.subtechnique.reference': { + type: 'keyword', + array: false, + required: true, + }, 'kibana.alert.rule.threat_filters': { type: 'keyword', array: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh index 4b8371c3d07a..47c5cb4eda2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh @@ -33,12 +33,12 @@ curl -X POST ${KIBANA_URL}${SPACE_URL}/api/alerts/alert \ "severityMapping": [], "threat": [], "threshold": { - "field": ["host.id", "host.ip"], - "value": 5, + "field": ["source.ip"], + "value": 2, "cardinality": [ { "field": "source.ip", - "value": 11, + "value": 1 } ] }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index ff49fb5892f5..7b30f70701e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -75,9 +75,11 @@ export const singleSearchAfter = async ({ searchAfterQuery as estypes.SearchRequest ); const end = performance.now(); + const searchErrors = createErrorsFromShard({ errors: nextSearchAfterResult._shards.failures ?? [], }); + return { searchResult: nextSearchAfterResult, searchDuration: makeFloatString(end - start), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts index 927a8c1f0005..740ba281cfcf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts @@ -90,9 +90,11 @@ export const findThresholdSignals = async ({ const thresholdFields = threshold.field; // order buckets by cardinality (https://github.com/elastic/kibana/issues/95258) - const orderByCardinality = threshold.cardinality?.length - ? { order: { cardinality_count: 'desc' } } - : {}; + const thresholdFieldCount = thresholdFields.length; + const orderByCardinality = (i: number = 0) => + (thresholdFieldCount === 0 || i === thresholdFieldCount - 1) && threshold.cardinality?.length + ? { order: { cardinality_count: 'desc' } } + : {}; // Generate a nested terms aggregation for each threshold grouping field provided, appending leaf // aggregations to 1) filter out buckets that don't meet the cardinality threshold, if provided, and @@ -109,7 +111,7 @@ export const findThresholdSignals = async ({ set(acc, aggPath, { terms: { field, - ...orderByCardinality, + ...orderByCardinality(i), min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set size: 10000, // max 10k buckets }, @@ -127,7 +129,7 @@ export const findThresholdSignals = async ({ source: '""', // Group everything in the same bucket lang: 'painless', }, - ...orderByCardinality, + ...orderByCardinality(), min_doc_count: threshold.value, }, aggs: leafAggs, From b29aee992db0ca1b40b49fa0962504a07e235268 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 9 Sep 2021 13:50:22 -0400 Subject: [PATCH 06/15] Fix threshold signal generation --- .../factories/bulk_create_factory.ts | 16 ++++++++++----- .../factories/utils/flatten_with_prefix.ts | 20 ++++++++++++++----- .../threshold/create_threshold_alert_type.ts | 5 ++++- .../signals/executors/threshold.ts | 19 ++++++++++++++---- .../signals/threshold/build_signal_history.ts | 2 +- 5 files changed, 46 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts index c600e187bc8f..cd7a5150ad38 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; + import { performance } from 'perf_hooks'; import { countBy, isEmpty } from 'lodash'; @@ -62,11 +64,15 @@ export const bulkCreateFactory = ( ); const createdItems = wrappedDocs - .map((doc, index) => ({ - _id: response.body.items[index].index?._id ?? '', - _index: response.body.items[index].index?._index ?? '', - ...doc._source, - })) + .map((doc, index) => { + const responseIndex = response.body.items[index].index; + return { + _id: responseIndex?._id ?? '', + _index: responseIndex?._index ?? '', + [ALERT_INSTANCE_ID]: responseIndex?._id ?? '', + ...doc._source, + }; + }) .filter((_, index) => response.body.items[index].index?.status === 201); const createdItemsCount = createdItems.length; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts index d472dc5885e5..02f418a15188 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts @@ -5,16 +5,26 @@ * 2.0. */ +import { isPlainObject } from 'lodash'; import { SearchTypes } from '../../../../../../common/detection_engine/types'; export const flattenWithPrefix = ( prefix: string, - obj: Record + maybeObj: unknown ): Record => { - return Object.keys(obj).reduce((acc: Record, key) => { + if (maybeObj != null && isPlainObject(maybeObj)) { + return Object.keys(maybeObj as Record).reduce( + (acc: Record, key) => { + return { + ...acc, + ...flattenWithPrefix(`${prefix}.${key}`, (maybeObj as Record)[key]), + }; + }, + {} + ); + } else { return { - ...acc, - [`${prefix}.${key}`]: obj[key], + [prefix]: maybeObj as SearchTypes, }; - }, {}); + } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts index f0663d9d15f8..570660a432e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts @@ -72,6 +72,8 @@ export const createThresholdAlertType = (createOptions: CreateRuleOptions) => { state, } = execOptions; + // console.log(JSON.stringify(state)); + const result = await thresholdExecutor({ buildRuleMessage, bulkCreate, @@ -86,7 +88,8 @@ export const createThresholdAlertType = (createOptions: CreateRuleOptions) => { version, wrapHits, }); - return { ...result, state }; + + return result; }, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index 301c0df760b3..887c88f4e3b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -5,12 +5,12 @@ * 2.0. */ -import { Logger } from 'src/core/server'; -import { SavedObject } from 'src/core/types'; - import { SearchHit } from '@elastic/elasticsearch/api/types'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { Logger } from 'src/core/server'; +import { SavedObject } from 'src/core/types'; + import { AlertInstanceContext, AlertInstanceState, @@ -184,8 +184,19 @@ export const thresholdExecutor = async ({ }), ]); + const createdAlerts = createdItems.map((alert) => { + const { _id, _index, ...source } = alert as { _id: string; _index: string }; + return { + _id, + _index, + _source: { + ...source, + }, + } as SearchHit; + }); + const newSignalHistory = await buildThresholdSignalHistory({ - alerts: result.createdSignals as Array>, + alerts: createdAlerts, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts index 1a057aabd08b..e108174aff5b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts @@ -16,7 +16,7 @@ interface GetThresholdSignalHistoryParams { const getTerms = (alert: SimpleHit) => { if (isWrappedRACAlert(alert)) { - return (alert._source['kibana.alert.threshold.field'] as string[]).map((field) => ({ + return (alert._source['kibana.alert.rule.threshold.field'] as string[]).map((field) => ({ field, value: alert._source[field] as string, })); From 851301b2a545016e4dde1606a4620e8cae539269 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 9 Sep 2021 15:29:29 -0400 Subject: [PATCH 07/15] Fix tests --- .../rule_types/factories/utils/build_alert.test.ts | 6 ++---- .../signals/threshold/find_threshold_signals.test.ts | 2 ++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts index a95da275fc53..a8e350cd3b5e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts @@ -76,7 +76,7 @@ describe('buildAlert', () => { [ALERT_WORKFLOW_STATUS]: 'open', ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { author: [], - id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + uuid: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', created_at: new Date(ANCHOR_DATE).toISOString(), updated_at: new Date(ANCHOR_DATE).toISOString(), created_by: 'elastic', @@ -100,7 +100,6 @@ describe('buildAlert', () => { status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], @@ -153,7 +152,7 @@ describe('buildAlert', () => { [ALERT_WORKFLOW_STATUS]: 'open', ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { author: [], - id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + uuid: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', created_at: new Date(ANCHOR_DATE).toISOString(), updated_at: new Date(ANCHOR_DATE).toISOString(), created_by: 'elastic', @@ -177,7 +176,6 @@ describe('buildAlert', () => { status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.test.ts index e84b4f31fb15..41d46925770b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.test.ts @@ -227,6 +227,7 @@ describe('findThresholdSignals', () => { 'threshold_1:user.name': { terms: { field: 'user.name', + order: { cardinality_count: 'desc' }, min_doc_count: 100, size: 10000, }, @@ -302,6 +303,7 @@ describe('findThresholdSignals', () => { lang: 'painless', }, min_doc_count: 200, + order: { cardinality_count: 'desc' }, }, aggs: { cardinality_count: { From 847876f4986393ae849631683bf8426177d51508 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 9 Sep 2021 15:51:19 -0400 Subject: [PATCH 08/15] Update mappings --- .../rule_types/field_maps/alerts.ts | 17 +----------- .../rule_types/field_maps/rules.ts | 26 +++---------------- 2 files changed, 4 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts index c3d6a79a59e4..f21fc5b6ad39 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts @@ -43,11 +43,6 @@ export const alertsFieldMap: FieldMap = { array: false, required: true, }, - 'kibana.alert.group': { - type: 'object', - array: false, - required: false, - }, 'kibana.alert.group.id': { type: 'keyword', array: false, @@ -58,11 +53,6 @@ export const alertsFieldMap: FieldMap = { array: false, required: false, }, - 'kibana.alert.original_event': { - type: 'object', - array: false, - required: false, - }, 'kibana.alert.original_event.action': { type: 'keyword', array: false, @@ -198,11 +188,6 @@ export const alertsFieldMap: FieldMap = { array: false, required: false, }, - 'kibana.alert.threshold_result': { - type: 'object', - array: false, - required: false, - }, 'kibana.alert.threshold_result.cardinality': { type: 'object', array: false, @@ -230,7 +215,7 @@ export const alertsFieldMap: FieldMap = { }, 'kibana.alert.threshold_result.terms': { type: 'object', - array: false, + array: true, required: false, }, 'kibana.alert.threshold_result.terms.field': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts index 00b1db13829d..87b55e092ec5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts @@ -56,11 +56,6 @@ export const rulesFieldMap = { array: false, required: true, }, - 'kibana.alert.rule.threat.tactic': { - type: 'object', - array: false, - required: true, - }, 'kibana.alert.rule.threat.tactic.id': { type: 'keyword', array: false, @@ -76,11 +71,6 @@ export const rulesFieldMap = { array: false, required: true, }, - 'kibana.alert.rule.threat.technique': { - type: 'object', - array: false, - required: true, - }, 'kibana.alert.rule.threat.technique.id': { type: 'keyword', array: false, @@ -96,11 +86,6 @@ export const rulesFieldMap = { array: false, required: true, }, - 'kibana.alert.rule.threat.technique.subtechnique': { - type: 'object', - array: false, - required: true, - }, 'kibana.alert.rule.threat.technique.subtechnique.id': { type: 'keyword', array: false, @@ -161,11 +146,6 @@ export const rulesFieldMap = { array: true, required: false, }, - 'kibana.alert.rule.threshold': { - type: 'object', - array: true, - required: false, - }, 'kibana.alert.rule.threshold.field': { type: 'keyword', array: true, @@ -173,7 +153,7 @@ export const rulesFieldMap = { }, 'kibana.alert.rule.threshold.value': { type: 'float', // TODO: should be 'long' (eventually, after we stabilize) - array: true, + array: false, required: false, }, 'kibana.alert.rule.threshold.cardinality': { @@ -183,12 +163,12 @@ export const rulesFieldMap = { }, 'kibana.alert.rule.threshold.cardinality.field': { type: 'keyword', - array: true, + array: false, required: false, }, 'kibana.alert.rule.threshold.cardinality.value': { type: 'long', - array: true, + array: false, required: false, }, 'kibana.alert.rule.timeline_id': { From 0e7676d14895acb3ab7048c431058a77fe22b77c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 9 Sep 2021 22:23:38 -0400 Subject: [PATCH 09/15] ALERT_TYPE_ID => RULE_TYPE_ID --- .../rule_types/threshold/create_threshold_alert_type.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts index 570660a432e8..a503cf5aedbe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts @@ -8,7 +8,7 @@ import { validateNonExact } from '@kbn/securitysolution-io-ts-utils'; import { PersistenceServices } from '../../../../../../rule_registry/server'; -import { THRESHOLD_ALERT_TYPE_ID } from '../../../../../common/constants'; +import { THRESHOLD_RULE_TYPE_ID } from '../../../../../common/constants'; import { thresholdRuleParams, ThresholdRuleParams } from '../../schemas/rule_schemas'; import { thresholdExecutor } from '../../signals/executors/threshold'; import { ThresholdAlertState } from '../../signals/types'; @@ -35,7 +35,7 @@ export const createThresholdAlertType = (createOptions: CreateRuleOptions) => { ruleDataService, }); return createSecurityRuleType({ - id: THRESHOLD_ALERT_TYPE_ID, + id: THRESHOLD_RULE_TYPE_ID, name: 'Threshold Rule', validate: { params: { From 2082b3bc23c9a2082e285d4c8dd00a98f41f8c35 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 13 Sep 2021 10:18:57 -0400 Subject: [PATCH 10/15] Add tests --- .../rule_types/__mocks__/threshold.ts | 104 ++++++++++++++++++ .../rule_types/field_maps/field_names.ts | 5 +- .../create_threshold_alert_type.test.ts | 26 ++++- .../signals/executors/threshold.ts | 2 +- .../threshold/build_signal_history.test.ts | 25 ++++- .../signals/threshold/build_signal_history.ts | 12 +- .../threshold/get_threshold_signal_history.ts | 2 +- 7 files changed, 167 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts index 40d2ed37a557..25d53f833b08 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts @@ -5,7 +5,35 @@ * 2.0. */ +import { + SPACE_IDS, + ALERT_RULE_CONSUMER, + ALERT_REASON, + ALERT_STATUS, + ALERT_STATUS_ACTIVE, + ALERT_WORKFLOW_STATUS, + ALERT_RULE_NAMESPACE, + ALERT_INSTANCE_ID, + ALERT_UUID, + ALERT_RULE_TYPE_ID, + ALERT_RULE_PRODUCER, + ALERT_RULE_CATEGORY, + ALERT_RULE_UUID, + ALERT_RULE_NAME, +} from '@kbn/rule-data-utils'; +import { TypeOfFieldMap } from '../../../../../../rule_registry/common/field_map'; +import { SERVER_APP_ID } from '../../../../../common/constants'; +import { ANCHOR_DATE } from '../../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock'; import { sampleDocNoSortId } from '../../signals/__mocks__/es_results'; +import { flattenWithPrefix } from '../factories/utils/flatten_with_prefix'; +import { RulesFieldMap } from '../field_maps'; +import { + ALERT_ANCESTORS, + ALERT_ORIGINAL_TIME, + ALERT_ORIGINAL_EVENT, +} from '../field_maps/field_names'; +import { RACAlert, WrappedRACAlert } from '../types'; export const mockThresholdResults = { rawResponse: { @@ -59,3 +87,79 @@ export const mockThresholdResults = { }, }, }; + +export const sampleThresholdAlert: WrappedRACAlert = { + _id: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d', + _index: 'some-index', + _source: { + '@timestamp': '2020-04-20T21:26:30.000Z', + [SPACE_IDS]: ['default'], + [ALERT_INSTANCE_ID]: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d', + [ALERT_UUID]: '310158f7-994d-4a38-8cdc-152139ac4d29', + [ALERT_RULE_CONSUMER]: SERVER_APP_ID, + [ALERT_ANCESTORS]: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', + [ALERT_ORIGINAL_EVENT]: { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }, + [ALERT_REASON]: 'alert reasonable reason', + [ALERT_STATUS]: ALERT_STATUS_ACTIVE, + [ALERT_WORKFLOW_STATUS]: 'open', + 'source.ip': '127.0.0.1', + 'host.name': 'garden-gnomes', + [ALERT_RULE_CATEGORY]: 'security', + [ALERT_RULE_NAME]: 'a threshold rule', + [ALERT_RULE_PRODUCER]: 'siem', + [ALERT_RULE_TYPE_ID]: 'query-rule-id', + [ALERT_RULE_UUID]: '151af49f-2e82-4b6f-831b-7f8cb341a5ff', + ...(flattenWithPrefix(ALERT_RULE_NAMESPACE, { + author: [], + uuid: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), + created_by: 'elastic', + description: 'some description', + enabled: true, + false_positives: ['false positive 1', 'false positive 2'], + from: 'now-6m', + immutable: false, + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + references: ['test 1', 'test 2'], + severity: 'high', + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [], + threshold: { + field: ['source.ip', 'host.name'], + value: 1, + }, + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'f88a544c-1d4e-4652-ae2a-c953b38da5d0', + interval: '5m', + exceptions_list: getListArrayMock(), + }) as TypeOfFieldMap), + 'kibana.alert.depth': 1, + }, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts index 41b7e6b02c9c..7ffbf029fbd0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts @@ -5,9 +5,12 @@ * 2.0. */ -import { ALERT_NAMESPACE } from '@kbn/rule-data-utils'; +import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from '@kbn/rule-data-utils'; export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors`; export const ALERT_DEPTH = `${ALERT_NAMESPACE}.depth`; export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event`; export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time`; + +const ALERT_RULE_THRESHOLD = `${ALERT_RULE_NAMESPACE}.threshold`; +export const ALERT_RULE_THRESHOLD_FIELD = `${ALERT_RULE_THRESHOLD}.field`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts index f3eb3154dfe3..74435cb30047 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts @@ -5,6 +5,30 @@ * 2.0. */ +import { allowedExperimentalValues } from '../../../../../common/experimental_features'; +import { createThresholdAlertType } from './create_threshold_alert_type'; +import { createRuleTypeMocks } from '../__mocks__/rule_type'; +import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock'; + +jest.mock('../../rule_execution_log/rule_execution_log_client'); + describe('Threshold Alerts', () => { - it('TODO', async () => {}); + it('does not send an alert when no events found', async () => { + const params = getThresholdRuleParams(); + const { dependencies, executor } = createRuleTypeMocks('threshold', params); + const thresholdAlertTpe = createThresholdAlertType({ + experimentalFeatures: allowedExperimentalValues, + lists: dependencies.lists, + logger: dependencies.logger, + mergeStrategy: 'allFields', + ignoreFields: [], + ruleDataClient: dependencies.ruleDataClient, + ruleDataService: dependencies.ruleDataService, + version: '1.0.0', + }); + dependencies.alerting.registerType(thresholdAlertTpe); + + await executor({ params }); + expect(dependencies.ruleDataClient.getWriter).not.toBeCalled(); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index 887c88f4e3b6..524bc6a6c524 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -195,7 +195,7 @@ export const thresholdExecutor = async ({ } as SearchHit; }); - const newSignalHistory = await buildThresholdSignalHistory({ + const newSignalHistory = buildThresholdSignalHistory({ alerts: createdAlerts, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts index b15e025391e2..8362942af15b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts @@ -5,6 +5,29 @@ * 2.0. */ +import { ALERT_ORIGINAL_TIME } from '../../rule_types/field_maps/field_names'; +import { sampleThresholdAlert } from '../../rule_types/__mocks__/threshold'; +import { buildThresholdSignalHistory } from './build_signal_history'; + describe('buildSignalHistory', () => { - it('TODO', () => {}); + it('builds a signal history from an alert', () => { + const signalHistory = buildThresholdSignalHistory({ alerts: [sampleThresholdAlert] }); + expect(signalHistory).toEqual({ + '7a75c5c2db61f57ec166c669cb8244b91f812f0b2f1d4f8afd528d4f8b4e199b': { + lastSignalTimestamp: Date.parse( + sampleThresholdAlert._source[ALERT_ORIGINAL_TIME] as string + ), + terms: [ + { + field: 'host.name', + value: 'garden-gnomes', + }, + { + field: 'source.ip', + value: '127.0.0.1', + }, + ], + }, + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts index e108174aff5b..81b12d2d4f22 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts @@ -6,6 +6,10 @@ */ import { SearchHit } from '@elastic/elasticsearch/api/types'; +import { + ALERT_ORIGINAL_TIME, + ALERT_RULE_THRESHOLD_FIELD, +} from '../../rule_types/field_maps/field_names'; import { SimpleHit, ThresholdSignalHistory } from '../types'; import { getThresholdTermsHash, isWrappedRACAlert, isWrappedSignalHit } from '../utils'; @@ -16,7 +20,7 @@ interface GetThresholdSignalHistoryParams { const getTerms = (alert: SimpleHit) => { if (isWrappedRACAlert(alert)) { - return (alert._source['kibana.alert.rule.threshold.field'] as string[]).map((field) => ({ + return (alert._source[ALERT_RULE_THRESHOLD_FIELD] as string[]).map((field) => ({ field, value: alert._source[field] as string, })); @@ -30,7 +34,7 @@ const getTerms = (alert: SimpleHit) => { const getOriginalTime = (alert: SimpleHit) => { if (isWrappedRACAlert(alert)) { - const originalTime = alert._source['kibana.alert.original_time']; + const originalTime = alert._source[ALERT_ORIGINAL_TIME]; return originalTime != null ? new Date(originalTime as string).getTime() : undefined; } else if (isWrappedSignalHit(alert)) { const originalTime = alert._source.signal?.original_time; @@ -41,9 +45,9 @@ const getOriginalTime = (alert: SimpleHit) => { } }; -export const buildThresholdSignalHistory = async ({ +export const buildThresholdSignalHistory = ({ alerts, -}: GetThresholdSignalHistoryParams): Promise => { +}: GetThresholdSignalHistoryParams): ThresholdSignalHistory => { const signalHistory = alerts.reduce((acc, alert) => { if (!alert._source) { return acc; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts index 9ef782e3c0bf..276431c3bc92 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts @@ -56,7 +56,7 @@ export const getThresholdSignalHistory = async ({ }); return { - signalHistory: await buildThresholdSignalHistory({ + signalHistory: buildThresholdSignalHistory({ alerts: searchResult.hits.hits, }), searchErrors, From a352cbb14d2b076e5f1034891f23b2dbe7b250e2 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 13 Sep 2021 12:25:48 -0400 Subject: [PATCH 11/15] Fix types --- .../lib/detection_engine/rule_types/__mocks__/threshold.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts index 25d53f833b08..554672806c12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts @@ -33,7 +33,7 @@ import { ALERT_ORIGINAL_TIME, ALERT_ORIGINAL_EVENT, } from '../field_maps/field_names'; -import { RACAlert, WrappedRACAlert } from '../types'; +import { WrappedRACAlert } from '../types'; export const mockThresholdResults = { rawResponse: { From f25067684c5b22685461f1623c58d243356629a7 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 14 Sep 2021 10:19:43 -0400 Subject: [PATCH 12/15] Fix threshold tests (remove outputIndex) --- .../detection_engine/rule_types/factories/utils/build_alert.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index 5fbca2dc6178..d39d6fa2eb80 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -120,7 +120,7 @@ export const buildAlert = ( [] ); - const { id, ...mappedRule } = rule; + const { id, output_index: outputIndex, ...mappedRule } = rule; mappedRule.uuid = id; return ({ From bc969c20acce68fab77abf060722a8ba0df0d553 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 14 Sep 2021 10:21:24 -0400 Subject: [PATCH 13/15] Add threshold rule type to ruleTypeMappings --- .../server/lib/detection_engine/signals/utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index efd7200202b5..5663737f1df2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -67,6 +67,7 @@ import { ML_RULE_TYPE_ID, QUERY_RULE_TYPE_ID, SIGNALS_ID, + THRESHOLD_RULE_TYPE_ID, } from '../../../../common/constants'; interface SortExceptionsReturn { @@ -1018,5 +1019,5 @@ export const ruleTypeMappings = { query: QUERY_RULE_TYPE_ID, saved_query: SIGNALS_ID, threat_match: INDICATOR_RULE_TYPE_ID, - threshold: SIGNALS_ID, + threshold: THRESHOLD_RULE_TYPE_ID, }; From 9339cee121db3bca3959e77ce720344f48046594 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 14 Sep 2021 10:22:47 -0400 Subject: [PATCH 14/15] Add eql rule type to ruleTypeMappings --- .../server/lib/detection_engine/signals/utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 5663737f1df2..5993dd626729 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -63,6 +63,7 @@ import { WrappedRACAlert } from '../rule_types/types'; import { SearchTypes } from '../../../../common/detection_engine/types'; import { IRuleExecutionLogClient } from '../rule_execution_log/types'; import { + EQL_RULE_TYPE_ID, INDICATOR_RULE_TYPE_ID, ML_RULE_TYPE_ID, QUERY_RULE_TYPE_ID, @@ -1014,7 +1015,7 @@ export const getField = (event: SimpleHit, field: string) * Maps legacy rule types to RAC rule type IDs. */ export const ruleTypeMappings = { - eql: SIGNALS_ID, + eql: EQL_RULE_TYPE_ID, machine_learning: ML_RULE_TYPE_ID, query: QUERY_RULE_TYPE_ID, saved_query: SIGNALS_ID, From 653b608ec0becf4e43c2c2dbeb6a8896cdd2cd9b Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 14 Sep 2021 12:12:19 -0400 Subject: [PATCH 15/15] Fix tests with remaining rule types --- x-pack/plugins/security_solution/common/constants.ts | 1 - .../server/lib/detection_engine/rules/find_rules.test.ts | 8 ++++++-- .../server/lib/detection_engine/schemas/rule_schemas.ts | 7 ++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index 8076caf60f69..a93439b29069 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -197,7 +197,6 @@ export const EQL_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.eqlRule` as const; export const INDICATOR_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.indicatorRule` as const; export const ML_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.mlRule` as const; export const QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.queryRule` as const; -export const SAVED_QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.savedQueryRule` as const; export const THRESHOLD_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.thresholdRule` as const; /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/find_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/find_rules.test.ts index 0f7545c4df93..ebde1d0ad6df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/find_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/find_rules.test.ts @@ -7,15 +7,19 @@ import { getFilter } from './find_rules'; import { + EQL_RULE_TYPE_ID, INDICATOR_RULE_TYPE_ID, ML_RULE_TYPE_ID, QUERY_RULE_TYPE_ID, + THRESHOLD_RULE_TYPE_ID, SIGNALS_ID, } from '../../../../common/constants'; -const allAlertTypeIds = `(alert.attributes.alertTypeId: ${ML_RULE_TYPE_ID} +const allAlertTypeIds = `(alert.attributes.alertTypeId: ${EQL_RULE_TYPE_ID} + OR alert.attributes.alertTypeId: ${ML_RULE_TYPE_ID} OR alert.attributes.alertTypeId: ${QUERY_RULE_TYPE_ID} - OR alert.attributes.alertTypeId: ${INDICATOR_RULE_TYPE_ID})`.replace(/[\n\r]/g, ''); + OR alert.attributes.alertTypeId: ${INDICATOR_RULE_TYPE_ID} + OR alert.attributes.alertTypeId: ${THRESHOLD_RULE_TYPE_ID})`.replace(/[\n\r]/g, ''); describe('find_rules', () => { const fullFilterTestCases: Array<[boolean, string]> = [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts index e9215084614c..578d8c4926b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts @@ -69,6 +69,8 @@ import { INDICATOR_RULE_TYPE_ID, ML_RULE_TYPE_ID, QUERY_RULE_TYPE_ID, + EQL_RULE_TYPE_ID, + THRESHOLD_RULE_TYPE_ID, } from '../../../../common/constants'; const nonEqlLanguages = t.keyof({ kuery: null, lucene: null }); @@ -206,12 +208,11 @@ export const notifyWhen = t.union([ export const allRuleTypes = t.union([ t.literal(SIGNALS_ID), - // t.literal(EQL_RULE_TYPE_ID), + t.literal(EQL_RULE_TYPE_ID), t.literal(ML_RULE_TYPE_ID), t.literal(QUERY_RULE_TYPE_ID), - // t.literal(SAVED_QUERY_RULE_TYPE_ID), t.literal(INDICATOR_RULE_TYPE_ID), - // t.literal(THRESHOLD_RULE_TYPE_ID), + t.literal(THRESHOLD_RULE_TYPE_ID), ]); export type AllRuleTypes = t.TypeOf;