From 4b57ac5381b005418b3b86e174e51e5e7b1cbfed Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Tue, 15 Dec 2020 08:52:36 -0500 Subject: [PATCH 1/4] [Security Solution][Detections] Add alert source to detection rule action context (#85488) (#85910) * Adds context.alerts as available parameter for detection rule actions Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../security_solution/common/constants.ts | 1 + .../pages/detection_engine/rules/helpers.tsx | 1 + .../notifications/build_signals_query.ts | 10 ++- .../notifications/get_signals.ts | 43 +++++++++++ .../rules_notification_alert_type.test.ts | 33 ++++----- .../rules_notification_alert_type.ts | 24 +++++-- .../schedule_notification_actions.ts | 4 ++ .../signals/__mocks__/es_results.ts | 9 +++ .../signals/build_bulk_body.test.ts | 71 ++++++++++++++----- .../signals/build_bulk_body.ts | 5 +- .../signals/search_after_bulk_create.test.ts | 48 ------------- .../signals/search_after_bulk_create.ts | 2 + .../signals/signal_rule_alert_type.test.ts | 1 + .../signals/signal_rule_alert_type.ts | 12 +++- .../signals/single_bulk_create.ts | 40 +++++++++-- .../threat_mapping/create_threat_signals.ts | 1 + .../signals/threat_mapping/utils.test.ts | 40 ++++++++++- .../signals/threat_mapping/utils.ts | 3 + .../lib/detection_engine/signals/types.ts | 3 + .../detection_engine/signals/utils.test.ts | 11 +++ .../lib/detection_engine/signals/utils.ts | 15 +++- 21 files changed, 273 insertions(+), 104 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/notifications/get_signals.ts diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index 686c05e2fda51..6b0b7c62f5bfb 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -42,6 +42,7 @@ export const ENDPOINT_METADATA_INDEX = 'metrics-endpoint.metadata-*'; export const DEFAULT_RULE_REFRESH_INTERVAL_ON = true; export const DEFAULT_RULE_REFRESH_INTERVAL_VALUE = 60000; // ms export const DEFAULT_RULE_REFRESH_IDLE_VALUE = 2700000; // ms +export const DEFAULT_RULE_NOTIFICATION_QUERY_SIZE = 100; export enum SecurityPageName { detections = 'detections', diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 36a1248f94dd4..74549827b6fa2 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -382,6 +382,7 @@ export const getActionMessageParams = memoizeOne( description: 'context.results_link', useWithTripleBracesInTemplates: true, }, + { name: 'alerts', description: 'context.alerts' }, ...actionMessageRuleParams.map((param) => { const extendedParam = `rule.${param}`; return { name: extendedParam, description: `context.${extendedParam}` }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/build_signals_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/build_signals_query.ts index be0943c014225..b3cdf3afe683a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/build_signals_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/build_signals_query.ts @@ -9,11 +9,19 @@ interface BuildSignalsSearchQuery { index: string; from: string; to: string; + size?: number; } -export const buildSignalsSearchQuery = ({ ruleId, index, from, to }: BuildSignalsSearchQuery) => ({ +export const buildSignalsSearchQuery = ({ + ruleId, + index, + from, + to, + size, +}: BuildSignalsSearchQuery) => ({ index, body: { + size, query: { bool: { filter: [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/get_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/get_signals.ts new file mode 100644 index 0000000000000..4d80736ca6bb6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/get_signals.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { AlertServices } from '../../../../../alerts/server'; +import { SignalSearchResponse } from '../signals/types'; +import { buildSignalsSearchQuery } from './build_signals_query'; + +interface GetSignalsParams { + from?: string; + to?: string; + size?: number; + ruleId: string; + index: string; + callCluster: AlertServices['callCluster']; +} + +export const getSignals = async ({ + from, + to, + size, + ruleId, + index, + callCluster, +}: GetSignalsParams): Promise => { + if (from == null || to == null) { + throw Error('"from" or "to" was not provided to signals query'); + } + + const query = buildSignalsSearchQuery({ + index, + ruleId, + to, + from, + size, + }); + + const result: SignalSearchResponse = await callCluster('search', query); + + return result; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.test.ts index 593ada470b118..5c38461b124c6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.test.ts @@ -10,6 +10,12 @@ import { rulesNotificationAlertType } from './rules_notification_alert_type'; import { buildSignalsSearchQuery } from './build_signals_query'; import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; import { NotificationExecutorOptions } from './types'; +import { + sampleDocSearchResultsNoSortIdNoVersion, + sampleDocSearchResultsWithSortId, + sampleEmptyDocSearchResults, +} from '../signals/__mocks__/es_results'; +import { DEFAULT_RULE_NOTIFICATION_QUERY_SIZE } from '../../../../common/constants'; jest.mock('./build_signals_query'); describe('rules_notification_alert_type', () => { @@ -63,9 +69,7 @@ describe('rules_notification_alert_type', () => { references: [], attributes: ruleAlert, }); - alertServices.callCluster.mockResolvedValue({ - count: 0, - }); + alertServices.callCluster.mockResolvedValue(sampleDocSearchResultsWithSortId()); await alert.executor(payload); @@ -75,6 +79,7 @@ describe('rules_notification_alert_type', () => { index: '.siem-signals', ruleId: 'rule-1', to: '1576341633400', + size: DEFAULT_RULE_NOTIFICATION_QUERY_SIZE, }) ); }); @@ -88,9 +93,7 @@ describe('rules_notification_alert_type', () => { references: [], attributes: ruleAlert, }); - alertServices.callCluster.mockResolvedValue({ - count: 10, - }); + alertServices.callCluster.mockResolvedValue(sampleDocSearchResultsWithSortId()); await alert.executor(payload); expect(alertServices.alertInstanceFactory).toHaveBeenCalled(); @@ -114,9 +117,7 @@ describe('rules_notification_alert_type', () => { references: [], attributes: ruleAlert, }); - alertServices.callCluster.mockResolvedValue({ - count: 10, - }); + alertServices.callCluster.mockResolvedValue(sampleDocSearchResultsWithSortId()); await alert.executor(payload); expect(alertServices.alertInstanceFactory).toHaveBeenCalled(); @@ -141,9 +142,7 @@ describe('rules_notification_alert_type', () => { references: [], attributes: ruleAlert, }); - alertServices.callCluster.mockResolvedValue({ - count: 10, - }); + alertServices.callCluster.mockResolvedValue(sampleDocSearchResultsWithSortId()); await alert.executor(payload); expect(alertServices.alertInstanceFactory).toHaveBeenCalled(); @@ -165,9 +164,7 @@ describe('rules_notification_alert_type', () => { references: [], attributes: ruleAlert, }); - alertServices.callCluster.mockResolvedValue({ - count: 0, - }); + alertServices.callCluster.mockResolvedValue(sampleEmptyDocSearchResults()); await alert.executor(payload); @@ -182,9 +179,7 @@ describe('rules_notification_alert_type', () => { references: [], attributes: ruleAlert, }); - alertServices.callCluster.mockResolvedValue({ - count: 10, - }); + alertServices.callCluster.mockResolvedValue(sampleDocSearchResultsNoSortIdNoVersion()); await alert.executor(payload); @@ -192,7 +187,7 @@ describe('rules_notification_alert_type', () => { const [{ value: alertInstanceMock }] = alertServices.alertInstanceFactory.mock.results; expect(alertInstanceMock.replaceState).toHaveBeenCalledWith( - expect.objectContaining({ signals_count: 10 }) + expect.objectContaining({ signals_count: 100 }) ); expect(alertInstanceMock.scheduleActions).toHaveBeenCalledWith( 'default', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts index 802b9472a4487..d60e5dbd5ab4c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts @@ -6,15 +6,19 @@ import { Logger } from 'src/core/server'; import { schema } from '@kbn/config-schema'; -import { NOTIFICATIONS_ID, SERVER_APP_ID } from '../../../../common/constants'; +import { + DEFAULT_RULE_NOTIFICATION_QUERY_SIZE, + NOTIFICATIONS_ID, + SERVER_APP_ID, +} from '../../../../common/constants'; import { NotificationAlertTypeDefinition } from './types'; -import { getSignalsCount } from './get_signals_count'; import { RuleAlertAttributes } from '../signals/types'; import { siemRuleActionGroups } from '../signals/siem_rule_action_groups'; import { scheduleNotificationActions } from './schedule_notification_actions'; import { getNotificationResultsLink } from './utils'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; +import { getSignals } from './get_signals'; export const rulesNotificationAlertType = ({ logger, @@ -52,14 +56,20 @@ export const rulesNotificationAlertType = ({ )?.format('x'); const toInMs = parseScheduleDates(startedAt.toISOString())?.format('x'); - const signalsCount = await getSignalsCount({ + const results = await getSignals({ from: fromInMs, to: toInMs, + size: DEFAULT_RULE_NOTIFICATION_QUERY_SIZE, index: ruleParams.outputIndex, ruleId: ruleParams.ruleId, callCluster: services.callCluster, }); + const signals = results.hits.hits.map((hit) => hit._source); + + const signalsCount = + typeof results.hits.total === 'number' ? results.hits.total : results.hits.total.value; + const resultsLink = getNotificationResultsLink({ from: fromInMs, to: toInMs, @@ -74,7 +84,13 @@ export const rulesNotificationAlertType = ({ if (signalsCount !== 0) { const alertInstance = services.alertInstanceFactory(alertId); - scheduleNotificationActions({ alertInstance, signalsCount, resultsLink, ruleParams }); + scheduleNotificationActions({ + alertInstance, + signalsCount, + resultsLink, + ruleParams, + signals, + }); } }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts index a26ddfc90434a..bc697b08c5d29 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts @@ -6,6 +6,7 @@ import { mapKeys, snakeCase } from 'lodash/fp'; import { AlertInstance } from '../../../../../alerts/server'; +import { SignalSource } from '../signals/types'; import { RuleTypeParams } from '../types'; export type NotificationRuleTypeParams = RuleTypeParams & { @@ -18,6 +19,7 @@ interface ScheduleNotificationActions { signalsCount: number; resultsLink: string; ruleParams: NotificationRuleTypeParams; + signals: SignalSource[]; } export const scheduleNotificationActions = ({ @@ -25,6 +27,7 @@ export const scheduleNotificationActions = ({ signalsCount, resultsLink = '', ruleParams, + signals, }: ScheduleNotificationActions): AlertInstance => alertInstance .replaceState({ @@ -33,4 +36,5 @@ export const scheduleNotificationActions = ({ .scheduleActions('default', { results_link: resultsLink, rule: mapKeys(snakeCase, ruleParams), + alerts: signals, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 764604a793788..6718ff2d1f15f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -12,6 +12,7 @@ import { BulkItem, RuleAlertAttributes, SignalHit, + WrappedSignalHit, } from '../types'; import { Logger, @@ -240,6 +241,14 @@ export const sampleEmptyDocSearchResults = (): SignalSearchResponse => ({ }, }); +export const sampleWrappedSignalHit = (): WrappedSignalHit => { + return { + _index: 'myFakeSignalIndex', + _id: sampleIdGuid, + _source: sampleSignalHit(), + }; +}; + export const sampleDocWithAncestors = (): SignalSearchResponse => { const sampleDoc = sampleDocNoSortId(); delete sampleDoc.sort; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index f81000091749a..6f04811feaef4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -11,7 +11,7 @@ import { sampleIdGuid, sampleDocWithAncestors, sampleRuleSO, - sampleDocNoSortIdNoVersion, + sampleWrappedSignalHit, } from './__mocks__/es_results'; import { buildBulkBody, @@ -776,10 +776,10 @@ describe('buildBulkBody', () => { describe('buildSignalFromSequence', () => { test('builds a basic signal from a sequence of building blocks', () => { - const block1 = sampleDocWithAncestors().hits.hits[0]; + const block1 = sampleWrappedSignalHit(); block1._source.new_key = 'new_key_value'; block1._source.new_key2 = 'new_key2_value'; - const block2 = sampleDocWithAncestors().hits.hits[0]; + const block2 = sampleWrappedSignalHit(); block2._source.new_key = 'new_key_value'; const blocks = [block1, block2]; const ruleSO = sampleRuleSO(); @@ -787,8 +787,7 @@ describe('buildSignalFromSequence', () => { // Timestamp will potentially always be different so remove it for the test // @ts-expect-error delete signal['@timestamp']; - const expected: Omit & { someKey: string; new_key: string } = { - someKey: 'someValue', + const expected: Omit & { new_key: string } = { new_key: 'new_key_value', event: { kind: 'signal', @@ -800,14 +799,14 @@ describe('buildSignalFromSequence', () => { parents: [ { id: sampleIdGuid, - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', type: 'signal', index: 'myFakeSignalIndex', depth: 1, }, { id: sampleIdGuid, - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', type: 'signal', index: 'myFakeSignalIndex', depth: 1, @@ -820,9 +819,15 @@ describe('buildSignalFromSequence', () => { index: 'myFakeSignalIndex', depth: 0, }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, { id: sampleIdGuid, - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', type: 'signal', index: 'myFakeSignalIndex', depth: 1, @@ -833,9 +838,15 @@ describe('buildSignalFromSequence', () => { index: 'myFakeSignalIndex', depth: 0, }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, { id: sampleIdGuid, - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', type: 'signal', index: 'myFakeSignalIndex', depth: 1, @@ -889,8 +900,8 @@ describe('buildSignalFromSequence', () => { }); test('builds a basic signal if there is no overlap between source events', () => { - const block1 = sampleDocNoSortIdNoVersion(); - const block2 = sampleDocNoSortIdNoVersion(); + const block1 = sampleWrappedSignalHit(); + const block2 = sampleWrappedSignalHit(); block2._source['@timestamp'] = '2021-05-20T22:28:46+0000'; block2._source.someKey = 'someOtherValue'; const ruleSO = sampleRuleSO(); @@ -909,30 +920,58 @@ describe('buildSignalFromSequence', () => { parents: [ { id: sampleIdGuid, + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + rule: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + }, + { + id: sampleIdGuid, + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + rule: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', depth: 0, }, { - id: sampleIdGuid, + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', depth: 0, }, - ], - ancestors: [ { id: sampleIdGuid, + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + rule: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + }, + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', depth: 0, }, { - id: sampleIdGuid, + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', depth: 0, }, + { + id: sampleIdGuid, + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + rule: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + }, ], status: 'open', rule: { @@ -972,7 +1011,7 @@ describe('buildSignalFromSequence', () => { throttle: 'no_actions', exceptions_list: getListArrayMock(), }, - depth: 1, + depth: 2, group: { id: '269c1f5754bff92fb8040283b687258e99b03e8b2ab1262cc20c82442e5de5ea', }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index e6e6d2e0f5fa9..5f103d25e5ace 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -12,6 +12,7 @@ import { RuleAlertAttributes, BaseSignalHit, SignalSource, + WrappedSignalHit, } from './types'; import { buildRule, buildRuleWithoutOverrides, buildRuleWithOverrides } from './build_rule'; import { additionalSignalFields, buildSignal } from './build_signal'; @@ -94,7 +95,7 @@ export const buildSignalGroupFromSequence = ( sequence: EqlSequence, ruleSO: SavedObject, outputIndex: string -): BaseSignalHit[] => { +): WrappedSignalHit[] => { const wrappedBuildingBlocks = wrapBuildingBlocks( sequence.events.map((event) => { const signal = buildSignalFromEvent(event, ruleSO, false); @@ -132,7 +133,7 @@ export const buildSignalGroupFromSequence = ( }; export const buildSignalFromSequence = ( - events: BaseSignalHit[], + events: WrappedSignalHit[], ruleSO: SavedObject ): SignalHit => { const rule = buildRuleWithoutOverrides(ruleSO); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 46722c69e53e3..67246a830ce90 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -53,9 +53,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -68,9 +65,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -83,9 +77,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -98,9 +89,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -164,9 +152,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -179,9 +164,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -194,9 +176,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -210,9 +189,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -225,9 +201,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -290,9 +263,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -503,9 +473,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -587,9 +554,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -767,9 +731,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -862,9 +823,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -877,9 +835,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, @@ -892,9 +847,6 @@ describe('searchAfterAndBulkCreate', () => { took: 100, errors: false, items: [ - { - fakeItemValue: 'fakeItemKey', - }, { create: { status: 201, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 32865e117cba9..b79f758cd7503 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -156,6 +156,7 @@ export const searchAfterAndBulkCreate = async ({ const { bulkCreateDuration: bulkDuration, createdItemsCount: createdCount, + createdItems, success: bulkSuccess, errors: bulkErrors, } = await singleBulkCreate({ @@ -183,6 +184,7 @@ export const searchAfterAndBulkCreate = async ({ createSearchAfterReturnType({ success: bulkSuccess, createdSignalsCount: createdCount, + createdSignals: createdItems, bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined, errors: bulkErrors, }), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index dc68e3949eb36..9a40573095a1a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -518,6 +518,7 @@ describe('rules_notification_alert_type', () => { bulkCreateTimes: [], lastLookBackDate: null, createdSignalsCount: 0, + createdSignals: [], errors: ['Error that bubbled up.'], }; (searchAfterAndBulkCreate as jest.Mock).mockResolvedValue(result); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 8d4dd877996db..fdd1fed2e6e89 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -29,8 +29,8 @@ import { SignalRuleAlertTypeDefinition, RuleAlertAttributes, EqlSignalSearchResponse, - BaseSignalHit, ThresholdQueryBucket, + WrappedSignalHit, } from './types'; import { getGapBetweenRuns, @@ -265,6 +265,7 @@ export const signalRulesAlertType = ({ errors, bulkCreateDuration, createdItemsCount, + createdItems, } = await bulkCreateMlSignals({ actions, throttle, @@ -299,6 +300,7 @@ export const signalRulesAlertType = ({ success: success && filteredAnomalyResults._shards.failed === 0, errors: [...errors, ...searchErrors], createdSignalsCount: createdItemsCount, + createdSignals: createdItems, bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], }), ]); @@ -371,6 +373,7 @@ export const signalRulesAlertType = ({ success, bulkCreateDuration, createdItemsCount, + createdItems, errors, } = await bulkCreateThresholdSignals({ actions, @@ -406,6 +409,7 @@ export const signalRulesAlertType = ({ success, errors: [...errors, ...previousSearchErrors, ...searchErrors], createdSignalsCount: createdItemsCount, + createdSignals: createdItems, bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], }), ]); @@ -539,10 +543,10 @@ export const signalRulesAlertType = ({ 'transport.request', request ); - let newSignals: BaseSignalHit[] | undefined; + let newSignals: WrappedSignalHit[] | undefined; if (response.hits.sequences !== undefined) { newSignals = response.hits.sequences.reduce( - (acc: BaseSignalHit[], sequence) => + (acc: WrappedSignalHit[], sequence) => acc.concat(buildSignalGroupFromSequence(sequence, savedObject, outputIndex)), [] ); @@ -562,6 +566,7 @@ export const signalRulesAlertType = ({ const insertResult = await bulkInsertSignals(newSignals, logger, services, refresh); result.bulkCreateTimes.push(insertResult.bulkCreateDuration); result.createdSignalsCount += insertResult.createdItemsCount; + result.createdSignals = insertResult.createdItems; } result.success = true; } else { @@ -596,6 +601,7 @@ export const signalRulesAlertType = ({ scheduleNotificationActions({ alertInstance, signalsCount: result.createdSignalsCount, + signals: result.createdSignals, resultsLink, ruleParams: notificationRuleParams, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index 8c1d4210a7b36..943b70794a9b1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { countBy, isEmpty } from 'lodash'; +import { countBy, isEmpty, get } from 'lodash'; import { performance } from 'perf_hooks'; import { AlertServices } from '../../../../../alerts/server'; -import { SignalSearchResponse, BulkResponse, BaseSignalHit } from './types'; +import { SignalSearchResponse, BulkResponse, SignalHit, WrappedSignalHit } from './types'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { generateId, makeFloatString, errorAggregator } from './utils'; @@ -68,7 +68,7 @@ export const filterDuplicateRules = ( * @param ruleId The rule id * @param signals The candidate new signals */ -export const filterDuplicateSignals = (ruleId: string, signals: BaseSignalHit[]) => { +export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => { return signals.filter( (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId) ); @@ -78,12 +78,14 @@ export interface SingleBulkCreateResponse { success: boolean; bulkCreateDuration?: string; createdItemsCount: number; + createdItems: SignalHit[]; errors: string[]; } export interface BulkInsertSignalsResponse { bulkCreateDuration: string; createdItemsCount: number; + createdItems: SignalHit[]; } // Bulk Index documents. @@ -111,7 +113,7 @@ export const singleBulkCreate = async ({ logger.debug(buildRuleMessage(`about to bulk create ${filteredEvents.hits.hits.length} events`)); if (filteredEvents.hits.hits.length === 0) { logger.debug(buildRuleMessage(`all events were duplicates`)); - return { success: true, createdItemsCount: 0, errors: [] }; + return { success: true, createdItemsCount: 0, createdItems: [], errors: [] }; } // index documents after creating an ID based on the // source documents' originating index, and the original @@ -164,7 +166,26 @@ export const singleBulkCreate = async ({ ); logger.debug(buildRuleMessage(`took property says bulk took: ${response.took} milliseconds`)); - const createdItemsCount = countBy(response.items, 'create.status')['201'] ?? 0; + const createdItems = filteredEvents.hits.hits + .map((doc) => + buildBulkBody({ + doc, + ruleParams, + id, + actions, + name, + createdAt, + createdBy, + updatedAt, + updatedBy, + interval, + enabled, + tags, + throttle, + }) + ) + .filter((_, index) => get(response.items[index], 'create.status') === 201); + const createdItemsCount = createdItems.length; const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; const errorCountByMessage = errorAggregator(response, [409]); @@ -184,6 +205,7 @@ export const singleBulkCreate = async ({ success: false, bulkCreateDuration: makeFloatString(end - start), createdItemsCount, + createdItems, }; } else { return { @@ -191,13 +213,14 @@ export const singleBulkCreate = async ({ success: true, bulkCreateDuration: makeFloatString(end - start), createdItemsCount, + createdItems, }; } }; // Bulk Index new signals. export const bulkInsertSignals = async ( - signals: BaseSignalHit[], + signals: WrappedSignalHit[], logger: Logger, services: AlertServices, refresh: RefreshTypes @@ -234,6 +257,9 @@ export const bulkInsertSignals = async ( } const createdItemsCount = countBy(response.items, 'create.status')['201'] ?? 0; + const createdItems = signals + .map((doc) => doc._source) + .filter((_, index) => get(response.items[index], 'create.status') === 201); logger.debug(`bulk created ${createdItemsCount} signals`); - return { bulkCreateDuration: makeFloatString(end - start), createdItemsCount }; + return { bulkCreateDuration: makeFloatString(end - start), createdItems, createdItemsCount }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index e90c45d40de95..b9398b4aa2af0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -59,6 +59,7 @@ export const createThreatSignals = async ({ searchAfterTimes: [], lastLookBackDate: null, createdSignalsCount: 0, + createdSignals: [], errors: [], }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 840d64381c793..4bb7b9e07578d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -5,6 +5,7 @@ */ import { SearchAfterAndBulkCreateReturnType } from '../types'; +import { sampleSignalHit } from '../__mocks__/es_results'; import { calculateAdditiveMax, @@ -50,6 +51,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; @@ -59,6 +61,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const combinedResults = combineResults(existingResult, newResult); @@ -72,6 +75,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; @@ -81,6 +85,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const combinedResults = combineResults(existingResult, newResult); @@ -94,6 +99,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; @@ -103,6 +109,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const combinedResults = combineResults(existingResult, newResult); @@ -116,6 +123,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; @@ -125,6 +133,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const combinedResults = combineResults(existingResult, newResult); @@ -143,6 +152,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 1', 'error 2', 'error 3'], }; @@ -152,6 +162,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 4', 'error 1', 'error 3', 'error 5'], }; const combinedResults = combineResults(existingResult, newResult); @@ -261,6 +272,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { @@ -269,19 +281,21 @@ describe('utils', () => { bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const combinedResults = combineConcurrentResults(existingResult, []); expect(combinedResults).toEqual(expectedResult); }); - test('it should work with empty arrays for searchAfterTimes and bulkCreateTimes', () => { + test('it should work with empty arrays for searchAfterTimes and bulkCreateTimes and createdSignals', () => { const existingResult: SearchAfterAndBulkCreateReturnType = { success: true, searchAfterTimes: ['10', '20', '30'], bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -290,6 +304,7 @@ describe('utils', () => { bulkCreateTimes: [], lastLookBackDate: undefined, createdSignalsCount: 0, + createdSignals: [], errors: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { @@ -298,6 +313,7 @@ describe('utils', () => { bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; @@ -312,6 +328,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], // max is 25 lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const newResult1: SearchAfterAndBulkCreateReturnType = { @@ -320,6 +337,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), createdSignalsCount: 5, + createdSignals: Array(5).fill(sampleSignalHit()), errors: [], }; const newResult2: SearchAfterAndBulkCreateReturnType = { @@ -328,6 +346,7 @@ describe('utils', () => { bulkCreateTimes: ['50', '5', '15'], lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), createdSignalsCount: 8, + createdSignals: Array(8).fill(sampleSignalHit()), errors: [], }; @@ -337,6 +356,7 @@ describe('utils', () => { bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + createdSignals: Array(16).fill(sampleSignalHit()), errors: [], }; @@ -351,6 +371,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], // max is 25 lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const newResult1: SearchAfterAndBulkCreateReturnType = { @@ -359,6 +380,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), createdSignalsCount: 5, + createdSignals: Array(5).fill(sampleSignalHit()), errors: [], }; const newResult2: SearchAfterAndBulkCreateReturnType = { @@ -367,6 +389,7 @@ describe('utils', () => { bulkCreateTimes: ['50', '5', '15'], lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), createdSignalsCount: 8, + createdSignals: Array(8).fill(sampleSignalHit()), errors: [], }; @@ -376,6 +399,7 @@ describe('utils', () => { bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + createdSignals: Array(16).fill(sampleSignalHit()), errors: [], }; @@ -390,6 +414,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], // max is 25 lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const newResult1: SearchAfterAndBulkCreateReturnType = { @@ -398,6 +423,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), createdSignalsCount: 5, + createdSignals: Array(5).fill(sampleSignalHit()), errors: [], }; const newResult2: SearchAfterAndBulkCreateReturnType = { @@ -406,6 +432,7 @@ describe('utils', () => { bulkCreateTimes: ['50', '5', '15'], lastLookBackDate: null, createdSignalsCount: 8, + createdSignals: Array(8).fill(sampleSignalHit()), errors: [], }; @@ -415,6 +442,7 @@ describe('utils', () => { bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), // max lastLookBackDate createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + createdSignals: Array(16).fill(sampleSignalHit()), errors: [], }; @@ -429,6 +457,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; @@ -438,6 +467,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); @@ -451,6 +481,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; @@ -460,6 +491,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); @@ -473,6 +505,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; @@ -482,6 +515,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); @@ -495,6 +529,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; @@ -504,6 +539,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); @@ -522,6 +558,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: undefined, createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 1', 'error 2', 'error 3'], }; @@ -531,6 +568,7 @@ describe('utils', () => { bulkCreateTimes: ['5', '15', '25'], lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 4', 'error 1', 'error 3', 'error 5'], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index d6c91fad6d9cb..653b01f11f552 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -71,6 +71,7 @@ export const combineResults = ( ), lastLookBackDate: newResult.lastLookBackDate, createdSignalsCount: currentResult.createdSignalsCount + newResult.createdSignalsCount, + createdSignals: [...currentResult.createdSignals, ...newResult.createdSignals], errors: [...new Set([...currentResult.errors, ...newResult.errors])], }); @@ -94,6 +95,7 @@ export const combineConcurrentResults = ( bulkCreateTimes: [maxBulkCreateTimes], lastLookBackDate, createdSignalsCount: accum.createdSignalsCount + item.createdSignalsCount, + createdSignals: [...accum.createdSignals, ...item.createdSignals], errors: [...new Set([...accum.errors, ...item.errors])], }; }, @@ -103,6 +105,7 @@ export const combineConcurrentResults = ( bulkCreateTimes: [], lastLookBackDate: undefined, createdSignalsCount: 0, + createdSignals: [], errors: [], } ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 66f3a21dfe680..4167d056df885 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -123,6 +123,7 @@ export interface GetResponse { export type EventSearchResponse = SearchResponse; export type SignalSearchResponse = SearchResponse; export type SignalSourceHit = SignalSearchResponse['hits']['hits'][number]; +export type WrappedSignalHit = BaseHit; export type BaseSignalHit = BaseHit; export type EqlSignalSearchResponse = EqlSearchResponse; @@ -174,6 +175,7 @@ export interface SignalHit { '@timestamp': string; event: object; signal: Signal; + [key: string]: SearchTypes; } export interface AlertAttributes { @@ -242,6 +244,7 @@ export interface SearchAfterAndBulkCreateReturnType { bulkCreateTimes: string[]; lastLookBackDate: Date | null | undefined; createdSignalsCount: number; + createdSignals: SignalHit[]; errors: string[]; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 894e934ff0247..dd936776f691a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -979,6 +979,7 @@ describe('utils', () => { const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: [], createdSignalsCount: 0, + createdSignals: [], errors: [], lastLookBackDate: null, searchAfterTimes: [], @@ -996,6 +997,7 @@ describe('utils', () => { const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: [], createdSignalsCount: 0, + createdSignals: [], errors: [], lastLookBackDate: new Date('2020-04-20T21:27:45.000Z'), searchAfterTimes: [], @@ -1147,6 +1149,7 @@ describe('utils', () => { const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: [], createdSignalsCount: 0, + createdSignals: [], errors: [], lastLookBackDate: null, searchAfterTimes: [], @@ -1159,6 +1162,7 @@ describe('utils', () => { const searchAfterReturnType = createSearchAfterReturnType({ bulkCreateTimes: ['123'], createdSignalsCount: 5, + createdSignals: Array(5).fill(sampleSignalHit()), errors: ['error 1'], lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), searchAfterTimes: ['123'], @@ -1167,6 +1171,7 @@ describe('utils', () => { const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: ['123'], createdSignalsCount: 5, + createdSignals: Array(5).fill(sampleSignalHit()), errors: ['error 1'], lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), searchAfterTimes: ['123'], @@ -1178,11 +1183,13 @@ describe('utils', () => { test('createSearchAfterReturnType can override select values', () => { const searchAfterReturnType = createSearchAfterReturnType({ createdSignalsCount: 5, + createdSignals: Array(5).fill(sampleSignalHit()), errors: ['error 1'], }); const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: [], createdSignalsCount: 5, + createdSignals: Array(5).fill(sampleSignalHit()), errors: ['error 1'], lastLookBackDate: null, searchAfterTimes: [], @@ -1198,6 +1205,7 @@ describe('utils', () => { const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: [], createdSignalsCount: 0, + createdSignals: [], errors: [], lastLookBackDate: null, searchAfterTimes: [], @@ -1251,6 +1259,7 @@ describe('utils', () => { createSearchAfterReturnType({ bulkCreateTimes: ['123'], createdSignalsCount: 3, + createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 1', 'error 2'], lastLookBackDate: new Date('2020-08-21T18:51:25.193Z'), searchAfterTimes: ['123'], @@ -1259,6 +1268,7 @@ describe('utils', () => { createSearchAfterReturnType({ bulkCreateTimes: ['456'], createdSignalsCount: 2, + createdSignals: Array(2).fill(sampleSignalHit()), errors: ['error 3'], lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), searchAfterTimes: ['567'], @@ -1268,6 +1278,7 @@ describe('utils', () => { const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: ['123', '456'], // concatenates the prev and next together createdSignalsCount: 5, // Adds the 3 and 2 together + createdSignals: Array(5).fill(sampleSignalHit()), errors: ['error 1', 'error 2', 'error 3'], // concatenates the prev and next together lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), // takes the next lastLookBackDate searchAfterTimes: ['123', '567'], // concatenates the searchAfterTimes together diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index e2b39b8d0a8c8..2114f21d9cead 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -18,10 +18,10 @@ import { BulkResponseErrorAggregation, isValidUnit, SignalHit, - BaseSignalHit, SearchAfterAndBulkCreateReturnType, SignalSearchResponse, Signal, + WrappedSignalHit, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; @@ -247,7 +247,10 @@ export const generateBuildingBlockIds = (buildingBlocks: SignalHit[]): string[] ); }; -export const wrapBuildingBlocks = (buildingBlocks: SignalHit[], index: string): BaseSignalHit[] => { +export const wrapBuildingBlocks = ( + buildingBlocks: SignalHit[], + index: string +): WrappedSignalHit[] => { const blockIds = generateBuildingBlockIds(buildingBlocks); return buildingBlocks.map((block, idx) => { return { @@ -260,7 +263,7 @@ export const wrapBuildingBlocks = (buildingBlocks: SignalHit[], index: string): }); }; -export const wrapSignal = (signal: SignalHit, index: string): BaseSignalHit => { +export const wrapSignal = (signal: SignalHit, index: string): WrappedSignalHit => { return { _id: generateSignalId(signal.signal), _index: index, @@ -589,6 +592,7 @@ export const createSearchAfterReturnType = ({ bulkCreateTimes, lastLookBackDate, createdSignalsCount, + createdSignals, errors, }: { success?: boolean | undefined; @@ -596,6 +600,7 @@ export const createSearchAfterReturnType = ({ bulkCreateTimes?: string[] | undefined; lastLookBackDate?: Date | undefined; createdSignalsCount?: number | undefined; + createdSignals?: SignalHit[] | undefined; errors?: string[] | undefined; } = {}): SearchAfterAndBulkCreateReturnType => { return { @@ -604,6 +609,7 @@ export const createSearchAfterReturnType = ({ bulkCreateTimes: bulkCreateTimes ?? [], lastLookBackDate: lastLookBackDate ?? null, createdSignalsCount: createdSignalsCount ?? 0, + createdSignals: createdSignals ?? [], errors: errors ?? [], }; }; @@ -618,6 +624,7 @@ export const mergeReturns = ( bulkCreateTimes: existingBulkCreateTimes, lastLookBackDate: existingLastLookBackDate, createdSignalsCount: existingCreatedSignalsCount, + createdSignals: existingCreatedSignals, errors: existingErrors, } = prev; @@ -627,6 +634,7 @@ export const mergeReturns = ( bulkCreateTimes: newBulkCreateTimes, lastLookBackDate: newLastLookBackDate, createdSignalsCount: newCreatedSignalsCount, + createdSignals: newCreatedSignals, errors: newErrors, } = next; @@ -636,6 +644,7 @@ export const mergeReturns = ( bulkCreateTimes: [...existingBulkCreateTimes, ...newBulkCreateTimes], lastLookBackDate: newLastLookBackDate ?? existingLastLookBackDate, createdSignalsCount: existingCreatedSignalsCount + newCreatedSignalsCount, + createdSignals: [...existingCreatedSignals, ...newCreatedSignals], errors: [...new Set([...existingErrors, ...newErrors])], }; }); From 27afabd272824f629390f7a41c64ed636c2ff392 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 15 Dec 2020 15:02:26 +0000 Subject: [PATCH 2/4] skip flaky suite (#85714) --- .../public/resolver/view/panels/node_events_of_type.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.test.tsx b/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.test.tsx index 4d3ccaf278c91..c462bd1e3553e 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.test.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/panels/node_events_of_type.test.tsx @@ -15,7 +15,7 @@ import { urlSearch } from '../../test_utilities/url_search'; const resolverComponentInstanceID = 'resolverComponentInstanceID'; // FLAKY: https://github.com/elastic/kibana/issues/85714 -describe(`Resolver: when analyzing a tree with only the origin and paginated related events, and when the component instance ID is ${resolverComponentInstanceID}`, () => { +describe.skip(`Resolver: when analyzing a tree with only the origin and paginated related events, and when the component instance ID is ${resolverComponentInstanceID}`, () => { /** * Get (or lazily create and get) the simulator. */ From 815d19d410c9b173c0566441ff01bbdd5346c5c0 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 15 Dec 2020 15:05:39 +0000 Subject: [PATCH 3/4] skip flaky suite (#85911) --- test/api_integration/apis/saved_objects/find.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/api_integration/apis/saved_objects/find.js b/test/api_integration/apis/saved_objects/find.js index 8e8730b1e574a..b22d73fc30d43 100644 --- a/test/api_integration/apis/saved_objects/find.js +++ b/test/api_integration/apis/saved_objects/find.js @@ -78,7 +78,8 @@ export default function ({ getService }) { })); }); - describe('page beyond total', () => { + // FLAKY: https://github.com/elastic/kibana/issues/85911 + describe.skip('page beyond total', () => { it('should return 200 with empty response', async () => await supertest .get('/api/saved_objects/_find?type=visualization&page=100&per_page=100') From 53cb05388a4e3dbca22b7d3389a2ed75c0e2ecaf Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 15 Dec 2020 15:15:11 +0000 Subject: [PATCH 4/4] skip flaky suite (#85899) --- .../monitor/synthetics/__tests__/executed_step.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/uptime/public/components/monitor/synthetics/__tests__/executed_step.test.tsx b/x-pack/plugins/uptime/public/components/monitor/synthetics/__tests__/executed_step.test.tsx index 0155a06db6825..52ff0425a7bf7 100644 --- a/x-pack/plugins/uptime/public/components/monitor/synthetics/__tests__/executed_step.test.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/synthetics/__tests__/executed_step.test.tsx @@ -9,7 +9,8 @@ import { ExecutedStep } from '../executed_step'; import { Ping } from '../../../../../common/runtime_types'; import { mountWithRouter } from '../../../../lib'; -describe('ExecutedStep', () => { +// FLAKY: https://github.com/elastic/kibana/issues/85899 +describe.skip('ExecutedStep', () => { let step: Ping; beforeEach(() => {