From 3b78aafab2ff23745d6fcd8cf86b395893e725b6 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Tue, 26 Nov 2019 16:19:13 -0700 Subject: [PATCH] [SIEM][Detection Engine] Adds signal to ECS event.kind and fixes status in signals (#51772) ## Summary * Adds signal to the ECS event.kind when it copies over a signal * Creates a `original_event` if needed within signal so additional look ups don't have to happen * Fixes a bug with `signal.status` where it was not plumbed correctly * Adds extra unit tests around the filter * Adds missing unit tests around utils I didn't add before * Fixes a typing issue with output of a signal Example signal output: Original event turns into this: ```ts "event" : { "dataset" : "socket", "kind" : "signal", "action" : "existing_socket", "id" : "ffec6797-b92f-4436-bb40-69bac2c21874", "module" : "system" }, ``` Signal amplification turns into this where it contains original_event looks like this: ```ts "signal" : { "parent" : { "id" : "xNRlqW4BHe9nqdOi2358", "type" : "event", "index" : "auditbeat", "depth" : 1 }, "original_time" : "2019-11-26T20:27:11.169Z", "status" : "open", "rule" : { "id" : "643fbd2f-a3c9-449e-ba95-e3d89000a72a", "rule_id" : "rule-1", "false_positives" : [ ], "max_signals" : 100, "risk_score" : 1, "description" : "Detecting root and admin users", "from" : "now-6m", "immutable" : false, "interval" : "5m", "language" : "kuery", "name" : "Detect Root/Admin Users", "query" : "user.name: root or user.name: admin", "references" : [ "http://www.example.com", "https://ww.example.com" ], "severity" : "high", "tags" : [ ], "type" : "query", "to" : "now", "enabled" : true, "created_by" : "elastic_some_user", "updated_by" : "elastic_some_user" }, "original_event" : { "dataset" : "socket", "kind" : "state", "action" : "existing_socket", "id" : "ffec6797-b92f-4436-bb40-69bac2c21874", "module" : "system" } } ``` ### Checklist Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR. ~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~ ~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~ ~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~ - [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios ~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~ ### For maintainers ~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ - [x] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process) --- .../alerts/__mocks__/es_results.ts | 57 +- .../alerts/get_filter.test.ts | 98 ++++ .../lib/detection_engine/alerts/types.ts | 4 - .../lib/detection_engine/alerts/utils.test.ts | 536 +++++++++++++++++- .../lib/detection_engine/alerts/utils.ts | 26 +- .../lib/detection_engine/signals_mapping.json | 72 +++ .../legacy/plugins/siem/server/lib/types.ts | 7 +- 7 files changed, 750 insertions(+), 50 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts index 079d3658461f..8080bd5ddd91 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -4,10 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalSearchResponse, RuleTypeParams } from '../types'; +import { + SignalSourceHit, + SignalSearchResponse, + RuleTypeParams, + OutputRuleAlertRest, +} from '../types'; export const sampleRuleAlertParams = ( - maxSignals: number | undefined, + maxSignals?: number | undefined, riskScore?: number | undefined ): RuleTypeParams => ({ ruleId: 'rule-1', @@ -32,7 +37,7 @@ export const sampleRuleAlertParams = ( meta: undefined, }); -export const sampleDocNoSortId = (someUuid: string): SignalSourceHit => ({ +export const sampleDocNoSortId = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -44,7 +49,7 @@ export const sampleDocNoSortId = (someUuid: string): SignalSourceHit => ({ }, }); -export const sampleDocNoSortIdNoVersion = (someUuid: string): SignalSourceHit => ({ +export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -55,7 +60,7 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string): SignalSourceHit => }, }); -export const sampleDocWithSortId = (someUuid: string): SignalSourceHit => ({ +export const sampleDocWithSortId = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -138,7 +143,9 @@ export const sampleBulkCreateDuplicateResult = { ], }; -export const sampleDocSearchResultsNoSortId = (someUuid: string): SignalSearchResponse => ({ +export const sampleDocSearchResultsNoSortId = ( + someUuid: string = sampleIdGuid +): SignalSearchResponse => ({ took: 10, timed_out: false, _shards: { @@ -159,7 +166,7 @@ export const sampleDocSearchResultsNoSortId = (someUuid: string): SignalSearchRe }); export const sampleDocSearchResultsNoSortIdNoVersion = ( - someUuid: string + someUuid: string = sampleIdGuid ): SignalSearchResponse => ({ took: 10, timed_out: false, @@ -180,7 +187,9 @@ export const sampleDocSearchResultsNoSortIdNoVersion = ( }, }); -export const sampleDocSearchResultsNoSortIdNoHits = (someUuid: string): SignalSearchResponse => ({ +export const sampleDocSearchResultsNoSortIdNoHits = ( + someUuid: string = sampleIdGuid +): SignalSearchResponse => ({ took: 10, timed_out: false, _shards: { @@ -222,7 +231,9 @@ export const repeatedSearchResultsWithSortId = ( }, }); -export const sampleDocSearchResultsWithSortId = (someUuid: string): SignalSearchResponse => ({ +export const sampleDocSearchResultsWithSortId = ( + someUuid: string = sampleIdGuid +): SignalSearchResponse => ({ took: 10, timed_out: false, _shards: { @@ -243,3 +254,31 @@ export const sampleDocSearchResultsWithSortId = (someUuid: string): SignalSearch }); export const sampleRuleGuid = '04128c15-0d1b-4716-a4c5-46997ac7f3bd'; +export const sampleIdGuid = 'e1e08ddc-5e37-49ff-a258-5393aa44435a'; + +export const sampleRule = (): Partial => { + return { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + risk_score: 50, + rule_id: 'rule-1', + language: 'kuery', + max_signals: 100, + name: 'Detect Root/Admin Users', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://www.example.com', 'https://ww.example.com'], + severity: 'high', + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + }; +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.test.ts index 9f72da44e963..c55c99fb291c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.test.ts @@ -7,6 +7,7 @@ import { getQueryFilter, getFilter } from './get_filter'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { AlertServices } from '../../../../../alerting/server/types'; +import { PartialFilter } from './types'; describe('get_filter', () => { let savedObjectsClient = savedObjectsClientMock.create(); @@ -145,6 +146,103 @@ describe('get_filter', () => { }); }); + test('it should work with a simple filter as a kuery without meta information', () => { + const esQuery = getQueryFilter( + 'host.name: windows', + 'kuery', + [ + { + query: { + match_phrase: { + 'host.name': 'siem-windows', + }, + }, + }, + ], + ['auditbeat-*'] + ); + expect(esQuery).toEqual({ + bool: { + must: [], + filter: [ + { + bool: { + should: [ + { + match: { + 'host.name': 'windows', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + match_phrase: { + 'host.name': 'siem-windows', + }, + }, + ], + should: [], + must_not: [], + }, + }); + }); + + test('it should work with a simple filter as a kuery without meta information with an exists', () => { + const query: PartialFilter = { + query: { + match_phrase: { + 'host.name': 'siem-windows', + }, + }, + } as PartialFilter; + + const exists: PartialFilter = { + exists: { + field: 'host.hostname', + }, + } as PartialFilter; + + const esQuery = getQueryFilter( + 'host.name: windows', + 'kuery', + [query, exists], + ['auditbeat-*'] + ); + expect(esQuery).toEqual({ + bool: { + must: [], + filter: [ + { + bool: { + should: [ + { + match: { + 'host.name': 'windows', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + match_phrase: { + 'host.name': 'siem-windows', + }, + }, + { + exists: { + field: 'host.hostname', + }, + }, + ], + should: [], + must_not: [], + }, + }); + }); + test('it should work with a simple filter that is disabled as a kuery', () => { const esQuery = getQueryFilter( 'host.name: windows', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts index 28431b816526..462a9b7d65ee 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -65,10 +65,6 @@ export type OutputRuleAlertRest = RuleAlertParamsRest & { updated_by: string | undefined | null; }; -export type OutputRuleES = OutputRuleAlertRest & { - status: 'open' | 'closed'; -}; - export type UpdateRuleAlertParamsRest = Partial & { id: string | undefined; rule_id: RuleAlertParams['ruleId'] | undefined; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 19c8d5ccc87c..fc50e54e06e4 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -14,6 +14,9 @@ import { singleBulkCreate, singleSearchAfter, searchAfterAndBulkCreate, + buildEventTypeSignal, + buildSignal, + buildRule, } from './utils'; import { sampleDocNoSortId, @@ -26,8 +29,12 @@ import { repeatedSearchResultsWithSortId, sampleBulkCreateDuplicateResult, sampleRuleGuid, + sampleRule, + sampleIdGuid, } from './__mocks__/es_results'; import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; +import { OutputRuleAlertRest } from './types'; +import { Signal } from '../../types'; const mockLogger: Logger = { log: jest.fn(), @@ -51,10 +58,9 @@ describe('utils', () => { }); describe('buildBulkBody', () => { test('if bulk body builds well-defined body', () => { - const fakeUuid = uuid.v4(); - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const fakeSignalSourceHit = buildBulkBody({ - doc: sampleDocNoSortId(fakeUuid), + doc: sampleDocNoSortId(), ruleParams: sampleParams, id: sampleRuleGuid, name: 'rule-name', @@ -65,18 +71,225 @@ describe('utils', () => { }); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; - if (fakeSignalSourceHit.signal.parent) { - delete fakeSignalSourceHit.signal.parent.id; - } expect(fakeSignalSourceHit).toEqual({ someKey: 'someValue', + event: { + kind: 'signal', + }, signal: { parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + status: 'open', + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + tags: ['some fake tag'], + type: 'query', + to: 'now', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + }, + }, + }); + }); + + test('if bulk body builds original_event if it exists on the event to begin with', () => { + const sampleParams = sampleRuleAlertParams(); + const doc = sampleDocNoSortId(); + doc._source.event = { + action: 'socket_opened', + module: 'system', + dataset: 'socket', + kind: 'event', + }; + const fakeSignalSourceHit = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); + // Timestamp will potentially always be different so remove it for the test + delete fakeSignalSourceHit['@timestamp']; + expect(fakeSignalSourceHit).toEqual({ + someKey: 'someValue', + event: { + action: 'socket_opened', + dataset: 'socket', + kind: 'signal', + module: 'system', + }, + signal: { + original_event: { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + status: 'open', + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + tags: ['some fake tag'], + type: 'query', + to: 'now', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + }, + }, + }); + }); + + test('if bulk body builds original_event if it exists on the event to begin with but no kind information', () => { + const sampleParams = sampleRuleAlertParams(); + const doc = sampleDocNoSortId(); + doc._source.event = { + action: 'socket_opened', + module: 'system', + dataset: 'socket', + }; + const fakeSignalSourceHit = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); + // Timestamp will potentially always be different so remove it for the test + delete fakeSignalSourceHit['@timestamp']; + expect(fakeSignalSourceHit).toEqual({ + someKey: 'someValue', + event: { + action: 'socket_opened', + dataset: 'socket', + kind: 'signal', + module: 'system', + }, + signal: { + original_event: { + action: 'socket_opened', + dataset: 'socket', + module: 'system', + }, + parent: { + id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', depth: 1, }, original_time: 'someTimeStamp', + status: 'open', + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + tags: ['some fake tag'], + type: 'query', + to: 'now', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + }, + }, + }); + }); + + test('if bulk body builds original_event if it exists on the event to begin with with only kind information', () => { + const sampleParams = sampleRuleAlertParams(); + const doc = sampleDocNoSortId(); + doc._source.event = { + kind: 'event', + }; + const fakeSignalSourceHit = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); + // Timestamp will potentially always be different so remove it for the test + delete fakeSignalSourceHit['@timestamp']; + expect(fakeSignalSourceHit).toEqual({ + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_event: { + kind: 'event', + }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + status: 'open', rule: { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', rule_id: 'rule-1', @@ -96,7 +309,6 @@ describe('utils', () => { severity: 'high', tags: ['some fake tag'], type: 'query', - status: 'open', to: 'now', enabled: true, created_by: 'elastic', @@ -213,8 +425,7 @@ describe('utils', () => { }); }); test('create successful bulk create', async () => { - const fakeUuid = uuid.v4(); - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const sampleSearchResult = sampleDocSearchResultsNoSortId; mockService.callCluster.mockReturnValueOnce({ took: 100, @@ -226,7 +437,7 @@ describe('utils', () => { ], }); const successfulsingleBulkCreate = await singleBulkCreate({ - someResult: sampleSearchResult(fakeUuid), + someResult: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -241,8 +452,7 @@ describe('utils', () => { expect(successfulsingleBulkCreate).toEqual(true); }); test('create successful bulk create with docs with no versioning', async () => { - const fakeUuid = uuid.v4(); - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const sampleSearchResult = sampleDocSearchResultsNoSortIdNoVersion; mockService.callCluster.mockReturnValueOnce({ took: 100, @@ -254,7 +464,7 @@ describe('utils', () => { ], }); const successfulsingleBulkCreate = await singleBulkCreate({ - someResult: sampleSearchResult(fakeUuid), + someResult: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -269,7 +479,7 @@ describe('utils', () => { expect(successfulsingleBulkCreate).toEqual(true); }); test('create unsuccessful bulk create due to empty search results', async () => { - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const sampleSearchResult = sampleEmptyDocSearchResults; mockService.callCluster.mockReturnValue(false); const successfulsingleBulkCreate = await singleBulkCreate({ @@ -288,12 +498,11 @@ describe('utils', () => { expect(successfulsingleBulkCreate).toEqual(true); }); test('create successful bulk create when bulk create has errors', async () => { - const fakeUuid = uuid.v4(); - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const sampleSearchResult = sampleDocSearchResultsNoSortId; mockService.callCluster.mockReturnValue(sampleBulkCreateDuplicateResult); const successfulsingleBulkCreate = await singleBulkCreate({ - someResult: sampleSearchResult(fakeUuid), + someResult: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -312,7 +521,7 @@ describe('utils', () => { describe('singleSearchAfter', () => { test('if singleSearchAfter works without a given sort id', async () => { let searchAfterSortId; - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValue(sampleDocSearchResultsNoSortId); await expect( singleSearchAfter({ @@ -326,7 +535,7 @@ describe('utils', () => { }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValue(sampleDocSearchResultsWithSortId); const searchAfterResult = await singleSearchAfter({ searchAfterSortId, @@ -339,7 +548,7 @@ describe('utils', () => { }); test('if singleSearchAfter throws error', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockImplementation(async () => { throw Error('Fake Error'); }); @@ -356,7 +565,7 @@ describe('utils', () => { }); describe('searchAfterAndBulkCreate', () => { test('if successful with empty search results', async () => { - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const result = await searchAfterAndBulkCreate({ someResult: sampleEmptyDocSearchResults, ruleParams: sampleParams, @@ -446,8 +655,7 @@ describe('utils', () => { expect(result).toEqual(false); }); test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids', async () => { - const sampleParams = sampleRuleAlertParams(undefined); - const someUuid = uuid.v4(); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValueOnce({ took: 100, errors: false, @@ -458,7 +666,7 @@ describe('utils', () => { ], }); const result = await searchAfterAndBulkCreate({ - someResult: sampleDocSearchResultsNoSortId(someUuid), + someResult: sampleDocSearchResultsNoSortId(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -475,8 +683,7 @@ describe('utils', () => { expect(result).toEqual(false); }); test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids and 0 total hits', async () => { - const sampleParams = sampleRuleAlertParams(undefined); - const someUuid = uuid.v4(); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValueOnce({ took: 100, errors: false, @@ -487,7 +694,7 @@ describe('utils', () => { ], }); const result = await searchAfterAndBulkCreate({ - someResult: sampleDocSearchResultsNoSortIdNoHits(someUuid), + someResult: sampleDocSearchResultsNoSortIdNoHits(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -504,7 +711,6 @@ describe('utils', () => { }); test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => { const sampleParams = sampleRuleAlertParams(10); - const oneGuid = uuid.v4(); const someGuids = Array.from({ length: 4 }).map(x => uuid.v4()); mockService.callCluster .mockReturnValueOnce({ @@ -516,7 +722,7 @@ describe('utils', () => { }, ], }) - .mockReturnValueOnce(sampleDocSearchResultsNoSortId(oneGuid)); + .mockReturnValueOnce(sampleDocSearchResultsNoSortId()); const result = await searchAfterAndBulkCreate({ someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), ruleParams: sampleParams, @@ -596,4 +802,276 @@ describe('utils', () => { expect(result).toEqual(false); }); }); + + describe('buildEventTypeSignal', () => { + test('it returns the event appended of kind signal if it does not exist', () => { + const doc = sampleDocNoSortId(); + delete doc._source.event; + const eventType = buildEventTypeSignal(doc); + const expected: object = { kind: 'signal' }; + expect(eventType).toEqual(expected); + }); + + test('it returns the event appended of kind signal if it is an empty object', () => { + const doc = sampleDocNoSortId(); + doc._source.event = {}; + const eventType = buildEventTypeSignal(doc); + const expected: object = { kind: 'signal' }; + expect(eventType).toEqual(expected); + }); + + test('it returns the event with kind signal and other properties if they exist', () => { + const doc = sampleDocNoSortId(); + doc._source.event = { + action: 'socket_opened', + module: 'system', + dataset: 'socket', + }; + const eventType = buildEventTypeSignal(doc); + const expected: object = { + action: 'socket_opened', + module: 'system', + dataset: 'socket', + kind: 'signal', + }; + expect(eventType).toEqual(expected); + }); + }); + + describe('buildSignal', () => { + test('it builds a signal as expected without original_event if event does not exist', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + delete doc._source.event; + const rule: Partial = sampleRule(); + const signal = buildSignal(doc, rule); + const expected: Signal = { + parent: { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + status: 'open', + rule: { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + risk_score: 50, + rule_id: 'rule-1', + language: 'kuery', + max_signals: 100, + name: 'Detect Root/Admin Users', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://www.example.com', 'https://ww.example.com'], + severity: 'high', + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + }, + }; + expect(signal).toEqual(expected); + }); + + test('it builds a signal as expected with original_event if is present', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + doc._source.event = { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }; + const rule: Partial = sampleRule(); + const signal = buildSignal(doc, rule); + const expected: Signal = { + parent: { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + original_event: { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }, + status: 'open', + rule: { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + risk_score: 50, + rule_id: 'rule-1', + language: 'kuery', + max_signals: 100, + name: 'Detect Root/Admin Users', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://www.example.com', 'https://ww.example.com'], + severity: 'high', + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + }, + }; + expect(signal).toEqual(expected); + }); + }); + + describe('buildRule', () => { + test('it builds a rule as expected with filters present', () => { + const ruleParams = sampleRuleAlertParams(); + ruleParams.filters = [ + { + query: 'host.name: Rebecca', + }, + { + query: 'host.name: Evan', + }, + { + query: 'host.name: Braden', + }, + ]; + const rule = buildRule({ + ruleParams, + name: 'some-name', + id: sampleRuleGuid, + enabled: false, + createdBy: 'elastic', + updatedBy: 'elastic', + interval: 'some interval', + }); + const expected: Partial = { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: false, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: 'some interval', + language: 'kuery', + max_signals: 10000, + name: 'some-name', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + risk_score: 50, + rule_id: 'rule-1', + severity: 'high', + tags: ['some fake tag'], + to: 'now', + type: 'query', + updated_by: 'elastic', + filters: [ + { + query: 'host.name: Rebecca', + }, + { + query: 'host.name: Evan', + }, + { + query: 'host.name: Braden', + }, + ], + }; + expect(rule).toEqual(expected); + }); + + test('it omits a null value such as if enabled is null if is present', () => { + const ruleParams = sampleRuleAlertParams(); + ruleParams.filters = undefined; + const rule = buildRule({ + ruleParams, + name: 'some-name', + id: sampleRuleGuid, + enabled: true, + createdBy: 'elastic', + updatedBy: 'elastic', + interval: 'some interval', + }); + const expected: Partial = { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: 'some interval', + language: 'kuery', + max_signals: 10000, + name: 'some-name', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + risk_score: 50, + rule_id: 'rule-1', + severity: 'high', + tags: ['some fake tag'], + to: 'now', + type: 'query', + updated_by: 'elastic', + }; + expect(rule).toEqual(expected); + }); + + test('it omits a null value such as if filters is undefined if is present', () => { + const ruleParams = sampleRuleAlertParams(); + ruleParams.filters = undefined; + const rule = buildRule({ + ruleParams, + name: 'some-name', + id: sampleRuleGuid, + enabled: true, + createdBy: 'elastic', + updatedBy: 'elastic', + interval: 'some interval', + }); + const expected: Partial = { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: 'some interval', + language: 'kuery', + max_signals: 10000, + name: 'some-name', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + risk_score: 50, + rule_id: 'rule-1', + severity: 'high', + tags: ['some fake tag'], + to: 'now', + type: 'query', + updated_by: 'elastic', + }; + expect(rule).toEqual(expected); + }); + }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index ba3f310c886c..c3988b8fea45 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -14,7 +14,7 @@ import { SignalSearchResponse, BulkResponse, RuleTypeParams, - OutputRuleES, + OutputRuleAlertRest, } from './types'; import { buildEventsSearchQuery } from './build_events_query'; @@ -36,10 +36,9 @@ export const buildRule = ({ createdBy, updatedBy, interval, -}: BuildRuleParams): Partial => { - return pickBy((value: unknown) => value != null, { +}: BuildRuleParams): Partial => { + return pickBy((value: unknown) => value != null, { id, - status: 'open', rule_id: ruleParams.ruleId, false_positives: ruleParams.falsePositives, saved_id: ruleParams.savedId, @@ -68,8 +67,8 @@ export const buildRule = ({ }); }; -export const buildSignal = (doc: SignalSourceHit, rule: Partial): Signal => { - return { +export const buildSignal = (doc: SignalSourceHit, rule: Partial): Signal => { + const signal: Signal = { parent: { id: doc._id, type: 'event', @@ -77,8 +76,13 @@ export const buildSignal = (doc: SignalSourceHit, rule: Partial): depth: 1, }, original_time: doc._source['@timestamp'], + status: 'open', rule, }; + if (doc._source.event != null) { + return { ...signal, original_event: doc._source.event }; + } + return signal; }; interface BuildBulkBodyParams { @@ -92,6 +96,14 @@ interface BuildBulkBodyParams { enabled: boolean; } +export const buildEventTypeSignal = (doc: SignalSourceHit): object => { + if (doc._source.event != null && doc._source.event instanceof Object) { + return { ...doc._source.event, kind: 'signal' }; + } else { + return { kind: 'signal' }; + } +}; + // format search_after result for signals index. export const buildBulkBody = ({ doc, @@ -113,9 +125,11 @@ export const buildBulkBody = ({ interval, }); const signal = buildSignal(doc, rule); + const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, '@timestamp': new Date().toISOString(), + event, signal, }; return signalHit; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json index a95c9625a000..dfe3caed5b71 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json @@ -107,6 +107,78 @@ }, "original_time": { "type": "date" + }, + "original_event": { + "properties": { + "action": { + "type": "keyword" + }, + "category": { + "type": "keyword" + }, + "code": { + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "kind": { + "type": "keyword" + }, + "module": { + "type": "keyword" + }, + "original": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "outcome": { + "type": "keyword" + }, + "provider": { + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "status": { + "type": "keyword" } } }, diff --git a/x-pack/legacy/plugins/siem/server/lib/types.ts b/x-pack/legacy/plugins/siem/server/lib/types.ts index 9c0059d0d109..c53805dc95fe 100644 --- a/x-pack/legacy/plugins/siem/server/lib/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/types.ts @@ -23,7 +23,7 @@ import { Note } from './note/saved_object'; import { PinnedEvent } from './pinned_event/saved_object'; import { Timeline } from './timeline/saved_object'; import { TLS } from './tls'; -import { RuleAlertParamsRest } from './detection_engine/alerts/types'; +import { SearchTypes, OutputRuleAlertRest } from './detection_engine/alerts/types'; export * from './hosts'; @@ -66,7 +66,7 @@ export interface SiemContext { } export interface Signal { - rule: Partial; + rule: Partial; parent: { id: string; type: string; @@ -74,10 +74,13 @@ export interface Signal { depth: number; }; original_time: string; + original_event?: SearchTypes; + status: 'open' | 'closed'; } export interface SignalHit { '@timestamp': string; + event: object; signal: Partial; }