From f70bff299ef38bdd3dc3ac9ce876beb0c22a809e Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Tue, 26 Nov 2019 13:36:23 -0700 Subject: [PATCH 1/2] Added singal type to the event, copy off the event to original, and fixed status bug --- .../alerts/__mocks__/es_results.ts | 57 +- .../lib/detection_engine/alerts/types.ts | 4 - .../lib/detection_engine/alerts/utils.test.ts | 536 +++++++++++++++++- .../lib/detection_engine/alerts/utils.ts | 26 +- .../lib/detection_engine/signals_mapping.json | 72 +++ .../legacy/plugins/siem/server/lib/types.ts | 7 +- 6 files changed, 652 insertions(+), 50 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts index 079d3658461f..8080bd5ddd91 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -4,10 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalSearchResponse, RuleTypeParams } from '../types'; +import { + SignalSourceHit, + SignalSearchResponse, + RuleTypeParams, + OutputRuleAlertRest, +} from '../types'; export const sampleRuleAlertParams = ( - maxSignals: number | undefined, + maxSignals?: number | undefined, riskScore?: number | undefined ): RuleTypeParams => ({ ruleId: 'rule-1', @@ -32,7 +37,7 @@ export const sampleRuleAlertParams = ( meta: undefined, }); -export const sampleDocNoSortId = (someUuid: string): SignalSourceHit => ({ +export const sampleDocNoSortId = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -44,7 +49,7 @@ export const sampleDocNoSortId = (someUuid: string): SignalSourceHit => ({ }, }); -export const sampleDocNoSortIdNoVersion = (someUuid: string): SignalSourceHit => ({ +export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -55,7 +60,7 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string): SignalSourceHit => }, }); -export const sampleDocWithSortId = (someUuid: string): SignalSourceHit => ({ +export const sampleDocWithSortId = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -138,7 +143,9 @@ export const sampleBulkCreateDuplicateResult = { ], }; -export const sampleDocSearchResultsNoSortId = (someUuid: string): SignalSearchResponse => ({ +export const sampleDocSearchResultsNoSortId = ( + someUuid: string = sampleIdGuid +): SignalSearchResponse => ({ took: 10, timed_out: false, _shards: { @@ -159,7 +166,7 @@ export const sampleDocSearchResultsNoSortId = (someUuid: string): SignalSearchRe }); export const sampleDocSearchResultsNoSortIdNoVersion = ( - someUuid: string + someUuid: string = sampleIdGuid ): SignalSearchResponse => ({ took: 10, timed_out: false, @@ -180,7 +187,9 @@ export const sampleDocSearchResultsNoSortIdNoVersion = ( }, }); -export const sampleDocSearchResultsNoSortIdNoHits = (someUuid: string): SignalSearchResponse => ({ +export const sampleDocSearchResultsNoSortIdNoHits = ( + someUuid: string = sampleIdGuid +): SignalSearchResponse => ({ took: 10, timed_out: false, _shards: { @@ -222,7 +231,9 @@ export const repeatedSearchResultsWithSortId = ( }, }); -export const sampleDocSearchResultsWithSortId = (someUuid: string): SignalSearchResponse => ({ +export const sampleDocSearchResultsWithSortId = ( + someUuid: string = sampleIdGuid +): SignalSearchResponse => ({ took: 10, timed_out: false, _shards: { @@ -243,3 +254,31 @@ export const sampleDocSearchResultsWithSortId = (someUuid: string): SignalSearch }); export const sampleRuleGuid = '04128c15-0d1b-4716-a4c5-46997ac7f3bd'; +export const sampleIdGuid = 'e1e08ddc-5e37-49ff-a258-5393aa44435a'; + +export const sampleRule = (): Partial => { + return { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + risk_score: 50, + rule_id: 'rule-1', + language: 'kuery', + max_signals: 100, + name: 'Detect Root/Admin Users', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://www.example.com', 'https://ww.example.com'], + severity: 'high', + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + }; +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts index 28431b816526..462a9b7d65ee 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -65,10 +65,6 @@ export type OutputRuleAlertRest = RuleAlertParamsRest & { updated_by: string | undefined | null; }; -export type OutputRuleES = OutputRuleAlertRest & { - status: 'open' | 'closed'; -}; - export type UpdateRuleAlertParamsRest = Partial & { id: string | undefined; rule_id: RuleAlertParams['ruleId'] | undefined; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 19c8d5ccc87c..fc50e54e06e4 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -14,6 +14,9 @@ import { singleBulkCreate, singleSearchAfter, searchAfterAndBulkCreate, + buildEventTypeSignal, + buildSignal, + buildRule, } from './utils'; import { sampleDocNoSortId, @@ -26,8 +29,12 @@ import { repeatedSearchResultsWithSortId, sampleBulkCreateDuplicateResult, sampleRuleGuid, + sampleRule, + sampleIdGuid, } from './__mocks__/es_results'; import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; +import { OutputRuleAlertRest } from './types'; +import { Signal } from '../../types'; const mockLogger: Logger = { log: jest.fn(), @@ -51,10 +58,9 @@ describe('utils', () => { }); describe('buildBulkBody', () => { test('if bulk body builds well-defined body', () => { - const fakeUuid = uuid.v4(); - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const fakeSignalSourceHit = buildBulkBody({ - doc: sampleDocNoSortId(fakeUuid), + doc: sampleDocNoSortId(), ruleParams: sampleParams, id: sampleRuleGuid, name: 'rule-name', @@ -65,18 +71,225 @@ describe('utils', () => { }); // Timestamp will potentially always be different so remove it for the test delete fakeSignalSourceHit['@timestamp']; - if (fakeSignalSourceHit.signal.parent) { - delete fakeSignalSourceHit.signal.parent.id; - } expect(fakeSignalSourceHit).toEqual({ someKey: 'someValue', + event: { + kind: 'signal', + }, signal: { parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + status: 'open', + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + tags: ['some fake tag'], + type: 'query', + to: 'now', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + }, + }, + }); + }); + + test('if bulk body builds original_event if it exists on the event to begin with', () => { + const sampleParams = sampleRuleAlertParams(); + const doc = sampleDocNoSortId(); + doc._source.event = { + action: 'socket_opened', + module: 'system', + dataset: 'socket', + kind: 'event', + }; + const fakeSignalSourceHit = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); + // Timestamp will potentially always be different so remove it for the test + delete fakeSignalSourceHit['@timestamp']; + expect(fakeSignalSourceHit).toEqual({ + someKey: 'someValue', + event: { + action: 'socket_opened', + dataset: 'socket', + kind: 'signal', + module: 'system', + }, + signal: { + original_event: { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + status: 'open', + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + tags: ['some fake tag'], + type: 'query', + to: 'now', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + }, + }, + }); + }); + + test('if bulk body builds original_event if it exists on the event to begin with but no kind information', () => { + const sampleParams = sampleRuleAlertParams(); + const doc = sampleDocNoSortId(); + doc._source.event = { + action: 'socket_opened', + module: 'system', + dataset: 'socket', + }; + const fakeSignalSourceHit = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); + // Timestamp will potentially always be different so remove it for the test + delete fakeSignalSourceHit['@timestamp']; + expect(fakeSignalSourceHit).toEqual({ + someKey: 'someValue', + event: { + action: 'socket_opened', + dataset: 'socket', + kind: 'signal', + module: 'system', + }, + signal: { + original_event: { + action: 'socket_opened', + dataset: 'socket', + module: 'system', + }, + parent: { + id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', depth: 1, }, original_time: 'someTimeStamp', + status: 'open', + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + tags: ['some fake tag'], + type: 'query', + to: 'now', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + }, + }, + }); + }); + + test('if bulk body builds original_event if it exists on the event to begin with with only kind information', () => { + const sampleParams = sampleRuleAlertParams(); + const doc = sampleDocNoSortId(); + doc._source.event = { + kind: 'event', + }; + const fakeSignalSourceHit = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + }); + // Timestamp will potentially always be different so remove it for the test + delete fakeSignalSourceHit['@timestamp']; + expect(fakeSignalSourceHit).toEqual({ + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_event: { + kind: 'event', + }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + status: 'open', rule: { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', rule_id: 'rule-1', @@ -96,7 +309,6 @@ describe('utils', () => { severity: 'high', tags: ['some fake tag'], type: 'query', - status: 'open', to: 'now', enabled: true, created_by: 'elastic', @@ -213,8 +425,7 @@ describe('utils', () => { }); }); test('create successful bulk create', async () => { - const fakeUuid = uuid.v4(); - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const sampleSearchResult = sampleDocSearchResultsNoSortId; mockService.callCluster.mockReturnValueOnce({ took: 100, @@ -226,7 +437,7 @@ describe('utils', () => { ], }); const successfulsingleBulkCreate = await singleBulkCreate({ - someResult: sampleSearchResult(fakeUuid), + someResult: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -241,8 +452,7 @@ describe('utils', () => { expect(successfulsingleBulkCreate).toEqual(true); }); test('create successful bulk create with docs with no versioning', async () => { - const fakeUuid = uuid.v4(); - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const sampleSearchResult = sampleDocSearchResultsNoSortIdNoVersion; mockService.callCluster.mockReturnValueOnce({ took: 100, @@ -254,7 +464,7 @@ describe('utils', () => { ], }); const successfulsingleBulkCreate = await singleBulkCreate({ - someResult: sampleSearchResult(fakeUuid), + someResult: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -269,7 +479,7 @@ describe('utils', () => { expect(successfulsingleBulkCreate).toEqual(true); }); test('create unsuccessful bulk create due to empty search results', async () => { - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const sampleSearchResult = sampleEmptyDocSearchResults; mockService.callCluster.mockReturnValue(false); const successfulsingleBulkCreate = await singleBulkCreate({ @@ -288,12 +498,11 @@ describe('utils', () => { expect(successfulsingleBulkCreate).toEqual(true); }); test('create successful bulk create when bulk create has errors', async () => { - const fakeUuid = uuid.v4(); - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const sampleSearchResult = sampleDocSearchResultsNoSortId; mockService.callCluster.mockReturnValue(sampleBulkCreateDuplicateResult); const successfulsingleBulkCreate = await singleBulkCreate({ - someResult: sampleSearchResult(fakeUuid), + someResult: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -312,7 +521,7 @@ describe('utils', () => { describe('singleSearchAfter', () => { test('if singleSearchAfter works without a given sort id', async () => { let searchAfterSortId; - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValue(sampleDocSearchResultsNoSortId); await expect( singleSearchAfter({ @@ -326,7 +535,7 @@ describe('utils', () => { }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValue(sampleDocSearchResultsWithSortId); const searchAfterResult = await singleSearchAfter({ searchAfterSortId, @@ -339,7 +548,7 @@ describe('utils', () => { }); test('if singleSearchAfter throws error', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockImplementation(async () => { throw Error('Fake Error'); }); @@ -356,7 +565,7 @@ describe('utils', () => { }); describe('searchAfterAndBulkCreate', () => { test('if successful with empty search results', async () => { - const sampleParams = sampleRuleAlertParams(undefined); + const sampleParams = sampleRuleAlertParams(); const result = await searchAfterAndBulkCreate({ someResult: sampleEmptyDocSearchResults, ruleParams: sampleParams, @@ -446,8 +655,7 @@ describe('utils', () => { expect(result).toEqual(false); }); test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids', async () => { - const sampleParams = sampleRuleAlertParams(undefined); - const someUuid = uuid.v4(); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValueOnce({ took: 100, errors: false, @@ -458,7 +666,7 @@ describe('utils', () => { ], }); const result = await searchAfterAndBulkCreate({ - someResult: sampleDocSearchResultsNoSortId(someUuid), + someResult: sampleDocSearchResultsNoSortId(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -475,8 +683,7 @@ describe('utils', () => { expect(result).toEqual(false); }); test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids and 0 total hits', async () => { - const sampleParams = sampleRuleAlertParams(undefined); - const someUuid = uuid.v4(); + const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValueOnce({ took: 100, errors: false, @@ -487,7 +694,7 @@ describe('utils', () => { ], }); const result = await searchAfterAndBulkCreate({ - someResult: sampleDocSearchResultsNoSortIdNoHits(someUuid), + someResult: sampleDocSearchResultsNoSortIdNoHits(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -504,7 +711,6 @@ describe('utils', () => { }); test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => { const sampleParams = sampleRuleAlertParams(10); - const oneGuid = uuid.v4(); const someGuids = Array.from({ length: 4 }).map(x => uuid.v4()); mockService.callCluster .mockReturnValueOnce({ @@ -516,7 +722,7 @@ describe('utils', () => { }, ], }) - .mockReturnValueOnce(sampleDocSearchResultsNoSortId(oneGuid)); + .mockReturnValueOnce(sampleDocSearchResultsNoSortId()); const result = await searchAfterAndBulkCreate({ someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), ruleParams: sampleParams, @@ -596,4 +802,276 @@ describe('utils', () => { expect(result).toEqual(false); }); }); + + describe('buildEventTypeSignal', () => { + test('it returns the event appended of kind signal if it does not exist', () => { + const doc = sampleDocNoSortId(); + delete doc._source.event; + const eventType = buildEventTypeSignal(doc); + const expected: object = { kind: 'signal' }; + expect(eventType).toEqual(expected); + }); + + test('it returns the event appended of kind signal if it is an empty object', () => { + const doc = sampleDocNoSortId(); + doc._source.event = {}; + const eventType = buildEventTypeSignal(doc); + const expected: object = { kind: 'signal' }; + expect(eventType).toEqual(expected); + }); + + test('it returns the event with kind signal and other properties if they exist', () => { + const doc = sampleDocNoSortId(); + doc._source.event = { + action: 'socket_opened', + module: 'system', + dataset: 'socket', + }; + const eventType = buildEventTypeSignal(doc); + const expected: object = { + action: 'socket_opened', + module: 'system', + dataset: 'socket', + kind: 'signal', + }; + expect(eventType).toEqual(expected); + }); + }); + + describe('buildSignal', () => { + test('it builds a signal as expected without original_event if event does not exist', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + delete doc._source.event; + const rule: Partial = sampleRule(); + const signal = buildSignal(doc, rule); + const expected: Signal = { + parent: { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + status: 'open', + rule: { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + risk_score: 50, + rule_id: 'rule-1', + language: 'kuery', + max_signals: 100, + name: 'Detect Root/Admin Users', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://www.example.com', 'https://ww.example.com'], + severity: 'high', + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + }, + }; + expect(signal).toEqual(expected); + }); + + test('it builds a signal as expected with original_event if is present', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + doc._source.event = { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }; + const rule: Partial = sampleRule(); + const signal = buildSignal(doc, rule); + const expected: Signal = { + parent: { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 1, + }, + original_time: 'someTimeStamp', + original_event: { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }, + status: 'open', + rule: { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + risk_score: 50, + rule_id: 'rule-1', + language: 'kuery', + max_signals: 100, + name: 'Detect Root/Admin Users', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://www.example.com', 'https://ww.example.com'], + severity: 'high', + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + }, + }; + expect(signal).toEqual(expected); + }); + }); + + describe('buildRule', () => { + test('it builds a rule as expected with filters present', () => { + const ruleParams = sampleRuleAlertParams(); + ruleParams.filters = [ + { + query: 'host.name: Rebecca', + }, + { + query: 'host.name: Evan', + }, + { + query: 'host.name: Braden', + }, + ]; + const rule = buildRule({ + ruleParams, + name: 'some-name', + id: sampleRuleGuid, + enabled: false, + createdBy: 'elastic', + updatedBy: 'elastic', + interval: 'some interval', + }); + const expected: Partial = { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: false, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: 'some interval', + language: 'kuery', + max_signals: 10000, + name: 'some-name', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + risk_score: 50, + rule_id: 'rule-1', + severity: 'high', + tags: ['some fake tag'], + to: 'now', + type: 'query', + updated_by: 'elastic', + filters: [ + { + query: 'host.name: Rebecca', + }, + { + query: 'host.name: Evan', + }, + { + query: 'host.name: Braden', + }, + ], + }; + expect(rule).toEqual(expected); + }); + + test('it omits a null value such as if enabled is null if is present', () => { + const ruleParams = sampleRuleAlertParams(); + ruleParams.filters = undefined; + const rule = buildRule({ + ruleParams, + name: 'some-name', + id: sampleRuleGuid, + enabled: true, + createdBy: 'elastic', + updatedBy: 'elastic', + interval: 'some interval', + }); + const expected: Partial = { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: 'some interval', + language: 'kuery', + max_signals: 10000, + name: 'some-name', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + risk_score: 50, + rule_id: 'rule-1', + severity: 'high', + tags: ['some fake tag'], + to: 'now', + type: 'query', + updated_by: 'elastic', + }; + expect(rule).toEqual(expected); + }); + + test('it omits a null value such as if filters is undefined if is present', () => { + const ruleParams = sampleRuleAlertParams(); + ruleParams.filters = undefined; + const rule = buildRule({ + ruleParams, + name: 'some-name', + id: sampleRuleGuid, + enabled: true, + createdBy: 'elastic', + updatedBy: 'elastic', + interval: 'some interval', + }); + const expected: Partial = { + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: 'some interval', + language: 'kuery', + max_signals: 10000, + name: 'some-name', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + risk_score: 50, + rule_id: 'rule-1', + severity: 'high', + tags: ['some fake tag'], + to: 'now', + type: 'query', + updated_by: 'elastic', + }; + expect(rule).toEqual(expected); + }); + }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index ba3f310c886c..c3988b8fea45 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -14,7 +14,7 @@ import { SignalSearchResponse, BulkResponse, RuleTypeParams, - OutputRuleES, + OutputRuleAlertRest, } from './types'; import { buildEventsSearchQuery } from './build_events_query'; @@ -36,10 +36,9 @@ export const buildRule = ({ createdBy, updatedBy, interval, -}: BuildRuleParams): Partial => { - return pickBy((value: unknown) => value != null, { +}: BuildRuleParams): Partial => { + return pickBy((value: unknown) => value != null, { id, - status: 'open', rule_id: ruleParams.ruleId, false_positives: ruleParams.falsePositives, saved_id: ruleParams.savedId, @@ -68,8 +67,8 @@ export const buildRule = ({ }); }; -export const buildSignal = (doc: SignalSourceHit, rule: Partial): Signal => { - return { +export const buildSignal = (doc: SignalSourceHit, rule: Partial): Signal => { + const signal: Signal = { parent: { id: doc._id, type: 'event', @@ -77,8 +76,13 @@ export const buildSignal = (doc: SignalSourceHit, rule: Partial): depth: 1, }, original_time: doc._source['@timestamp'], + status: 'open', rule, }; + if (doc._source.event != null) { + return { ...signal, original_event: doc._source.event }; + } + return signal; }; interface BuildBulkBodyParams { @@ -92,6 +96,14 @@ interface BuildBulkBodyParams { enabled: boolean; } +export const buildEventTypeSignal = (doc: SignalSourceHit): object => { + if (doc._source.event != null && doc._source.event instanceof Object) { + return { ...doc._source.event, kind: 'signal' }; + } else { + return { kind: 'signal' }; + } +}; + // format search_after result for signals index. export const buildBulkBody = ({ doc, @@ -113,9 +125,11 @@ export const buildBulkBody = ({ interval, }); const signal = buildSignal(doc, rule); + const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, '@timestamp': new Date().toISOString(), + event, signal, }; return signalHit; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json index a95c9625a000..dfe3caed5b71 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals_mapping.json @@ -107,6 +107,78 @@ }, "original_time": { "type": "date" + }, + "original_event": { + "properties": { + "action": { + "type": "keyword" + }, + "category": { + "type": "keyword" + }, + "code": { + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "kind": { + "type": "keyword" + }, + "module": { + "type": "keyword" + }, + "original": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "outcome": { + "type": "keyword" + }, + "provider": { + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "status": { + "type": "keyword" } } }, diff --git a/x-pack/legacy/plugins/siem/server/lib/types.ts b/x-pack/legacy/plugins/siem/server/lib/types.ts index 9c0059d0d109..c53805dc95fe 100644 --- a/x-pack/legacy/plugins/siem/server/lib/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/types.ts @@ -23,7 +23,7 @@ import { Note } from './note/saved_object'; import { PinnedEvent } from './pinned_event/saved_object'; import { Timeline } from './timeline/saved_object'; import { TLS } from './tls'; -import { RuleAlertParamsRest } from './detection_engine/alerts/types'; +import { SearchTypes, OutputRuleAlertRest } from './detection_engine/alerts/types'; export * from './hosts'; @@ -66,7 +66,7 @@ export interface SiemContext { } export interface Signal { - rule: Partial; + rule: Partial; parent: { id: string; type: string; @@ -74,10 +74,13 @@ export interface Signal { depth: number; }; original_time: string; + original_event?: SearchTypes; + status: 'open' | 'closed'; } export interface SignalHit { '@timestamp': string; + event: object; signal: Partial; } From 875292403d20b1e8cd4b80aac66995e871d1b687 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Tue, 26 Nov 2019 13:39:39 -0700 Subject: [PATCH 2/2] Added more filter tests --- .../alerts/get_filter.test.ts | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.test.ts index 9f72da44e963..c55c99fb291c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.test.ts @@ -7,6 +7,7 @@ import { getQueryFilter, getFilter } from './get_filter'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { AlertServices } from '../../../../../alerting/server/types'; +import { PartialFilter } from './types'; describe('get_filter', () => { let savedObjectsClient = savedObjectsClientMock.create(); @@ -145,6 +146,103 @@ describe('get_filter', () => { }); }); + test('it should work with a simple filter as a kuery without meta information', () => { + const esQuery = getQueryFilter( + 'host.name: windows', + 'kuery', + [ + { + query: { + match_phrase: { + 'host.name': 'siem-windows', + }, + }, + }, + ], + ['auditbeat-*'] + ); + expect(esQuery).toEqual({ + bool: { + must: [], + filter: [ + { + bool: { + should: [ + { + match: { + 'host.name': 'windows', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + match_phrase: { + 'host.name': 'siem-windows', + }, + }, + ], + should: [], + must_not: [], + }, + }); + }); + + test('it should work with a simple filter as a kuery without meta information with an exists', () => { + const query: PartialFilter = { + query: { + match_phrase: { + 'host.name': 'siem-windows', + }, + }, + } as PartialFilter; + + const exists: PartialFilter = { + exists: { + field: 'host.hostname', + }, + } as PartialFilter; + + const esQuery = getQueryFilter( + 'host.name: windows', + 'kuery', + [query, exists], + ['auditbeat-*'] + ); + expect(esQuery).toEqual({ + bool: { + must: [], + filter: [ + { + bool: { + should: [ + { + match: { + 'host.name': 'windows', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + match_phrase: { + 'host.name': 'siem-windows', + }, + }, + { + exists: { + field: 'host.hostname', + }, + }, + ], + should: [], + must_not: [], + }, + }); + }); + test('it should work with a simple filter that is disabled as a kuery', () => { const esQuery = getQueryFilter( 'host.name: windows',