From 6cf620fdfc3e2d0c7877e02cdfaf0cb20d8c4edb Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Mon, 14 Sep 2020 18:18:57 -0400 Subject: [PATCH 01/26] First draft of EQL rules in detection engine --- .../schemas/common/schemas.ts | 1 + .../schemas/response/rules_schema.ts | 2 +- .../rules/description_step/helpers.tsx | 8 + .../scripts/rules/queries/query_eql.json | 69 ++++++++ .../signals/build_event_type_signal.ts | 4 +- .../signals/build_rule.test.ts | 20 +-- .../detection_engine/signals/build_rule.ts | 68 +++++++- .../signals/build_signal.test.ts | 94 ++++++---- .../detection_engine/signals/build_signal.ts | 10 +- .../detection_engine/signals/get_filter.ts | 3 + .../signals/signal_rule_alert_type.ts | 165 +++++++++++++++++- .../signals/single_bulk_create.ts | 68 +++++++- .../lib/detection_engine/signals/types.ts | 11 +- .../lib/detection_engine/signals/utils.ts | 12 +- .../security_solution/server/lib/types.ts | 55 ++++-- 15 files changed, 501 insertions(+), 89 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index 498b561a818f2..aaebcd11bf583 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -298,6 +298,7 @@ export const type = t.keyof({ query: null, saved_query: null, threshold: null, + eql_query: null, }); export type Type = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 04df25d805f9e..3f6ac5b2edf67 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -205,7 +205,7 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi }; export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { - if (['query', 'saved_query', 'threshold'].includes(typeAndTimelineOnly.type)) { + if (['query', 'saved_query', 'threshold', 'eql_query'].includes(typeAndTimelineOnly.type)) { return [ t.exact(t.type({ query: dependentRulesSchema.props.query })), t.exact(t.type({ language: dependentRulesSchema.props.language })), diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx index 680ca78fc222e..ce6b5b9307dcd 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/helpers.tsx @@ -383,6 +383,14 @@ export const buildRuleTypeDescription = (label: string, ruleType: Type): ListIte }, ]; } + case 'eql_query': { + return [ + { + title: label, + description: 'placeholder to get rid of typecheck error', + }, + ]; + } default: return assertUnreachable(ruleType); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json new file mode 100644 index 0000000000000..a77eebd708b39 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json @@ -0,0 +1,69 @@ +{ + "name": "EQL query rule", + "description": "Rule with an eql query", + "false_positives": [ + "https://www.example.com/some-article-about-a-false-positive", + "some text string about why another condition could be a false positive" + ], + "rule_id": "rule-id-eql", + "enabled": false, + "index": ["logs-endpoint.events.process-default"], + "interval": "30s", + "query": "process where process.name = \"mimikatz.exe\"", + "output_index": ".siem-signals-default", + "meta": { + "anything_you_want_ui_related_or_otherwise": { + "as_deep_structured_as_you_need": { + "any_data_type": {} + } + } + }, + "language": "kuery", + "risk_score": 1, + "max_signals": 100, + "tags": ["tag 1", "tag 2", "any tag you want"], + "to": "now", + "from": "now-30m", + "severity": "high", + "type": "eql_query", + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1499", + "name": "endpoint denial of service", + "reference": "https://attack.mitre.org/techniques/T1499/" + } + ] + }, + { + "framework": "Some other Framework you want", + "tactic": { + "id": "some-other-id", + "name": "Some other name", + "reference": "https://example.com" + }, + "technique": [ + { + "id": "some-other-id", + "name": "some other technique name", + "reference": "https://example.com" + } + ] + } + ], + "references": [ + "http://www.example.com/some-article-about-attack", + "Some plain text string here explaining why this is a valid thing to look out for" + ], + "timeline_id": "timeline_id", + "timeline_title": "timeline_title", + "note": "# note markdown", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts index 59cdc020c611d..81c9d1dedcc56 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit } from './types'; +import { BaseSignalHit } from './types'; -export const buildEventTypeSignal = (doc: SignalSourceHit): object => { +export const buildEventTypeSignal = (doc: BaseSignalHit): object => { if (doc._source.event != null && doc._source.event instanceof Object) { return { ...doc._source.event, kind: 'signal' }; } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index ba815a0b62f0d..dc50b18b977b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -9,7 +9,10 @@ import { sampleDocNoSortId, sampleRuleAlertParams, sampleRuleGuid } from './__mo import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; -import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { + getPartialRulesSchemaMock, + getRulesSchemaMock, +} from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildRule', () => { beforeEach(() => { @@ -274,7 +277,7 @@ describe('buildRule', () => { }); test('it removes internal tags from a typical rule', () => { - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); rule.tags = [ 'some fake tag 1', 'some fake tag 2', @@ -286,7 +289,7 @@ describe('buildRule', () => { }); test('it works with an empty array', () => { - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); rule.tags = []; const noInternals = removeInternalTagsFromRule(rule); const expected = getPartialRulesSchemaMock(); @@ -294,17 +297,8 @@ describe('buildRule', () => { expect(noInternals).toEqual(expected); }); - test('it works if tags does not exist', () => { - const rule = getPartialRulesSchemaMock(); - delete rule.tags; - const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); - delete expected.tags; - expect(noInternals).toEqual(expected); - }); - test('it works if tags contains normal values and no internal values', () => { - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); const noInternals = removeInternalTagsFromRule(rule); expect(noInternals).toEqual(rule); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index aacf9b8be31b4..301a71821cce1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -4,12 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { pickBy } from 'lodash/fp'; +import { SavedObject } from 'src/core/types'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; import { buildRiskScoreFromMapping } from './mappings/build_risk_score_from_mapping'; -import { SignalSourceHit } from './types'; +import { SignalSourceHit, RuleAlertAttributes } from './types'; import { buildSeverityFromMapping } from './mappings/build_severity_from_mapping'; import { buildRuleNameFromMapping } from './mappings/build_rule_name_from_mapping'; import { INTERNAL_IDENTIFIER } from '../../../../common/constants'; @@ -44,7 +44,7 @@ export const buildRule = ({ interval, tags, throttle, -}: BuildRuleParams): Partial => { +}: BuildRuleParams): RulesSchema => { const { riskScore, riskScoreMeta } = buildRiskScoreFromMapping({ doc, riskScore: ruleParams.riskScore, @@ -65,7 +65,7 @@ export const buildRule = ({ const meta = { ...ruleParams.meta, ...riskScoreMeta, ...severityMeta, ...ruleNameMeta }; - const rule = pickBy((value: unknown) => value != null, { + const rule = { id, rule_id: ruleParams.ruleId ?? '(unknown rule_id)', actions, @@ -111,15 +111,69 @@ export const buildRule = ({ machine_learning_job_id: ruleParams.machineLearningJobId, anomaly_threshold: ruleParams.anomalyThreshold, threshold: ruleParams.threshold, - }); + }; + return removeInternalTagsFromRule(rule); +}; + +export const buildRuleWithoutOverrides = ( + ruleSO: SavedObject +): RulesSchema => { + const ruleParams = ruleSO.attributes.params; + const rule = { + id: ruleSO.id, + rule_id: ruleParams.ruleId, + actions: ruleSO.attributes.actions, + author: ruleParams.author ?? [], + building_block_type: ruleParams.buildingBlockType, + false_positives: ruleParams.falsePositives, + saved_id: ruleParams.savedId, + timeline_id: ruleParams.timelineId, + timeline_title: ruleParams.timelineTitle, + meta: ruleParams.meta, + max_signals: ruleParams.maxSignals, + risk_score: ruleParams.riskScore, + risk_score_mapping: ruleParams.riskScoreMapping ?? [], + output_index: ruleParams.outputIndex, + description: ruleParams.description, + note: ruleParams.note, + from: ruleParams.from, + immutable: ruleParams.immutable, + index: ruleParams.index, + interval: ruleSO.attributes.schedule.interval, + language: ruleParams.language, + license: ruleParams.license, + name: ruleSO.attributes.name, + query: ruleParams.query, + references: ruleParams.references, + rule_name_override: ruleParams.ruleNameOverride, + severity: ruleParams.severity, + severity_mapping: ruleParams.severityMapping ?? [], + tags: ruleSO.attributes.tags, + type: ruleParams.type, + to: ruleParams.to, + enabled: ruleSO.attributes.enabled, + filters: ruleParams.filters, + created_by: ruleSO.attributes.createdBy, + updated_by: ruleSO.attributes.updatedBy, + threat: ruleParams.threat ?? [], + timestamp_override: ruleParams.timestampOverride, // TODO: Timestamp Override via timestamp_override + throttle: ruleSO.attributes.throttle, + version: ruleParams.version, + created_at: ruleSO.attributes.createdAt, + updated_at: ruleSO.updated_at ?? '', + exceptions_list: ruleParams.exceptionsList ?? [], + machine_learning_job_id: ruleParams.machineLearningJobId, + anomaly_threshold: ruleParams.anomalyThreshold, + threshold: ruleParams.threshold, + }; return removeInternalTagsFromRule(rule); }; -export const removeInternalTagsFromRule = (rule: Partial): Partial => { +export const removeInternalTagsFromRule = (rule: RulesSchema): RulesSchema => { if (rule.tags == null) { return rule; } else { - const ruleWithoutInternalTags: Partial = { + const ruleWithoutInternalTags: RulesSchema = { ...rule, tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index d684807a09126..ce434b2823110 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -7,7 +7,11 @@ import { sampleDocNoSortId } from './__mocks__/es_results'; import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal'; import { Signal, Ancestor } from './types'; -import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { + getRulesSchemaMock, + ANCHOR_DATE, +} from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; describe('buildSignal', () => { beforeEach(() => { @@ -17,7 +21,7 @@ describe('buildSignal', () => { test('it builds a signal as expected without original_event if event does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); delete doc._source.event; - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); const signal = { ...buildSignal([doc], rule), ...additionalSignalFields(doc), @@ -48,31 +52,39 @@ describe('buildSignal', () => { original_time: '2020-04-20T21:27:45+0000', status: 'open', rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), created_by: 'elastic', - description: 'Detecting root and admin users', + description: 'some description', enabled: true, - false_positives: [], + false_positives: ['false positive 1', 'false positive 2'], from: 'now-6m', - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', immutable: false, - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - risk_score: 50, - rule_id: 'rule-1', - language: 'kuery', - max_signals: 100, - name: 'Detect Root/Admin Users', - output_index: '.siem-signals', + name: 'Query with a rule id', query: 'user.name: root or user.name: admin', - references: ['http://www.example.com', 'https://ww.example.com'], + references: ['test 1', 'test 2'], severity: 'high', - updated_by: 'elastic', - tags: ['some fake tag 1', 'some fake tag 2'], + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: [], to: 'now', type: 'query', - note: '', - updated_at: signal.rule.updated_at, - created_at: signal.rule.created_at, + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-hassanabad-frank-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), }, depth: 1, }; @@ -87,7 +99,7 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); const signal = { ...buildSignal([doc], rule), ...additionalSignalFields(doc), @@ -124,31 +136,39 @@ describe('buildSignal', () => { }, status: 'open', rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), created_by: 'elastic', - description: 'Detecting root and admin users', + description: 'some description', enabled: true, - false_positives: [], + false_positives: ['false positive 1', 'false positive 2'], from: 'now-6m', - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', immutable: false, - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - risk_score: 50, - rule_id: 'rule-1', - language: 'kuery', - max_signals: 100, - name: 'Detect Root/Admin Users', - output_index: '.siem-signals', + name: 'Query with a rule id', query: 'user.name: root or user.name: admin', - references: ['http://www.example.com', 'https://ww.example.com'], + references: ['test 1', 'test 2'], severity: 'high', - updated_by: 'elastic', - tags: ['some fake tag 1', 'some fake tag 2'], + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: [], to: 'now', type: 'query', - note: '', - updated_at: signal.rule.updated_at, - created_at: signal.rule.created_at, + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-hassanabad-frank-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), }, depth: 1, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 78818779dd661..947938de6caca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,14 +5,14 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { SignalSourceHit, Signal, Ancestor } from './types'; +import { Signal, Ancestor, BaseSignalHit } from './types'; /** * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child * signal's `signal.parents` array. * @param doc The parent signal or event */ -export const buildParent = (doc: SignalSourceHit): Ancestor => { +export const buildParent = (doc: BaseSignalHit): Ancestor => { if (doc._source.signal != null) { return { rule: doc._source.signal.rule.id, @@ -38,7 +38,7 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => { * creating an array of N+1 ancestors. * @param doc The parent signal/event for which to extend the ancestry. */ -export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => { +export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => { const newAncestor = buildParent(doc); const existingAncestors = doc._source.signal?.ancestors; if (existingAncestors != null) { @@ -53,7 +53,7 @@ export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => { * @param docs The parent signals/events of the new signal to be built. * @param rule The rule that is generating the new signal. */ -export const buildSignal = (docs: SignalSourceHit[], rule: Partial): Signal => { +export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => { const parents = docs.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []); @@ -70,7 +70,7 @@ export const buildSignal = (docs: SignalSourceHit[], rule: Partial) * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. * @param doc The parent signal/event of the new signal to be built. */ -export const additionalSignalFields = (doc: SignalSourceHit) => { +export const additionalSignalFields = (doc: BaseSignalHit) => { return { parent: buildParent(doc), original_time: doc._source['@timestamp'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index f77485f39a98d..37632ce390c37 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -103,6 +103,9 @@ export const getFilter = async ({ 'Unsupported Rule of type "machine_learning" supplied to getFilter' ); } + case 'eql_query': { + throw new BadRequestError('Unsupported Rule of type "eql_query" supplied to getFilter'); + } default: { return assertUnreachable(type); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 7ee157beec789..71e6ce901d63b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,6 +8,7 @@ import { Logger, KibanaRequest } from 'src/core/server'; +import { SavedObject } from 'src/core/types'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, @@ -22,7 +23,15 @@ import { SearchAfterAndBulkCreateReturnType, } from './search_after_bulk_create'; import { getFilter } from './get_filter'; -import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types'; +import { + SignalRuleAlertTypeDefinition, + RuleAlertAttributes, + EqlSignalSearchResponse, + SignalSource, + Signal, + SignalHit, + BaseSignalHit, +} from './types'; import { getGapBetweenRuns, getListsClient, @@ -44,6 +53,16 @@ import { ruleStatusServiceFactory } from './rule_status_service'; import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; +import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; +import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; +import { buildExceptionListQueries } from '../../../../common/detection_engine/build_exceptions_query'; +import { buildExceptionFilter } from '../../../../common/detection_engine/get_query_filter'; +import { IIndexPattern, EsQueryConfig, Filter } from '../../../../../../../src/plugins/data/common'; +import { EqlSequence } from '../../types'; +import { buildRuleWithoutOverrides } from './build_rule'; +import { buildSignal, additionalSignalFields } from './build_signal'; +import { buildEventTypeSignal } from './build_event_type_signal'; +import { bulkInsertSignals, filterDuplicateSignals } from './single_bulk_create'; export const signalRulesAlertType = ({ logger, @@ -298,7 +317,7 @@ export const signalRulesAlertType = ({ if (bulkCreateDuration) { result.bulkCreateTimes.push(bulkCreateDuration); } - } else { + } else if (type === 'query' || type === 'saved_query') { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ type, @@ -337,6 +356,53 @@ export const signalRulesAlertType = ({ throttle, buildRuleMessage, }); + } else if (type === 'eql_query') { + if (query === undefined) { + throw new Error('eql query rule must have a query defined'); + } + const inputIndex = await getInputIndex(services, version, index); + const request = buildEqlQueryRequest({ + query, + index: inputIndex, + from: params.from, + to: params.to, + size: searchAfterSize, + timestampOverride: params.timestampOverride, + exceptionLists: exceptionItems ?? [], + }); + const response: EqlSignalSearchResponse = await services.callCluster( + 'transport.request', + request + ); + let newSignals: SignalHit[] | undefined; + if (response.hits.sequences !== undefined) { + newSignals = response.hits.sequences.map((sequence) => + buildSignalFromSequence(sequence, savedObject) + ); + } else if (response.hits.events !== undefined) { + newSignals = response.hits.events.map((event) => + buildSignalFromEvent(event, savedObject) + ); + } else { + throw new Error( + 'eql query response should have either `sequences` or `events` but had neither' + ); + } + const filteredSignals = filterDuplicateSignals(alertId, newSignals); + if (filteredSignals.length > 0) { + const insertResult = await bulkInsertSignals( + filteredSignals, + outputIndex, + logger, + services, + refresh + ); + result.bulkCreateTimes.push(insertResult.bulkCreateDuration); + result.createdSignalsCount += insertResult.createdItemsCount; + } + result.success = true; + } else { + throw new Error(`unknown rule type ${type}`); } if (result.success) { @@ -414,3 +480,98 @@ export const signalRulesAlertType = ({ }, }; }; + +interface BuildEqlSearchQuery { + query: string; + index: string[]; + from: string; + to: string; + size: number; + timestampOverride: TimestampOverrideOrUndefined; + exceptionLists: ExceptionListItemSchema[]; +} + +export const buildEqlQueryRequest = ({ + query, + index, + from, + to, + size, + timestampOverride, + exceptionLists, +}: BuildEqlSearchQuery) => { + const timestamp = timestampOverride ?? '@timestamp'; + // TODO: share this exception logic with getQueryFilter in a better way + const indexPattern: IIndexPattern = { + fields: [], + title: index.join(), + }; + const config: EsQueryConfig = { + allowLeadingWildcards: true, + queryStringOptions: { analyze_wildcard: true }, + ignoreFilterIfFieldNotInIndex: false, + dateFormatTZ: 'Zulu', + }; + const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists }); + let exceptionFilter: Filter | undefined; + if (exceptionQueries.length > 0) { + // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), + // allowing us to make 1024-item chunks of exception list items. + // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a + // very conservative value. + exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024); + } + const indexString = index.join(); + return { + method: 'POST', + path: `/${indexString}/_eql/search`, + body: { + size, + query, + filter: { + range: { + [timestamp]: { + gte: from, + lte: to, + }, + }, + bool: exceptionFilter?.query.bool, + }, + }, + }; +}; + +export const buildSignalFromSequence = ( + sequence: EqlSequence, + ruleSO: SavedObject +): SignalHit => { + const rule = buildRuleWithoutOverrides(ruleSO); + const signal: Signal = buildSignal(sequence.events, rule); + return { + '@timestamp': new Date().toISOString(), + event: { + kind: 'signal', + }, + signal, + }; +}; + +export const buildSignalFromEvent = ( + event: BaseSignalHit, + ruleSO: SavedObject +): SignalHit => { + const rule = buildRuleWithoutOverrides(ruleSO); + const signal: Signal = { + ...buildSignal([event], rule), + ...additionalSignalFields(event), + }; + const eventFields = buildEventTypeSignal(event); + // TODO: better naming for SignalHit - it's really a new signal to be inserted + const signalHit: SignalHit = { + ...event._source, + '@timestamp': new Date().toISOString(), + event: eventFields, + signal, + }; + return signalHit; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index be71c67615a4c..15aaffb7d44ce 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -7,10 +7,10 @@ import { countBy, isEmpty } from 'lodash'; import { performance } from 'perf_hooks'; import { AlertServices } from '../../../../../alerts/server'; -import { SignalSearchResponse, BulkResponse } from './types'; +import { SignalSearchResponse, BulkResponse, SignalHit } from './types'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; -import { generateId, makeFloatString, errorAggregator } from './utils'; +import { generateId, makeFloatString, errorAggregator, generateSignalId } from './utils'; import { buildBulkBody } from './build_bulk_body'; import { Logger } from '../../../../../../../src/core/server'; @@ -59,12 +59,29 @@ export const filterDuplicateRules = ( }); }; +/** + * Similar to filterDuplicateRules, but operates on candidate signal documents rather than events that matched + * the detection query. This means we only have to compare the ruleId against the ancestors array. + * @param ruleId The rule id + * @param signals The candidate new signals + */ +export const filterDuplicateSignals = (ruleId: string, signals: SignalHit[]) => { + return signals.filter( + (doc) => !doc.signal.ancestors.some((ancestor) => ancestor.rule === ruleId) + ); +}; + export interface SingleBulkCreateResponse { success: boolean; bulkCreateDuration?: string; createdItemsCount: number; } +export interface BulkInsertSignalsResponse { + bulkCreateDuration: string; + createdItemsCount: number; +} + // Bulk Index documents. export const singleBulkCreate = async ({ filteredEvents, @@ -153,3 +170,50 @@ export const singleBulkCreate = async ({ logger.debug(`bulk created ${createdItemsCount} signals`); return { success: true, bulkCreateDuration: makeFloatString(end - start), createdItemsCount }; }; + +// Bulk Index new signals. +export const bulkInsertSignals = async ( + filteredSignals: SignalHit[], + signalsIndex: string, + logger: Logger, + services: AlertServices, + refresh: RefreshTypes +): Promise => { + logger.debug(`about to bulk create ${filteredSignals.length} signals`); + + // index documents after creating an ID based on the + // id and index of each parent and the rule ID + const bulkBody = filteredSignals.flatMap((doc) => [ + { + create: { + _index: signalsIndex, + _id: generateSignalId(doc), + }, + }, + doc, + ]); + const start = performance.now(); + const response: BulkResponse = await services.callCluster('bulk', { + index: signalsIndex, + refresh, + body: bulkBody, + }); + const end = performance.now(); + logger.debug(`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`); + logger.debug(`took property says bulk took: ${response.took} milliseconds`); + + if (response.errors) { + const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; + logger.debug(`ignored ${duplicateSignalsCount} duplicate signals`); + const errorCountByMessage = errorAggregator(response, [409]); + if (!isEmpty(errorCountByMessage)) { + logger.error( + `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` + ); + } + } + + const createdItemsCount = countBy(response.items, 'create.status')['201'] ?? 0; + logger.debug(`bulk created ${createdItemsCount} signals`); + return { bulkCreateDuration: makeFloatString(end - start), createdItemsCount }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 700a8fb5022d7..f02b6de762e8f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -9,7 +9,7 @@ import { RulesSchema } from '../../../../common/detection_engine/schemas/respons import { AlertType, AlertTypeState, AlertExecutorOptions } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; -import { SearchResponse } from '../../types'; +import { SearchResponse, EqlSearchResponse, BaseHit } from '../../types'; // used for gap detection code // eslint-disable-next-line @typescript-eslint/naming-convention @@ -42,6 +42,8 @@ export type SearchTypes = export interface SignalSource { [key: string]: SearchTypes; + // TODO: SignalSource is being used as the type for documents matching detection engine queries, but they may not + // actually have @timestamp if a timestamp override is used '@timestamp': string; signal?: { // parent is deprecated: new signals should populate parents instead @@ -105,6 +107,9 @@ export interface GetResponse { export type EventSearchResponse = SearchResponse; export type SignalSearchResponse = SearchResponse; export type SignalSourceHit = SignalSearchResponse['hits']['hits'][number]; +export type BaseSignalHit = BaseHit; + +export type EqlSignalSearchResponse = EqlSearchResponse; export type RuleExecutorOptions = Omit & { params: RuleTypeParams; @@ -129,7 +134,7 @@ export interface Ancestor { } export interface Signal { - rule: Partial; + rule: RulesSchema; // DEPRECATED: use parents instead of parent parent?: Ancestor; parents: Ancestor[]; @@ -144,7 +149,7 @@ export interface Signal { export interface SignalHit { '@timestamp': string; event: object; - signal: Partial; + signal: Signal; } export interface AlertAttributes { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index dc09c6d5386fc..866f25bc04051 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -12,7 +12,7 @@ import { AlertServices, parseDuration } from '../../../../../alerts/server'; import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types/lists'; -import { BulkResponse, BulkResponseErrorAggregation, isValidUnit } from './types'; +import { BulkResponse, BulkResponseErrorAggregation, isValidUnit, SignalHit } from './types'; import { BuildRuleMessage } from './rule_messages'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; import { hasLargeValueList } from '../../../../common/detection_engine/utils'; @@ -213,6 +213,16 @@ export const generateId = ( ruleId: string ): string => createHash('sha256').update(docIndex.concat(docId, version, ruleId)).digest('hex'); +// TODO: do we need to include version in the id? If it does matter then we should include it in signal.parents as well +export const generateSignalId = (newSignal: SignalHit) => + createHash('sha256') + .update( + newSignal.signal.parents + .reduce((acc, parent) => acc.concat(parent.id, parent.index), '') + .concat(newSignal.signal.rule.id) + ) + .digest('hex'); + export const parseInterval = (intervalString: string): moment.Duration | null => { try { return moment.duration(parseDuration(intervalString)); diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index ff89512124b66..4548786614a5d 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -26,6 +26,7 @@ import { PinnedEvent } from './pinned_event/saved_object'; import { Timeline } from './timeline/saved_object'; import { TLS } from './tls'; import { MatrixHistogram } from './matrix_histogram'; +import { SearchTypes } from './detection_engine/signals/types'; export * from './hosts'; @@ -64,6 +65,12 @@ export interface TotalValue { relation: string; } +export interface BaseHit { + _index: string; + _id: string; + _source: T; +} + export interface SearchResponse { took: number; timed_out: boolean; @@ -72,27 +79,43 @@ export interface SearchResponse { hits: { total: TotalValue | number; max_score: number; - hits: Array<{ - _index: string; - _type: string; - _id: string; - _score: number; - _source: T; - _version?: number; - _explanation?: Explanation; - fields?: string[]; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - highlight?: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - inner_hits?: any; - matched_queries?: string[]; - sort?: string[]; - }>; + hits: Array< + BaseHit & { + _type: string; + _score: number; + _version?: number; + _explanation?: Explanation; + fields?: string[]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + highlight?: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + inner_hits?: any; + matched_queries?: string[]; + sort?: string[]; + } + >; }; // eslint-disable-next-line @typescript-eslint/no-explicit-any aggregations?: any; } +export interface EqlSequence { + join_keys: SearchTypes[]; + events: Array>; +} + +export interface EqlSearchResponse { + is_partial: boolean; + is_running: boolean; + took: number; + timed_out: boolean; + hits: { + total: TotalValue; + sequences?: Array>; + events?: Array>; + }; +} + export interface ShardsResponse { total: number; successful: number; From 82ae0c2e0a2df907e8c06911c2995947a8abadb8 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Mon, 14 Sep 2020 21:38:15 -0400 Subject: [PATCH 02/26] Reorganize functions to separate files --- .../detection_engine/get_query_filter.ts | 56 +++++++- .../signals/build_bulk_body.ts | 48 ++++++- .../signals/signal_rule_alert_type.ts | 126 ++---------------- 3 files changed, 111 insertions(+), 119 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 466a004c14c66..03be0206121a2 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -17,7 +17,12 @@ import { CreateExceptionListItemSchema, } from '../../../lists/common/schemas'; import { buildExceptionListQueries } from './build_exceptions_query'; -import { Query as QueryString, Language, Index } from './schemas/common/schemas'; +import { + Query as QueryString, + Language, + Index, + TimestampOverrideOrUndefined, +} from './schemas/common/schemas'; export const getQueryFilter = ( query: QueryString, @@ -67,6 +72,55 @@ export const getQueryFilter = ( return buildEsQuery(indexPattern, initialQuery, enabledFilters, config); }; +export const buildEqlSearchRequest = ( + query: string, + index: string[], + from: string, + to: string, + size: number, + timestampOverride: TimestampOverrideOrUndefined, + exceptionLists: ExceptionListItemSchema[] +) => { + const timestamp = timestampOverride ?? '@timestamp'; + const indexPattern: IIndexPattern = { + fields: [], + title: index.join(), + }; + const config: EsQueryConfig = { + allowLeadingWildcards: true, + queryStringOptions: { analyze_wildcard: true }, + ignoreFilterIfFieldNotInIndex: false, + dateFormatTZ: 'Zulu', + }; + const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists }); + let exceptionFilter: Filter | undefined; + if (exceptionQueries.length > 0) { + // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), + // allowing us to make 1024-item chunks of exception list items. + // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a + // very conservative value. + exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024); + } + const indexString = index.join(); + return { + method: 'POST', + path: `/${indexString}/_eql/search`, + body: { + size, + query, + filter: { + range: { + [timestamp]: { + gte: from, + lte: to, + }, + }, + bool: exceptionFilter?.query.bool, + }, + }, + }; +}; + export const buildExceptionFilter = ( exceptionQueries: Query[], indexPattern: IIndexPattern, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 7be97e46f91f2..6ada756180377 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -4,12 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalHit, Signal } from './types'; -import { buildRule } from './build_rule'; +import { SavedObject } from 'src/core/types'; +import { + SignalSourceHit, + SignalHit, + Signal, + SignalSource, + RuleAlertAttributes, + BaseSignalHit, +} from './types'; +import { buildRule, buildRuleWithoutOverrides } from './build_rule'; import { additionalSignalFields, buildSignal } from './build_signal'; import { buildEventTypeSignal } from './build_event_type_signal'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; +import { EqlSequence } from '../../types'; interface BuildBulkBodyParams { doc: SignalSourceHit; @@ -71,3 +80,38 @@ export const buildBulkBody = ({ }; return signalHit; }; + +export const buildSignalFromSequence = ( + sequence: EqlSequence, + ruleSO: SavedObject +): SignalHit => { + const rule = buildRuleWithoutOverrides(ruleSO); + const signal: Signal = buildSignal(sequence.events, rule); + return { + '@timestamp': new Date().toISOString(), + event: { + kind: 'signal', + }, + signal, + }; +}; + +export const buildSignalFromEvent = ( + event: BaseSignalHit, + ruleSO: SavedObject +): SignalHit => { + const rule = buildRuleWithoutOverrides(ruleSO); + const signal: Signal = { + ...buildSignal([event], rule), + ...additionalSignalFields(event), + }; + const eventFields = buildEventTypeSignal(event); + // TODO: better naming for SignalHit - it's really a new signal to be inserted + const signalHit: SignalHit = { + ...event._source, + '@timestamp': new Date().toISOString(), + event: eventFields, + signal, + }; + return signalHit; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 71e6ce901d63b..f7f3ce4ebf107 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,7 +8,6 @@ import { Logger, KibanaRequest } from 'src/core/server'; -import { SavedObject } from 'src/core/types'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, @@ -27,10 +26,7 @@ import { SignalRuleAlertTypeDefinition, RuleAlertAttributes, EqlSignalSearchResponse, - SignalSource, - Signal, SignalHit, - BaseSignalHit, } from './types'; import { getGapBetweenRuns, @@ -53,16 +49,9 @@ import { ruleStatusServiceFactory } from './rule_status_service'; import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; -import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; -import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; -import { buildExceptionListQueries } from '../../../../common/detection_engine/build_exceptions_query'; -import { buildExceptionFilter } from '../../../../common/detection_engine/get_query_filter'; -import { IIndexPattern, EsQueryConfig, Filter } from '../../../../../../../src/plugins/data/common'; -import { EqlSequence } from '../../types'; -import { buildRuleWithoutOverrides } from './build_rule'; -import { buildSignal, additionalSignalFields } from './build_signal'; -import { buildEventTypeSignal } from './build_event_type_signal'; +import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter'; import { bulkInsertSignals, filterDuplicateSignals } from './single_bulk_create'; +import { buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body'; export const signalRulesAlertType = ({ logger, @@ -361,15 +350,15 @@ export const signalRulesAlertType = ({ throw new Error('eql query rule must have a query defined'); } const inputIndex = await getInputIndex(services, version, index); - const request = buildEqlQueryRequest({ + const request = buildEqlSearchRequest( query, - index: inputIndex, - from: params.from, - to: params.to, - size: searchAfterSize, - timestampOverride: params.timestampOverride, - exceptionLists: exceptionItems ?? [], - }); + inputIndex, + params.from, + params.to, + searchAfterSize, + params.timestampOverride, + exceptionItems ?? [] + ); const response: EqlSignalSearchResponse = await services.callCluster( 'transport.request', request @@ -480,98 +469,3 @@ export const signalRulesAlertType = ({ }, }; }; - -interface BuildEqlSearchQuery { - query: string; - index: string[]; - from: string; - to: string; - size: number; - timestampOverride: TimestampOverrideOrUndefined; - exceptionLists: ExceptionListItemSchema[]; -} - -export const buildEqlQueryRequest = ({ - query, - index, - from, - to, - size, - timestampOverride, - exceptionLists, -}: BuildEqlSearchQuery) => { - const timestamp = timestampOverride ?? '@timestamp'; - // TODO: share this exception logic with getQueryFilter in a better way - const indexPattern: IIndexPattern = { - fields: [], - title: index.join(), - }; - const config: EsQueryConfig = { - allowLeadingWildcards: true, - queryStringOptions: { analyze_wildcard: true }, - ignoreFilterIfFieldNotInIndex: false, - dateFormatTZ: 'Zulu', - }; - const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists }); - let exceptionFilter: Filter | undefined; - if (exceptionQueries.length > 0) { - // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), - // allowing us to make 1024-item chunks of exception list items. - // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a - // very conservative value. - exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024); - } - const indexString = index.join(); - return { - method: 'POST', - path: `/${indexString}/_eql/search`, - body: { - size, - query, - filter: { - range: { - [timestamp]: { - gte: from, - lte: to, - }, - }, - bool: exceptionFilter?.query.bool, - }, - }, - }; -}; - -export const buildSignalFromSequence = ( - sequence: EqlSequence, - ruleSO: SavedObject -): SignalHit => { - const rule = buildRuleWithoutOverrides(ruleSO); - const signal: Signal = buildSignal(sequence.events, rule); - return { - '@timestamp': new Date().toISOString(), - event: { - kind: 'signal', - }, - signal, - }; -}; - -export const buildSignalFromEvent = ( - event: BaseSignalHit, - ruleSO: SavedObject -): SignalHit => { - const rule = buildRuleWithoutOverrides(ruleSO); - const signal: Signal = { - ...buildSignal([event], rule), - ...additionalSignalFields(event), - }; - const eventFields = buildEventTypeSignal(event); - // TODO: better naming for SignalHit - it's really a new signal to be inserted - const signalHit: SignalHit = { - ...event._source, - '@timestamp': new Date().toISOString(), - event: eventFields, - signal, - }; - return signalHit; -}; From 9b450bfd8307feb29e16fea2fb1488434577e5cd Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 17 Sep 2020 21:05:12 -0400 Subject: [PATCH 03/26] Start adding eventCategoryOverride option for EQL rules --- .../detection_engine/get_query_filter.ts | 22 ++++++++++++++++--- .../schemas/common/schemas.ts | 6 +++++ .../request/add_prepackaged_rules_schema.ts | 2 ++ .../schemas/request/create_rules_schema.ts | 2 ++ .../schemas/request/import_rules_schema.ts | 2 ++ .../schemas/request/patch_rules_schema.ts | 2 ++ .../schemas/request/update_rules_schema.ts | 2 ++ .../schemas/response/rules_schema.ts | 20 ++++++++++++++++- .../routes/__mocks__/request_responses.ts | 1 + .../routes/rules/create_rules_bulk_route.ts | 2 ++ .../routes/rules/create_rules_route.ts | 4 +++- .../routes/rules/import_rules_route.ts | 3 +++ .../routes/rules/patch_rules_bulk_route.ts | 2 ++ .../routes/rules/patch_rules_route.ts | 2 ++ .../routes/rules/update_rules_bulk_route.ts | 2 ++ .../routes/rules/update_rules_route.ts | 2 ++ .../detection_engine/routes/rules/utils.ts | 1 + .../rules/create_rules.mock.ts | 2 ++ .../detection_engine/rules/create_rules.ts | 2 ++ .../rules/install_prepacked_rules.ts | 2 ++ .../rules/patch_rules.mock.ts | 2 ++ .../lib/detection_engine/rules/patch_rules.ts | 2 ++ .../lib/detection_engine/rules/types.ts | 4 ++++ .../rules/update_prepacked_rules.ts | 2 ++ .../rules/update_rules.mock.ts | 2 ++ .../detection_engine/rules/update_rules.ts | 2 ++ .../lib/detection_engine/rules/utils.test.ts | 3 +++ .../lib/detection_engine/rules/utils.ts | 2 ++ .../signals/__mocks__/es_results.ts | 1 + .../signals/signal_params_schema.mock.ts | 1 + .../signals/signal_params_schema.ts | 1 + .../signals/signal_rule_alert_type.ts | 3 ++- .../server/lib/detection_engine/types.ts | 2 ++ 33 files changed, 104 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 03be0206121a2..238bb3e7d7ecd 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -72,6 +72,13 @@ export const getQueryFilter = ( return buildEsQuery(indexPattern, initialQuery, enabledFilters, config); }; +interface EqlSearchRequest { + method: string; + path: string; + body: object; + event_category_field?: string; +} + export const buildEqlSearchRequest = ( query: string, index: string[], @@ -79,8 +86,9 @@ export const buildEqlSearchRequest = ( to: string, size: number, timestampOverride: TimestampOverrideOrUndefined, - exceptionLists: ExceptionListItemSchema[] -) => { + exceptionLists: ExceptionListItemSchema[], + eventCategoryOverride: string | undefined +): EqlSearchRequest => { const timestamp = timestampOverride ?? '@timestamp'; const indexPattern: IIndexPattern = { fields: [], @@ -102,7 +110,7 @@ export const buildEqlSearchRequest = ( exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024); } const indexString = index.join(); - return { + const baseRequest = { method: 'POST', path: `/${indexString}/_eql/search`, body: { @@ -119,6 +127,14 @@ export const buildEqlSearchRequest = ( }, }, }; + if (eventCategoryOverride) { + return { + ...baseRequest, + event_category_field: eventCategoryOverride, + }; + } else { + return baseRequest; + } }; export const buildExceptionFilter = ( diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index aaebcd11bf583..0d621a45d3e89 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -40,6 +40,12 @@ export type Enabled = t.TypeOf; export const enabledOrUndefined = t.union([enabled, t.undefined]); export type EnabledOrUndefined = t.TypeOf; +export const event_category_override = t.string; +export type EventCategoryOverride = t.TypeOf; + +export const eventCategoryOverrideOrUndefined = t.union([event_category_override, t.undefined]); +export type EventCategoryOverrideOrUndefined = t.TypeOf; + export const false_positives = t.array(t.string); export type FalsePositives = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index 9b90cf9fdf782..d4eb2ff2ccf9f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -44,6 +44,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { @@ -90,6 +91,7 @@ export const addPrepackagedRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanFalse, // defaults to false if not set during decode + event_category_override, // defaults to "undefined" if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index 7b6b98383cc33..057c90e5b04fa 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -45,6 +45,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { @@ -82,6 +83,7 @@ export const createRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode + event_category_override, // defaults to "undefined" if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index 698716fea696e..f5f8a8fc7cb85 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -51,6 +51,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { @@ -101,6 +102,7 @@ export const importRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode + event_category_override, // defaults to "undefined" if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts index a674ac86af87b..40e79d96a9e6b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts @@ -46,6 +46,7 @@ import { timestamp_override, risk_score_mapping, severity_mapping, + event_category_override, } from '../common/schemas'; import { listArrayOrUndefined } from '../types/lists'; @@ -65,6 +66,7 @@ export const patchRulesSchema = t.exact( actions, anomaly_threshold, enabled, + event_category_override, false_positives, filters, from, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts index 1299dada065e1..8a13dd2f4e908 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts @@ -47,6 +47,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { @@ -90,6 +91,7 @@ export const updateRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode + event_category_override, false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 3f6ac5b2edf67..a4b64345606ac 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -16,6 +16,7 @@ import { anomaly_threshold, description, enabled, + event_category_override, false_positives, from, id, @@ -113,6 +114,9 @@ export const dependentRulesSchema = t.partial({ language, query, + // eql fields + event_category_override, + // when type = saved_query, saved_is is required saved_id, @@ -205,7 +209,7 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi }; export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { - if (['query', 'saved_query', 'threshold', 'eql_query'].includes(typeAndTimelineOnly.type)) { + if (['query', 'saved_query', 'threshold'].includes(typeAndTimelineOnly.type)) { return [ t.exact(t.type({ query: dependentRulesSchema.props.query })), t.exact(t.type({ language: dependentRulesSchema.props.language })), @@ -239,6 +243,19 @@ export const addThresholdFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t. } }; +export const addEqlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { + if (typeAndTimelineOnly.type === 'eql_query') { + return [ + t.exact( + t.partial({ event_category_override: dependentRulesSchema.props.event_category_override }) + ), + t.exact(t.type({ query: dependentRulesSchema.props.query })), + ]; + } else { + return []; + } +}; + export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed => { const dependents: t.Mixed[] = [ t.exact(requiredRulesSchema), @@ -248,6 +265,7 @@ export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed ...addQueryFields(typeAndTimelineOnly), ...addMlFields(typeAndTimelineOnly), ...addThresholdFields(typeAndTimelineOnly), + ...addEqlFields(typeAndTimelineOnly), ]; if (dependents.length > 1) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 29c56e8ed80b1..265ce7be45495 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -348,6 +348,7 @@ export const getResult = (): RuleAlertType => ({ description: 'Detecting root and admin users', ruleId: 'rule-1', index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + eventCategoryOverride: undefined, falsePositives: [], from: 'now-6m', immutable: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index acd800e54040c..a1f1131e0b1a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -68,6 +68,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -151,6 +152,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 482edb9925557..13cae7461775c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -52,6 +52,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -90,7 +91,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void type !== 'machine_learning' && queryOrUndefined == null ? '' : queryOrUndefined; const language = - type !== 'machine_learning' && languageOrUndefined == null + type !== 'machine_learning' && type !== 'eql_query' && languageOrUndefined == null ? 'kuery' : languageOrUndefined; @@ -136,6 +137,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index 8a7215e5a5bad..afb52e7252f05 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -138,6 +138,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, immutable, @@ -196,6 +197,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable, @@ -240,6 +242,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP savedObjectsClient, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts index 5099cf5de958f..39bbe9ee686a4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts @@ -59,6 +59,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -119,6 +120,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index 3b3efd2ed166d..879bd8d5b8a1d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -50,6 +50,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -117,6 +118,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index 518024387fed3..16cc26866cafc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -61,6 +61,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -129,6 +130,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index 299b99c4d37b0..361879ac6e003 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -51,6 +51,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -119,6 +120,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index ee83ea91578c5..efede63aa7afb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -114,6 +114,7 @@ export const transformAlertToRule = ( description: alert.params.description, enabled: alert.enabled, anomaly_threshold: alert.params.anomalyThreshold, + event_category_override: alert.params.eventCategoryOverride, false_positives: alert.params.falsePositives, filters: alert.params.filters, from: alert.params.from, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index 1117f34b6f8c5..b70d74694c79f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -14,6 +14,7 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ anomalyThreshold: undefined, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: 'user.name: root or user.name: admin', @@ -57,6 +58,7 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ anomalyThreshold: 55, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 0c67d9ca77146..dad23de5b930e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -17,6 +17,7 @@ export const createRules = async ({ buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, @@ -65,6 +66,7 @@ export const createRules = async ({ description, ruleId, index, + eventCategoryOverride, falsePositives, from, immutable, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index 3af0c3f55b485..5b850f75a6a23 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -22,6 +22,7 @@ export const installPrepackagedRules = ( building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -66,6 +67,7 @@ export const installPrepackagedRules = ( buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable: true, // At the moment we force all prepackaged rules to be immutable diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts index cfb40056eb85d..aeb136a969aa1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts @@ -120,6 +120,7 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ anomalyThreshold: undefined, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: 'user.name: root or user.name: admin', @@ -163,6 +164,7 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ anomalyThreshold: 55, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts index e0814647b4c39..852ff06bdc736 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts @@ -18,6 +18,7 @@ export const patchRules = async ({ buildingBlockType, savedObjectsClient, description, + eventCategoryOverride, falsePositives, enabled, query, @@ -62,6 +63,7 @@ export const patchRules = async ({ author, buildingBlockType, description, + eventCategoryOverride, falsePositives, query, language, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index b845990fd94ef..e56353c7fac82 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -84,6 +84,7 @@ import { TimestampOverrideOrUndefined, BuildingBlockTypeOrUndefined, RuleNameOverrideOrUndefined, + EventCategoryOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { AlertsClient, PartialAlert } from '../../../../../alerts/server'; import { Alert, SanitizedAlert } from '../../../../../alerts/common'; @@ -180,6 +181,7 @@ export interface CreateRulesOptions { buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; enabled: Enabled; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; query: QueryOrUndefined; @@ -225,6 +227,7 @@ export interface UpdateRulesOptions { buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; enabled: Enabled; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; query: QueryOrUndefined; @@ -268,6 +271,7 @@ export interface PatchRulesOptions { buildingBlockType: BuildingBlockTypeOrUndefined; description: DescriptionOrUndefined; enabled: EnabledOrUndefined; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositivesOrUndefined; from: FromOrUndefined; query: QueryOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts index bf97784e8d917..01a481ed7b2d9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts @@ -23,6 +23,7 @@ export const updatePrepackagedRules = async ( author, building_block_type: buildingBlockType, description, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -69,6 +70,7 @@ export const updatePrepackagedRules = async ( author, buildingBlockType, description, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts index 650b59fb85bc0..8cdc904a861c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts @@ -17,6 +17,7 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ anomalyThreshold: undefined, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: 'user.name: root or user.name: admin', @@ -61,6 +62,7 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ anomalyThreshold: 55, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts index 494a4e221d862..08df785884b76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts @@ -18,6 +18,7 @@ export const updateRules = async ({ buildingBlockType, savedObjectsClient, description, + eventCategoryOverride, falsePositives, enabled, query, @@ -64,6 +65,7 @@ export const updateRules = async ({ author, buildingBlockType, description, + eventCategoryOverride, falsePositives, query, language, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts index 17505a4478261..227f574bc4e4b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts @@ -31,6 +31,7 @@ describe('utils', () => { author: [], buildingBlockType: undefined, description: 'some description change', + eventCategoryOverride: undefined, falsePositives: undefined, query: undefined, language: undefined, @@ -73,6 +74,7 @@ describe('utils', () => { author: [], buildingBlockType: undefined, description: 'some description change', + eventCategoryOverride: undefined, falsePositives: undefined, query: undefined, language: undefined, @@ -115,6 +117,7 @@ describe('utils', () => { author: [], buildingBlockType: undefined, description: 'some description change', + eventCategoryOverride: undefined, falsePositives: undefined, query: undefined, language: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts index 49c02f92ff336..d9f953f2803a6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts @@ -39,6 +39,7 @@ import { RuleNameOverrideOrUndefined, SeverityMappingOrUndefined, TimestampOverrideOrUndefined, + EventCategoryOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { PartialFilter } from '../types'; import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types'; @@ -60,6 +61,7 @@ export interface UpdateProperties { author: AuthorOrUndefined; buildingBlockType: BuildingBlockTypeOrUndefined; description: DescriptionOrUndefined; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositivesOrUndefined; from: FromOrUndefined; query: QueryOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 9d3eb29be08dd..46c4638a0597e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -24,6 +24,7 @@ export const sampleRuleAlertParams = ( buildingBlockType: 'default', ruleId: 'rule-1', description: 'Detecting root and admin users', + eventCategoryOverride: undefined, falsePositives: [], immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts index 0c56ed300cb48..07bd825f4c65c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts @@ -22,6 +22,7 @@ export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({ author: [], buildingBlockType: null, description: 'Detecting root and admin users', + eventCategoryOverride: undefined, falsePositives: [], filters: null, from: 'now-6m', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index d08ca90f3e353..c95fc93d67f0f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -14,6 +14,7 @@ const signalSchema = schema.object({ buildingBlockType: schema.nullable(schema.string()), description: schema.string(), note: schema.nullable(schema.string()), + eventCategoryOverride: schema.maybe(schema.string()), falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }), from: schema.string(), ruleId: schema.string(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index f7f3ce4ebf107..97d1bc116de35 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -357,7 +357,8 @@ export const signalRulesAlertType = ({ params.to, searchAfterSize, params.timestampOverride, - exceptionItems ?? [] + exceptionItems ?? [], + params.eventCategoryOverride ); const response: EqlSignalSearchResponse = await services.callCluster( 'transport.request', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index cbe756064b72b..9a1ee8ea14f51 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -37,6 +37,7 @@ import { SeverityMappingOrUndefined, TimestampOverrideOrUndefined, Type, + EventCategoryOverrideOrUndefined, } from '../../../common/detection_engine/schemas/common/schemas'; import { LegacyCallAPIOptions } from '../../../../../../src/core/server'; import { Filter } from '../../../../../../src/plugins/data/server'; @@ -50,6 +51,7 @@ export interface RuleTypeParams { buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; note: NoteOrUndefined; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; ruleId: RuleId; From d3fce8ac453ed3f9ff1d0f386f28f883dff1e7e5 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Mon, 21 Sep 2020 16:32:10 -0400 Subject: [PATCH 04/26] Add building block alerts for each event within sequence --- .../routes/index/signals_mapping.json | 19 ++++ .../signals/build_bulk_body.ts | 4 +- .../signals/build_rule.test.ts | 9 +- .../signals/signal_rule_alert_type.ts | 88 ++++++++++++++++--- .../signals/single_bulk_create.ts | 18 ++-- .../lib/detection_engine/signals/types.ts | 1 + .../lib/detection_engine/signals/utils.ts | 21 +++++ 7 files changed, 128 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index cfce019910071..e0d448d374243 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -41,6 +41,25 @@ } } }, + "child": { + "properties": { + "rule": { + "type": "keyword" + }, + "index": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "depth": { + "type": "long" + } + } + }, "ancestors": { "properties": { "rule": { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 6ada756180377..fb6b56b64d93e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -82,11 +82,11 @@ export const buildBulkBody = ({ }; export const buildSignalFromSequence = ( - sequence: EqlSequence, + events: BaseSignalHit[], ruleSO: SavedObject ): SignalHit => { const rule = buildRuleWithoutOverrides(ruleSO); - const signal: Signal = buildSignal(sequence.events, rule); + const signal: Signal = buildSignal(events, rule); return { '@timestamp': new Date().toISOString(), event: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index dc50b18b977b4..aa870271d8c74 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -9,10 +9,7 @@ import { sampleDocNoSortId, sampleRuleAlertParams, sampleRuleGuid } from './__mo import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; -import { - getPartialRulesSchemaMock, - getRulesSchemaMock, -} from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { getRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildRule', () => { beforeEach(() => { @@ -285,14 +282,14 @@ describe('buildRule', () => { `${INTERNAL_IMMUTABLE_KEY}:true`, ]; const noInternals = removeInternalTagsFromRule(rule); - expect(noInternals).toEqual(getPartialRulesSchemaMock()); + expect(noInternals).toEqual(getRulesSchemaMock()); }); test('it works with an empty array', () => { const rule = getRulesSchemaMock(); rule.tags = []; const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); + const expected = getRulesSchemaMock(); expected.tags = []; expect(noInternals).toEqual(expected); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 97d1bc116de35..11f49ab718e6b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -27,6 +27,7 @@ import { RuleAlertAttributes, EqlSignalSearchResponse, SignalHit, + BaseSignalHit, } from './types'; import { getGapBetweenRuns, @@ -34,6 +35,8 @@ import { getExceptions, getGapMaxCatchupRatio, MAX_RULE_GAP_RATIO, + generateBuildingBlockIds, + generateSignalId, } from './utils'; import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; @@ -52,6 +55,12 @@ import { getNotificationResultsLink } from '../notifications/utils'; import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter'; import { bulkInsertSignals, filterDuplicateSignals } from './single_bulk_create'; import { buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body'; +import { buildParent } from './build_signal'; + +interface Sequence { + buildingBlocks: BaseSignalHit[]; + sequenceSignal: BaseSignalHit; +} export const signalRulesAlertType = ({ logger, @@ -364,29 +373,59 @@ export const signalRulesAlertType = ({ 'transport.request', request ); - let newSignals: SignalHit[] | undefined; + let newSignals: BaseSignalHit[] | undefined; if (response.hits.sequences !== undefined) { - newSignals = response.hits.sequences.map((sequence) => - buildSignalFromSequence(sequence, savedObject) + // 2D array: For each sequence, we make an array containing building block signals for each + // event within the sequence. We also wrap each building block with a doc id and index so they're ready + // to be used in the next step - creating the signal that links them all together. + const buildingBlockArrays = response.hits.sequences.map((sequence) => + wrapBuildingBlocks( + sequence.events.map((event) => { + const signal = buildSignalFromEvent(event, savedObject); + signal.signal.rule.building_block_type = 'default'; + return signal; + }), + outputIndex + ) ); + + // Now that we have an array of building blocks for each matching sequence, + // we can build the signal that links the building blocks of a sequence together + // and also insert references to this signal in each building block + const sequences: Sequence[] = buildingBlockArrays.map((blocks) => { + const sequenceSignal = wrapSignal( + buildSignalFromSequence(blocks, savedObject), + outputIndex + ); + blocks.forEach((block) => { + // TODO: fix type of blocks so we don't have to check existence of _source.signal + if (block._source.signal) { + block._source.signal.child = buildParent(sequenceSignal); + } + }); + return { + buildingBlocks: blocks, + sequenceSignal, + }; + }); + newSignals = sequences.reduce((acc: BaseSignalHit[], sequence): BaseSignalHit[] => { + acc.push(...sequence.buildingBlocks); + acc.push(sequence.sequenceSignal); + return acc; + }, []); } else if (response.hits.events !== undefined) { newSignals = response.hits.events.map((event) => - buildSignalFromEvent(event, savedObject) + wrapSignal(buildSignalFromEvent(event, savedObject), outputIndex) ); } else { throw new Error( 'eql query response should have either `sequences` or `events` but had neither' ); } - const filteredSignals = filterDuplicateSignals(alertId, newSignals); - if (filteredSignals.length > 0) { - const insertResult = await bulkInsertSignals( - filteredSignals, - outputIndex, - logger, - services, - refresh - ); + // TODO: replace with code that filters out recursive rule signals while allowing sequences and their building blocks + // const filteredSignals = filterDuplicateSignals(alertId, newSignals); + if (newSignals.length > 0) { + const insertResult = await bulkInsertSignals(newSignals, logger, services, refresh); result.bulkCreateTimes.push(insertResult.bulkCreateDuration); result.createdSignalsCount += insertResult.createdItemsCount; } @@ -470,3 +509,26 @@ export const signalRulesAlertType = ({ }, }; }; + +export const wrapBuildingBlocks = (buildingBlocks: SignalHit[], index: string): BaseSignalHit[] => { + const blockIds = generateBuildingBlockIds(buildingBlocks); + return buildingBlocks.map((block, idx) => { + return { + _id: blockIds[idx], + _index: index, + _source: { + ...block, + }, + }; + }); +}; + +export const wrapSignal = (signal: SignalHit, index: string): BaseSignalHit => { + return { + _id: generateSignalId(signal), + _index: index, + _source: { + ...signal, + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index 15aaffb7d44ce..9f3ddb7eb5c25 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -7,10 +7,10 @@ import { countBy, isEmpty } from 'lodash'; import { performance } from 'perf_hooks'; import { AlertServices } from '../../../../../alerts/server'; -import { SignalSearchResponse, BulkResponse, SignalHit } from './types'; +import { SignalSearchResponse, BulkResponse, SignalHit, BaseSignalHit } from './types'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; -import { generateId, makeFloatString, errorAggregator, generateSignalId } from './utils'; +import { generateId, makeFloatString, errorAggregator } from './utils'; import { buildBulkBody } from './build_bulk_body'; import { Logger } from '../../../../../../../src/core/server'; @@ -173,28 +173,24 @@ export const singleBulkCreate = async ({ // Bulk Index new signals. export const bulkInsertSignals = async ( - filteredSignals: SignalHit[], - signalsIndex: string, + signals: BaseSignalHit[], logger: Logger, services: AlertServices, refresh: RefreshTypes ): Promise => { - logger.debug(`about to bulk create ${filteredSignals.length} signals`); - // index documents after creating an ID based on the // id and index of each parent and the rule ID - const bulkBody = filteredSignals.flatMap((doc) => [ + const bulkBody = signals.flatMap((doc) => [ { create: { - _index: signalsIndex, - _id: generateSignalId(doc), + _index: doc._index, + _id: doc._id, }, }, - doc, + doc._source, ]); const start = performance.now(); const response: BulkResponse = await services.callCluster('bulk', { - index: signalsIndex, refresh, body: bulkBody, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index f02b6de762e8f..2813c13de32cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -51,6 +51,7 @@ export interface SignalSource { parent?: Ancestor; parents?: Ancestor[]; ancestors: Ancestor[]; + child?: Ancestor; rule: { id: string; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 866f25bc04051..3e1167414838e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -223,6 +223,27 @@ export const generateSignalId = (newSignal: SignalHit) => ) .digest('hex'); +/** + * Generates unique doc ids for each building block signal within a sequence. The id of each building block + * depends on the parents of every building block, so that a signal which appears in multiple different sequences + * (e.g. if multiple rules build sequences that share a common event/signal) will get a unique id per sequence. + * @param buildingBlocks The full list of building blocks in the sequence. + */ +export const generateBuildingBlockIds = (buildingBlocks: SignalHit[]): string[] => { + const baseHashString = buildingBlocks.reduce( + (baseString, block) => + baseString + .concat( + block.signal.parents.reduce((acc, parent) => acc.concat(parent.id, parent.index), '') + ) + .concat(block.signal.rule.id), + '' + ); + return buildingBlocks.map((block, idx) => + createHash('sha256').update(baseHashString).update(String(idx)).digest('hex') + ); +}; + export const parseInterval = (intervalString: string): moment.Duration | null => { try { return moment.duration(parseDuration(intervalString)); From 922a21e153fe6c2b2e08daaa06a4055b29f3f0a6 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Mon, 21 Sep 2020 17:58:56 -0400 Subject: [PATCH 05/26] Use eql instead of eql_query for rule type --- .../common/detection_engine/schemas/common/schemas.ts | 1 - .../detection_engine/schemas/response/rules_schema.ts | 2 +- .../detection_engine/routes/rules/create_rules_route.ts | 4 +++- .../scripts/rules/queries/query_eql.json | 9 ++++----- .../server/lib/detection_engine/signals/get_filter.ts | 5 ++--- .../detection_engine/signals/signal_rule_alert_type.ts | 2 +- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index e6d6659bbb05a..e8d7f409de20a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -305,7 +305,6 @@ export const type = t.keyof({ query: null, saved_query: null, threshold: null, - eql_query: null, threat_match: null, }); export type Type = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index bad52e7a6d89d..8b29c00a44506 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -260,7 +260,7 @@ export const addThresholdFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t. }; export const addEqlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { - if (typeAndTimelineOnly.type === 'eql_query') { + if (typeAndTimelineOnly.type === 'eql') { return [ t.exact( t.partial({ event_category_override: dependentRulesSchema.props.event_category_override }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 4e7b812ca92c2..eea0df8963845 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -95,7 +95,9 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void const query = !isMlRule(type) && queryOrUndefined == null ? '' : queryOrUndefined; const language = - !isMlRule(type) && type !== 'eql_query' && languageOrUndefined == null ? 'kuery' : languageOrUndefined; + !isMlRule(type) && type !== 'eql' && languageOrUndefined == null + ? 'kuery' + : languageOrUndefined; // TODO: Fix these either with an is conversion or by better typing them within io-ts const actions: RuleAlertAction[] = actionsRest as RuleAlertAction[]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json index a77eebd708b39..598f2182002c1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json @@ -7,9 +7,9 @@ ], "rule_id": "rule-id-eql", "enabled": false, - "index": ["logs-endpoint.events.process-default"], + "index": [".ds-logs-endpoint.events.process-default-000001"], "interval": "30s", - "query": "process where process.name = \"mimikatz.exe\"", + "query": "sequence [process where process.name = \"mimikatz.exe\"] [process where process.name = \"explorer.exe\"]", "output_index": ".siem-signals-default", "meta": { "anything_you_want_ui_related_or_otherwise": { @@ -18,14 +18,13 @@ } } }, - "language": "kuery", "risk_score": 1, "max_signals": 100, "tags": ["tag 1", "tag 2", "any tag you want"], "to": "now", - "from": "now-30m", + "from": "now-300m", "severity": "high", - "type": "eql_query", + "type": "eql", "threat": [ { "framework": "MITRE ATT&CK", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index 372a650315331..522f4bfa5ef98 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -90,7 +90,6 @@ export const getFilter = async ({ }; switch (type) { - case 'eql': case 'threat_match': case 'threshold': { return savedId != null ? savedQueryFilter() : queryFilter(); @@ -106,8 +105,8 @@ export const getFilter = async ({ 'Unsupported Rule of type "machine_learning" supplied to getFilter' ); } - case 'eql_query': { - throw new BadRequestError('Unsupported Rule of type "eql_query" supplied to getFilter'); + case 'eql': { + throw new BadRequestError('Unsupported Rule of type "eql" supplied to getFilter'); } default: { return assertUnreachable(type); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index a5420bc26cf22..8d8c5547cf086 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -426,7 +426,7 @@ export const signalRulesAlertType = ({ throttle, buildRuleMessage, }); - } else if (type === 'eql_query') { + } else if (type === 'eql') { if (query === undefined) { throw new Error('eql query rule must have a query defined'); } From ecedb3f588a207b87c6f646999fa91238c169783 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Mon, 21 Sep 2020 18:18:04 -0400 Subject: [PATCH 06/26] Remove unused imports --- .../lib/detection_engine/signals/build_bulk_body.ts | 10 +--------- .../detection_engine/signals/signal_rule_alert_type.ts | 2 +- .../server/lib/detection_engine/signals/utils.ts | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index fb6b56b64d93e..3d18937f0faab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -5,20 +5,12 @@ */ import { SavedObject } from 'src/core/types'; -import { - SignalSourceHit, - SignalHit, - Signal, - SignalSource, - RuleAlertAttributes, - BaseSignalHit, -} from './types'; +import { SignalSourceHit, SignalHit, Signal, RuleAlertAttributes, BaseSignalHit } from './types'; import { buildRule, buildRuleWithoutOverrides } from './build_rule'; import { additionalSignalFields, buildSignal } from './build_signal'; import { buildEventTypeSignal } from './build_event_type_signal'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; -import { EqlSequence } from '../../types'; interface BuildBulkBodyParams { doc: SignalSourceHit; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 8d8c5547cf086..ff41b07463463 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -58,7 +58,7 @@ import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter'; -import { bulkInsertSignals, filterDuplicateSignals } from './single_bulk_create'; +import { bulkInsertSignals } from './single_bulk_create'; import { buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body'; import { buildParent } from './build_signal'; import { createThreatSignals } from './threat_mapping/create_threat_signals'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index a17f378f5a326..a6e6de20c71d2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -11,7 +11,7 @@ import { Logger, SavedObjectsClientContract } from '../../../../../../../src/cor import { AlertServices, parseDuration } from '../../../../../alerts/server'; import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; -import { ListArray, ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types/lists'; +import { ListArray } from '../../../../common/detection_engine/schemas/types/lists'; import { BulkResponse, BulkResponseErrorAggregation, isValidUnit, SignalHit } from './types'; import { BuildRuleMessage } from './rule_messages'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; From 3e45bc8ec8644f12b33b58cb07ce4595981d38b1 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Mon, 21 Sep 2020 18:40:09 -0400 Subject: [PATCH 07/26] Fix tests --- .../detection_engine/schemas/response/rules_schema.mocks.ts | 2 +- .../server/lib/detection_engine/signals/build_signal.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts index a462b297d37f8..4770f0b7abb53 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts @@ -52,7 +52,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem severity: 'high', severity_mapping: [], updated_by: 'elastic_kibana', - tags: [], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', threat: [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index ce434b2823110..d820e3365bbfb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -68,7 +68,7 @@ describe('buildSignal', () => { severity: 'high', severity_mapping: [], updated_by: 'elastic_kibana', - tags: [], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', threat: [], @@ -152,7 +152,7 @@ describe('buildSignal', () => { severity: 'high', severity_mapping: [], updated_by: 'elastic_kibana', - tags: [], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', threat: [], From 4faa592553e20ea557f6b8daa3cf07f04ac40ecc Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Sep 2020 01:53:36 -0400 Subject: [PATCH 08/26] Add basic tests for buildEqlSearchRequest --- .../detection_engine/get_query_filter.test.ts | 136 +++++++++++++++++- .../detection_engine/get_query_filter.ts | 9 +- 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index 72ef230a42342..380ded1660457 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getQueryFilter, buildExceptionFilter } from './get_query_filter'; +import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter'; import { Filter, EsQueryConfig } from 'src/plugins/data/public'; import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock'; @@ -1085,4 +1085,138 @@ describe('get_filter', () => { }); }); }); + + describe('buildEqlSearchRequest', () => { + test('should build a basic request with time range', () => { + const request = buildEqlSearchRequest( + 'process where true', + ['testindex1', 'testindex2'], + 'now-5m', + 'now', + 100, + undefined, + [], + undefined + ); + expect(request).toEqual({ + method: 'POST', + path: `/testindex1,testindex2/_eql/search`, + body: { + size: 100, + query: 'process where true', + filter: { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'now', + }, + }, + }, + }, + }); + }); + + test('should build a request with timestamp and event category overrides', () => { + const request = buildEqlSearchRequest( + 'process where true', + ['testindex1', 'testindex2'], + 'now-5m', + 'now', + 100, + 'event.ingested', + [], + 'event.other_category' + ); + expect(request).toEqual({ + method: 'POST', + path: `/testindex1,testindex2/_eql/search`, + event_category_field: 'event.other_category', + body: { + size: 100, + query: 'process where true', + filter: { + range: { + 'event.ingested': { + gte: 'now-5m', + lte: 'now', + }, + }, + }, + }, + }); + }); + + test('should build a request with exceptions', () => { + const request = buildEqlSearchRequest( + 'process where true', + ['testindex1', 'testindex2'], + 'now-5m', + 'now', + 100, + undefined, + [getExceptionListItemSchemaMock()], + undefined + ); + expect(request).toEqual({ + method: 'POST', + path: `/testindex1,testindex2/_eql/search`, + body: { + size: 100, + query: 'process where true', + filter: { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'now', + }, + }, + bool: { + must_not: { + bool: { + should: [ + { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], + }, + }, + score_mode: 'none', + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + }, + }, + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 238bb3e7d7ecd..84c73adb96ec4 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -123,7 +123,14 @@ export const buildEqlSearchRequest = ( lte: to, }, }, - bool: exceptionFilter?.query.bool, + bool: + exceptionFilter !== undefined + ? { + must_not: { + bool: exceptionFilter?.query.bool, + }, + } + : undefined, }, }, }; From f9e26b0ad60f6be8d55a64065cd49c2f31076620 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Sep 2020 02:16:35 -0400 Subject: [PATCH 09/26] Add rulesSchema tests for eql --- .../schemas/response/rules_schema.mocks.ts | 8 ++++ .../schemas/response/rules_schema.test.ts | 46 +++++++++++++++++-- .../schemas/response/rules_schema.ts | 4 +- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts index 4770f0b7abb53..54ab85641261a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts @@ -110,3 +110,11 @@ export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): R ], }; }; + +export const getRulesEqlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => { + const baseRule = getRulesSchemaMock(anchorDate); + delete baseRule.language; + baseRule.type = 'eql'; + baseRule.query = 'process where true'; + return baseRule; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts index 3a47d4af6ac14..1e1c783b64334 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts @@ -18,6 +18,7 @@ import { addTimelineTitle, addMlFields, addThreatMatchFields, + addEqlFields, } from './rules_schema'; import { exactCheck } from '../../../exact_check'; import { foldLeftRight, getPaths } from '../../../test_utils'; @@ -26,6 +27,7 @@ import { getRulesSchemaMock, getRulesMlSchemaMock, getThreatMatchingSchemaMock, + getRulesEqlSchemaMock, } from './rules_schema.mocks'; import { ListArray } from '../types/lists'; @@ -628,6 +630,32 @@ describe('rules_schema', () => { ]); expect(message.schema).toEqual({}); }); + + test('it validates an eql rule response', () => { + const payload = getRulesEqlSchemaMock(); + + const dependents = getDependents(payload); + const decoded = dependents.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected = getRulesEqlSchemaMock(); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); + + test('it rejects an eql rule response that has a language defined', () => { + const payload = getRulesEqlSchemaMock(); + payload.language = 'kuery'; + + const dependents = getDependents(payload); + const decoded = dependents.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual(['invalid keys "language"']); + expect(message.schema).toEqual({}); + }); }); describe('addSavedId', () => { @@ -668,11 +696,6 @@ describe('rules_schema', () => { expect(fields.length).toEqual(2); }); - test('should return two fields for a rule of type "eql"', () => { - const fields = addQueryFields({ type: 'eql' }); - expect(fields.length).toEqual(2); - }); - test('should return two fields for a rule of type "threshold"', () => { const fields = addQueryFields({ type: 'threshold' }); expect(fields.length).toEqual(2); @@ -757,4 +780,17 @@ describe('rules_schema', () => { expect(fields.length).toEqual(5); }); }); + + describe('addEqlFields', () => { + test('should return empty array if type is not "eql"', () => { + const fields = addEqlFields({ type: 'query' }); + const expected: t.Mixed[] = []; + expect(fields).toEqual(expected); + }); + + test('should return 2 fields for a rule of type "eql"', () => { + const fields = addEqlFields({ type: 'eql' }); + expect(fields.length).toEqual(2); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 8b29c00a44506..669e649f03230 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -223,9 +223,7 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi }; export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { - if ( - ['eql', 'query', 'saved_query', 'threshold', 'threat_match'].includes(typeAndTimelineOnly.type) - ) { + if (['query', 'saved_query', 'threshold', 'threat_match'].includes(typeAndTimelineOnly.type)) { return [ t.exact(t.type({ query: dependentRulesSchema.props.query })), t.exact(t.type({ language: dependentRulesSchema.props.language })), From 3fd19f012e68707aa2430ff0bc9003f235829a5d Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Sep 2020 03:01:18 -0400 Subject: [PATCH 10/26] Add buildSignalFromSequence test --- .../signals/__mocks__/es_results.ts | 32 +++++- .../signals/build_bulk_body.test.ts | 104 +++++++++++++++++- 2 files changed, 134 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index c558806106fe4..c2160c458c9d1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -4,7 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalSearchResponse, BulkResponse, BulkItem } from '../types'; +import { + SignalSourceHit, + SignalSearchResponse, + BulkResponse, + BulkItem, + RuleAlertAttributes, +} from '../types'; import { Logger, SavedObject, @@ -61,6 +67,30 @@ export const sampleRuleAlertParams = ( exceptionsList: getListArrayMock(), }); +export const sampleRuleSO = (): SavedObject => { + return { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'alert', + version: '1', + updated_at: '2020-03-27T22:55:59.577Z', + attributes: { + actions: [], + enabled: true, + name: 'rule-name', + tags: ['some fake tag 1', 'some fake tag 2'], + createdBy: 'sample user', + createdAt: '2020-03-27T22:55:59.577Z', + updatedBy: 'sample user', + schedule: { + interval: '5m', + }, + throttle: 'no_actions', + params: sampleRuleAlertParams(), + }, + references: [], + }; +}; + export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 967dc5331e46b..c8d0bdc781209 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -9,8 +9,10 @@ import { sampleDocNoSortId, sampleRuleGuid, sampleIdGuid, + sampleDocWithAncestors, + sampleRuleSO, } from './__mocks__/es_results'; -import { buildBulkBody } from './build_bulk_body'; +import { buildBulkBody, buildSignalFromSequence } from './build_bulk_body'; import { SignalHit } from './types'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; @@ -433,3 +435,103 @@ describe('buildBulkBody', () => { expect(fakeSignalSourceHit).toEqual(expected); }); }); + +describe('buildSignalFromSequence', () => { + test('builds a basic signal from a sequence of building blocks', () => { + const blocks = [sampleDocWithAncestors().hits.hits[0], sampleDocWithAncestors().hits.hits[0]]; + const ruleSO = sampleRuleSO(); + const signal = buildSignalFromSequence(blocks, ruleSO); + delete signal['@timestamp']; + const expected: Omit = { + event: { + kind: 'signal', + }, + signal: { + parents: [ + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 2, + }, + }; + expect(signal).toEqual(expected); + }); +}); From bcdde6214017d288d94f45703e53f964f835a3fb Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Sep 2020 10:38:16 -0400 Subject: [PATCH 11/26] Add threat rule fields to buildRuleWithoutOverrides --- .../server/lib/detection_engine/signals/build_rule.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index 301a71821cce1..e5370735333bc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -119,7 +119,7 @@ export const buildRuleWithoutOverrides = ( ruleSO: SavedObject ): RulesSchema => { const ruleParams = ruleSO.attributes.params; - const rule = { + const rule: RulesSchema = { id: ruleSO.id, rule_id: ruleParams.ruleId, actions: ruleSO.attributes.actions, @@ -165,6 +165,10 @@ export const buildRuleWithoutOverrides = ( machine_learning_job_id: ruleParams.machineLearningJobId, anomaly_threshold: ruleParams.anomalyThreshold, threshold: ruleParams.threshold, + threat_filters: ruleParams.threatFilters, + threat_index: ruleParams.threatIndex, + threat_query: ruleParams.threatQuery, + threat_mapping: ruleParams.threatMapping, }; return removeInternalTagsFromRule(rule); }; From 162256a42b03435b711fa9721472ca549d019427 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Sep 2020 10:38:56 -0400 Subject: [PATCH 12/26] Fix buildSignalFromSequence typecheck error --- .../server/lib/detection_engine/signals/build_bulk_body.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index c8d0bdc781209..4386616a7719a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -441,6 +441,8 @@ describe('buildSignalFromSequence', () => { const blocks = [sampleDocWithAncestors().hits.hits[0], sampleDocWithAncestors().hits.hits[0]]; const ruleSO = sampleRuleSO(); const signal = buildSignalFromSequence(blocks, ruleSO); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error delete signal['@timestamp']; const expected: Omit = { event: { From 3f89fb9e1a533bb8e377f39f84586be12fc2b0c9 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Sep 2020 11:41:54 -0400 Subject: [PATCH 13/26] Add more tests --- .../signals/build_bulk_body.test.ts | 94 +++++++++++++++- .../signals/build_rule.test.ts | 103 +++++++++++++++++- 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 4386616a7719a..220373659f58c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -12,7 +12,7 @@ import { sampleDocWithAncestors, sampleRuleSO, } from './__mocks__/es_results'; -import { buildBulkBody, buildSignalFromSequence } from './build_bulk_body'; +import { buildBulkBody, buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body'; import { SignalHit } from './types'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; @@ -537,3 +537,95 @@ describe('buildSignalFromSequence', () => { expect(signal).toEqual(expected); }); }); + +describe('buildSignalFromEvent', () => { + test('builds a basic signal from a single event', () => { + const ancestor = sampleDocWithAncestors().hits.hits[0]; + delete ancestor._source.source; + const ruleSO = sampleRuleSO(); + const signal = buildSignalFromEvent(ancestor, ruleSO); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error + delete signal['@timestamp']; + const expected: Omit & { someKey: 'someValue' } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_time: '2020-04-20T21:27:45+0000', + parent: { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + parents: [ + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 2, + }, + }; + expect(signal).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index aa870271d8c74..62e5854037d9e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -4,8 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { buildRule, removeInternalTagsFromRule } from './build_rule'; -import { sampleDocNoSortId, sampleRuleAlertParams, sampleRuleGuid } from './__mocks__/es_results'; +import { buildRule, removeInternalTagsFromRule, buildRuleWithoutOverrides } from './build_rule'; +import { + sampleDocNoSortId, + sampleRuleAlertParams, + sampleRuleGuid, + sampleRuleSO, +} from './__mocks__/es_results'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; @@ -272,7 +277,9 @@ describe('buildRule', () => { }; expect(rule).toEqual(expected); }); +}); +describe('removeInternalTagsFromRule', () => { test('it removes internal tags from a typical rule', () => { const rule = getRulesSchemaMock(); rule.tags = [ @@ -300,3 +307,95 @@ describe('buildRule', () => { expect(noInternals).toEqual(rule); }); }); + +describe('buildRuleWithoutOverrides', () => { + test('builds a rule using rule SO', () => { + const ruleSO = sampleRuleSO(); + const rule = buildRuleWithoutOverrides(ruleSO); + expect(rule).toEqual({ + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }); + }); + + test('builds a rule using rule SO and removes internal tags', () => { + const ruleSO = sampleRuleSO(); + ruleSO.attributes.tags = [ + 'some fake tag 1', + 'some fake tag 2', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:true`, + ]; + const rule = buildRuleWithoutOverrides(ruleSO); + expect(rule).toEqual({ + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }); + }); +}); From 8b00aa71e7361bba5ef49be7728afb529e951967 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Sep 2020 17:34:47 -0400 Subject: [PATCH 14/26] Add tests for wrapBuildingBlock and generateSignalId --- .../signals/__mocks__/es_results.ts | 75 +++++++++++++++++++ .../signals/signal_rule_alert_type.ts | 28 +------ .../detection_engine/signals/utils.test.ts | 53 +++++++++++++ .../lib/detection_engine/signals/utils.ts | 31 +++++++- 4 files changed, 160 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index c2160c458c9d1..b86dbd52e2734 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -10,6 +10,7 @@ import { BulkResponse, BulkItem, RuleAlertAttributes, + SignalHit, } from '../types'; import { Logger, @@ -220,6 +221,80 @@ export const sampleDocWithAncestors = (): SignalSearchResponse => { }; }; +export const sampleSignalHit = (): SignalHit => ({ + '@timestamp': '2020-04-20T21:27:45+0000', + event: { + kind: 'signal', + }, + signal: { + parents: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + status: 'open', + rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: '2020-04-20T21:27:45+0000', + updated_at: '2020-04-20T21:27:45+0000', + created_by: 'elastic', + description: 'some description', + enabled: true, + false_positives: ['false positive 1', 'false positive 2'], + from: 'now-6m', + immutable: false, + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + references: ['test 1', 'test 2'], + severity: 'high', + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-hassanabad-frank-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, +}); + export const sampleBulkCreateDuplicateResult = { took: 60, errors: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index ff41b07463463..d415b687ca1cf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -31,7 +31,6 @@ import { SignalRuleAlertTypeDefinition, RuleAlertAttributes, EqlSignalSearchResponse, - SignalHit, BaseSignalHit, } from './types'; import { @@ -40,8 +39,8 @@ import { getExceptions, getGapMaxCatchupRatio, MAX_RULE_GAP_RATIO, - generateBuildingBlockIds, - generateSignalId, + wrapBuildingBlocks, + wrapSignal, } from './utils'; import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; @@ -582,26 +581,3 @@ export const signalRulesAlertType = ({ }, }; }; - -export const wrapBuildingBlocks = (buildingBlocks: SignalHit[], index: string): BaseSignalHit[] => { - const blockIds = generateBuildingBlockIds(buildingBlocks); - return buildingBlocks.map((block, idx) => { - return { - _id: blockIds[idx], - _index: index, - _source: { - ...block, - }, - }; - }); -}; - -export const wrapSignal = (signal: SignalHit, index: string): BaseSignalHit => { - return { - _id: generateSignalId(signal), - _index: index, - _source: { - ...signal, - }, - }; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 123b9c9bdffa2..ac9d8b183fae4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -25,6 +25,8 @@ import { getListsClient, getSignalTimeTuples, getExceptions, + wrapBuildingBlocks, + generateSignalId, } from './utils'; import { BulkResponseErrorAggregation } from './types'; import { @@ -33,6 +35,7 @@ import { sampleBulkError, sampleBulkErrorItem, mockLogger, + sampleSignalHit, } from './__mocks__/es_results'; const buildRuleMessage = buildRuleMessageFactory({ @@ -783,4 +786,54 @@ describe('utils', () => { expect(exceptions).toEqual([]); }); }); + + describe('wrapBuildingBlocks', () => { + it('should generate a unique id for each building block', () => { + const wrappedBlocks = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + const blockIds: string[] = []; + wrappedBlocks.forEach((block) => { + expect(blockIds.includes(block._id)).toEqual(false); + blockIds.push(block._id); + }); + }); + + it('should generate different ids for identical documents in different sequences', () => { + const wrappedBlockSequence1 = wrapBuildingBlocks([sampleSignalHit()], 'test-index'); + const wrappedBlockSequence2 = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + const blockId = wrappedBlockSequence1[0]._id; + wrappedBlockSequence2.forEach((block) => { + expect(block._id).not.toEqual(blockId); + }); + }); + + it('should generate the same ids when given the same sequence twice', () => { + const wrappedBlockSequence1 = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + const wrappedBlockSequence2 = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + wrappedBlockSequence1.forEach((block, idx) => { + expect(block._id).toEqual(wrappedBlockSequence2[idx]._id); + }); + }); + }); + + describe('generateSignalId', () => { + it('generates a unique signal id for same signal with different rule id', () => { + const signalId1 = generateSignalId(sampleSignalHit()); + const modifiedSignal = sampleSignalHit(); + modifiedSignal.signal.rule.id = 'some other rule id'; + const signalIdModified = generateSignalId(modifiedSignal); + expect(signalId1).not.toEqual(signalIdModified); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index a6e6de20c71d2..349d2d5eb029d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -12,7 +12,13 @@ import { AlertServices, parseDuration } from '../../../../../alerts/server'; import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { ListArray } from '../../../../common/detection_engine/schemas/types/lists'; -import { BulkResponse, BulkResponseErrorAggregation, isValidUnit, SignalHit } from './types'; +import { + BulkResponse, + BulkResponseErrorAggregation, + isValidUnit, + SignalHit, + BaseSignalHit, +} from './types'; import { BuildRuleMessage } from './rule_messages'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; import { hasLargeValueList } from '../../../../common/detection_engine/utils'; @@ -236,6 +242,29 @@ export const generateBuildingBlockIds = (buildingBlocks: SignalHit[]): string[] ); }; +export const wrapBuildingBlocks = (buildingBlocks: SignalHit[], index: string): BaseSignalHit[] => { + const blockIds = generateBuildingBlockIds(buildingBlocks); + return buildingBlocks.map((block, idx) => { + return { + _id: blockIds[idx], + _index: index, + _source: { + ...block, + }, + }; + }); +}; + +export const wrapSignal = (signal: SignalHit, index: string): BaseSignalHit => { + return { + _id: generateSignalId(signal), + _index: index, + _source: { + ...signal, + }, + }; +}; + export const parseInterval = (intervalString: string): moment.Duration | null => { try { return moment.duration(parseDuration(intervalString)); From 1011f073e900b226b3cf5b330c3aa4a3014ae4b3 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Sep 2020 23:55:26 -0400 Subject: [PATCH 15/26] Use isEqlRule function and fix import error --- .../lib/detection_engine/routes/rules/create_rules_route.ts | 3 ++- .../lib/detection_engine/signals/signal_rule_alert_type.ts | 2 +- .../server/lib/detection_engine/signals/types.ts | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index eea0df8963845..54df87ca17787 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { isEqlRule } from '../../../../../common/detection_engine/utils'; import { createRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/create_rules_type_dependents'; import { RuleAlertAction } from '../../../../../common/detection_engine/types'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; @@ -95,7 +96,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void const query = !isMlRule(type) && queryOrUndefined == null ? '' : queryOrUndefined; const language = - !isMlRule(type) && type !== 'eql' && languageOrUndefined == null + !isMlRule(type) && !isEqlRule(type) && languageOrUndefined == null ? 'kuery' : languageOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index f66c6c9c627eb..539fe8412b8b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -433,7 +433,7 @@ export const signalRulesAlertType = ({ throttle, buildRuleMessage, }); - } else if (type === 'eql') { + } else if (isEqlRule(type)) { if (query === undefined) { throw new Error('eql query rule must have a query defined'); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 726fbdd9b87ae..a1cc768e77ba3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -15,7 +15,7 @@ import { AlertServices, } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; -import { RuleTypeParams } from '../types'; +import { RuleTypeParams, RefreshTypes } from '../types'; import { SearchResponse, EqlSearchResponse, BaseHit } from '../../types'; import { ListClient } from '../../../../../lists/server'; import { Logger } from '../../../../../../../src/core/server'; From cf0f9fb8af63072b1b54cf40ad6b6e0b22e0ca5a Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 00:08:08 -0400 Subject: [PATCH 16/26] delete frank --- .../detection_engine/schemas/response/rules_schema.mocks.ts | 2 +- .../lib/detection_engine/signals/__mocks__/es_results.ts | 2 +- .../server/lib/detection_engine/signals/build_bulk_body.ts | 2 +- .../server/lib/detection_engine/signals/build_signal.test.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts index 54ab85641261a..ef3f3e763570f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts @@ -61,7 +61,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-hassanabad-frank-default', + output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index c8b2ee9601fe1..b37bc7d0fab69 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -282,7 +282,7 @@ export const sampleSignalHit = (): SignalHit => ({ status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-hassanabad-frank-default', + output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 3d18937f0faab..63ee908902f6c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -93,7 +93,7 @@ export const buildSignalFromEvent = ( ruleSO: SavedObject ): SignalHit => { const rule = buildRuleWithoutOverrides(ruleSO); - const signal: Signal = { + const signal = { ...buildSignal([event], rule), ...additionalSignalFields(event), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index d820e3365bbfb..d0c451bbdf2e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -77,7 +77,7 @@ describe('buildSignal', () => { status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-hassanabad-frank-default', + output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], @@ -161,7 +161,7 @@ describe('buildSignal', () => { status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-hassanabad-frank-default', + output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], From 1d680a71010e5b0d1528988adb103a21d25eef78 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 00:11:30 -0400 Subject: [PATCH 17/26] Move sequence interface to types.ts --- .../lib/detection_engine/signals/signal_rule_alert_type.ts | 5 ----- .../server/lib/detection_engine/signals/types.ts | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 539fe8412b8b6..1ce77330d9356 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -63,11 +63,6 @@ import { buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body import { buildParent } from './build_signal'; import { createThreatSignals } from './threat_mapping/create_threat_signals'; -interface Sequence { - buildingBlocks: BaseSignalHit[]; - sequenceSignal: BaseSignalHit; -} - export const signalRulesAlertType = ({ logger, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index a1cc768e77ba3..9246aa9ef03ca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -28,6 +28,11 @@ export type unitType = 's' | 'm' | 'h'; export const isValidUnit = (unitParam: string): unitParam is unitType => ['s', 'm', 'h'].includes(unitParam); +export interface Sequence { + buildingBlocks: BaseSignalHit[]; + sequenceSignal: BaseSignalHit; +} + export interface SignalsParams { signalIds: string[] | undefined | null; query: object | undefined | null; From 5ebe577e9cd6ddb72565786c3f219c8079f22314 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 01:47:59 -0400 Subject: [PATCH 18/26] Fix import --- .../lib/detection_engine/signals/signal_rule_alert_type.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 1ce77330d9356..8be283bb18276 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -29,6 +29,7 @@ import { RuleAlertAttributes, EqlSignalSearchResponse, BaseSignalHit, + Sequence, } from './types'; import { getGapBetweenRuns, From f445d87a79040aa541ae2fc6af629ed88e209d29 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 11:14:45 -0400 Subject: [PATCH 19/26] Remove EQL execution placeholder, add back language to eql rule type --- .../common/detection_engine/schemas/response/rules_schema.ts | 3 ++- .../lib/detection_engine/signals/signal_rule_alert_type.ts | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 669e649f03230..908425a7496d0 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -122,7 +122,7 @@ export const dependentRulesSchema = t.partial({ language, query, - // eql fields + // eql only fields event_category_override, // when type = saved_query, saved_id is required @@ -264,6 +264,7 @@ export const addEqlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[ t.partial({ event_category_override: dependentRulesSchema.props.event_category_override }) ), t.exact(t.type({ query: dependentRulesSchema.props.query })), + t.exact(t.type({ language: dependentRulesSchema.props.language })), ]; } else { return []; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 8be283bb18276..7f84b3cb53ff6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -277,8 +277,6 @@ export const signalRulesAlertType = ({ bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], }), ]); - } else if (isEqlRule(type)) { - throw new Error('EQL Rules are under development, execution is not yet implemented'); } else if (isThresholdRule(type) && threshold) { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ From 1c5afd93b860006c2e519ef4679cd053b32ca5eb Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 12:07:15 -0400 Subject: [PATCH 20/26] allow no indices for eql search --- .../common/detection_engine/get_query_filter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 84c73adb96ec4..05c706164ab44 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -112,7 +112,7 @@ export const buildEqlSearchRequest = ( const indexString = index.join(); const baseRequest = { method: 'POST', - path: `/${indexString}/_eql/search`, + path: `/${indexString}/_eql/search?allow_no_indices=true`, body: { size, query, From 506a3d6f6b4f933ff7adf6a715380db9f57b5974 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 12:15:41 -0400 Subject: [PATCH 21/26] Fix unit tests for language update --- .../schemas/response/rules_schema.mocks.ts | 11 ++++++----- .../schemas/response/rules_schema.test.ts | 17 ++--------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts index ef3f3e763570f..aaa246c82d9d7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts @@ -112,9 +112,10 @@ export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): R }; export const getRulesEqlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => { - const baseRule = getRulesSchemaMock(anchorDate); - delete baseRule.language; - baseRule.type = 'eql'; - baseRule.query = 'process where true'; - return baseRule; + return { + ...getRulesSchemaMock(anchorDate), + language: 'eql', + type: 'eql', + query: 'process where true', + }; }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts index 1e1c783b64334..c5bad3c55066b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts @@ -643,19 +643,6 @@ describe('rules_schema', () => { expect(getPaths(left(message.errors))).toEqual([]); expect(message.schema).toEqual(expected); }); - - test('it rejects an eql rule response that has a language defined', () => { - const payload = getRulesEqlSchemaMock(); - payload.language = 'kuery'; - - const dependents = getDependents(payload); - const decoded = dependents.decode(payload); - const checked = exactCheck(payload, decoded); - const message = pipe(checked, foldLeftRight); - - expect(getPaths(left(message.errors))).toEqual(['invalid keys "language"']); - expect(message.schema).toEqual({}); - }); }); describe('addSavedId', () => { @@ -788,9 +775,9 @@ describe('rules_schema', () => { expect(fields).toEqual(expected); }); - test('should return 2 fields for a rule of type "eql"', () => { + test('should return 3 fields for a rule of type "eql"', () => { const fields = addEqlFields({ type: 'eql' }); - expect(fields.length).toEqual(2); + expect(fields.length).toEqual(3); }); }); }); From aacc60541ecaebcb89d87d594003698d03cff5d3 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 13:12:41 -0400 Subject: [PATCH 22/26] Fix buildEqlSearchRequest tests --- .../common/detection_engine/get_query_filter.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index 380ded1660457..0224caafb41a8 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -1100,7 +1100,7 @@ describe('get_filter', () => { ); expect(request).toEqual({ method: 'POST', - path: `/testindex1,testindex2/_eql/search`, + path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, body: { size: 100, query: 'process where true', @@ -1129,7 +1129,7 @@ describe('get_filter', () => { ); expect(request).toEqual({ method: 'POST', - path: `/testindex1,testindex2/_eql/search`, + path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, event_category_field: 'event.other_category', body: { size: 100, @@ -1159,7 +1159,7 @@ describe('get_filter', () => { ); expect(request).toEqual({ method: 'POST', - path: `/testindex1,testindex2/_eql/search`, + path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, body: { size: 100, query: 'process where true', From 19cf9e25e8ef9baeffae869640c39cb6d19910d0 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 17:54:06 -0400 Subject: [PATCH 23/26] Replace signal.child with signal.group --- .../routes/index/signals_mapping.json | 17 ++++------------- .../signals/build_bulk_body.test.ts | 3 +++ .../detection_engine/signals/build_bulk_body.ts | 10 +++++++++- .../signals/signal_rule_alert_type.ts | 7 +++++-- .../lib/detection_engine/signals/types.ts | 9 ++++++++- .../lib/detection_engine/signals/utils.test.ts | 4 ++-- .../lib/detection_engine/signals/utils.ts | 9 +++++---- 7 files changed, 36 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index e0d448d374243..7255325358baf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -41,7 +41,7 @@ } } }, - "child": { + "ancestors": { "properties": { "rule": { "type": "keyword" @@ -60,22 +60,13 @@ } } }, - "ancestors": { + "group": { "properties": { - "rule": { - "type": "keyword" - }, - "index": { - "type": "keyword" - }, "id": { "type": "keyword" }, - "type": { - "type": "keyword" - }, - "depth": { - "type": "long" + "index": { + "type": "integer" } } }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 220373659f58c..f45a408cd32b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -532,6 +532,9 @@ describe('buildSignalFromSequence', () => { exceptions_list: getListArrayMock(), }, depth: 2, + group: { + id: '269c1f5754bff92fb8040283b687258e99b03e8b2ab1262cc20c82442e5de5ea', + }, }, }; expect(signal).toEqual(expected); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 63ee908902f6c..10c72b3211e67 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -11,6 +11,7 @@ import { additionalSignalFields, buildSignal } from './build_signal'; import { buildEventTypeSignal } from './build_event_type_signal'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; +import { generateSignalId } from './utils'; interface BuildBulkBodyParams { doc: SignalSourceHit; @@ -84,7 +85,14 @@ export const buildSignalFromSequence = ( event: { kind: 'signal', }, - signal, + signal: { + ...signal, + group: { + // This is the same function that is used later to generate the _id for the sequence signal document, + // so _id should equal signal.group.id for the "shell" document + id: generateSignalId(signal), + }, + }, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 7f84b3cb53ff6..c1f3d86fb9017 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -470,10 +470,13 @@ export const signalRulesAlertType = ({ buildSignalFromSequence(blocks, savedObject), outputIndex ); - blocks.forEach((block) => { + blocks.forEach((block, idx) => { // TODO: fix type of blocks so we don't have to check existence of _source.signal if (block._source.signal) { - block._source.signal.child = buildParent(sequenceSignal); + block._source.signal.group = { + id: sequenceSignal._id, + index: idx, + }; } }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 9246aa9ef03ca..c3e2316614290 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -67,7 +67,10 @@ export interface SignalSource { parent?: Ancestor; parents?: Ancestor[]; ancestors: Ancestor[]; - child?: Ancestor; + group?: { + id: string; + index?: number; + }; rule: { id: string; }; @@ -156,6 +159,10 @@ export interface Signal { parent?: Ancestor; parents: Ancestor[]; ancestors: Ancestor[]; + group?: { + id: string; + index?: number; + }; original_time?: string; original_event?: SearchTypes; status: Status; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 6e8fb957aff18..14e12b2ea4632 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -839,10 +839,10 @@ describe('utils', () => { describe('generateSignalId', () => { it('generates a unique signal id for same signal with different rule id', () => { - const signalId1 = generateSignalId(sampleSignalHit()); + const signalId1 = generateSignalId(sampleSignalHit().signal); const modifiedSignal = sampleSignalHit(); modifiedSignal.signal.rule.id = 'some other rule id'; - const signalIdModified = generateSignalId(modifiedSignal); + const signalIdModified = generateSignalId(modifiedSignal.signal); expect(signalId1).not.toEqual(signalIdModified); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 7b6bb6a7590cd..53089b7f1ca2b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -20,6 +20,7 @@ import { BaseSignalHit, SearchAfterAndBulkCreateReturnType, SignalSearchResponse, + Signal, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; @@ -215,12 +216,12 @@ export const generateId = ( ): string => createHash('sha256').update(docIndex.concat(docId, version, ruleId)).digest('hex'); // TODO: do we need to include version in the id? If it does matter then we should include it in signal.parents as well -export const generateSignalId = (newSignal: SignalHit) => +export const generateSignalId = (signal: Signal) => createHash('sha256') .update( - newSignal.signal.parents + signal.parents .reduce((acc, parent) => acc.concat(parent.id, parent.index), '') - .concat(newSignal.signal.rule.id) + .concat(signal.rule.id) ) .digest('hex'); @@ -260,7 +261,7 @@ export const wrapBuildingBlocks = (buildingBlocks: SignalHit[], index: string): export const wrapSignal = (signal: SignalHit, index: string): BaseSignalHit => { return { - _id: generateSignalId(signal), + _id: generateSignalId(signal.signal), _index: index, _source: { ...signal, From b42cf73e77ad100dc911e0cffb48a9757e62abe7 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 17:55:37 -0400 Subject: [PATCH 24/26] remove unused import --- .../lib/detection_engine/signals/signal_rule_alert_type.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index c1f3d86fb9017..00a042e82f9e7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -61,7 +61,6 @@ import { getNotificationResultsLink } from '../notifications/utils'; import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter'; import { bulkInsertSignals } from './single_bulk_create'; import { buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body'; -import { buildParent } from './build_signal'; import { createThreatSignals } from './threat_mapping/create_threat_signals'; export const signalRulesAlertType = ({ From 1942bf95670d723997706ece6e7ac5f717816e55 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 23 Sep 2020 18:55:49 -0400 Subject: [PATCH 25/26] Move sequence signal group building to separate testable function --- .../signals/build_bulk_body.ts | 53 ++++++++++++++++++- .../signals/signal_rule_alert_type.ts | 48 ++--------------- .../lib/detection_engine/signals/types.ts | 5 -- 3 files changed, 56 insertions(+), 50 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 10c72b3211e67..01a6b0e7aefad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -5,13 +5,21 @@ */ import { SavedObject } from 'src/core/types'; -import { SignalSourceHit, SignalHit, Signal, RuleAlertAttributes, BaseSignalHit } from './types'; +import { + SignalSourceHit, + SignalHit, + Signal, + RuleAlertAttributes, + BaseSignalHit, + SignalSource, +} from './types'; import { buildRule, buildRuleWithoutOverrides } from './build_rule'; import { additionalSignalFields, buildSignal } from './build_signal'; import { buildEventTypeSignal } from './build_event_type_signal'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; -import { generateSignalId } from './utils'; +import { generateSignalId, wrapBuildingBlocks, wrapSignal } from './utils'; +import { EqlSequence } from '../../types'; interface BuildBulkBodyParams { doc: SignalSourceHit; @@ -74,6 +82,47 @@ export const buildBulkBody = ({ return signalHit; }; +/** + * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed - + * one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals + * share the same signal.group.id to make it easy to query them. + * @param sequence The raw ES documents that make up the sequence + * @param ruleSO SavedObject representing the rule that found the sequence + * @param outputIndex Index to write the resulting signals to + */ +export const buildSignalGroupFromSequence = ( + sequence: EqlSequence, + ruleSO: SavedObject, + outputIndex: string +): BaseSignalHit[] => { + const wrappedBuildingBlocks = wrapBuildingBlocks( + sequence.events.map((event) => { + const signal = buildSignalFromEvent(event, ruleSO); + signal.signal.rule.building_block_type = 'default'; + return signal; + }), + outputIndex + ); + + // Now that we have an array of building blocks for the events in the sequence, + // we can build the signal that links the building blocks together + // and also insert the group id (which is also the "shell" signal _id) in each building block + const sequenceSignal = wrapSignal( + buildSignalFromSequence(wrappedBuildingBlocks, ruleSO), + outputIndex + ); + wrappedBuildingBlocks.forEach((block, idx) => { + // TODO: fix type of blocks so we don't have to check existence of _source.signal + if (block._source.signal) { + block._source.signal.group = { + id: sequenceSignal._id, + index: idx, + }; + } + }); + return [...wrappedBuildingBlocks, sequenceSignal]; +}; + export const buildSignalFromSequence = ( events: BaseSignalHit[], ruleSO: SavedObject diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 00a042e82f9e7..f7b56f42755ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -29,7 +29,6 @@ import { RuleAlertAttributes, EqlSignalSearchResponse, BaseSignalHit, - Sequence, } from './types'; import { getGapBetweenRuns, @@ -37,7 +36,6 @@ import { getExceptions, getGapMaxCatchupRatio, MAX_RULE_GAP_RATIO, - wrapBuildingBlocks, wrapSignal, createErrorsFromShard, createSearchAfterReturnType, @@ -60,7 +58,7 @@ import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects import { getNotificationResultsLink } from '../notifications/utils'; import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter'; import { bulkInsertSignals } from './single_bulk_create'; -import { buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body'; +import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk_body'; import { createThreatSignals } from './threat_mapping/create_threat_signals'; export const signalRulesAlertType = ({ @@ -447,47 +445,11 @@ export const signalRulesAlertType = ({ ); let newSignals: BaseSignalHit[] | undefined; if (response.hits.sequences !== undefined) { - // 2D array: For each sequence, we make an array containing building block signals for each - // event within the sequence. We also wrap each building block with a doc id and index so they're ready - // to be used in the next step - creating the signal that links them all together. - const buildingBlockArrays = response.hits.sequences.map((sequence) => - wrapBuildingBlocks( - sequence.events.map((event) => { - const signal = buildSignalFromEvent(event, savedObject); - signal.signal.rule.building_block_type = 'default'; - return signal; - }), - outputIndex - ) + newSignals = response.hits.sequences.reduce( + (acc: BaseSignalHit[], sequence) => + acc.concat(buildSignalGroupFromSequence(sequence, savedObject, outputIndex)), + [] ); - - // Now that we have an array of building blocks for each matching sequence, - // we can build the signal that links the building blocks of a sequence together - // and also insert references to this signal in each building block - const sequences: Sequence[] = buildingBlockArrays.map((blocks) => { - const sequenceSignal = wrapSignal( - buildSignalFromSequence(blocks, savedObject), - outputIndex - ); - blocks.forEach((block, idx) => { - // TODO: fix type of blocks so we don't have to check existence of _source.signal - if (block._source.signal) { - block._source.signal.group = { - id: sequenceSignal._id, - index: idx, - }; - } - }); - return { - buildingBlocks: blocks, - sequenceSignal, - }; - }); - newSignals = sequences.reduce((acc: BaseSignalHit[], sequence): BaseSignalHit[] => { - acc.push(...sequence.buildingBlocks); - acc.push(sequence.sequenceSignal); - return acc; - }, []); } else if (response.hits.events !== undefined) { newSignals = response.hits.events.map((event) => wrapSignal(buildSignalFromEvent(event, savedObject), outputIndex) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index c3e2316614290..2f6ed0c1e3a8e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -28,11 +28,6 @@ export type unitType = 's' | 'm' | 'h'; export const isValidUnit = (unitParam: string): unitParam is unitType => ['s', 'm', 'h'].includes(unitParam); -export interface Sequence { - buildingBlocks: BaseSignalHit[]; - sequenceSignal: BaseSignalHit; -} - export interface SignalsParams { signalIds: string[] | undefined | null; query: object | undefined | null; From 45ecb4292c3fedb3ffd01bb7880ffac1cd9e0dd1 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 24 Sep 2020 13:07:43 -0400 Subject: [PATCH 26/26] Unbork the merge conflict resolution --- x-pack/plugins/security_solution/server/lib/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index 8ee7f834b5145..117cffd844cfb 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -80,7 +80,7 @@ export interface SearchResponse { } export interface EqlSequence { - join_keys: []; + join_keys: SearchTypes[]; events: Array>; }