From 5b97d522b6cfbebddbdfd1e08ef3d390c3c9b45a Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 16 Feb 2021 18:25:31 -0600 Subject: [PATCH] [Security Solution][Detections] Adds Indicator path config for indicator match rules (#91260) * Add new field for overriding threat indicator path There is no UI for this currently, nor is it used during rule execution. * Adds form field for indicator path parameter Also adds missing plumbing that was preventing the new field from being persisted to the alert/returned in the response. * Wire up our indicator path config to enrichment * Add unit test for enriching from a custom indicator path We always persist to `threat.indicator.*` on the signal, but this allows users to specify where the enrichment fields can be found on the matched indicator document. * Wire up the missing piece of our indicator path config We were not passing this from the rule itself into the threat matching logic, and so were merely getting the default value. An integration test will fix this. Incoming! * Move indicator path defaulting outside of helper functions This happens closer to where we pass data from the rule to our helpers, and will prevent errors/bugs due to defaulting logic down the road. It makes tests a little more verbose, but that's okay. * Fix remaining type errors around new rule field * Make threat indicator path a conditional field Always sending along this field, but only allowing it for threat match rules was implicitly breaking the workflow of otther rule types. By making the field conditional on the rule type, this field only impacts threat match rules. This also fixes some types and tests accordingly. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../request/add_prepackaged_rules_schema.ts | 2 + .../schemas/request/import_rules_schema.ts | 2 + .../schemas/request/patch_rules_schema.ts | 2 + .../schemas/request/rule_schemas.mock.ts | 1 + .../schemas/request/rule_schemas.test.ts | 2 +- .../schemas/request/rule_schemas.ts | 2 + .../schemas/response/rules_schema.mocks.ts | 1 + .../schemas/response/rules_schema.test.ts | 4 +- .../schemas/response/rules_schema.ts | 5 + .../schemas/types/threat_mapping.ts | 5 + .../common/detection_engine/utils.ts | 3 +- .../rules/step_about_rule/default_value.ts | 1 + .../rules/step_about_rule/index.tsx | 18 ++++ .../rules/step_about_rule/schema.tsx | 17 ++++ .../detection_engine/rules/types.ts | 2 + .../rules/create/helpers.test.ts | 9 ++ .../detection_engine/rules/create/helpers.ts | 2 + .../detection_engine/rules/helpers.test.tsx | 1 + .../pages/detection_engine/rules/helpers.tsx | 2 + .../pages/detection_engine/rules/types.ts | 2 + .../routes/__mocks__/request_responses.ts | 1 + .../routes/rules/import_rules_route.ts | 2 + .../detection_engine/routes/rules/utils.ts | 1 + .../rules/create_rules.mock.ts | 2 + .../detection_engine/rules/create_rules.ts | 2 + .../rules/install_prepacked_rules.ts | 2 + .../rules/patch_rules.mock.ts | 1 + .../lib/detection_engine/rules/types.ts | 2 + .../schemas/rule_converters.ts | 1 + .../detection_engine/schemas/rule_schemas.ts | 2 + .../signals/__mocks__/es_results.ts | 1 + .../detection_engine/signals/build_rule.ts | 1 + .../signals/signal_params_schema.ts | 1 + .../signals/signal_rule_alert_type.ts | 2 + .../threat_mapping/build_threat_enrichment.ts | 6 +- .../threat_mapping/create_threat_signals.ts | 2 + .../enrich_signal_threat_matches.test.ts | 91 +++++++++++++++++-- .../enrich_signal_threat_matches.ts | 14 ++- .../signals/threat_mapping/types.ts | 3 + .../server/lib/detection_engine/types.ts | 2 + 40 files changed, 204 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index b76a762ca6cbf..981a5422a0594 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -55,6 +55,7 @@ import { threat_filters, threat_mapping, threat_language, + threat_indicator_path, } from '../types/threat_mapping'; import { @@ -133,6 +134,7 @@ export const addPrepackagedRulesSchema = t.intersection([ threat_query, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + threat_indicator_path, // defaults "undefined" if not set during decode concurrent_searches, // defaults to "undefined" if not set during decode items_per_search, // defaults to "undefined" if not set during decode }) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index 0a7b8b120ba7e..8fa5809abe68b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -62,6 +62,7 @@ import { threat_filters, threat_mapping, threat_language, + threat_indicator_path, } from '../types/threat_mapping'; import { @@ -152,6 +153,7 @@ export const importRulesSchema = t.intersection([ threat_query, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + threat_indicator_path, // defaults to "undefined" if not set during decode concurrent_searches, // defaults to "undefined" if not set during decode items_per_search, // defaults to "undefined" if not set during decode }) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts index 9d5331aeab8e4..920fbaf4915c5 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts @@ -57,6 +57,7 @@ import { threat_filters, threat_mapping, threat_language, + threat_indicator_path, } from '../types/threat_mapping'; import { listArrayOrUndefined } from '../types/lists'; @@ -112,6 +113,7 @@ export const patchRulesSchema = t.exact( threat_filters, threat_mapping, threat_language, + threat_indicator_path, concurrent_searches, items_per_search, }) diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts index 87e5acb5428df..fb29e37a53fdb 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts @@ -56,6 +56,7 @@ export const getCreateThreatMatchRulesSchemaMock = ( rule_id: ruleId, threat_query: '*:*', threat_index: ['list-index'], + threat_indicator_path: 'threat.indicator', threat_mapping: [ { entries: [ diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts index 14b47c8b2b328..6b8211b23088c 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts @@ -1152,7 +1152,7 @@ describe('create rules schema', () => { }); }); - describe('threat_mapping', () => { + describe('threat_match', () => { test('You can set a threat query, index, mapping, filters when creating a rule', () => { const payload = getCreateThreatMatchRulesSchemaMock(); const decoded = createRulesSchema.decode(payload); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts index 1c9ebe0033315..5cf2b6242b2f8 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts @@ -13,6 +13,7 @@ import { threat_query, threat_mapping, threat_index, + threat_indicator_path, concurrent_searches, items_per_search, } from '../types/threat_mapping'; @@ -213,6 +214,7 @@ const threatMatchRuleParams = { filters, saved_id, threat_filters, + threat_indicator_path, threat_language: t.keyof({ kuery: null, lucene: null }), concurrent_searches, items_per_search, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts index b14c646e862d3..cf07389e207b3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts @@ -150,6 +150,7 @@ export const getThreatMatchingSchemaPartialMock = (enabled = false): Partial { expect(fields).toEqual(expected); }); - test('should return 8 fields for a rule of type "threat_match"', () => { + test('should return nine (9) fields for a rule of type "threat_match"', () => { const fields = addThreatMatchFields({ type: 'threat_match' }); - expect(fields.length).toEqual(8); + expect(fields.length).toEqual(9); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index bcdb0fa9b085d..6bd54973e064f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -70,6 +70,7 @@ import { threat_filters, threat_mapping, threat_language, + threat_indicator_path, } from '../types/threat_mapping'; import { DefaultListArray } from '../types/lists_default_array'; @@ -151,6 +152,7 @@ export const dependentRulesSchema = t.partial({ items_per_search, threat_mapping, threat_language, + threat_indicator_path, }); /** @@ -286,6 +288,9 @@ export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.exact(t.type({ threat_mapping: dependentRulesSchema.props.threat_mapping })), t.exact(t.partial({ threat_language: dependentRulesSchema.props.threat_language })), t.exact(t.partial({ threat_filters: dependentRulesSchema.props.threat_filters })), + t.exact( + t.partial({ threat_indicator_path: dependentRulesSchema.props.threat_indicator_path }) + ), t.exact(t.partial({ saved_id: dependentRulesSchema.props.saved_id })), t.exact(t.partial({ concurrent_searches: dependentRulesSchema.props.concurrent_searches })), t.exact( diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts index d3975df488de9..aab06941686c2 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts @@ -18,6 +18,11 @@ export type ThreatQuery = t.TypeOf; export const threatQueryOrUndefined = t.union([threat_query, t.undefined]); export type ThreatQueryOrUndefined = t.TypeOf; +export const threat_indicator_path = t.string; +export type ThreatIndicatorPath = t.TypeOf; +export const threatIndicatorPathOrUndefined = t.union([threat_indicator_path, t.undefined]); +export type ThreatIndicatorPathOrUndefined = t.TypeOf; + export const threat_filters = t.array(t.unknown); // Filters are not easily type-able yet export type ThreatFilters = t.TypeOf; export const threatFiltersOrUndefined = t.union([threat_filters, t.undefined]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.ts index 080b704e9c193..725a2eb9fea7b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.ts @@ -30,4 +30,5 @@ export const isEqlRule = (ruleType: Type | undefined): boolean => ruleType === ' export const isThresholdRule = (ruleType: Type | undefined): boolean => ruleType === 'threshold'; export const isQueryRule = (ruleType: Type | undefined): boolean => ruleType === 'query' || ruleType === 'saved_query'; -export const isThreatMatchRule = (ruleType: Type): boolean => ruleType === 'threat_match'; +export const isThreatMatchRule = (ruleType: Type | undefined): boolean => + ruleType === 'threat_match'; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/default_value.ts b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/default_value.ts index 08feb5f2e5166..f73b2ccfb02ae 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/default_value.ts +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/default_value.ts @@ -29,6 +29,7 @@ export const stepAboutDefaultValue: AboutStepRule = { license: '', ruleNameOverride: '', tags: [], + threatIndicatorPath: '', timestampOverride: '', threat: threatDefault, note: '', diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx index 209071d27536d..25295a823ea66 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx @@ -40,6 +40,7 @@ import { SeverityField } from '../severity_mapping'; import { RiskScoreField } from '../risk_score_mapping'; import { AutocompleteField } from '../autocomplete_field'; import { useFetchIndex } from '../../../../common/containers/source'; +import { isThreatMatchRule } from '../../../../../common/detection_engine/utils'; const CommonUseField = getUseField({ component: Field }); @@ -298,6 +299,23 @@ const StepAboutRuleComponent: FC = ({ /> + {isThreatMatchRule(defineRuleData?.ruleType) && ( + <> + + + )} + = { ), labelAppend: OptionalFieldLabel, }, + threatIndicatorPath: { + type: FIELD_TYPES.TEXT, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathLabel', + { + defaultMessage: 'Threat Indicator Path', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathHelpText', + { + defaultMessage: + 'Specify the document path containing your threat indicator fields. Used for enrichment of indicator match alerts. Defaults to threat.indicator unless otherwise specified.', + } + ), + labelAppend: OptionalFieldLabel, + }, timestampOverride: { type: FIELD_TYPES.TEXT, label: i18n.translate( diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts index 591432829d90a..b703bc3aa8bcb 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts @@ -24,6 +24,7 @@ import { listArray, threat_query, threat_index, + threat_indicator_path, threat_mapping, threat_language, threat_filters, @@ -132,6 +133,7 @@ export const RuleSchema = t.intersection([ threat_query, threat_filters, threat_index, + threat_indicator_path, threat_mapping, threat_language, timeline_id: t.string, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts index 5e2aeb4ead934..fdb0513d7b708 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts @@ -493,6 +493,15 @@ describe('helpers', () => { expect(result.exceptions_list).toEqual([getListMock()]); }); + test('returns a threat indicator path', () => { + mockData = { + ...mockData, + threatIndicatorPath: 'my_custom.path', + }; + const result = formatAboutStepData(mockData); + expect(result.threat_indicator_path).toEqual('my_custom.path'); + }); + test('returns formatted object with both exceptions_lists', () => { const result = formatAboutStepData( { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index c09f85ce7edcc..7c447214cfdeb 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -288,6 +288,7 @@ export const formatAboutStepData = ( isBuildingBlock, note, ruleNameOverride, + threatIndicatorPath, timestampOverride, ...rest } = aboutStepData; @@ -330,6 +331,7 @@ export const formatAboutStepData = ( ...singleThreat, framework: 'MITRE ATT&CK', })), + threat_indicator_path: threatIndicatorPath, timestamp_override: timestampOverride !== '' ? timestampOverride : undefined, ...(!isEmpty(note) ? { note } : {}), ...rest, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index f0511602bd67f..111eb8a5594a8 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -116,6 +116,7 @@ describe('rule helpers', () => { severity: { value: 'low', mapping: fillEmptySeverityMappings([]), isMappingChecked: false }, tags: ['tag1', 'tag2'], threat: getThreatMock(), + threatIndicatorPath: '', timestampOverride: 'event.ingested', }; const scheduleRuleStepData = { from: '0s', interval: '5m' }; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 35f9f0c658a6a..d37c2d9141f5d 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -153,6 +153,7 @@ export const getAboutStepsData = (rule: Rule, detailsView: boolean): AboutStepRu risk_score: riskScore, tags, threat, + threat_indicator_path: threatIndicatorPath, } = rule; return { @@ -179,6 +180,7 @@ export const getAboutStepsData = (rule: Rule, detailsView: boolean): AboutStepRu }, falsePositives, threat: threat as Threats, + threatIndicatorPath: threatIndicatorPath ?? '', }; }; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index 218d8c0178a2b..94fdcc4069fc2 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -101,6 +101,7 @@ export interface AboutStepRule { ruleNameOverride: string; tags: string[]; timestampOverride: string; + threatIndicatorPath?: string; threat: Threats; note: string; } @@ -186,6 +187,7 @@ export interface AboutStepRuleJson { rule_name_override?: RuleNameOverride; tags: string[]; threat: Threats; + threat_indicator_path?: string; timestamp_override?: TimestampOverride; note?: string; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index dc6f7d35a6395..cf6ea572aa856 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -393,6 +393,7 @@ export const getResult = (): RuleAlertType => ({ threatMapping: undefined, threatLanguage: undefined, threatIndex: undefined, + threatIndicatorPath: undefined, threatQuery: undefined, references: ['http://www.example.com', 'https://ww.example.com'], note: '# Investigative notes', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index fb62a8bc6a14b..27231ab896b7e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -174,6 +174,7 @@ export const importRulesRoute = ( threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + threat_indicator_path: threatIndicatorPath, concurrent_searches: concurrentSearches, items_per_search: itemsPerSearch, threshold, @@ -239,6 +240,7 @@ export const importRulesRoute = ( threshold, threatFilters, threatIndex, + threatIndicatorPath, threatQuery, threatMapping, threatLanguage, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index defff7235dcba..45665c61ea3f0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -150,6 +150,7 @@ export const transformAlertToRule = ( threshold: alert.params.threshold, threat_filters: alert.params.threatFilters, threat_index: alert.params.threatIndex, + threat_indicator_path: alert.params.threatIndicatorPath, threat_query: alert.params.threatQuery, threat_mapping: alert.params.threatMapping, threat_language: alert.params.threatLanguage, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index e36b7b3079eb5..1232971c7baf8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -48,6 +48,7 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ itemsPerSearch: undefined, threatQuery: undefined, threatIndex: undefined, + threatIndicatorPath: undefined, threshold: undefined, timestampOverride: undefined, to: 'now', @@ -94,6 +95,7 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ threat: [], threatFilters: undefined, threatIndex: undefined, + threatIndicatorPath: undefined, threatMapping: undefined, threatQuery: undefined, threatLanguage: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 9726df176e93b..3683cd377e672 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -47,6 +47,7 @@ export const createRules = async ({ threat, threatFilters, threatIndex, + threatIndicatorPath, threatLanguage, concurrentSearches, itemsPerSearch, @@ -102,6 +103,7 @@ export const createRules = async ({ */ threatFilters: threatFilters as PartialFilter[] | undefined, threatIndex, + threatIndicatorPath, threatQuery, concurrentSearches, itemsPerSearch, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index cd1935ef50c10..0d046bb6ab211 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -56,6 +56,7 @@ export const installPrepackagedRules = ( items_per_search: itemsPerSearch, threat_query: threatQuery, threat_index: threatIndex, + threat_indicator_path: threatIndicatorPath, threshold, timestamp_override: timestampOverride, references, @@ -110,6 +111,7 @@ export const installPrepackagedRules = ( itemsPerSearch, threatQuery, threatIndex, + threatIndicatorPath, threshold, timestampOverride, references, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts index 07eb665c8cbd6..22c7dcc3a8616 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts @@ -119,6 +119,7 @@ const rule: SanitizedAlert = { threshold: undefined, threatFilters: undefined, threatIndex: undefined, + threatIndicatorPath: undefined, threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 72798c3533982..e8be32111a0e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -96,6 +96,7 @@ import { ThreatLanguageOrUndefined, ConcurrentSearchesOrUndefined, ItemsPerSearchOrUndefined, + ThreatIndicatorPathOrUndefined, } from '../../../../common/detection_engine/schemas/types/threat_mapping'; import { AlertsClient, PartialAlert } from '../../../../../alerts/server'; @@ -238,6 +239,7 @@ export interface CreateRulesOptions { threshold: ThresholdOrUndefined; threatFilters: ThreatFiltersOrUndefined; threatIndex: ThreatIndexOrUndefined; + threatIndicatorPath: ThreatIndicatorPathOrUndefined; threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; concurrentSearches: ConcurrentSearchesOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts index 691ac818100a2..e9a75af14310e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts @@ -53,6 +53,7 @@ export const typeSpecificSnakeToCamel = (params: CreateTypeSpecific): TypeSpecif threatMapping: params.threat_mapping, threatLanguage: params.threat_language, threatIndex: params.threat_index, + threatIndicatorPath: params.threat_indicator_path, concurrentSearches: params.concurrent_searches, itemsPerSearch: params.items_per_search, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts index f31d6af1a0d7a..abbcfcaa79107 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts @@ -14,6 +14,7 @@ import { threat_query, concurrentSearchesOrUndefined, itemsPerSearchOrUndefined, + threatIndicatorPathOrUndefined, } from '../../../../common/detection_engine/schemas/types/threat_mapping'; import { authorOrUndefined, @@ -116,6 +117,7 @@ const threatSpecificRuleParams = t.type({ threatMapping: threat_mapping, threatLanguage: t.union([nonEqlLanguages, t.undefined]), threatIndex: threat_index, + threatIndicatorPath: threatIndicatorPathOrUndefined, concurrentSearches: concurrentSearchesOrUndefined, itemsPerSearch: itemsPerSearchOrUndefined, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 6011c67376973..6177fc4cd4661 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -59,6 +59,7 @@ export const sampleRuleAlertParams = ( threatQuery: undefined, threatMapping: undefined, threatIndex: undefined, + threatIndicatorPath: undefined, threatLanguage: undefined, timelineId: undefined, timelineTitle: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index 5586d9e19f7c1..8f3fda800d726 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -170,6 +170,7 @@ export const buildRuleWithoutOverrides = ( threat_query: ruleParams.threatQuery, threat_mapping: ruleParams.threatMapping, threat_language: ruleParams.threatLanguage, + threat_indicator_path: ruleParams.threatIndicatorPath, }; return removeInternalTagsFromRule(rule); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index 2599f7db49f57..da7ee8796afbf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -52,6 +52,7 @@ export const signalSchema = schema.object({ exceptionsList: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threatFilters: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threatIndex: schema.maybe(schema.arrayOf(schema.string())), + threatIndicatorPath: schema.maybe(schema.string()), threatQuery: schema.maybe(schema.string()), threatMapping: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threatLanguage: schema.maybe(schema.string()), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index d7fce9d83a490..8e63633cd49fd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -137,6 +137,7 @@ export const signalRulesAlertType = ({ threatFilters, threatQuery, threatIndex, + threatIndicatorPath, threatMapping, threatLanguage, timestampOverride, @@ -508,6 +509,7 @@ export const signalRulesAlertType = ({ threatLanguage, buildRuleMessage, threatIndex, + threatIndicatorPath, concurrentSearches: concurrentSearches ?? 1, itemsPerSearch: itemsPerSearch ?? 9000, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts index b14d148218938..4f38f2db9230a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts @@ -10,6 +10,8 @@ import { enrichSignalThreatMatches } from './enrich_signal_threat_matches'; import { getThreatList } from './get_threat_list'; import { BuildThreatEnrichmentOptions, GetMatchedThreats } from './types'; +const DEFAULT_INDICATOR_PATH = 'threat.indicator'; + export const buildThreatEnrichment = ({ buildRuleMessage, exceptionItems, @@ -18,6 +20,7 @@ export const buildThreatEnrichment = ({ services, threatFilters, threatIndex, + threatIndicatorPath, threatLanguage, threatQuery, }: BuildThreatEnrichmentOptions): SignalsEnrichment => { @@ -50,6 +53,7 @@ export const buildThreatEnrichment = ({ return threatResponse.hits.hits; }; + const defaultedIndicatorPath = threatIndicatorPath ? threatIndicatorPath : DEFAULT_INDICATOR_PATH; return (signals: SignalSearchResponse): Promise => - enrichSignalThreatMatches(signals, getMatchedThreats); + enrichSignalThreatMatches(signals, getMatchedThreats, defaultedIndicatorPath); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 7690eb5eb1d55..e45aea29c423f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -48,6 +48,7 @@ export const createThreatSignals = async ({ threatLanguage, buildRuleMessage, threatIndex, + threatIndicatorPath, name, concurrentSearches, itemsPerSearch, @@ -99,6 +100,7 @@ export const createThreatSignals = async ({ services, threatFilters, threatIndex, + threatIndicatorPath, threatLanguage, threatQuery, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index 3c0765b56ae20..fada314116871 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -93,6 +93,7 @@ describe('buildMatchedIndicator', () => { const indicators = buildMatchedIndicator({ queries: [], threats, + indicatorPath: 'threat.indicator', }); expect(indicators).toEqual([]); @@ -102,6 +103,7 @@ describe('buildMatchedIndicator', () => { const [indicator] = buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }); expect(get(indicator, 'matched.atomic')).toEqual('domain_1'); @@ -111,6 +113,7 @@ describe('buildMatchedIndicator', () => { const [indicator] = buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }); expect(get(indicator, 'matched.field')).toEqual('event.field'); @@ -120,6 +123,7 @@ describe('buildMatchedIndicator', () => { const [indicator] = buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }); expect(get(indicator, 'matched.type')).toEqual('type_1'); @@ -148,6 +152,7 @@ describe('buildMatchedIndicator', () => { const indicators = buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }); expect(indicators).toHaveLength(queries.length); @@ -157,6 +162,7 @@ describe('buildMatchedIndicator', () => { const indicators = buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }); expect(indicators).toEqual([ @@ -192,9 +198,9 @@ describe('buildMatchedIndicator', () => { ]; const indicators = buildMatchedIndicator({ - indicatorPath: 'custom.indicator.path', queries, threats, + indicatorPath: 'custom.indicator.path', }); expect(indicators).toEqual([ @@ -221,6 +227,7 @@ describe('buildMatchedIndicator', () => { const indicators = buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }); expect(indicators).toEqual([ @@ -245,6 +252,7 @@ describe('buildMatchedIndicator', () => { const indicators = buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }); expect(indicators).toEqual([ @@ -276,6 +284,7 @@ describe('buildMatchedIndicator', () => { const indicators = buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }); expect(indicators).toEqual([ @@ -307,6 +316,7 @@ describe('buildMatchedIndicator', () => { buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }) ).toThrowError('Expected indicator field to be an object, but found: not an object'); }); @@ -327,6 +337,7 @@ describe('buildMatchedIndicator', () => { buildMatchedIndicator({ queries, threats, + indicatorPath: 'threat.indicator', }) ).toThrowError('Expected indicator field to be an object, but found: not an object'); }); @@ -352,7 +363,11 @@ describe('enrichSignalThreatMatches', () => { it('performs no enrichment if there are no signals', async () => { const signals = getSignalsResponseMock([]); - const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const enrichedSignals = await enrichSignalThreatMatches( + signals, + getMatchedThreats, + 'threat.indicator' + ); expect(enrichedSignals.hits.hits).toEqual([]); }); @@ -363,7 +378,11 @@ describe('enrichSignalThreatMatches', () => { matched_queries: [matchedQuery], }); const signals = getSignalsResponseMock([signalHit]); - const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const enrichedSignals = await enrichSignalThreatMatches( + signals, + getMatchedThreats, + 'threat.indicator' + ); const [enrichedHit] = enrichedSignals.hits.hits; const indicators = get(enrichedHit._source, 'threat.indicator'); @@ -384,7 +403,11 @@ describe('enrichSignalThreatMatches', () => { matched_queries: [matchedQuery], }); const signals = getSignalsResponseMock([signalHit]); - const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const enrichedSignals = await enrichSignalThreatMatches( + signals, + getMatchedThreats, + 'threat.indicator' + ); const [enrichedHit] = enrichedSignals.hits.hits; const indicators = get(enrichedHit._source, 'threat.indicator'); @@ -401,7 +424,11 @@ describe('enrichSignalThreatMatches', () => { matched_queries: [matchedQuery], }); const signals = getSignalsResponseMock([signalHit]); - const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const enrichedSignals = await enrichSignalThreatMatches( + signals, + getMatchedThreats, + 'threat.indicator' + ); const [enrichedHit] = enrichedSignals.hits.hits; const indicators = get(enrichedHit._source, 'threat.indicator'); @@ -422,9 +449,53 @@ describe('enrichSignalThreatMatches', () => { matched_queries: [matchedQuery], }); const signals = getSignalsResponseMock([signalHit]); - await expect(() => enrichSignalThreatMatches(signals, getMatchedThreats)).rejects.toThrowError( - 'Expected threat field to be an object, but found: whoops' + await expect(() => + enrichSignalThreatMatches(signals, getMatchedThreats, 'threat.indicator') + ).rejects.toThrowError('Expected threat field to be an object, but found: whoops'); + }); + + it('enriches from a configured indicator path, if specified', async () => { + getMatchedThreats = async () => [ + getThreatListItemMock({ + _id: '123', + _source: { + custom_threat: { + custom_indicator: { + domain: 'custom_domain', + other: 'custom_other', + type: 'custom_type', + }, + }, + }, + }), + ]; + matchedQuery = encodeThreatMatchNamedQuery( + getNamedQueryMock({ + id: '123', + field: 'event.field', + value: 'custom_threat.custom_indicator.domain', + }) + ); + const signalHit = getSignalHitMock({ + matched_queries: [matchedQuery], + }); + const signals = getSignalsResponseMock([signalHit]); + const enrichedSignals = await enrichSignalThreatMatches( + signals, + getMatchedThreats, + 'custom_threat.custom_indicator' ); + const [enrichedHit] = enrichedSignals.hits.hits; + const indicators = get(enrichedHit._source, 'threat.indicator'); + + expect(indicators).toEqual([ + { + domain: 'custom_domain', + matched: { atomic: 'custom_domain', field: 'event.field', type: 'custom_type' }, + other: 'custom_other', + type: 'custom_type', + }, + ]); }); it('merges duplicate matched signals into a single signal with multiple indicators', async () => { @@ -455,7 +526,11 @@ describe('enrichSignalThreatMatches', () => { ], }); const signals = getSignalsResponseMock([signalHit, otherSignalHit]); - const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const enrichedSignals = await enrichSignalThreatMatches( + signals, + getMatchedThreats, + 'threat.indicator' + ); expect(enrichedSignals.hits.total).toEqual(expect.objectContaining({ value: 1 })); expect(enrichedSignals.hits.hits).toHaveLength(1); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index c298ef98ebcd5..c5b032207f1c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -16,7 +16,6 @@ import type { } from './types'; import { extractNamedQueries } from './utils'; -const DEFAULT_INDICATOR_PATH = 'threat.indicator'; const getSignalId = (signal: SignalSourceHit): string => signal._id; export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): SignalSourceHit[] => { @@ -43,11 +42,11 @@ export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): Signa export const buildMatchedIndicator = ({ queries, threats, - indicatorPath = DEFAULT_INDICATOR_PATH, + indicatorPath, }: { queries: ThreatMatchNamedQuery[]; threats: ThreatListItem[]; - indicatorPath?: string; + indicatorPath: string; }): ThreatIndicator[] => queries.map((query) => { const matchedThreat = threats.find((threat) => threat._id === query.id); @@ -67,7 +66,8 @@ export const buildMatchedIndicator = ({ export const enrichSignalThreatMatches = async ( signals: SignalSearchResponse, - getMatchedThreats: GetMatchedThreats + getMatchedThreats: GetMatchedThreats, + indicatorPath: string ): Promise => { const signalHits = signals.hits.hits; if (signalHits.length === 0) { @@ -79,7 +79,11 @@ export const enrichSignalThreatMatches = async ( const matchedThreatIds = [...new Set(signalMatches.flat().map(({ id }) => id))]; const matchedThreats = await getMatchedThreats(matchedThreatIds); const matchedIndicators = signalMatches.map((queries) => - buildMatchedIndicator({ queries, threats: matchedThreats }) + buildMatchedIndicator({ + indicatorPath, + queries, + threats: matchedThreats, + }) ); const enrichedSignals: SignalSourceHit[] = uniqueHits.map((signalHit, i) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index b80d3faf9b61c..a022cbbdd4042 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -21,6 +21,7 @@ import { ThreatLanguageOrUndefined, ConcurrentSearches, ItemsPerSearch, + ThreatIndicatorPathOrUndefined, } from '../../../../../common/detection_engine/schemas/types/threat_mapping'; import { PartialFilter, RuleTypeParams } from '../../types'; import { @@ -70,6 +71,7 @@ export interface CreateThreatSignalsOptions { threatQuery: ThreatQuery; buildRuleMessage: BuildRuleMessage; threatIndex: ThreatIndex; + threatIndicatorPath: ThreatIndicatorPathOrUndefined; threatLanguage: ThreatLanguageOrUndefined; name: string; concurrentSearches: ConcurrentSearches; @@ -214,6 +216,7 @@ export interface BuildThreatEnrichmentOptions { services: AlertServices; threatFilters: PartialFilter[]; threatIndex: ThreatIndex; + threatIndicatorPath: ThreatIndicatorPathOrUndefined; threatLanguage: ThreatLanguageOrUndefined; threatQuery: ThreatQuery; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index 4b59fcddcb51f..a8721d82285f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -47,6 +47,7 @@ import { ThreatLanguageOrUndefined, ConcurrentSearchesOrUndefined, ItemsPerSearchOrUndefined, + ThreatIndicatorPathOrUndefined, } from '../../../common/detection_engine/schemas/types/threat_mapping'; import { LegacyCallAPIOptions } from '../../../../../../src/core/server'; @@ -88,6 +89,7 @@ export interface RuleTypeParams extends AlertTypeParams { threshold: ThresholdOrUndefined; threatFilters: PartialFilter[] | undefined; threatIndex: ThreatIndexOrUndefined; + threatIndicatorPath: ThreatIndicatorPathOrUndefined; threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; threatLanguage: ThreatLanguageOrUndefined;