[Security Solution][Detections] Adds Indicator path config for indica…

…tor match rules (#91260)

* Add new field for overriding threat indicator path

There is no UI for this currently, nor is it used during rule execution.

* Adds form field for indicator path parameter

Also adds missing plumbing that was preventing the new field from being
persisted to the alert/returned in the response.

* Wire up our indicator path config to enrichment

* Add unit test for enriching from a custom indicator path

We always persist to `threat.indicator.*` on the signal, but this allows
users to specify where the enrichment fields can be found on the matched
indicator document.

* Wire up the missing piece of our indicator path config

We were not passing this from the rule itself into the threat matching
logic, and so were merely getting the default value.

An integration test will fix this. Incoming!

* Move indicator path defaulting outside of helper functions

This happens closer to where we pass data from the rule to our helpers,
and will prevent errors/bugs due to defaulting logic down the road.

It makes tests a little more verbose, but that's okay.

* Fix remaining type errors around new rule field

* Make threat indicator path a conditional field

Always sending along this field, but only allowing it for threat match
rules was implicitly breaking the workflow of otther rule types. By
making the field conditional on the rule type, this field only impacts
threat match rules.

This also fixes some types and tests accordingly.

Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts

@@ -55,6 +55,7 @@ import {
   threat_filters,
   threat_mapping,
   threat_language,
+  threat_indicator_path,
 } from '../types/threat_mapping';
 
 import {
@@ -133,6 +134,7 @@ export const addPrepackagedRulesSchema = t.intersection([
       threat_query, // defaults to "undefined" if not set during decode
       threat_index, // defaults to "undefined" if not set during decode
       threat_language, // defaults "undefined" if not set during decode
+      threat_indicator_path, // defaults "undefined" if not set during decode
       concurrent_searches, // defaults to "undefined" if not set during decode
       items_per_search, // defaults to "undefined" if not set during decode
     })

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts

@@ -62,6 +62,7 @@ import {
   threat_filters,
   threat_mapping,
   threat_language,
+  threat_indicator_path,
 } from '../types/threat_mapping';
 
 import {
@@ -152,6 +153,7 @@ export const importRulesSchema = t.intersection([
       threat_query, // defaults to "undefined" if not set during decode
       threat_index, // defaults to "undefined" if not set during decode
       threat_language, // defaults "undefined" if not set during decode
+      threat_indicator_path, // defaults to "undefined" if not set during decode
       concurrent_searches, // defaults to "undefined" if not set during decode
       items_per_search, // defaults to "undefined" if not set during decode
     })

x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts

@@ -57,6 +57,7 @@ import {
   threat_filters,
   threat_mapping,
   threat_language,
+  threat_indicator_path,
 } from '../types/threat_mapping';
 import { listArrayOrUndefined } from '../types/lists';
 
@@ -112,6 +113,7 @@ export const patchRulesSchema = t.exact(
     threat_filters,
     threat_mapping,
     threat_language,
+    threat_indicator_path,
     concurrent_searches,
     items_per_search,
   })

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock.ts

@@ -56,6 +56,7 @@ export const getCreateThreatMatchRulesSchemaMock = (
   rule_id: ruleId,
   threat_query: '*:*',
   threat_index: ['list-index'],
+  threat_indicator_path: 'threat.indicator',
   threat_mapping: [
     {
       entries: [

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.test.ts

@@ -1152,7 +1152,7 @@ describe('create rules schema', () => {
     });
   });
 
-  describe('threat_mapping', () => {
+  describe('threat_match', () => {
     test('You can set a threat query, index, mapping, filters when creating a rule', () => {
       const payload = getCreateThreatMatchRulesSchemaMock();
       const decoded = createRulesSchema.decode(payload);

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts

@@ -13,6 +13,7 @@ import {
   threat_query,
   threat_mapping,
   threat_index,
+  threat_indicator_path,
   concurrent_searches,
   items_per_search,
 } from '../types/threat_mapping';
@@ -213,6 +214,7 @@ const threatMatchRuleParams = {
     filters,
     saved_id,
     threat_filters,
+    threat_indicator_path,
     threat_language: t.keyof({ kuery: null, lucene: null }),
     concurrent_searches,
     items_per_search,

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts

@@ -150,6 +150,7 @@ export const getThreatMatchingSchemaPartialMock = (enabled = false): Partial<Rul
     language: 'kuery',
     threat_query: '*:*',
     threat_index: ['list-index'],
+    threat_indicator_path: 'threat.indicator',
     threat_mapping: [
       {
         entries: [

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts

@@ -763,9 +763,9 @@ describe('rules_schema', () => {
       expect(fields).toEqual(expected);
     });
 
-    test('should return 8 fields for a rule of type "threat_match"', () => {
+    test('should return nine (9) fields for a rule of type "threat_match"', () => {
       const fields = addThreatMatchFields({ type: 'threat_match' });
-      expect(fields.length).toEqual(8);
+      expect(fields.length).toEqual(9);
     });
   });
 

x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts

@@ -70,6 +70,7 @@ import {
   threat_filters,
   threat_mapping,
   threat_language,
+  threat_indicator_path,
 } from '../types/threat_mapping';
 
 import { DefaultListArray } from '../types/lists_default_array';
@@ -151,6 +152,7 @@ export const dependentRulesSchema = t.partial({
   items_per_search,
   threat_mapping,
   threat_language,
+  threat_indicator_path,
 });
 
 /**
@@ -286,6 +288,9 @@ export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly):
       t.exact(t.type({ threat_mapping: dependentRulesSchema.props.threat_mapping })),
       t.exact(t.partial({ threat_language: dependentRulesSchema.props.threat_language })),
       t.exact(t.partial({ threat_filters: dependentRulesSchema.props.threat_filters })),
+      t.exact(
+        t.partial({ threat_indicator_path: dependentRulesSchema.props.threat_indicator_path })
+      ),
       t.exact(t.partial({ saved_id: dependentRulesSchema.props.saved_id })),
       t.exact(t.partial({ concurrent_searches: dependentRulesSchema.props.concurrent_searches })),
       t.exact(

x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts

@@ -18,6 +18,11 @@ export type ThreatQuery = t.TypeOf<typeof threat_query>;
 export const threatQueryOrUndefined = t.union([threat_query, t.undefined]);
 export type ThreatQueryOrUndefined = t.TypeOf<typeof threatQueryOrUndefined>;
 
+export const threat_indicator_path = t.string;
+export type ThreatIndicatorPath = t.TypeOf<typeof threat_indicator_path>;
+export const threatIndicatorPathOrUndefined = t.union([threat_indicator_path, t.undefined]);
+export type ThreatIndicatorPathOrUndefined = t.TypeOf<typeof threatIndicatorPathOrUndefined>;
+
 export const threat_filters = t.array(t.unknown); // Filters are not easily type-able yet
 export type ThreatFilters = t.TypeOf<typeof threat_filters>;
 export const threatFiltersOrUndefined = t.union([threat_filters, t.undefined]);

x-pack/plugins/security_solution/common/detection_engine/utils.ts

@@ -30,4 +30,5 @@ export const isEqlRule = (ruleType: Type | undefined): boolean => ruleType === '
 export const isThresholdRule = (ruleType: Type | undefined): boolean => ruleType === 'threshold';
 export const isQueryRule = (ruleType: Type | undefined): boolean =>
   ruleType === 'query' || ruleType === 'saved_query';
-export const isThreatMatchRule = (ruleType: Type): boolean => ruleType === 'threat_match';
+export const isThreatMatchRule = (ruleType: Type | undefined): boolean =>
+  ruleType === 'threat_match';

x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/default_value.ts

@@ -29,6 +29,7 @@ export const stepAboutDefaultValue: AboutStepRule = {
   license: '',
   ruleNameOverride: '',
   tags: [],
+  threatIndicatorPath: '',
   timestampOverride: '',
   threat: threatDefault,
   note: '',

x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx

@@ -40,6 +40,7 @@ import { SeverityField } from '../severity_mapping';
 import { RiskScoreField } from '../risk_score_mapping';
 import { AutocompleteField } from '../autocomplete_field';
 import { useFetchIndex } from '../../../../common/containers/source';
+import { isThreatMatchRule } from '../../../../../common/detection_engine/utils';
 
 const CommonUseField = getUseField({ component: Field });
 
@@ -298,6 +299,23 @@ const StepAboutRuleComponent: FC<StepAboutRuleProps> = ({
               />
             </EuiFormRow>
             <EuiSpacer size="l" />
+            {isThreatMatchRule(defineRuleData?.ruleType) && (
+              <>
+                <CommonUseField
+                  path="threatIndicatorPath"
+                  componentProps={{
+                    idAria: 'detectionEngineStepAboutThreatIndicatorPath',
+                    'data-test-subj': 'detectionEngineStepAboutThreatIndicatorPath',
+                    euiFieldProps: {
+                      fullWidth: true,
+                      disabled: isLoading,
+                      placeholder: 'threat.indicator',
+                    },
+                  }}
+                />
+              </>
+            )}
+            <EuiSpacer size="l" />
             <UseField
               path="ruleNameOverride"
               component={AutocompleteField}

x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx

@@ -193,6 +193,23 @@ export const schema: FormSchema<AboutStepRule> = {
     ),
     labelAppend: OptionalFieldLabel,
   },
+  threatIndicatorPath: {
+    type: FIELD_TYPES.TEXT,
+    label: i18n.translate(
+      'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathLabel',
+      {
+        defaultMessage: 'Threat Indicator Path',
+      }
+    ),
+    helpText: i18n.translate(
+      'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathHelpText',
+      {
+        defaultMessage:
+          'Specify the document path containing your threat indicator fields. Used for enrichment of indicator match alerts. Defaults to threat.indicator unless otherwise specified.',
+      }
+    ),
+    labelAppend: OptionalFieldLabel,
+  },
   timestampOverride: {
     type: FIELD_TYPES.TEXT,
     label: i18n.translate(

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts

@@ -24,6 +24,7 @@ import {
   listArray,
   threat_query,
   threat_index,
+  threat_indicator_path,
   threat_mapping,
   threat_language,
   threat_filters,
@@ -132,6 +133,7 @@ export const RuleSchema = t.intersection([
     threat_query,
     threat_filters,
     threat_index,
+    threat_indicator_path,
     threat_mapping,
     threat_language,
     timeline_id: t.string,

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts

@@ -493,6 +493,15 @@ describe('helpers', () => {
       expect(result.exceptions_list).toEqual([getListMock()]);
     });
 
+    test('returns a threat indicator path', () => {
+      mockData = {
+        ...mockData,
+        threatIndicatorPath: 'my_custom.path',
+      };
+      const result = formatAboutStepData(mockData);
+      expect(result.threat_indicator_path).toEqual('my_custom.path');
+    });
+
     test('returns formatted object with both exceptions_lists', () => {
       const result = formatAboutStepData(
         {

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts

@@ -288,6 +288,7 @@ export const formatAboutStepData = (
     isBuildingBlock,
     note,
     ruleNameOverride,
+    threatIndicatorPath,
     timestampOverride,
     ...rest
   } = aboutStepData;
@@ -330,6 +331,7 @@ export const formatAboutStepData = (
       ...singleThreat,
       framework: 'MITRE ATT&CK',
     })),
+    threat_indicator_path: threatIndicatorPath,
     timestamp_override: timestampOverride !== '' ? timestampOverride : undefined,
     ...(!isEmpty(note) ? { note } : {}),
     ...rest,

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx

@@ -116,6 +116,7 @@ describe('rule helpers', () => {
         severity: { value: 'low', mapping: fillEmptySeverityMappings([]), isMappingChecked: false },
         tags: ['tag1', 'tag2'],
         threat: getThreatMock(),
+        threatIndicatorPath: '',
         timestampOverride: 'event.ingested',
       };
       const scheduleRuleStepData = { from: '0s', interval: '5m' };

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx

@@ -153,6 +153,7 @@ export const getAboutStepsData = (rule: Rule, detailsView: boolean): AboutStepRu
     risk_score: riskScore,
     tags,
     threat,
+    threat_indicator_path: threatIndicatorPath,
   } = rule;
 
   return {
@@ -179,6 +180,7 @@ export const getAboutStepsData = (rule: Rule, detailsView: boolean): AboutStepRu
     },
     falsePositives,
     threat: threat as Threats,
+    threatIndicatorPath: threatIndicatorPath ?? '',
   };
 };
 

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts

@@ -101,6 +101,7 @@ export interface AboutStepRule {
   ruleNameOverride: string;
   tags: string[];
   timestampOverride: string;
+  threatIndicatorPath?: string;
   threat: Threats;
   note: string;
 }
@@ -186,6 +187,7 @@ export interface AboutStepRuleJson {
   rule_name_override?: RuleNameOverride;
   tags: string[];
   threat: Threats;
+  threat_indicator_path?: string;
   timestamp_override?: TimestampOverride;
   note?: string;
 }

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts

@@ -393,6 +393,7 @@ export const getResult = (): RuleAlertType => ({
     threatMapping: undefined,
     threatLanguage: undefined,
     threatIndex: undefined,
+    threatIndicatorPath: undefined,
     threatQuery: undefined,
     references: ['http://www.example.com', 'https://ww.example.com'],
     note: '# Investigative notes',

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts

@@ -174,6 +174,7 @@ export const importRulesRoute = (
                   threat_query: threatQuery,
                   threat_mapping: threatMapping,
                   threat_language: threatLanguage,
+                  threat_indicator_path: threatIndicatorPath,
                   concurrent_searches: concurrentSearches,
                   items_per_search: itemsPerSearch,
                   threshold,
@@ -239,6 +240,7 @@ export const importRulesRoute = (
                       threshold,
                       threatFilters,
                       threatIndex,
+                      threatIndicatorPath,
                       threatQuery,
                       threatMapping,
                       threatLanguage,

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts

@@ -150,6 +150,7 @@ export const transformAlertToRule = (
     threshold: alert.params.threshold,
     threat_filters: alert.params.threatFilters,
     threat_index: alert.params.threatIndex,
+    threat_indicator_path: alert.params.threatIndicatorPath,
     threat_query: alert.params.threatQuery,
     threat_mapping: alert.params.threatMapping,
     threat_language: alert.params.threatLanguage,

x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts

@@ -48,6 +48,7 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({
   itemsPerSearch: undefined,
   threatQuery: undefined,
   threatIndex: undefined,
+  threatIndicatorPath: undefined,
   threshold: undefined,
   timestampOverride: undefined,
   to: 'now',
@@ -94,6 +95,7 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({
   threat: [],
   threatFilters: undefined,
   threatIndex: undefined,
+  threatIndicatorPath: undefined,
   threatMapping: undefined,
   threatQuery: undefined,
   threatLanguage: undefined,

x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts

@@ -47,6 +47,7 @@ export const createRules = async ({
   threat,
   threatFilters,
   threatIndex,
+  threatIndicatorPath,
   threatLanguage,
   concurrentSearches,
   itemsPerSearch,
@@ -102,6 +103,7 @@ export const createRules = async ({
          */
         threatFilters: threatFilters as PartialFilter[] | undefined,
         threatIndex,
+        threatIndicatorPath,
         threatQuery,
         concurrentSearches,
         itemsPerSearch,

x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts

@@ -56,6 +56,7 @@ export const installPrepackagedRules = (
       items_per_search: itemsPerSearch,
       threat_query: threatQuery,
       threat_index: threatIndex,
+      threat_indicator_path: threatIndicatorPath,
       threshold,
       timestamp_override: timestampOverride,
       references,
@@ -110,6 +111,7 @@ export const installPrepackagedRules = (
         itemsPerSearch,
         threatQuery,
         threatIndex,
+        threatIndicatorPath,
         threshold,
         timestampOverride,
         references,

x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts

@@ -119,6 +119,7 @@ const rule: SanitizedAlert<RuleTypeParams> = {
     threshold: undefined,
     threatFilters: undefined,
     threatIndex: undefined,
+    threatIndicatorPath: undefined,
     threatQuery: undefined,
     threatMapping: undefined,
     threatLanguage: undefined,