diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts index 53cdc7239f69d..b2e0461b0b9b8 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts @@ -26,10 +26,12 @@ export interface EventsActionGroupData { doc_count: number; } +export type Fields = Record; + export interface EventHit extends SearchHit { sort: string[]; _source: EventSource; - fields: Record; + fields: Fields; aggregations: { // eslint-disable-next-line @typescript-eslint/no-explicit-any [agg: string]: any; diff --git a/x-pack/plugins/security_solution/common/utils/mock_event_details.ts b/x-pack/plugins/security_solution/common/utils/mock_event_details.ts index 13b7fe7051246..7dc257ebb3fef 100644 --- a/x-pack/plugins/security_solution/common/utils/mock_event_details.ts +++ b/x-pack/plugins/security_solution/common/utils/mock_event_details.ts @@ -40,7 +40,7 @@ export const eventHit = { 'source.geo.location': [{ coordinates: [118.7778, 32.0617], type: 'Point' }], 'threat.indicator': [ { - 'matched.field': ['matched_field'], + 'matched.field': ['matched_field', 'other_matched_field'], first_seen: ['2021-02-22T17:29:25.195Z'], provider: ['yourself'], type: ['custom'], @@ -259,8 +259,8 @@ export const eventDetailsFormattedFields = [ { category: 'threat', field: 'threat.indicator.matched.field', - values: ['matched_field', 'matched_field_2'], - originalValue: ['matched_field', 'matched_field_2'], + values: ['matched_field', 'other_matched_field', 'matched_field_2'], + originalValue: ['matched_field', 'other_matched_field', 'matched_field_2'], isObjectArray: false, }, { diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts index 15d0e2d5494b8..29b0df9e4bbf7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts @@ -5,6 +5,15 @@ * 2.0. */ +export const TIMELINE_CTI_FIELDS = [ + 'threat.indicator.event.dataset', + 'threat.indicator.event.reference', + 'threat.indicator.matched.atomic', + 'threat.indicator.matched.field', + 'threat.indicator.matched.type', + 'threat.indicator.provider', +]; + export const TIMELINE_EVENTS_FIELDS = [ '@timestamp', 'signal.status', @@ -230,4 +239,5 @@ export const TIMELINE_EVENTS_FIELDS = [ 'zeek.ssl.established', 'zeek.ssl.resumed', 'zeek.ssl.version', + ...TIMELINE_CTI_FIELDS, ]; diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts index 405ddba137dae..da19df32ac87a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts @@ -5,10 +5,10 @@ * 2.0. */ +import { eventHit } from '../../../../../../common/utils/mock_event_details'; import { EventHit } from '../../../../../../common/search_strategy'; import { TIMELINE_EVENTS_FIELDS } from './constants'; -import { formatTimelineData } from './helpers'; -import { eventHit } from '../../../../../../common/utils/mock_event_details'; +import { buildObjectForFieldPath, formatTimelineData } from './helpers'; describe('#formatTimelineData', () => { it('happy path', async () => { @@ -42,12 +42,12 @@ describe('#formatTimelineData', () => { value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], }, { - field: 'source.geo.location', - value: [`{"lon":118.7778,"lat":32.0617}`], + field: 'threat.indicator.matched.field', + value: ['matched_field', 'other_matched_field', 'matched_field_2'], }, { - field: 'threat.indicator.matched.field', - value: ['matched_field', 'matched_field_2'], + field: 'source.geo.location', + value: [`{"lon":118.7778,"lat":32.0617}`], }, ], ecs: { @@ -94,6 +94,34 @@ describe('#formatTimelineData', () => { user: { name: ['jenkins'], }, + threat: { + indicator: [ + { + event: { + dataset: [], + reference: [], + }, + matched: { + atomic: ['matched_atomic'], + field: ['matched_field', 'other_matched_field'], + type: [], + }, + provider: ['yourself'], + }, + { + event: { + dataset: [], + reference: [], + }, + matched: { + atomic: ['matched_atomic_2'], + field: ['matched_field_2'], + type: [], + }, + provider: ['other_you'], + }, + ], + }, }, }, }); @@ -371,4 +399,173 @@ describe('#formatTimelineData', () => { }, }); }); + + describe('buildObjectForFieldPath', () => { + it('builds an object from a single non-nested field', () => { + expect(buildObjectForFieldPath('@timestamp', eventHit)).toEqual({ + '@timestamp': ['2020-11-17T14:48:08.922Z'], + }); + }); + + it('builds an object with no fields response', () => { + const { fields, ...fieldLessHit } = eventHit; + // @ts-expect-error fieldLessHit is intentionally missing fields + expect(buildObjectForFieldPath('@timestamp', fieldLessHit)).toEqual({ + '@timestamp': [], + }); + }); + + it('does not misinterpret non-nested fields with a common prefix', () => { + // @ts-expect-error hit is minimal + const hit: EventHit = { + fields: { + 'foo.bar': ['baz'], + 'foo.barBaz': ['foo'], + }, + }; + + expect(buildObjectForFieldPath('foo.barBaz', hit)).toEqual({ + foo: { barBaz: ['foo'] }, + }); + }); + + it('builds an array of objects from a nested field', () => { + // @ts-expect-error hit is minimal + const hit: EventHit = { + fields: { + foo: [{ bar: ['baz'] }], + }, + }; + expect(buildObjectForFieldPath('foo.bar', hit)).toEqual({ + foo: [{ bar: ['baz'] }], + }); + }); + + it('builds intermediate objects for nested fields', () => { + // @ts-expect-error nestedHit is minimal + const nestedHit: EventHit = { + fields: { + 'foo.bar': [ + { + baz: ['host.name'], + }, + ], + }, + }; + expect(buildObjectForFieldPath('foo.bar.baz', nestedHit)).toEqual({ + foo: { + bar: [ + { + baz: ['host.name'], + }, + ], + }, + }); + }); + + it('builds intermediate objects at multiple levels', () => { + expect(buildObjectForFieldPath('threat.indicator.matched.atomic', eventHit)).toEqual({ + threat: { + indicator: [ + { + matched: { + atomic: ['matched_atomic'], + }, + }, + { + matched: { + atomic: ['matched_atomic_2'], + }, + }, + ], + }, + }); + }); + + it('preserves multiple values for a single leaf', () => { + expect(buildObjectForFieldPath('threat.indicator.matched.field', eventHit)).toEqual({ + threat: { + indicator: [ + { + matched: { + field: ['matched_field', 'other_matched_field'], + }, + }, + { + matched: { + field: ['matched_field_2'], + }, + }, + ], + }, + }); + }); + + describe('multiple levels of nested fields', () => { + let nestedHit: EventHit; + + beforeEach(() => { + // @ts-expect-error nestedHit is minimal + nestedHit = { + fields: { + 'nested_1.foo': [ + { + 'nested_2.bar': [ + { leaf: ['leaf_value'], leaf_2: ['leaf_2_value'] }, + { leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] }, + ], + }, + { + 'nested_2.bar': [ + { leaf: ['leaf_value_2'], leaf_2: ['leaf_2_value_4'] }, + { leaf: ['leaf_value_3'], leaf_2: ['leaf_2_value_5'] }, + ], + }, + ], + }, + }; + }); + + it('includes objects without the field', () => { + expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf', nestedHit)).toEqual({ + nested_1: { + foo: [ + { + nested_2: { + bar: [{ leaf: ['leaf_value'] }, { leaf: [] }], + }, + }, + { + nested_2: { + bar: [{ leaf: ['leaf_value_2'] }, { leaf: ['leaf_value_3'] }], + }, + }, + ], + }, + }); + }); + + it('groups multiple leaf values', () => { + expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf_2', nestedHit)).toEqual({ + nested_1: { + foo: [ + { + nested_2: { + bar: [ + { leaf_2: ['leaf_2_value'] }, + { leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] }, + ], + }, + }, + { + nested_2: { + bar: [{ leaf_2: ['leaf_2_value_4'] }, { leaf_2: ['leaf_2_value_5'] }], + }, + }, + ], + }, + }); + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts index 2c18fb2840865..6c20843058ff1 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts @@ -5,9 +5,12 @@ * 2.0. */ +import { set } from '@elastic/safer-lodash-set'; import { get, has, merge, uniq } from 'lodash/fp'; +import { Ecs } from '../../../../../../common/ecs'; import { EventHit, + Fields, TimelineEdges, TimelineNonEcsData, } from '../../../../../../common/search_strategy'; @@ -78,18 +81,13 @@ const getValuesFromFields = async ( [fieldName]: get(fieldName, hit._source), }; } else { - if (nestedParentFieldName == null || nestedParentFieldName === fieldName) { + if (nestedParentFieldName == null) { fieldToEval = { [fieldName]: hit.fields[fieldName], }; - } else if (nestedParentFieldName != null) { - fieldToEval = { - [nestedParentFieldName]: hit.fields[nestedParentFieldName], - }; } else { - // fallback, should never hit fieldToEval = { - [fieldName]: [], + [nestedParentFieldName]: hit.fields[nestedParentFieldName], }; } } @@ -102,6 +100,37 @@ const getValuesFromFields = async ( ); }; +const buildObjectRecursive = (fieldPath: string, fields: Fields): Partial => { + const nestedParentPath = getNestedParentPath(fieldPath, fields); + if (!nestedParentPath) { + return set({}, fieldPath, toStringArray(get(fieldPath, fields))); + } + + const subPath = fieldPath.replace(`${nestedParentPath}.`, ''); + const subFields = (get(nestedParentPath, fields) ?? []) as Fields[]; + return set( + {}, + nestedParentPath, + subFields.map((subField) => buildObjectRecursive(subPath, subField)) + ); +}; + +export const buildObjectForFieldPath = (fieldPath: string, hit: EventHit): Partial => { + if (has(fieldPath, hit._source)) { + const value = get(fieldPath, hit._source); + return set({}, fieldPath, toStringArray(value)); + } + + return buildObjectRecursive(fieldPath, hit.fields); +}; + +/** + * If a prefix of our full field path is present as a field, we know that our field is nested + */ +const getNestedParentPath = (fieldPath: string, fields: Fields | undefined): string | undefined => + fields && + Object.keys(fields).find((field) => field !== fieldPath && fieldPath.startsWith(`${field}.`)); + const mergeTimelineFieldsWithHit = async ( fieldName: string, flattenedFields: T, @@ -109,15 +138,12 @@ const mergeTimelineFieldsWithHit = async ( dataFields: readonly string[], ecsFields: readonly string[] ) => { - if (fieldName != null || dataFields.includes(fieldName)) { - const fieldNameAsArray = fieldName.split('.'); - const nestedParentFieldName = Object.keys(hit.fields ?? []).find((f) => { - return f === fieldNameAsArray.slice(0, f.split('.').length).join('.'); - }); + if (fieldName != null) { + const nestedParentPath = getNestedParentPath(fieldName, hit.fields); if ( + nestedParentPath != null || has(fieldName, hit._source) || has(fieldName, hit.fields) || - nestedParentFieldName != null || specialFields.includes(fieldName) ) { const objectWithProperty = { @@ -126,22 +152,13 @@ const mergeTimelineFieldsWithHit = async ( data: dataFields.includes(fieldName) ? [ ...get('node.data', flattenedFields), - ...(await getValuesFromFields(fieldName, hit, nestedParentFieldName)), + ...(await getValuesFromFields(fieldName, hit, nestedParentPath)), ] : get('node.data', flattenedFields), ecs: ecsFields.includes(fieldName) ? { ...get('node.ecs', flattenedFields), - // @ts-expect-error - ...fieldName.split('.').reduceRight( - // @ts-expect-error - (obj, next) => ({ [next]: obj }), - toStringArray( - has(fieldName, hit._source) - ? get(fieldName, hit._source) - : hit.fields[fieldName] - ) - ), + ...buildObjectForFieldPath(fieldName, hit), } : get('node.ecs', flattenedFields), },