From c17044e6c5e443a772516700080d6b1c47c1d4f2 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 1 Mar 2023 06:30:29 -0500 Subject: [PATCH 1/3] [8.7] [Security Solution][Alerts] addresses IM performance PR feedback (#150677) (#152426) # Backport This will backport the following commits from `main` to `8.7`: - [[Security Solution][Alerts] addresses IM performance PR feedback (#150677)](https://github.com/elastic/kibana/pull/150677) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> --- .../threat_mapping/build_threat_enrichment.ts | 1 + .../threat_mapping/create_event_signal.ts | 1 + .../get_signals_map_from_threat_index.test.ts | 8 +++ .../get_signals_map_from_threat_index.ts | 54 +++++++++++-------- 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts index 18cf4240d0b18..35e839d5297d2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts @@ -61,6 +61,7 @@ export const buildThreatEnrichment = ({ const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ threatSearchParams, eventsCount: signals.length, + termsQueryAllowed: false, }); const enrichment = threatEnrichmentFactory({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts index c006ab528b3c5..0e16caeb5a36e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts @@ -86,6 +86,7 @@ export const createEventSignal = async ({ threatSearchParams, eventsCount: currentEventList.length, signalValueMap: getSignalValueMap({ eventList: currentEventList, threatMatchedFields }), + termsQueryAllowed: true, }); const ids = Array.from(signalsQueryMap.keys()); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.test.ts index 38a6947beebcb..4a48db4816b48 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.test.ts @@ -53,6 +53,7 @@ describe('getSignalsQueryMapFromThreatIndex', () => { await getSignalsQueryMapFromThreatIndex({ threatSearchParams: threatSearchParamsMock, eventsCount: 50, + termsQueryAllowed: false, }); expect(getThreatListMock).toHaveBeenCalledTimes(1); @@ -65,6 +66,7 @@ describe('getSignalsQueryMapFromThreatIndex', () => { const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ threatSearchParams: threatSearchParamsMock, eventsCount: 50, + termsQueryAllowed: false, }); expect(signalsQueryMap).toEqual(new Map()); @@ -98,6 +100,7 @@ describe('getSignalsQueryMapFromThreatIndex', () => { const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ threatSearchParams: threatSearchParamsMock, eventsCount: 50, + termsQueryAllowed: false, }); expect(signalsQueryMap).toEqual( @@ -153,6 +156,7 @@ describe('getSignalsQueryMapFromThreatIndex', () => { const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ threatSearchParams: threatSearchParamsMock, eventsCount: 50, + termsQueryAllowed: false, }); expect(signalsQueryMap.get('source-1')).toHaveLength(MAX_NUMBER_OF_SIGNAL_MATCHES); @@ -168,6 +172,7 @@ describe('getSignalsQueryMapFromThreatIndex', () => { const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ threatSearchParams: threatSearchParamsMock, eventsCount: 50, + termsQueryAllowed: false, }); expect(signalsQueryMap).toEqual(new Map()); @@ -201,6 +206,7 @@ describe('getSignalsQueryMapFromThreatIndex', () => { threatSearchParams: threatSearchParamsMock, eventsCount: 50, signalValueMap, + termsQueryAllowed: true, }); expect(signalsQueryMap).toEqual(new Map()); @@ -234,6 +240,7 @@ describe('getSignalsQueryMapFromThreatIndex', () => { threatSearchParams: threatSearchParamsMock, eventsCount: 50, signalValueMap, + termsQueryAllowed: true, }); const queries = [ @@ -283,6 +290,7 @@ describe('getSignalsQueryMapFromThreatIndex', () => { threatSearchParams: threatSearchParamsMock, eventsCount: 50, signalValueMap, + termsQueryAllowed: true, }); const queries = [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.ts index 0deb3beeee2e8..7d0f49b548f37 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.ts @@ -21,20 +21,34 @@ import { MAX_NUMBER_OF_SIGNAL_MATCHES } from './enrich_signal_threat_matches'; export type SignalsQueryMap = Map; -interface GetSignalsMatchesFromThreatIndexOptions { +interface GetSignalsQueryMapFromThreatIndexOptionsTerms { threatSearchParams: Omit; eventsCount: number; - signalValueMap?: SignalValuesMap; + signalValueMap: SignalValuesMap; + termsQueryAllowed: true; +} + +interface GetSignalsQueryMapFromThreatIndexOptionsMatch { + threatSearchParams: Omit; + eventsCount: number; + termsQueryAllowed: false; } /** * fetches threats and creates signals map from results, that matches signal is with list of threat queries */ -export const getSignalsQueryMapFromThreatIndex = async ({ - threatSearchParams, - eventsCount, - signalValueMap, -}: GetSignalsMatchesFromThreatIndexOptions): Promise => { +/** + * fetches threats and creates signals map from results, that matches signal is with list of threat queries + * @param options.termsQueryAllowed - if terms query allowed to be executed, then signalValueMap should be provided + * @param options.signalValueMap - map of signal values from terms query results + */ +export async function getSignalsQueryMapFromThreatIndex( + options: + | GetSignalsQueryMapFromThreatIndexOptionsTerms + | GetSignalsQueryMapFromThreatIndexOptionsMatch +): Promise { + const { threatSearchParams, eventsCount, termsQueryAllowed } = options; + let threatList: Awaited> | undefined; const signalsQueryMap = new Map(); // number of threat matches per signal is limited by MAX_NUMBER_OF_SIGNAL_MATCHES. Once it hits this number, threats stop to be processed for a signal @@ -50,9 +64,6 @@ export const getSignalsQueryMapFromThreatIndex = async ({ decodedQuery: ThreatMatchNamedQuery | ThreatTermNamedQuery; }) => { const signalMatch = signalsQueryMap.get(signalId); - if (!signalMatch) { - signalsQueryMap.set(signalId, []); - } const threatQuery = { id: threatHit._id, @@ -74,15 +85,9 @@ export const getSignalsQueryMapFromThreatIndex = async ({ } }; - while ( - maxThreatsReachedMap.size < eventsCount && - (threatList ? threatList?.hits.hits.length > 0 : true) - ) { - threatList = await getThreatList({ - ...threatSearchParams, - searchAfter: threatList?.hits.hits[threatList.hits.hits.length - 1].sort || undefined, - }); + threatList = await getThreatList({ ...threatSearchParams, searchAfter: undefined }); + while (maxThreatsReachedMap.size < eventsCount && threatList?.hits.hits.length > 0) { threatList.hits.hits.forEach((threatHit) => { const matchedQueries = threatHit?.matched_queries || []; @@ -90,13 +95,13 @@ export const getSignalsQueryMapFromThreatIndex = async ({ const decodedQuery = decodeThreatMatchNamedQuery(matchedQuery); const signalId = decodedQuery.id; - if (decodedQuery.queryType === ThreatMatchQueryType.term) { + if (decodedQuery.queryType === ThreatMatchQueryType.term && termsQueryAllowed) { const threatValue = get(threatHit?._source, decodedQuery.value); const values = Array.isArray(threatValue) ? threatValue : [threatValue]; values.forEach((value) => { - if (value && signalValueMap) { - const ids = signalValueMap[decodedQuery.field][value?.toString()]; + if (value && options.signalValueMap) { + const ids = options.signalValueMap[decodedQuery.field][value?.toString()]; ids?.forEach((id: string) => { addSignalValueToMap({ @@ -120,7 +125,12 @@ export const getSignalsQueryMapFromThreatIndex = async ({ } }); }); + + threatList = await getThreatList({ + ...threatSearchParams, + searchAfter: threatList.hits.hits[threatList.hits.hits.length - 1].sort, + }); } return signalsQueryMap; -}; +} From de22cd9361a0dbf429f9648d3c7b7c45aa862e90 Mon Sep 17 00:00:00 2001 From: Miriam <31922082+MiriamAparicio@users.noreply.github.com> Date: Wed, 1 Mar 2023 11:55:27 +0000 Subject: [PATCH 2/3] [8.7] [APM] Manage state for enviroments toggle in preview alerts chart (#150918) (#152436) # Backport This will backport the following commits from `main` to `8.7`: - [[APM] Manage state for enviroments toggle in preview alerts chart (#150918)](https://github.com/elastic/kibana/pull/150918) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --- .../alerting/ui_components/chart_preview/index.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/apm/public/components/alerting/ui_components/chart_preview/index.tsx b/x-pack/plugins/apm/public/components/alerting/ui_components/chart_preview/index.tsx index 21cee514fc363..0e0b09109387c 100644 --- a/x-pack/plugins/apm/public/components/alerting/ui_components/chart_preview/index.tsx +++ b/x-pack/plugins/apm/public/components/alerting/ui_components/chart_preview/index.tsx @@ -20,7 +20,7 @@ import { TickFormatter, } from '@elastic/charts'; import { EuiSpacer } from '@elastic/eui'; -import React from 'react'; +import React, { useState } from 'react'; import { IUiSettingsClient } from '@kbn/core/public'; import { Coordinate } from '../../../../../typings/timeseries'; import { useTheme } from '../../../../hooks/use_theme'; @@ -39,6 +39,8 @@ export function ChartPreview({ uiSettings, series, }: ChartPreviewProps) { + const [yMax, setYMax] = useState(threshold); + const theme = useTheme(); const thresholdOpacity = 0.3; const timestamps = series.flatMap(({ data }) => data.map(({ x }) => x)); @@ -46,6 +48,12 @@ export function ChartPreview({ const xMax = Math.max(...timestamps); const xFormatter = niceTimeFormatter([xMin, xMax]); + function updateYMax() { + // Make the maximum Y value either the actual max or 20% more than the threshold + const values = series.flatMap(({ data }) => data.map((d) => d.y ?? 0)); + setYMax(Math.max(...values, threshold * 1.2)); + } + const style = { fill: theme.eui.euiColorVis2, line: { @@ -85,6 +93,7 @@ export function ChartPreview({ showLegend={series.length > 1} legendPosition={'bottom'} legendSize={legendSize} + onLegendItemClick={updateYMax} /> {series.map(({ name, data }, index) => ( Date: Wed, 1 Mar 2023 13:43:24 +0000 Subject: [PATCH 3/3] [8.7] [Security Solution][Alerts] fixes merge fields with source in Detection Engine on Alerts creation (#151004) (#152449) # Backport This will backport the following commits from `main` to `8.7`: - [[Security Solution][Alerts] fixes merge fields with source in Detection Engine on Alerts creation (#151004)](https://github.com/elastic/kibana/pull/151004) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --- .../merge_all_fields_with_source.test.ts | 55 ++++++++++++++ .../merge_all_fields_with_source.ts | 4 +- .../merge_missing_fields_with_source.test.ts | 62 ++++++++++++++++ .../merge_missing_fields_with_source.ts | 4 +- .../utils/is_path_valid.test.ts | 74 +++++++++++++++++++ .../utils/is_path_valid.ts | 30 ++++++++ .../rule_execution_logic/non_ecs_fields.ts | 58 +++++++++++++++ .../ecs_non_compliant/mappings.json | 18 +++++ 8 files changed, 301 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts index 74445d4be790a..25583ae8aca05 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts @@ -1327,6 +1327,35 @@ describe('merge_all_fields_with_source', () => { }); }); + test('does not add multi field values such as "process.command_line.text" to nested source when "process.command_line" has value', () => { + const _source: SignalSourceHit['_source'] = { + process: { + command_line: 'string longer than 10 characters', + }, + }; + const fields: SignalSourceHit['fields'] = { + 'process.command_line.text': ['string longer than 10 characters'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + expect(merged).toEqual(_source); + }); + + test('does not add multi field values such as "process.command_line.text" to nested source when "process.command_line" has array value', () => { + const _source: SignalSourceHit['_source'] = { + process: { + command_line: ['string longer than 10 characters'], + }, + }; + const fields: SignalSourceHit['fields'] = { + 'process.command_line.text': ['string longer than 10 characters'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + + expect(merged).toEqual(_source); + }); + test('multi-field values mixed with regular values will not be merged accidentally"', () => { const _source: SignalSourceHit['_source'] = {}; const fields: SignalSourceHit['fields'] = { @@ -1393,6 +1422,32 @@ describe('merge_all_fields_with_source', () => { foo: 'other_value_1', }); }); + + test('does not add multi field values such as "process.command_line.text" to flattened source when "process.command_line" has value', () => { + const _source: SignalSourceHit['_source'] = { + 'process.command_line': 'string longer than 10 characters', + }; + + const fields: SignalSourceHit['fields'] = { + 'process.command_line.text': ['string longer than 10 characters'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + expect(merged).toEqual(_source); + }); + + test('does not add multi field values such as "process.command_line.text" to flattened source when "process.command_line" has array value', () => { + const _source: SignalSourceHit['_source'] = { + 'process.command_line': ['string longer than 10 characters'], + }; + + const fields: SignalSourceHit['fields'] = { + 'process.command_line.text': ['string longer than 10 characters'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + expect(merged).toEqual(_source); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts index e3c7f8f5ee50e..bd88fb6be38e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts @@ -15,8 +15,8 @@ import { isNestedObject } from '../utils/is_nested_object'; import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields'; import { isPrimitive } from '../utils/is_primitive'; import { isArrayOfPrimitives } from '../utils/is_array_of_primitives'; -import { arrayInPathExists } from '../utils/array_in_path_exists'; import { isTypeObject } from '../utils/is_type_object'; +import { isPathValid } from '../utils/is_path_valid'; /** * Merges all of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information @@ -107,7 +107,7 @@ const hasEarlyReturnConditions = ({ const valueInMergedDocument = get(fieldsKey, merged); return ( fieldsValue.length === 0 || - (valueInMergedDocument === undefined && arrayInPathExists(fieldsKey, merged)) || + (valueInMergedDocument === undefined && !isPathValid(fieldsKey, merged)) || (isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && !isNestedObject(fieldsValue) && !isTypeObject(fieldsValue)) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts index f5863533ea283..8b794f5304cf8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts @@ -1283,6 +1283,38 @@ describe('merge_missing_fields_with_source', () => { foo: 'other_value_1', }); }); + + test('does not add multi field values such as "process.command_line.text" to nested source when "process.command_line" has value', () => { + const _source: SignalSourceHit['_source'] = { + '@timestamp': '2023-02-10T10:15:50Z', + process: { + command_line: 'string longer than 10 characters', + }, + }; + const fields: SignalSourceHit['fields'] = { + 'process.command_line.text': ['string longer than 10 characters'], + '@timestamp': ['2023-02-10T10:15:50.000Z'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + expect(merged).toEqual(_source); + }); + + test('does not add multi field values such as "process.command_line.text" to nested source when "process.command_line" has array value', () => { + const _source: SignalSourceHit['_source'] = { + '@timestamp': '2023-02-10T10:15:50Z', + process: { + command_line: ['string longer than 10 characters'], + }, + }; + const fields: SignalSourceHit['fields'] = { + 'process.command_line.text': ['string longer than 10 characters'], + '@timestamp': ['2023-02-10T10:15:50.000Z'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + expect(merged).toEqual(_source); + }); }); describe('flattened keys for the _source', () => { @@ -1331,6 +1363,36 @@ describe('merge_missing_fields_with_source', () => { foo: 'other_value_1', }); }); + + test('does not add multi field values such as "process.command_line.text" to flattened source when "process.command_line" has value', () => { + const _source: SignalSourceHit['_source'] = { + '@timestamp': '2023-02-10T10:15:50Z', + 'process.command_line': 'string longer than 10 characters', + }; + + const fields: SignalSourceHit['fields'] = { + 'process.command_line.text': ['string longer than 10 characters'], + '@timestamp': ['2023-02-10T10:15:50.000Z'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + expect(merged).toEqual(_source); + }); + + test('does not add multi field values such as "process.command_line.text" to flattened source when "process.command_line" has array value', () => { + const _source: SignalSourceHit['_source'] = { + '@timestamp': '2023-02-10T10:15:50Z', + 'process.command_line': ['string longer than 10 characters'], + }; + + const fields: SignalSourceHit['fields'] = { + 'process.command_line.text': ['string longer than 10 characters'], + '@timestamp': ['2023-02-10T10:15:50.000Z'], + }; + const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields }; + const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + expect(merged).toEqual(_source); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts index c20f6b55301bd..997fbc920431c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts @@ -12,8 +12,8 @@ import { filterFieldEntries } from '../utils/filter_field_entries'; import type { FieldsType, MergeStrategyFunction } from '../types'; import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields'; import { isTypeObject } from '../utils/is_type_object'; -import { arrayInPathExists } from '../utils/array_in_path_exists'; import { isNestedObject } from '../utils/is_nested_object'; +import { isPathValid } from '../utils/is_path_valid'; /** * Merges only missing sections of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information @@ -79,7 +79,7 @@ const hasEarlyReturnConditions = ({ return ( fieldsValue.length === 0 || valueInMergedDocument !== undefined || - arrayInPathExists(fieldsKey, merged) || + !isPathValid(fieldsKey, merged) || isNestedObject(fieldsValue) || isTypeObject(fieldsValue) ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.test.ts new file mode 100644 index 0000000000000..e899142bb7352 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isPathValid } from './is_path_valid'; + +describe('isPathValid', () => { + test('not valid when empty string and empty object', () => { + expect(isPathValid('', {})).toEqual(false); + }); + + test('valid when a path and empty object', () => { + expect(isPathValid('a.b.c', {})).toEqual(true); + }); + + test('not valid when a path and an array exists', () => { + expect(isPathValid('a', { a: [] })).toEqual(false); + }); + + test('not valid when a path and primitive value exists', () => { + expect(isPathValid('a', { a: 'test' })).toEqual(false); + expect(isPathValid('a', { a: 1 })).toEqual(false); + expect(isPathValid('a', { a: true })).toEqual(false); + }); + + test('valid when a path and object value exists', () => { + expect(isPathValid('a', { a: {} })).toEqual(true); + }); + + test('not valid when a path and an array exists within the parent path at level 1', () => { + expect(isPathValid('a.b', { a: [] })).toEqual(false); + }); + + test('not valid when a path and primitive value exists within the parent path at level 1', () => { + expect(isPathValid('a.b', { a: 'test' })).toEqual(false); + expect(isPathValid('a.b', { a: 1 })).toEqual(false); + expect(isPathValid('a.b', { a: true })).toEqual(false); + }); + + test('valid when a path and object value exists within the parent path at level 1', () => { + expect(isPathValid('a.b', { a: {} })).toEqual(true); + }); + + test('not valid when a path and an array exists within the parent path at level 2', () => { + expect(isPathValid('a.b.c', { a: { b: [] } })).toEqual(false); + }); + + test('not valid when a path and primitive value exists within the parent path at level 2', () => { + expect(isPathValid('a.b', { a: { b: 'test' } })).toEqual(false); + expect(isPathValid('a.b', { a: { b: 1 } })).toEqual(false); + expect(isPathValid('a.b', { a: { b: true } })).toEqual(false); + }); + + test('valid when a path and object value exists within the parent path at level 2', () => { + expect(isPathValid('a.b', { a: { b: {} } })).toEqual(true); + }); + + test('not valid when a path and an array exists within the parent path at level 3', () => { + expect(isPathValid('a.b.c', { a: { b: { c: [] } } })).toEqual(false); + }); + + test('not valid when a path and primitive value exists within the parent path at level 3', () => { + expect(isPathValid('a.b.c', { a: { b: { c: 'test' } } })).toEqual(false); + expect(isPathValid('a.b.c', { a: { b: { c: 1 } } })).toEqual(false); + expect(isPathValid('a.b.c', { a: { b: { c: true } } })).toEqual(false); + }); + + test('valid when a path and object value exists within the parent path at level 3', () => { + expect(isPathValid('a.b.c', { a: { b: { c: {} } } })).toEqual(true); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.ts new file mode 100644 index 0000000000000..a54094100ed18 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get, isPlainObject } from 'lodash/fp'; +import type { SignalSource } from '../../types'; + +/** + * Returns true if path in SignalSource object is valid + * Path is valid if each field in hierarchy is object or undefined + * Path is not valid if ANY of field in hierarchy is not object or undefined + * @param path in source to check within source + * @param source The source document + * @returns boolean + */ +export const isPathValid = (path: string, source: SignalSource): boolean => { + if (!path) { + return false; + } + const splitPath = path.split('.'); + + return splitPath.every((_, index, array) => { + const newPath = [...array].splice(0, index + 1).join('.'); + const valueToCheck = get(newPath, source); + return valueToCheck === undefined || isPlainObject(valueToCheck); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts index 3c5368b7a23ad..0bcb05b5e1c40 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts @@ -323,5 +323,63 @@ export default ({ getService }: FtrProviderContext) => { // invalid ECS field is getting removed expect(alertSource).not.toHaveProperty('dll.code_signature.valid'); }); + + describe('multi-fields', () => { + it('should not add multi field .text to ecs compliant nested source', async () => { + const document = { + process: { + command_line: 'string longer than 10 characters', + }, + }; + + const { errors, alertSource } = await indexAndCreatePreviewAlert(document); + + expect(errors).toEqual([]); + + expect(alertSource).toHaveProperty('process', document.process); + expect(alertSource).not.toHaveProperty('process.command_line.text'); + }); + + it('should not add multi field .text to ecs compliant flattened source', async () => { + const document = { + 'process.command_line': 'string longer than 10 characters', + }; + + const { errors, alertSource } = await indexAndCreatePreviewAlert(document); + + expect(errors).toEqual([]); + + expect(alertSource?.['process.command_line']).toEqual(document['process.command_line']); + expect(alertSource).not.toHaveProperty('process.command_line.text'); + }); + + it('should not add multi field .text to ecs non compliant nested source', async () => { + const document = { + nonEcs: { + command_line: 'string longer than 10 characters', + }, + }; + + const { errors, alertSource } = await indexAndCreatePreviewAlert(document); + + expect(errors).toEqual([]); + + expect(alertSource).toHaveProperty('nonEcs', document.nonEcs); + expect(alertSource).not.toHaveProperty('nonEcs.command_line.text'); + }); + + it('should not add multi field .text to ecs non compliant flattened source', async () => { + const document = { + 'nonEcs.command_line': 'string longer than 10 characters', + }; + + const { errors, alertSource } = await indexAndCreatePreviewAlert(document); + + expect(errors).toEqual([]); + + expect(alertSource?.['nonEcs.command_line']).toEqual(document['nonEcs.command_line']); + expect(alertSource).not.toHaveProperty('nonEcs.command_line.text'); + }); + }); }); }; diff --git a/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json b/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json index 42d23e794ba23..40408d65b6d89 100644 --- a/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json @@ -55,6 +55,24 @@ } } } + }, + "process.command_line": { + "type": "keyword", + "ignore_above": 10, + "fields": { + "text": { + "type": "text" + } + } + }, + "nonEcs.command_line": { + "type": "keyword", + "ignore_above": 10, + "fields": { + "text": { + "type": "text" + } + } } } },