Merge branch '8.7' into backport/8.7/pr-152075

x-pack/plugins/apm/public/components/alerting/ui_components/chart_preview/index.tsx

@@ -20,7 +20,7 @@ import {
   TickFormatter,
 } from '@elastic/charts';
 import { EuiSpacer } from '@elastic/eui';
-import React from 'react';
+import React, { useState } from 'react';
 import { IUiSettingsClient } from '@kbn/core/public';
 import { Coordinate } from '../../../../../typings/timeseries';
 import { useTheme } from '../../../../hooks/use_theme';
@@ -39,13 +39,21 @@ export function ChartPreview({
   uiSettings,
   series,
 }: ChartPreviewProps) {
+  const [yMax, setYMax] = useState(threshold);
+
   const theme = useTheme();
   const thresholdOpacity = 0.3;
   const timestamps = series.flatMap(({ data }) => data.map(({ x }) => x));
   const xMin = Math.min(...timestamps);
   const xMax = Math.max(...timestamps);
   const xFormatter = niceTimeFormatter([xMin, xMax]);
 
+  function updateYMax() {
+    // Make the maximum Y value either the actual max or 20% more than the threshold
+    const values = series.flatMap(({ data }) => data.map((d) => d.y ?? 0));
+    setYMax(Math.max(...values, threshold * 1.2));
+  }
+
   const style = {
     fill: theme.eui.euiColorVis2,
     line: {
@@ -85,6 +93,7 @@ export function ChartPreview({
           showLegend={series.length > 1}
           legendPosition={'bottom'}
           legendSize={legendSize}
+          onLegendItemClick={updateYMax}
         />
         <LineAnnotation
           dataValues={[{ dataValue: threshold }]}
@@ -110,6 +119,7 @@ export function ChartPreview({
           position={Position.Left}
           tickFormat={yTickFormat}
           ticks={5}
+          domain={{ max: yMax, min: 0 }}
         />
         {series.map(({ name, data }, index) => (
           <BarSeries

x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.test.ts

@@ -1327,6 +1327,35 @@ describe('merge_all_fields_with_source', () => {
         });
       });
 
+      test('does not add multi field values such as "process.command_line.text" to nested source when "process.command_line" has value', () => {
+        const _source: SignalSourceHit['_source'] = {
+          process: {
+            command_line: 'string longer than 10 characters',
+          },
+        };
+        const fields: SignalSourceHit['fields'] = {
+          'process.command_line.text': ['string longer than 10 characters'],
+        };
+        const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields };
+        const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source;
+        expect(merged).toEqual<ReturnTypeMergeFieldsWithSource>(_source);
+      });
+
+      test('does not add multi field values such as "process.command_line.text" to nested source when "process.command_line" has array value', () => {
+        const _source: SignalSourceHit['_source'] = {
+          process: {
+            command_line: ['string longer than 10 characters'],
+          },
+        };
+        const fields: SignalSourceHit['fields'] = {
+          'process.command_line.text': ['string longer than 10 characters'],
+        };
+        const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields };
+        const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source;
+
+        expect(merged).toEqual<ReturnTypeMergeFieldsWithSource>(_source);
+      });
+
       test('multi-field values mixed with regular values will not be merged accidentally"', () => {
         const _source: SignalSourceHit['_source'] = {};
         const fields: SignalSourceHit['fields'] = {
@@ -1393,6 +1422,32 @@ describe('merge_all_fields_with_source', () => {
           foo: 'other_value_1',
         });
       });
+
+      test('does not add multi field values such as "process.command_line.text" to flattened source when "process.command_line" has value', () => {
+        const _source: SignalSourceHit['_source'] = {
+          'process.command_line': 'string longer than 10 characters',
+        };
+
+        const fields: SignalSourceHit['fields'] = {
+          'process.command_line.text': ['string longer than 10 characters'],
+        };
+        const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields };
+        const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source;
+        expect(merged).toEqual<ReturnTypeMergeFieldsWithSource>(_source);
+      });
+
+      test('does not add multi field values such as "process.command_line.text" to flattened source when "process.command_line" has array value', () => {
+        const _source: SignalSourceHit['_source'] = {
+          'process.command_line': ['string longer than 10 characters'],
+        };
+
+        const fields: SignalSourceHit['fields'] = {
+          'process.command_line.text': ['string longer than 10 characters'],
+        };
+        const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields };
+        const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source;
+        expect(merged).toEqual<ReturnTypeMergeFieldsWithSource>(_source);
+      });
     });
   });
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_all_fields_with_source.ts

@@ -15,8 +15,8 @@ import { isNestedObject } from '../utils/is_nested_object';
 import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields';
 import { isPrimitive } from '../utils/is_primitive';
 import { isArrayOfPrimitives } from '../utils/is_array_of_primitives';
-import { arrayInPathExists } from '../utils/array_in_path_exists';
 import { isTypeObject } from '../utils/is_type_object';
+import { isPathValid } from '../utils/is_path_valid';
 
 /**
  * Merges all of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information
@@ -107,7 +107,7 @@ const hasEarlyReturnConditions = ({
   const valueInMergedDocument = get(fieldsKey, merged);
   return (
     fieldsValue.length === 0 ||
-    (valueInMergedDocument === undefined && arrayInPathExists(fieldsKey, merged)) ||
+    (valueInMergedDocument === undefined && !isPathValid(fieldsKey, merged)) ||
     (isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) &&
       !isNestedObject(fieldsValue) &&
       !isTypeObject(fieldsValue))

x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts

@@ -1283,6 +1283,38 @@ describe('merge_missing_fields_with_source', () => {
           foo: 'other_value_1',
         });
       });
+
+      test('does not add multi field values such as "process.command_line.text" to nested source when "process.command_line" has value', () => {
+        const _source: SignalSourceHit['_source'] = {
+          '@timestamp': '2023-02-10T10:15:50Z',
+          process: {
+            command_line: 'string longer than 10 characters',
+          },
+        };
+        const fields: SignalSourceHit['fields'] = {
+          'process.command_line.text': ['string longer than 10 characters'],
+          '@timestamp': ['2023-02-10T10:15:50.000Z'],
+        };
+        const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields };
+        const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source;
+        expect(merged).toEqual<ReturnTypeMergeFieldsWithSource>(_source);
+      });
+
+      test('does not add multi field values such as "process.command_line.text" to nested source when "process.command_line" has array value', () => {
+        const _source: SignalSourceHit['_source'] = {
+          '@timestamp': '2023-02-10T10:15:50Z',
+          process: {
+            command_line: ['string longer than 10 characters'],
+          },
+        };
+        const fields: SignalSourceHit['fields'] = {
+          'process.command_line.text': ['string longer than 10 characters'],
+          '@timestamp': ['2023-02-10T10:15:50.000Z'],
+        };
+        const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields };
+        const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source;
+        expect(merged).toEqual<ReturnTypeMergeFieldsWithSource>(_source);
+      });
     });
 
     describe('flattened keys for the _source', () => {
@@ -1331,6 +1363,36 @@ describe('merge_missing_fields_with_source', () => {
           foo: 'other_value_1',
         });
       });
+
+      test('does not add multi field values such as "process.command_line.text" to flattened source when "process.command_line" has value', () => {
+        const _source: SignalSourceHit['_source'] = {
+          '@timestamp': '2023-02-10T10:15:50Z',
+          'process.command_line': 'string longer than 10 characters',
+        };
+
+        const fields: SignalSourceHit['fields'] = {
+          'process.command_line.text': ['string longer than 10 characters'],
+          '@timestamp': ['2023-02-10T10:15:50.000Z'],
+        };
+        const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields };
+        const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source;
+        expect(merged).toEqual<ReturnTypeMergeFieldsWithSource>(_source);
+      });
+
+      test('does not add multi field values such as "process.command_line.text" to flattened source when "process.command_line" has array value', () => {
+        const _source: SignalSourceHit['_source'] = {
+          '@timestamp': '2023-02-10T10:15:50Z',
+          'process.command_line': ['string longer than 10 characters'],
+        };
+
+        const fields: SignalSourceHit['fields'] = {
+          'process.command_line.text': ['string longer than 10 characters'],
+          '@timestamp': ['2023-02-10T10:15:50.000Z'],
+        };
+        const doc: SignalSourceHit = { ...emptyEsResult(), _source, fields };
+        const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source;
+        expect(merged).toEqual<ReturnTypeMergeFieldsWithSource>(_source);
+      });
     });
   });
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/strategies/merge_missing_fields_with_source.ts

@@ -12,8 +12,8 @@ import { filterFieldEntries } from '../utils/filter_field_entries';
 import type { FieldsType, MergeStrategyFunction } from '../types';
 import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields';
 import { isTypeObject } from '../utils/is_type_object';
-import { arrayInPathExists } from '../utils/array_in_path_exists';
 import { isNestedObject } from '../utils/is_nested_object';
+import { isPathValid } from '../utils/is_path_valid';
 
 /**
  * Merges only missing sections of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information
@@ -79,7 +79,7 @@ const hasEarlyReturnConditions = ({
   return (
     fieldsValue.length === 0 ||
     valueInMergedDocument !== undefined ||
-    arrayInPathExists(fieldsKey, merged) ||
+    !isPathValid(fieldsKey, merged) ||
     isNestedObject(fieldsValue) ||
     isTypeObject(fieldsValue)
   );

x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.test.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isPathValid } from './is_path_valid';
+
+describe('isPathValid', () => {
+  test('not valid when empty string and empty object', () => {
+    expect(isPathValid('', {})).toEqual(false);
+  });
+
+  test('valid when a path and empty object', () => {
+    expect(isPathValid('a.b.c', {})).toEqual(true);
+  });
+
+  test('not valid when a path and an array exists', () => {
+    expect(isPathValid('a', { a: [] })).toEqual(false);
+  });
+
+  test('not valid when a path and primitive value exists', () => {
+    expect(isPathValid('a', { a: 'test' })).toEqual(false);
+    expect(isPathValid('a', { a: 1 })).toEqual(false);
+    expect(isPathValid('a', { a: true })).toEqual(false);
+  });
+
+  test('valid when a path and object value exists', () => {
+    expect(isPathValid('a', { a: {} })).toEqual(true);
+  });
+
+  test('not valid when a path and an array exists within the parent path at level 1', () => {
+    expect(isPathValid('a.b', { a: [] })).toEqual(false);
+  });
+
+  test('not valid when a path and primitive value exists within the parent path at level 1', () => {
+    expect(isPathValid('a.b', { a: 'test' })).toEqual(false);
+    expect(isPathValid('a.b', { a: 1 })).toEqual(false);
+    expect(isPathValid('a.b', { a: true })).toEqual(false);
+  });
+
+  test('valid when a path and object value exists within the parent path at level 1', () => {
+    expect(isPathValid('a.b', { a: {} })).toEqual(true);
+  });
+
+  test('not valid when a path and an array exists within the parent path at level 2', () => {
+    expect(isPathValid('a.b.c', { a: { b: [] } })).toEqual(false);
+  });
+
+  test('not valid when a path and primitive value exists within the parent path at level 2', () => {
+    expect(isPathValid('a.b', { a: { b: 'test' } })).toEqual(false);
+    expect(isPathValid('a.b', { a: { b: 1 } })).toEqual(false);
+    expect(isPathValid('a.b', { a: { b: true } })).toEqual(false);
+  });
+
+  test('valid when a path and object value exists within the parent path at level 2', () => {
+    expect(isPathValid('a.b', { a: { b: {} } })).toEqual(true);
+  });
+
+  test('not valid when a path and an array exists within the parent path at level 3', () => {
+    expect(isPathValid('a.b.c', { a: { b: { c: [] } } })).toEqual(false);
+  });
+
+  test('not valid when a path and primitive value exists within the parent path at level 3', () => {
+    expect(isPathValid('a.b.c', { a: { b: { c: 'test' } } })).toEqual(false);
+    expect(isPathValid('a.b.c', { a: { b: { c: 1 } } })).toEqual(false);
+    expect(isPathValid('a.b.c', { a: { b: { c: true } } })).toEqual(false);
+  });
+
+  test('valid when a path and object value exists within the parent path at level 3', () => {
+    expect(isPathValid('a.b.c', { a: { b: { c: {} } } })).toEqual(true);
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/signals/source_fields_merging/utils/is_path_valid.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { get, isPlainObject } from 'lodash/fp';
+import type { SignalSource } from '../../types';
+
+/**
+ * Returns true if path in SignalSource object is valid
+ * Path is valid if each field in hierarchy is object or undefined
+ * Path is not valid if ANY of field in hierarchy is not object or undefined
+ * @param path in source to check within source
+ * @param source The source document
+ * @returns boolean
+ */
+export const isPathValid = (path: string, source: SignalSource): boolean => {
+  if (!path) {
+    return false;
+  }
+  const splitPath = path.split('.');
+
+  return splitPath.every((_, index, array) => {
+    const newPath = [...array].splice(0, index + 1).join('.');
+    const valueToCheck = get(newPath, source);
+    return valueToCheck === undefined || isPlainObject(valueToCheck);
+  });
+};

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts

@@ -61,6 +61,7 @@ export const buildThreatEnrichment = ({
     const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({
       threatSearchParams,
       eventsCount: signals.length,
+      termsQueryAllowed: false,
     });
 
     const enrichment = threatEnrichmentFactory({

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts

@@ -86,6 +86,7 @@ export const createEventSignal = async ({
       threatSearchParams,
       eventsCount: currentEventList.length,
       signalValueMap: getSignalValueMap({ eventList: currentEventList, threatMatchedFields }),
+      termsQueryAllowed: true,
     });
 
     const ids = Array.from(signalsQueryMap.keys());