From 09e326e1368f14647fb79d2fc8077aa87aaaaff9 Mon Sep 17 00:00:00 2001
From: Joe Reuter <johannes.reuter@elastic.co>
Date: Wed, 25 Nov 2020 12:12:43 +0100
Subject: [PATCH 01/37] [Lens] Fix label input debouncing (#84121)

---
 .../dimension_panel/dimension_editor.tsx                    | 6 +-----
 .../operations/definitions/filters/filter_popover.tsx       | 4 ----
 .../definitions/shared_components/label_input.tsx           | 6 +-----
 3 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx
index e5c05a1cf8c7a..0a67c157bd837 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx
@@ -6,7 +6,7 @@
 
 import './dimension_editor.scss';
 import _ from 'lodash';
-import React, { useState, useMemo, useEffect } from 'react';
+import React, { useState, useMemo } from 'react';
 import { i18n } from '@kbn/i18n';
 import {
   EuiListGroup,
@@ -46,10 +46,6 @@ export interface DimensionEditorProps extends IndexPatternDimensionEditorProps {
 const LabelInput = ({ value, onChange }: { value: string; onChange: (value: string) => void }) => {
   const [inputValue, setInputValue] = useState(value);
 
-  useEffect(() => {
-    setInputValue(value);
-  }, [value, setInputValue]);
-
   const onChangeDebounced = useMemo(() => _.debounce(onChange, 256), [onChange]);
 
   const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx
index b9d9d6306b9ae..ca84c072be5ce 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx
@@ -110,10 +110,6 @@ export const QueryInput = ({
 }) => {
   const [inputValue, setInputValue] = useState(value);
 
-  React.useEffect(() => {
-    setInputValue(value);
-  }, [value, setInputValue]);
-
   useDebounce(() => onChange(inputValue), 256, [inputValue]);
 
   const handleInputChange = (input: Query) => {
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/shared_components/label_input.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/shared_components/label_input.tsx
index f0ee30bb4331b..ddcb5633b376f 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/shared_components/label_input.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/shared_components/label_input.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useState, useEffect } from 'react';
+import React, { useState } from 'react';
 import useDebounce from 'react-use/lib/useDebounce';
 import { EuiFieldText, keys } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
@@ -28,10 +28,6 @@ export const LabelInput = ({
 }) => {
   const [inputValue, setInputValue] = useState(value);
 
-  useEffect(() => {
-    setInputValue(value);
-  }, [value, setInputValue]);
-
   useDebounce(() => onChange(inputValue), 256, [inputValue]);
 
   const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {

From c1b807eda3d34bff02cc54b91f2648392da2734c Mon Sep 17 00:00:00 2001
From: Pete Harverson <peteharverson@users.noreply.github.com>
Date: Wed, 25 Nov 2020 11:36:31 +0000
Subject: [PATCH 02/37] [ML] Fix Anomaly Explorer population charts when
 multiple causes in anomaly (#84254)

---
 x-pack/plugins/ml/public/application/util/chart_utils.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/util/chart_utils.js b/x-pack/plugins/ml/public/application/util/chart_utils.js
index f2dec5f16df1d..d142d2e246659 100644
--- a/x-pack/plugins/ml/public/application/util/chart_utils.js
+++ b/x-pack/plugins/ml/public/application/util/chart_utils.js
@@ -77,7 +77,10 @@ export function chartExtendedLimits(data = [], functionDescription) {
       metricValue = actualValue;
     }
 
-    if (d.anomalyScore !== undefined) {
+    // Check for both an anomaly and for an actual value as anomalies in detectors with
+    // by and over fields and more than one cause will not have actual / typical values
+    // at the top level of the anomaly record.
+    if (d.anomalyScore !== undefined && actualValue !== undefined) {
       _min = Math.min(_min, metricValue, actualValue, typicalValue);
       _max = Math.max(_max, metricValue, actualValue, typicalValue);
     } else {

From 4367a10d1b2f12d5f8f891b83ec4563c53d1f03b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= <sorenlouv@gmail.com>
Date: Wed, 25 Nov 2020 12:44:02 +0100
Subject: [PATCH 03/37] [APM] Rename `ChartsSyncContext` to
 `PointerEventContext` (#84272)

---
 .../app/TransactionDetails/index.tsx           |  6 +++---
 .../components/app/service_metrics/index.tsx   |  6 +++---
 .../app/service_node_metrics/index.tsx         | 10 +++++-----
 .../components/app/service_overview/index.tsx  |  6 +++---
 .../shared/charts/timeseries_chart.tsx         | 14 ++++++--------
 .../transaction_breakdown_chart_contents.tsx   | 18 ++++++++++--------
 .../shared/charts/transaction_charts/index.tsx |  6 +++---
 ...ext.tsx => chart_pointer_event_context.tsx} | 16 +++++++++-------
 ...ts_sync.tsx => use_chart_pointer_event.tsx} |  8 ++++----
 9 files changed, 46 insertions(+), 44 deletions(-)
 rename x-pack/plugins/apm/public/context/{charts_sync_context.tsx => chart_pointer_event_context.tsx} (52%)
 rename x-pack/plugins/apm/public/hooks/{use_charts_sync.tsx => use_chart_pointer_event.tsx} (56%)

diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx
index 8a99773a97baf..8f335ddc71c72 100644
--- a/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/index.tsx
@@ -25,7 +25,7 @@ import { TransactionCharts } from '../../shared/charts/transaction_charts';
 import { TransactionDistribution } from './Distribution';
 import { WaterfallWithSummmary } from './WaterfallWithSummmary';
 import { FETCH_STATUS } from '../../../hooks/useFetcher';
-import { ChartsSyncContextProvider } from '../../../context/charts_sync_context';
+import { ChartPointerEventContextProvider } from '../../../context/chart_pointer_event_context';
 import { useTrackPageview } from '../../../../../observability/public';
 import { Projection } from '../../../../common/projections';
 import { fromQuery, toQuery } from '../../shared/Links/url_helpers';
@@ -125,13 +125,13 @@ export function TransactionDetails({
             <LocalUIFilters {...localUIFiltersConfig} />
           </EuiFlexItem>
           <EuiFlexItem grow={7}>
-            <ChartsSyncContextProvider>
+            <ChartPointerEventContextProvider>
               <TransactionCharts
                 fetchStatus={transactionChartsStatus}
                 charts={transactionChartsData}
                 urlParams={urlParams}
               />
-            </ChartsSyncContextProvider>
+            </ChartPointerEventContextProvider>
 
             <EuiHorizontalRule size="full" margin="l" />
 
diff --git a/x-pack/plugins/apm/public/components/app/service_metrics/index.tsx b/x-pack/plugins/apm/public/components/app/service_metrics/index.tsx
index 5dc1645a1760d..d0f8fc1e61332 100644
--- a/x-pack/plugins/apm/public/components/app/service_metrics/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/service_metrics/index.tsx
@@ -16,7 +16,7 @@ import React, { useMemo } from 'react';
 import { useServiceMetricCharts } from '../../../hooks/useServiceMetricCharts';
 import { MetricsChart } from '../../shared/charts/metrics_chart';
 import { useUrlParams } from '../../../hooks/useUrlParams';
-import { ChartsSyncContextProvider } from '../../../context/charts_sync_context';
+import { ChartPointerEventContextProvider } from '../../../context/chart_pointer_event_context';
 import { Projection } from '../../../../common/projections';
 import { LocalUIFilters } from '../../shared/LocalUIFilters';
 import { SearchBar } from '../../shared/search_bar';
@@ -57,7 +57,7 @@ export function ServiceMetrics({
             <LocalUIFilters {...localFiltersConfig} />
           </EuiFlexItem>
           <EuiFlexItem grow={7}>
-            <ChartsSyncContextProvider>
+            <ChartPointerEventContextProvider>
               <EuiFlexGrid columns={2} gutterSize="s">
                 {data.charts.map((chart) => (
                   <EuiFlexItem key={chart.key}>
@@ -73,7 +73,7 @@ export function ServiceMetrics({
                 ))}
               </EuiFlexGrid>
               <EuiSpacer size="xxl" />
-            </ChartsSyncContextProvider>
+            </ChartPointerEventContextProvider>
           </EuiFlexItem>
         </EuiFlexGroup>
       </EuiPage>
diff --git a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx
index 59e919199be76..a74ff574bc0c8 100644
--- a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx
@@ -22,7 +22,7 @@ import React from 'react';
 import { RouteComponentProps } from 'react-router-dom';
 import styled from 'styled-components';
 import { SERVICE_NODE_NAME_MISSING } from '../../../../common/service_nodes';
-import { ChartsSyncContextProvider } from '../../../context/charts_sync_context';
+import { ChartPointerEventContextProvider } from '../../../context/chart_pointer_event_context';
 import { useAgentName } from '../../../hooks/useAgentName';
 import { FETCH_STATUS, useFetcher } from '../../../hooks/useFetcher';
 import { useServiceMetricCharts } from '../../../hooks/useServiceMetricCharts';
@@ -178,7 +178,7 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) {
         </MetadataFlexGroup>
       )}
       {agentName && (
-        <ChartsSyncContextProvider>
+        <ChartPointerEventContextProvider>
           <EuiFlexGrid columns={2} gutterSize="s">
             {data.charts.map((chart) => (
               <EuiFlexItem key={chart.key}>
@@ -194,12 +194,12 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) {
             ))}
           </EuiFlexGrid>
           <EuiSpacer size="xxl" />
-        </ChartsSyncContextProvider>
+        </ChartPointerEventContextProvider>
       )}
       <SearchBar />
       <EuiPage>
         {agentName && (
-          <ChartsSyncContextProvider>
+          <ChartPointerEventContextProvider>
             <EuiFlexGrid columns={2} gutterSize="s">
               {data.charts.map((chart) => (
                 <EuiFlexItem key={chart.key}>
@@ -215,7 +215,7 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) {
               ))}
             </EuiFlexGrid>
             <EuiSpacer size="xxl" />
-          </ChartsSyncContextProvider>
+          </ChartPointerEventContextProvider>
         )}
       </EuiPage>
     </>
diff --git a/x-pack/plugins/apm/public/components/app/service_overview/index.tsx b/x-pack/plugins/apm/public/components/app/service_overview/index.tsx
index 33027f3946d1f..ddf3107a8ab1e 100644
--- a/x-pack/plugins/apm/public/components/app/service_overview/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/service_overview/index.tsx
@@ -15,7 +15,7 @@ import { i18n } from '@kbn/i18n';
 import React from 'react';
 import { useTrackPageview } from '../../../../../observability/public';
 import { isRumAgentName } from '../../../../common/agent_name';
-import { ChartsSyncContextProvider } from '../../../context/charts_sync_context';
+import { ChartPointerEventContextProvider } from '../../../context/chart_pointer_event_context';
 import { TransactionBreakdownChart } from '../../shared/charts/transaction_breakdown_chart';
 import { TransactionErrorRateChart } from '../../shared/charts/transaction_error_rate_chart';
 import { ServiceMapLink } from '../../shared/Links/apm/ServiceMapLink';
@@ -43,7 +43,7 @@ export function ServiceOverview({
   useTrackPageview({ app: 'apm', path: 'service_overview', delay: 15000 });
 
   return (
-    <ChartsSyncContextProvider>
+    <ChartPointerEventContextProvider>
       <SearchBar />
       <EuiPage>
         <EuiFlexGroup direction="column" gutterSize="s">
@@ -170,6 +170,6 @@ export function ServiceOverview({
           </EuiFlexItem>
         </EuiFlexGroup>
       </EuiPage>
-    </ChartsSyncContextProvider>
+    </ChartPointerEventContextProvider>
   );
 }
diff --git a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx
index fe3d9a1edc1fb..c4f5abe104aa9 100644
--- a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx
+++ b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx
@@ -24,7 +24,7 @@ import { useChartTheme } from '../../../../../observability/public';
 import { TimeSeries } from '../../../../typings/timeseries';
 import { FETCH_STATUS } from '../../../hooks/useFetcher';
 import { useUrlParams } from '../../../hooks/useUrlParams';
-import { useChartsSync } from '../../../hooks/use_charts_sync';
+import { useChartPointerEvent } from '../../../hooks/use_chart_pointer_event';
 import { unit } from '../../../style/variables';
 import { Annotations } from './annotations';
 import { ChartContainer } from './chart_container';
@@ -60,15 +60,15 @@ export function TimeseriesChart({
   const history = useHistory();
   const chartRef = React.createRef<Chart>();
   const chartTheme = useChartTheme();
-  const { event, setEvent } = useChartsSync();
+  const { pointerEvent, setPointerEvent } = useChartPointerEvent();
   const { urlParams } = useUrlParams();
   const { start, end } = urlParams;
 
   useEffect(() => {
-    if (event.chartId !== id && chartRef.current) {
-      chartRef.current.dispatchExternalPointerEvent(event);
+    if (pointerEvent && pointerEvent?.chartId !== id && chartRef.current) {
+      chartRef.current.dispatchExternalPointerEvent(pointerEvent);
     }
-  }, [event, chartRef, id]);
+  }, [pointerEvent, chartRef, id]);
 
   const min = moment.utc(start).valueOf();
   const max = moment.utc(end).valueOf();
@@ -89,9 +89,7 @@ export function TimeseriesChart({
         <Settings
           onBrushEnd={({ x }) => onBrushEnd({ x, history })}
           theme={chartTheme}
-          onPointerUpdate={(currEvent: any) => {
-            setEvent(currEvent);
-          }}
+          onPointerUpdate={setPointerEvent}
           externalPointerEvents={{
             tooltip: { visible: true, placement: Placement.Bottom },
           }}
diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx
index 8070868f831b2..04c07c01442a4 100644
--- a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx
+++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx
@@ -23,7 +23,7 @@ import { asPercent } from '../../../../../common/utils/formatters';
 import { TimeSeries } from '../../../../../typings/timeseries';
 import { FETCH_STATUS } from '../../../../hooks/useFetcher';
 import { useUrlParams } from '../../../../hooks/useUrlParams';
-import { useChartsSync as useChartsSync2 } from '../../../../hooks/use_charts_sync';
+import { useChartPointerEvent } from '../../../../hooks/use_chart_pointer_event';
 import { unit } from '../../../../style/variables';
 import { Annotations } from '../../charts/annotations';
 import { ChartContainer } from '../../charts/chart_container';
@@ -45,15 +45,19 @@ export function TransactionBreakdownChartContents({
   const history = useHistory();
   const chartRef = React.createRef<Chart>();
   const chartTheme = useChartTheme();
-  const { event, setEvent } = useChartsSync2();
+  const { pointerEvent, setPointerEvent } = useChartPointerEvent();
   const { urlParams } = useUrlParams();
   const { start, end } = urlParams;
 
   useEffect(() => {
-    if (event.chartId !== 'timeSpentBySpan' && chartRef.current) {
-      chartRef.current.dispatchExternalPointerEvent(event);
+    if (
+      pointerEvent &&
+      pointerEvent.chartId !== 'timeSpentBySpan' &&
+      chartRef.current
+    ) {
+      chartRef.current.dispatchExternalPointerEvent(pointerEvent);
     }
-  }, [chartRef, event]);
+  }, [chartRef, pointerEvent]);
 
   const min = moment.utc(start).valueOf();
   const max = moment.utc(end).valueOf();
@@ -71,9 +75,7 @@ export function TransactionBreakdownChartContents({
           theme={chartTheme}
           xDomain={{ min, max }}
           flatLegend
-          onPointerUpdate={(currEvent: any) => {
-            setEvent(currEvent);
-          }}
+          onPointerUpdate={setPointerEvent}
           externalPointerEvents={{
             tooltip: { visible: true, placement: Placement.Bottom },
           }}
diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx
index 3af081c11c9b1..221f17bb9e1d5 100644
--- a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx
+++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx
@@ -20,7 +20,7 @@ import {
   TRANSACTION_ROUTE_CHANGE,
 } from '../../../../../common/transaction_types';
 import { asTransactionRate } from '../../../../../common/utils/formatters';
-import { ChartsSyncContextProvider } from '../../../../context/charts_sync_context';
+import { ChartPointerEventContextProvider } from '../../../../context/chart_pointer_event_context';
 import { LicenseContext } from '../../../../context/LicenseContext';
 import { IUrlParams } from '../../../../context/UrlParamsContext/types';
 import { FETCH_STATUS } from '../../../../hooks/useFetcher';
@@ -51,7 +51,7 @@ export function TransactionCharts({
 
   return (
     <>
-      <ChartsSyncContextProvider>
+      <ChartPointerEventContextProvider>
         <EuiFlexGrid columns={2} gutterSize="s">
           <EuiFlexItem data-cy={`transaction-duration-charts`}>
             <EuiPanel>
@@ -109,7 +109,7 @@ export function TransactionCharts({
             <TransactionBreakdownChart />
           </EuiFlexItem>
         </EuiFlexGrid>
-      </ChartsSyncContextProvider>
+      </ChartPointerEventContextProvider>
     </>
   );
 }
diff --git a/x-pack/plugins/apm/public/context/charts_sync_context.tsx b/x-pack/plugins/apm/public/context/chart_pointer_event_context.tsx
similarity index 52%
rename from x-pack/plugins/apm/public/context/charts_sync_context.tsx
rename to x-pack/plugins/apm/public/context/chart_pointer_event_context.tsx
index d983a857a26ec..ea60206463258 100644
--- a/x-pack/plugins/apm/public/context/charts_sync_context.tsx
+++ b/x-pack/plugins/apm/public/context/chart_pointer_event_context.tsx
@@ -12,21 +12,23 @@ import React, {
   useState,
 } from 'react';
 
-export const ChartsSyncContext = createContext<{
-  event: any;
-  setEvent: Dispatch<SetStateAction<{}>>;
+import { PointerEvent } from '@elastic/charts';
+
+export const ChartPointerEventContext = createContext<{
+  pointerEvent: PointerEvent | null;
+  setPointerEvent: Dispatch<SetStateAction<PointerEvent | null>>;
 } | null>(null);
 
-export function ChartsSyncContextProvider({
+export function ChartPointerEventContextProvider({
   children,
 }: {
   children: ReactNode;
 }) {
-  const [event, setEvent] = useState({});
+  const [pointerEvent, setPointerEvent] = useState<PointerEvent | null>(null);
 
   return (
-    <ChartsSyncContext.Provider
-      value={{ event, setEvent }}
+    <ChartPointerEventContext.Provider
+      value={{ pointerEvent, setPointerEvent }}
       children={children}
     />
   );
diff --git a/x-pack/plugins/apm/public/hooks/use_charts_sync.tsx b/x-pack/plugins/apm/public/hooks/use_chart_pointer_event.tsx
similarity index 56%
rename from x-pack/plugins/apm/public/hooks/use_charts_sync.tsx
rename to x-pack/plugins/apm/public/hooks/use_chart_pointer_event.tsx
index cde5c84a6097b..058ec594e2d22 100644
--- a/x-pack/plugins/apm/public/hooks/use_charts_sync.tsx
+++ b/x-pack/plugins/apm/public/hooks/use_chart_pointer_event.tsx
@@ -5,13 +5,13 @@
  */
 
 import { useContext } from 'react';
-import { ChartsSyncContext } from '../context/charts_sync_context';
+import { ChartPointerEventContext } from '../context/chart_pointer_event_context';
 
-export function useChartsSync() {
-  const context = useContext(ChartsSyncContext);
+export function useChartPointerEvent() {
+  const context = useContext(ChartPointerEventContext);
 
   if (!context) {
-    throw new Error('Missing ChartsSync context provider');
+    throw new Error('Missing ChartPointerEventContext provider');
   }
 
   return context;

From 0f780229b5c195f41e4182f13c3a2d952356a816 Mon Sep 17 00:00:00 2001
From: Chris Roberson <chrisronline@gmail.com>
Date: Wed, 25 Nov 2020 08:49:17 -0500
Subject: [PATCH 04/37] Put the cluster_uuid in quotes (#83987)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../public/lib/get_safe_for_external_link.test.ts     |  4 ++--
 .../public/lib/get_safe_for_external_link.ts          | 11 ++++++-----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.test.ts b/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.test.ts
index 4b40c7f4a88dc..8c1b9d0c475dd 100644
--- a/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.test.ts
+++ b/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.test.ts
@@ -51,7 +51,7 @@ describe('getSafeForExternalLink', () => {
         location
       )
     ).toBe(
-      `#/overview?_g=(cluster_uuid:NDKg6VXAT6-TaGzEK2Zy7g,filters:!(),refreshInterval:(pause:!t,value:10000),time:(from:'2017-09-07T20:12:04.011Z',to:'2017-09-07T20:18:55.733Z'))`
+      `#/overview?_g=(cluster_uuid:'NDKg6VXAT6-TaGzEK2Zy7g',filters:!(),refreshInterval:(pause:!t,value:10000),time:(from:'2017-09-07T20:12:04.011Z',to:'2017-09-07T20:18:55.733Z'))`
     );
   });
 
@@ -68,7 +68,7 @@ describe('getSafeForExternalLink', () => {
         location
       )
     ).toBe(
-      `#/overview?_g=(filters:!(),refreshInterval:(pause:!t,value:10000),time:(from:'2017-09-07T20:12:04.011Z',to:'2017-09-07T20:18:55.733Z'),cluster_uuid:NDKg6VXAT6-TaGzEK2Zy7g)`
+      `#/overview?_g=(filters:!(),refreshInterval:(pause:!t,value:10000),time:(from:'2017-09-07T20:12:04.011Z',to:'2017-09-07T20:18:55.733Z'),cluster_uuid:'NDKg6VXAT6-TaGzEK2Zy7g')`
     );
   });
 });
diff --git a/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.ts b/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.ts
index 3730ed6411227..86d571b87bc94 100644
--- a/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.ts
+++ b/x-pack/plugins/monitoring/public/lib/get_safe_for_external_link.ts
@@ -24,15 +24,16 @@ export function getSafeForExternalLink(
 
   let newGlobalState = globalStateExecResult[1];
   Object.keys(globalState).forEach((globalStateKey) => {
+    let value = globalState[globalStateKey];
+    if (globalStateKey === 'cluster_uuid') {
+      value = `'${value}'`;
+    }
     const keyRegExp = new RegExp(`${globalStateKey}:([^,]+)`);
     const execResult = keyRegExp.exec(newGlobalState);
     if (execResult && execResult.length) {
-      newGlobalState = newGlobalState.replace(
-        execResult[0],
-        `${globalStateKey}:${globalState[globalStateKey]}`
-      );
+      newGlobalState = newGlobalState.replace(execResult[0], `${globalStateKey}:${value}`);
     } else {
-      newGlobalState += `,${globalStateKey}:${globalState[globalStateKey]}`;
+      newGlobalState += `,${globalStateKey}:${value}`;
     }
   });
 

From ced1dadb8a5c2dcf50f590dbe3c5db38d512e0c1 Mon Sep 17 00:00:00 2001
From: Chris Roberson <chrisronline@gmail.com>
Date: Wed, 25 Nov 2020 08:50:03 -0500
Subject: [PATCH 05/37] [Monitoring] Only look at ES for the missing data alert
 for now (#83839)

* Only look at ES for the missing data alert for now

* PR feedback

* Fix tests
---
 .../missing_monitoring_data_alert.test.ts     |  67 +------
 .../fetch_missing_monitoring_data.test.ts     |  89 ---------
 .../alerts/fetch_missing_monitoring_data.ts   | 178 +++---------------
 3 files changed, 35 insertions(+), 299 deletions(-)

diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts
index 57d01dc6a1100..1332148a61cdd 100644
--- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts
+++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts
@@ -65,13 +65,6 @@ describe('MissingMonitoringDataAlert', () => {
         clusterUuid,
         gapDuration,
       },
-      {
-        stackProduct: 'kibana',
-        stackProductUuid: 'kibanaUuid1',
-        stackProductName: 'kibanaInstance1',
-        clusterUuid,
-        gapDuration: gapDuration + 10,
-      },
     ];
     const getUiSettingsService = () => ({
       asScopedToClient: jest.fn(),
@@ -140,7 +133,7 @@ describe('MissingMonitoringDataAlert', () => {
         // @ts-ignore
         params: alert.defaultParams,
       } as any);
-      const count = 2;
+      const count = 1;
       expect(replaceState).toHaveBeenCalledWith({
         alertStates: [
           {
@@ -187,61 +180,17 @@ describe('MissingMonitoringDataAlert', () => {
               lastCheckedMS: 0,
             },
           },
-          {
-            ccs: undefined,
-            cluster: { clusterUuid, clusterName },
-            gapDuration: gapDuration + 10,
-            stackProduct: 'kibana',
-            stackProductName: 'kibanaInstance1',
-            stackProductUuid: 'kibanaUuid1',
-            ui: {
-              isFiring: true,
-              message: {
-                text:
-                  'For the past an hour, we have not detected any monitoring data from the Kibana instance: kibanaInstance1, starting at #absolute',
-                nextSteps: [
-                  {
-                    text: '#start_linkView all Kibana instances#end_link',
-                    tokens: [
-                      {
-                        startToken: '#start_link',
-                        endToken: '#end_link',
-                        type: 'link',
-                        url: 'kibana/instances',
-                      },
-                    ],
-                  },
-                  {
-                    text: 'Verify monitoring settings on the instance',
-                  },
-                ],
-                tokens: [
-                  {
-                    startToken: '#absolute',
-                    type: 'time',
-                    isAbsolute: true,
-                    isRelative: false,
-                    timestamp: 1,
-                  },
-                ],
-              },
-              severity: 'danger',
-              resolvedMS: 0,
-              triggeredMS: 1,
-              lastCheckedMS: 0,
-            },
-          },
         ],
       });
       expect(scheduleActions).toHaveBeenCalledWith('default', {
-        internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`,
-        internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`,
+        internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`,
+        internalShortMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`,
         action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`,
         actionPlain:
           'Verify these stack products are up and running, then double check the monitoring settings.',
         clusterName,
         count,
-        stackProducts: 'Elasticsearch node: esName1, Kibana instance: kibanaInstance1',
+        stackProducts: 'Elasticsearch node: esName1',
         state: 'firing',
       });
     });
@@ -442,16 +391,16 @@ describe('MissingMonitoringDataAlert', () => {
         // @ts-ignore
         params: alert.defaultParams,
       } as any);
-      const count = 2;
+      const count = 1;
       expect(scheduleActions).toHaveBeenCalledWith('default', {
-        internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`,
-        internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`,
+        internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`,
+        internalShortMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`,
         action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`,
         actionPlain:
           'Verify these stack products are up and running, then double check the monitoring settings.',
         clusterName,
         count,
-        stackProducts: 'Elasticsearch node: esName1, Kibana instance: kibanaInstance1',
+        stackProducts: 'Elasticsearch node: esName1',
         state: 'firing',
       });
     });
diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts
index b09f5a88dba9c..7edd7496805a0 100644
--- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts
+++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts
@@ -75,63 +75,6 @@ describe('fetchMissingMonitoringData', () => {
                   timestamp: 2,
                 },
               ]),
-              kibana_uuids: getResponse('.monitoring-kibana-*', [
-                {
-                  uuid: 'kibanaUuid1',
-                  nameSource: {
-                    kibana_stats: {
-                      kibana: {
-                        name: 'kibanaName1',
-                      },
-                    },
-                  },
-                  timestamp: 4,
-                },
-              ]),
-              logstash_uuids: getResponse('.monitoring-logstash-*', [
-                {
-                  uuid: 'logstashUuid1',
-                  nameSource: {
-                    logstash_stats: {
-                      logstash: {
-                        host: 'logstashName1',
-                      },
-                    },
-                  },
-                  timestamp: 2,
-                },
-              ]),
-              beats: {
-                beats_uuids: getResponse('.monitoring-beats-*', [
-                  {
-                    uuid: 'beatUuid1',
-                    nameSource: {
-                      beats_stats: {
-                        beat: {
-                          name: 'beatName1',
-                        },
-                      },
-                    },
-                    timestamp: 0,
-                  },
-                ]),
-              },
-              apms: {
-                apm_uuids: getResponse('.monitoring-beats-*', [
-                  {
-                    uuid: 'apmUuid1',
-                    nameSource: {
-                      beats_stats: {
-                        beat: {
-                          name: 'apmName1',
-                          type: 'apm-server',
-                        },
-                      },
-                    },
-                    timestamp: 1,
-                  },
-                ]),
-              },
             })),
           },
         },
@@ -162,38 +105,6 @@ describe('fetchMissingMonitoringData', () => {
         gapDuration: 8,
         ccs: null,
       },
-      {
-        stackProduct: 'kibana',
-        stackProductUuid: 'kibanaUuid1',
-        stackProductName: 'kibanaName1',
-        clusterUuid: 'clusterUuid1',
-        gapDuration: 6,
-        ccs: null,
-      },
-      {
-        stackProduct: 'logstash',
-        stackProductUuid: 'logstashUuid1',
-        stackProductName: 'logstashName1',
-        clusterUuid: 'clusterUuid1',
-        gapDuration: 8,
-        ccs: null,
-      },
-      {
-        stackProduct: 'beats',
-        stackProductUuid: 'beatUuid1',
-        stackProductName: 'beatName1',
-        clusterUuid: 'clusterUuid1',
-        gapDuration: 10,
-        ccs: null,
-      },
-      {
-        stackProduct: 'apm',
-        stackProductUuid: 'apmUuid1',
-        stackProductName: 'apmName1',
-        clusterUuid: 'clusterUuid1',
-        gapDuration: 9,
-        ccs: null,
-      },
     ]);
   });
 
diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts
index 0fa90e1d6fb39..b4e12e5d86139 100644
--- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts
+++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts
@@ -5,25 +5,11 @@
  */
 import { get } from 'lodash';
 import { AlertCluster, AlertMissingData } from '../../../common/types/alerts';
-import {
-  KIBANA_SYSTEM_ID,
-  BEATS_SYSTEM_ID,
-  APM_SYSTEM_ID,
-  LOGSTASH_SYSTEM_ID,
-  ELASTICSEARCH_SYSTEM_ID,
-} from '../../../common/constants';
+import { ELASTICSEARCH_SYSTEM_ID } from '../../../common/constants';
 
 interface ClusterBucketESResponse {
   key: string;
-  kibana_uuids?: UuidResponse;
-  logstash_uuids?: UuidResponse;
-  es_uuids?: UuidResponse;
-  beats?: {
-    beats_uuids: UuidResponse;
-  };
-  apms?: {
-    apm_uuids: UuidResponse;
-  };
+  es_uuids: UuidResponse;
 }
 
 interface UuidResponse {
@@ -48,44 +34,11 @@ interface TopHitESResponse {
     source_node?: {
       name: string;
     };
-    kibana_stats?: {
-      kibana: {
-        name: string;
-      };
-    };
-    logstash_stats?: {
-      logstash: {
-        host: string;
-      };
-    };
-    beats_stats?: {
-      beat: {
-        name: string;
-        type: string;
-      };
-    };
   };
 }
 
-function getStackProductFromIndex(index: string, beatType: string) {
-  if (index.includes('-kibana-')) {
-    return KIBANA_SYSTEM_ID;
-  }
-  if (index.includes('-beats-')) {
-    if (beatType === 'apm-server') {
-      return APM_SYSTEM_ID;
-    }
-    return BEATS_SYSTEM_ID;
-  }
-  if (index.includes('-logstash-')) {
-    return LOGSTASH_SYSTEM_ID;
-  }
-  if (index.includes('-es-')) {
-    return ELASTICSEARCH_SYSTEM_ID;
-  }
-  return '';
-}
-
+// TODO: only Elasticsearch until we can figure out how to handle upgrades for the rest of the stack
+// https://github.com/elastic/kibana/issues/83309
 export async function fetchMissingMonitoringData(
   callCluster: any,
   clusters: AlertCluster[],
@@ -95,37 +48,6 @@ export async function fetchMissingMonitoringData(
   startMs: number
 ): Promise<AlertMissingData[]> {
   const endMs = nowInMs;
-
-  const nameFields = [
-    'source_node.name',
-    'kibana_stats.kibana.name',
-    'logstash_stats.logstash.host',
-    'beats_stats.beat.name',
-    'beats_stats.beat.type',
-  ];
-  const subAggs = {
-    most_recent: {
-      max: {
-        field: 'timestamp',
-      },
-    },
-    document: {
-      top_hits: {
-        size: 1,
-        sort: [
-          {
-            timestamp: {
-              order: 'desc',
-            },
-          },
-        ],
-        _source: {
-          includes: ['_index', ...nameFields],
-        },
-      },
-    },
-  };
-
   const params = {
     index,
     filterPath: ['aggregations.clusters.buckets'],
@@ -163,61 +85,28 @@ export async function fetchMissingMonitoringData(
                 field: 'node_stats.node_id',
                 size,
               },
-              aggs: subAggs,
-            },
-            kibana_uuids: {
-              terms: {
-                field: 'kibana_stats.kibana.uuid',
-                size,
-              },
-              aggs: subAggs,
-            },
-            beats: {
-              filter: {
-                bool: {
-                  must_not: {
-                    term: {
-                      'beats_stats.beat.type': 'apm-server',
-                    },
-                  },
-                },
-              },
               aggs: {
-                beats_uuids: {
-                  terms: {
-                    field: 'beats_stats.beat.uuid',
-                    size,
+                most_recent: {
+                  max: {
+                    field: 'timestamp',
                   },
-                  aggs: subAggs,
                 },
-              },
-            },
-            apms: {
-              filter: {
-                bool: {
-                  must: {
-                    term: {
-                      'beats_stats.beat.type': 'apm-server',
+                document: {
+                  top_hits: {
+                    size: 1,
+                    sort: [
+                      {
+                        timestamp: {
+                          order: 'desc',
+                        },
+                      },
+                    ],
+                    _source: {
+                      includes: ['_index', 'source_node.name'],
                     },
                   },
                 },
               },
-              aggs: {
-                apm_uuids: {
-                  terms: {
-                    field: 'beats_stats.beat.uuid',
-                    size,
-                  },
-                  aggs: subAggs,
-                },
-              },
-            },
-            logstash_uuids: {
-              terms: {
-                field: 'logstash_stats.logstash.uuid',
-                size,
-              },
-              aggs: subAggs,
             },
           },
         },
@@ -234,33 +123,20 @@ export async function fetchMissingMonitoringData(
   const uniqueList: { [id: string]: AlertMissingData } = {};
   for (const clusterBucket of clusterBuckets) {
     const clusterUuid = clusterBucket.key;
-
-    const uuidBuckets = [
-      ...(clusterBucket.es_uuids?.buckets || []),
-      ...(clusterBucket.kibana_uuids?.buckets || []),
-      ...(clusterBucket.logstash_uuids?.buckets || []),
-      ...(clusterBucket.beats?.beats_uuids.buckets || []),
-      ...(clusterBucket.apms?.apm_uuids.buckets || []),
-    ];
+    const uuidBuckets = clusterBucket.es_uuids.buckets;
 
     for (const uuidBucket of uuidBuckets) {
       const stackProductUuid = uuidBucket.key;
       const indexName = get(uuidBucket, `document.hits.hits[0]._index`);
-      const stackProduct = getStackProductFromIndex(
-        indexName,
-        get(uuidBucket, `document.hits.hits[0]._source.beats_stats.beat.type`)
-      );
       const differenceInMs = nowInMs - uuidBucket.most_recent.value;
-      let stackProductName = stackProductUuid;
-      for (const nameField of nameFields) {
-        stackProductName = get(uuidBucket, `document.hits.hits[0]._source.${nameField}`);
-        if (stackProductName) {
-          break;
-        }
-      }
+      const stackProductName = get(
+        uuidBucket,
+        `document.hits.hits[0]._source.source_node.name`,
+        stackProductUuid
+      );
 
-      uniqueList[`${clusterUuid}${stackProduct}${stackProductUuid}`] = {
-        stackProduct,
+      uniqueList[`${clusterUuid}${stackProductUuid}`] = {
+        stackProduct: ELASTICSEARCH_SYSTEM_ID,
         stackProductUuid,
         stackProductName,
         clusterUuid,

From 5ee0104fe1d400135eda157f535996a22d63dec1 Mon Sep 17 00:00:00 2001
From: Rudolf Meijering <skaapgif@gmail.com>
Date: Wed, 25 Nov 2020 15:20:56 +0100
Subject: [PATCH 06/37] Use .kibana instead of .kibana_current to mark
 migration completion (#83373)

---
 rfcs/text/0013_saved_object_migrations.md | 92 ++++++++++++++++-------
 1 file changed, 63 insertions(+), 29 deletions(-)

diff --git a/rfcs/text/0013_saved_object_migrations.md b/rfcs/text/0013_saved_object_migrations.md
index 1a0967d110d06..6e125c28c04c0 100644
--- a/rfcs/text/0013_saved_object_migrations.md
+++ b/rfcs/text/0013_saved_object_migrations.md
@@ -13,6 +13,7 @@
   - [4.2.1 Idempotent migrations performed without coordination](#421-idempotent-migrations-performed-without-coordination)
     - [4.2.1.1 Restrictions](#4211-restrictions)
     - [4.2.1.2 Migration algorithm: Cloned index per version](#4212-migration-algorithm-cloned-index-per-version)
+      - [Known weaknesses:](#known-weaknesses)
     - [4.2.1.3 Upgrade and rollback procedure](#4213-upgrade-and-rollback-procedure)
     - [4.2.1.4 Handling documents that belong to a disabled plugin](#4214-handling-documents-that-belong-to-a-disabled-plugin)
 - [5. Alternatives](#5-alternatives)
@@ -192,26 +193,24 @@ id's deterministically with e.g. UUIDv5.
 ### 4.2.1.2 Migration algorithm: Cloned index per version
 Note:
 - The description below assumes the migration algorithm is released in 7.10.0.
-  So < 7.10.0 will use `.kibana` and >= 7.10.0 will use `.kibana_current`.
+  So >= 7.10.0 will use the new algorithm.
 - We refer to the alias and index that outdated nodes use as the source alias
   and source index.
 - Every version performs a migration even if mappings or documents aren't outdated.
 
-1. Locate the source index by fetching aliases (including `.kibana` for
-   versions prior to v7.10.0)
+1. Locate the source index by fetching kibana indices:
    
    ```
-   GET '/_alias/.kibana_current,.kibana_7.10.0,.kibana'
+   GET '/_indices/.kibana,.kibana_7.10.0'
    ```
    
    The source index is:
-   1. the index the `.kibana_current` alias points to, or if it doesn’t exist, 
-   2. the index the `.kibana` alias points to, or if it doesn't exist,
-   3. the v6.x `.kibana` index
+   1. the index the `.kibana` alias points to, or if it doesn't exist,
+   2. the v6.x `.kibana` index
    
    If none of the aliases exists, this is a new Elasticsearch cluster and no
    migrations are necessary. Create the `.kibana_7.10.0_001` index with the
-   following aliases: `.kibana_current` and `.kibana_7.10.0`.
+   following aliases: `.kibana` and `.kibana_7.10.0`.
 2. If the source is a < v6.5 `.kibana` index or < 7.4 `.kibana_task_manager`
    index prepare the legacy index for a migration:
    1. Mark the legacy index as read-only and wait for all in-flight operations to drain (requires https://github.com/elastic/elasticsearch/pull/58094). This prevents any further writes from outdated nodes. Assuming this API is similar to the existing `/<index>/_close` API, we expect to receive `"acknowledged" : true` and `"shards_acknowledged" : true`. If all shards don’t acknowledge within the timeout, retry the operation until it succeeds.
@@ -235,13 +234,13 @@ Note:
       atomically so that other Kibana instances will always see either a
       `.kibana` index or an alias, but never neither.
    6. Use the cloned `.kibana_pre6.5.0_001` as the source for the rest of the migration algorithm.
-3. If `.kibana_current` and `.kibana_7.10.0` both exists and are pointing to the same index this version's migration has already been completed.
+3. If `.kibana` and `.kibana_7.10.0` both exists and are pointing to the same index this version's migration has already been completed.
    1. Because the same version can have plugins enabled at any point in time,
       perform the mappings update in step (6) and migrate outdated documents
       with step (7).
    2. Skip to step (9) to start serving traffic.
 4. Fail the migration if:
-   1. `.kibana_current` is pointing to an index that belongs to a later version of Kibana .e.g. `.kibana_7.12.0_001`
+   1. `.kibana` is pointing to an index that belongs to a later version of Kibana .e.g. `.kibana_7.12.0_001`
    2. (Only in 8.x) The source index contains documents that belong to an unknown Saved Object type (from a disabled plugin). Log an error explaining that the plugin that created these documents needs to be enabled again or that these objects should be deleted. See section (4.2.1.4).
 5. Mark the source index as read-only and wait for all in-flight operations to drain (requires https://github.com/elastic/elasticsearch/pull/58094). This prevents any further writes from outdated nodes. Assuming this API is similar to the existing `/<index>/_close` API, we expect to receive `"acknowledged" : true` and `"shards_acknowledged" : true`. If all shards don’t acknowledge within the timeout, retry the operation until it succeeds.
 6. Clone the source index into a new target index which has writes enabled. All nodes on the same version will use the same fixed index name e.g. `.kibana_7.10.0_001`. The `001` postfix isn't used by Kibana, but allows for re-indexing an index should this be required by an Elasticsearch upgrade. E.g. re-index `.kibana_7.10.0_001` into `.kibana_7.10.0_002` and point the `.kibana_7.10.0` alias to `.kibana_7.10.0_002`.
@@ -257,24 +256,62 @@ Note:
 8. Transform documents by reading batches of outdated documents from the target index then transforming and updating them with optimistic concurrency control. 
    1. Ignore any version conflict errors.
    2. If a document transform throws an exception, add the document to a failure list and continue trying to transform all other documents. If any failures occured, log the complete list of documents that failed to transform. Fail the migration.
-9.  Mark the migration as complete by doing a single atomic operation (requires https://github.com/elastic/elasticsearch/pull/58100) that:
-   3. Checks that `.kibana_current` alias is still pointing to the source index
-   4. Points the `.kibana_7.10.0`  and `.kibana_current` aliases to the target index.
-   5. If this fails with a "required alias [.kibana_current] does not exist" error fetch `.kibana_current` again:
-      1. If `.kibana_current` is _not_ pointing to our target index fail the migration.
-      2. If `.kibana_current` is pointing to our target index the migration has succeeded and we can proceed to step (9).
-10. Start serving traffic.
-
-This algorithm shares a weakness with our existing migration algorithm
-(since v7.4). When the task manager index gets reindexed a reindex script is
-applied. Because we delete the original task manager index there is no way to
-rollback a failed task manager migration without a snapshot.
+9.  Mark the migration as complete. This is done as a single atomic
+    operation (requires https://github.com/elastic/elasticsearch/pull/58100)
+    to guarantees when multiple versions of Kibana are performing the
+    migration in parallel, only one version will win. E.g. if 7.11 and 7.12
+    are started in parallel and migrate from a 7.9 index, either 7.11 or 7.12
+    should succeed and accept writes, but not both.
+   3. Checks that `.kibana` alias is still pointing to the source index
+   4. Points the `.kibana_7.10.0` and `.kibana` aliases to the target index.
+   5. If this fails with a "required alias [.kibana] does not exist" error fetch `.kibana` again:
+      1. If `.kibana` is _not_ pointing to our target index fail the migration.
+      2. If `.kibana` is pointing to our target index the migration has succeeded and we can proceed to step (10).
+10. Start serving traffic. All saved object reads/writes happen through the
+    version-specific alias `.kibana_7.10.0`.
 
 Together with the limitations, this algorithm ensures that migrations are
 idempotent. If two nodes are started simultaneously, both of them will start
 transforming documents in that version's target index, but because migrations
 are idempotent, it doesn’t matter which node’s writes win.
 
+#### Known weaknesses:
+(Also present in our existing migration algorithm since v7.4)
+When the task manager index gets reindexed a reindex script is applied.
+Because we delete the original task manager index there is no way to rollback
+a failed task manager migration without a snapshot. Although losing the task
+manager data has a fairly low impact.
+
+(Also present in our existing migration algorithm since v6.5)
+If the outdated instance isn't shutdown before starting the migration, the
+following data-loss scenario is possible: 
+1. Upgrade a 7.9 index without shutting down the 7.9 nodes
+2. Kibana v7.10 performs a migration and after completing points `.kibana`
+   alias to `.kibana_7.11.0_001`
+3. Kibana v7.9 writes unmigrated documents into `.kibana`.
+4. Kibana v7.10 performs a query based on the updated mappings of documents so
+   results potentially don't match the acknowledged write from step (3).
+
+Note:
+ - Data loss won't occur if both nodes have the updated migration algorithm
+   proposed in this RFC. It is only when one of the nodes use the existing
+   algorithm that data loss is possible.
+ - Once v7.10 is restarted it will transform any outdated documents making
+   these visible to queries again.
+
+It is possible to work around this weakness by introducing a new alias such as
+`.kibana_current` so that after a migration the `.kibana` alias will continue
+to point to the outdated index. However, we decided to keep using the
+`.kibana` alias despite this weakness for the following reasons:
+  - Users might rely on `.kibana` alias for snapshots, so if this alias no
+    longer points to the latest index their snapshots would no longer backup
+    kibana's latest data.
+  - Introducing another alias introduces complexity for users and support.
+    The steps to diagnose, fix or rollback a failed migration will deviate
+    depending on the 7.x version of Kibana you are using.
+  - The existing Kibana documentation clearly states that outdated nodes should
+    be shutdown, this scenario has never been supported by Kibana.
+
 <details>
   <summary>In the future, this algorithm could enable (2.6) "read-only functionality during the downtime window" but this is outside of the scope of this RFC.</summary>
 
@@ -303,12 +340,9 @@ To rollback to a previous version of Kibana without a snapshot:
 (Assumes the migration to 7.11.0 failed)
 1. Shutdown all Kibana nodes.
 2. Remove the index created by the failed Kibana migration by using the version-specific alias e.g. `DELETE /.kibana_7.11.0` 
-3. Identify the rollback index:
-   1. If rolling back to a Kibana version < 7.10.0 use `.kibana`
-   2. If rolling back to a Kibana version >= 7.10.0 use the version alias of the Kibana version you wish to rollback to e.g. `.kibana_7.10.0`
-4. Point the `.kibana_current` alias to the rollback index.
-5. Remove the write block from the rollback index.
-6. Start the rollback Kibana nodes. All running Kibana nodes should be on the same rollback version, have the same plugins enabled and use the same configuration.
+3. Remove the write block from the rollback index using the `.kibana` alias
+   `PUT /.kibana/_settings {"index.blocks.write": false}`
+4. Start the rollback Kibana nodes. All running Kibana nodes should be on the same rollback version, have the same plugins enabled and use the same configuration.
 
 ### 4.2.1.4 Handling documents that belong to a disabled plugin
 It is possible for a plugin to create documents in one version of Kibana, but then when upgrading Kibana to a newer version, that plugin is disabled. Because the plugin is disabled it cannot register it's Saved Objects type including the mappings or any migration transformation functions. These "orphan" documents could cause future problems:
@@ -378,7 +412,7 @@ There are several approaches we could take to dealing with these orphan document
       deterministically perform the delete and re-clone operation without
       coordination.
 
-5. Transform outdated documents (step 7) on every startup
+5. Transform outdated documents (step 8) on every startup
     Advantages:
     - Outdated documents belonging to disabled plugins will be upgraded as soon
      as the plugin is enabled again.

From b3430e3f09deac0ed3e577d4f27ef7f9865f2b01 Mon Sep 17 00:00:00 2001
From: Liza Katz <lizka.k@gmail.com>
Date: Wed, 25 Nov 2020 16:32:05 +0200
Subject: [PATCH 07/37] [Search] Search batching using bfetch (again) (#84043)

Re-merging after cypress fixes
---
 ...plugin-plugins-data-public.plugin.setup.md |   4 +-
 ...ata-public.searchinterceptordeps.bfetch.md |  11 ++
 ...ugins-data-public.searchinterceptordeps.md |   1 +
 ...plugin-plugins-data-server.plugin.setup.md |   4 +-
 .../create_streaming_batched_function.test.ts | 127 +++++++++++++++++-
 .../create_streaming_batched_function.ts      |  87 ++++++++----
 src/plugins/bfetch/public/batching/types.ts   |  31 +++++
 src/plugins/bfetch/public/index.ts            |   2 +
 src/plugins/bfetch/public/plugin.ts           |   2 +-
 .../public/streaming/fetch_streaming.test.ts  |  27 ++++
 .../public/streaming/fetch_streaming.ts       |   4 +-
 .../streaming/from_streaming_xhr.test.ts      |  34 +++++
 .../public/streaming/from_streaming_xhr.ts    |  22 ++-
 src/plugins/data/kibana.json                  |   1 +
 src/plugins/data/public/plugin.ts             |   3 +-
 src/plugins/data/public/public.api.md         |   5 +-
 .../public/search/search_interceptor.test.ts  |  26 ++--
 .../data/public/search/search_interceptor.ts  |  37 +++--
 .../data/public/search/search_service.test.ts |   3 +
 .../data/public/search/search_service.ts      |   5 +-
 src/plugins/data/public/types.ts              |   2 +
 src/plugins/data/server/plugin.ts             |   5 +-
 .../data/server/search/search_service.test.ts |  18 ++-
 .../data/server/search/search_service.ts      |  56 +++++++-
 src/plugins/data/server/server.api.md         |   5 +-
 src/plugins/data/server/ui_settings.ts        |   7 +-
 src/plugins/embeddable/public/public.api.md   |   1 +
 x-pack/plugins/data_enhanced/kibana.json      |   1 +
 x-pack/plugins/data_enhanced/public/plugin.ts |   5 +-
 .../public/search/search_interceptor.test.ts  |  25 ++--
 .../security_solution/cypress/cypress.json    |   1 +
 .../fixtures/overview_search_strategy.json    |   6 +-
 .../cypress/integration/alerts.spec.ts        |  14 +-
 .../alerts_detection_rules.spec.ts            |   3 +-
 .../alerts_detection_rules_custom.spec.ts     |   2 +-
 .../alerts_detection_rules_export.spec.ts     |   3 +-
 .../cypress/integration/overview.spec.ts      |  16 +--
 .../cypress/support/commands.js               |  70 ++++++++--
 .../cypress/support/index.d.ts                |   7 +-
 .../es_archives/timeline_alerts/mappings.json |   5 +-
 40 files changed, 562 insertions(+), 126 deletions(-)
 create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md
 create mode 100644 src/plugins/bfetch/public/batching/types.ts

diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.plugin.setup.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.plugin.setup.md
index a0c9b38792825..1ed6059c23062 100644
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.plugin.setup.md
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.plugin.setup.md
@@ -7,7 +7,7 @@
 <b>Signature:</b>
 
 ```typescript
-setup(core: CoreSetup<DataStartDependencies, DataPublicPluginStart>, { expressions, uiActions, usageCollection }: DataSetupDependencies): DataPublicPluginSetup;
+setup(core: CoreSetup<DataStartDependencies, DataPublicPluginStart>, { bfetch, expressions, uiActions, usageCollection }: DataSetupDependencies): DataPublicPluginSetup;
 ```
 
 ## Parameters
@@ -15,7 +15,7 @@ setup(core: CoreSetup<DataStartDependencies, DataPublicPluginStart>, { expressio
 |  Parameter | Type | Description |
 |  --- | --- | --- |
 |  core | <code>CoreSetup&lt;DataStartDependencies, DataPublicPluginStart&gt;</code> |  |
-|  { expressions, uiActions, usageCollection } | <code>DataSetupDependencies</code> |  |
+|  { bfetch, expressions, uiActions, usageCollection } | <code>DataSetupDependencies</code> |  |
 
 <b>Returns:</b>
 
diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md
new file mode 100644
index 0000000000000..5b7c635c71529
--- /dev/null
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [SearchInterceptorDeps](./kibana-plugin-plugins-data-public.searchinterceptordeps.md) &gt; [bfetch](./kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md)
+
+## SearchInterceptorDeps.bfetch property
+
+<b>Signature:</b>
+
+```typescript
+bfetch: BfetchPublicSetup;
+```
diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.md
index 3653394d28b92..543566b783c23 100644
--- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.md
+++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptordeps.md
@@ -14,6 +14,7 @@ export interface SearchInterceptorDeps
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [bfetch](./kibana-plugin-plugins-data-public.searchinterceptordeps.bfetch.md) | <code>BfetchPublicSetup</code> |  |
 |  [http](./kibana-plugin-plugins-data-public.searchinterceptordeps.http.md) | <code>CoreSetup['http']</code> |  |
 |  [session](./kibana-plugin-plugins-data-public.searchinterceptordeps.session.md) | <code>ISessionService</code> |  |
 |  [startServices](./kibana-plugin-plugins-data-public.searchinterceptordeps.startservices.md) | <code>Promise&lt;[CoreStart, any, unknown]&gt;</code> |  |
diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.setup.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.setup.md
index 43129891c5412..b90018c3d9cdd 100644
--- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.setup.md
+++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.plugin.setup.md
@@ -7,7 +7,7 @@
 <b>Signature:</b>
 
 ```typescript
-setup(core: CoreSetup<DataPluginStartDependencies, DataPluginStart>, { expressions, usageCollection }: DataPluginSetupDependencies): {
+setup(core: CoreSetup<DataPluginStartDependencies, DataPluginStart>, { bfetch, expressions, usageCollection }: DataPluginSetupDependencies): {
         __enhance: (enhancements: DataEnhancements) => void;
         search: ISearchSetup;
         fieldFormats: {
@@ -21,7 +21,7 @@ setup(core: CoreSetup<DataPluginStartDependencies, DataPluginStart>, { expressio
 |  Parameter | Type | Description |
 |  --- | --- | --- |
 |  core | <code>CoreSetup&lt;DataPluginStartDependencies, DataPluginStart&gt;</code> |  |
-|  { expressions, usageCollection } | <code>DataPluginSetupDependencies</code> |  |
+|  { bfetch, expressions, usageCollection } | <code>DataPluginSetupDependencies</code> |  |
 
 <b>Returns:</b>
 
diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts
index d7dde8f1b93d3..3498f205b3286 100644
--- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts
+++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts
@@ -19,7 +19,7 @@
 
 import { createStreamingBatchedFunction } from './create_streaming_batched_function';
 import { fetchStreaming as fetchStreamingReal } from '../streaming/fetch_streaming';
-import { defer, of } from '../../../kibana_utils/public';
+import { AbortError, defer, of } from '../../../kibana_utils/public';
 import { Subject } from 'rxjs';
 
 const getPromiseState = (promise: Promise<unknown>): Promise<'resolved' | 'rejected' | 'pending'> =>
@@ -168,6 +168,28 @@ describe('createStreamingBatchedFunction()', () => {
       expect(fetchStreaming).toHaveBeenCalledTimes(1);
     });
 
+    test('ignores a request with an aborted signal', async () => {
+      const { fetchStreaming } = setup();
+      const fn = createStreamingBatchedFunction({
+        url: '/test',
+        fetchStreaming,
+        maxItemAge: 5,
+        flushOnMaxItems: 3,
+      });
+
+      const abortController = new AbortController();
+      abortController.abort();
+
+      of(fn({ foo: 'bar' }, abortController.signal));
+      fn({ baz: 'quix' });
+
+      await new Promise((r) => setTimeout(r, 6));
+      const { body } = fetchStreaming.mock.calls[0][0];
+      expect(JSON.parse(body)).toEqual({
+        batch: [{ baz: 'quix' }],
+      });
+    });
+
     test('sends POST request to correct endpoint with items in array batched sorted in call order', async () => {
       const { fetchStreaming } = setup();
       const fn = createStreamingBatchedFunction({
@@ -423,6 +445,73 @@ describe('createStreamingBatchedFunction()', () => {
       expect(result3).toEqual({ b: '3' });
     });
 
+    describe('when requests are aborted', () => {
+      test('aborts stream when all are aborted', async () => {
+        const { fetchStreaming } = setup();
+        const fn = createStreamingBatchedFunction({
+          url: '/test',
+          fetchStreaming,
+          maxItemAge: 5,
+          flushOnMaxItems: 3,
+        });
+
+        const abortController = new AbortController();
+        const promise = fn({ a: '1' }, abortController.signal);
+        const promise2 = fn({ a: '2' }, abortController.signal);
+        await new Promise((r) => setTimeout(r, 6));
+
+        expect(await isPending(promise)).toBe(true);
+        expect(await isPending(promise2)).toBe(true);
+
+        abortController.abort();
+        await new Promise((r) => setTimeout(r, 6));
+
+        expect(await isPending(promise)).toBe(false);
+        expect(await isPending(promise2)).toBe(false);
+        const [, error] = await of(promise);
+        const [, error2] = await of(promise2);
+        expect(error).toBeInstanceOf(AbortError);
+        expect(error2).toBeInstanceOf(AbortError);
+        expect(fetchStreaming.mock.calls[0][0].signal.aborted).toBeTruthy();
+      });
+
+      test('rejects promise on abort and lets others continue', async () => {
+        const { fetchStreaming, stream } = setup();
+        const fn = createStreamingBatchedFunction({
+          url: '/test',
+          fetchStreaming,
+          maxItemAge: 5,
+          flushOnMaxItems: 3,
+        });
+
+        const abortController = new AbortController();
+        const promise = fn({ a: '1' }, abortController.signal);
+        const promise2 = fn({ a: '2' });
+        await new Promise((r) => setTimeout(r, 6));
+
+        expect(await isPending(promise)).toBe(true);
+
+        abortController.abort();
+        await new Promise((r) => setTimeout(r, 6));
+
+        expect(await isPending(promise)).toBe(false);
+        const [, error] = await of(promise);
+        expect(error).toBeInstanceOf(AbortError);
+
+        stream.next(
+          JSON.stringify({
+            id: 1,
+            result: { b: '2' },
+          }) + '\n'
+        );
+
+        await new Promise((r) => setTimeout(r, 1));
+
+        const [result2] = await of(promise2);
+        expect(result2).toEqual({ b: '2' });
+      });
+    });
+
     describe('when stream closes prematurely', () => {
       test('rejects pending promises with CONNECTION error code', async () => {
         const { fetchStreaming, stream } = setup();
@@ -558,5 +647,41 @@ describe('createStreamingBatchedFunction()', () => {
         });
       });
     });
+
+    test('rejects with STREAM error on JSON parse error only pending promises', async () => {
+      const { fetchStreaming, stream } = setup();
+      const fn = createStreamingBatchedFunction({
+        url: '/test',
+        fetchStreaming,
+        maxItemAge: 5,
+        flushOnMaxItems: 3,
+      });
+
+      const promise1 = of(fn({ a: '1' }));
+      const promise2 = of(fn({ a: '2' }));
+
+      await new Promise((r) => setTimeout(r, 6));
+
+      stream.next(
+        JSON.stringify({
+          id: 1,
+          result: { b: '1' },
+        }) + '\n'
+      );
+
+      stream.next('Not a JSON\n');
+
+      await new Promise((r) => setTimeout(r, 1));
+
+      const [, error1] = await promise1;
+      const [result1] = await promise2;
+      expect(error1).toMatchObject({
+        message: 'Unexpected token N in JSON at position 0',
+        code: 'STREAM',
+      });
+      expect(result1).toMatchObject({
+        b: '1',
+      });
+    });
   });
 });
diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts
index 89793fff6b325..f3971ed04efa7 100644
--- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts
+++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.ts
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { defer, Defer } from '../../../kibana_utils/public';
+import { AbortError, abortSignalToPromise, defer } from '../../../kibana_utils/public';
 import {
   ItemBufferParams,
   TimedItemBufferParams,
@@ -27,13 +27,7 @@ import {
 } from '../../common';
 import { fetchStreaming, split } from '../streaming';
 import { normalizeError } from '../../common';
-
-export interface BatchItem<Payload, Result> {
-  payload: Payload;
-  future: Defer<Result>;
-}
-
-export type BatchedFunc<Payload, Result> = (payload: Payload) => Promise<Result>;
+import { BatchedFunc, BatchItem } from './types';
 
 export interface BatchedFunctionProtocolError extends ErrorLike {
   code: string;
@@ -82,43 +76,84 @@ export const createStreamingBatchedFunction = <Payload, Result extends object>(
     flushOnMaxItems = 25,
     maxItemAge = 10,
   } = params;
-  const [fn] = createBatchedFunction<BatchedFunc<Payload, Result>, BatchItem<Payload, Result>>({
-    onCall: (payload: Payload) => {
+  const [fn] = createBatchedFunction({
+    onCall: (payload: Payload, signal?: AbortSignal) => {
       const future = defer<Result>();
       const entry: BatchItem<Payload, Result> = {
         payload,
         future,
+        signal,
       };
       return [future.promise, entry];
     },
     onBatch: async (items) => {
       try {
-        let responsesReceived = 0;
-        const batch = items.map(({ payload }) => payload);
+        // Filter out any items whose signal is already aborted
+        items = items.filter((item) => {
+          if (item.signal?.aborted) item.future.reject(new AbortError());
+          return !item.signal?.aborted;
+        });
+
+        const donePromises: Array<Promise<any>> = items.map((item) => {
+          return new Promise<void>((resolve) => {
+            const { promise: abortPromise, cleanup } = item.signal
+              ? abortSignalToPromise(item.signal)
+              : {
+                  promise: undefined,
+                  cleanup: () => {},
+                };
+
+            const onDone = () => {
+              resolve();
+              cleanup();
+            };
+            if (abortPromise)
+              abortPromise.catch(() => {
+                item.future.reject(new AbortError());
+                onDone();
+              });
+            item.future.promise.then(onDone, onDone);
+          });
+        });
+
+        // abort when all items were either resolved, rejected or aborted
+        const abortController = new AbortController();
+        let isBatchDone = false;
+        Promise.all(donePromises).then(() => {
+          isBatchDone = true;
+          abortController.abort();
+        });
+        const batch = items.map((item) => item.payload);
+
         const { stream } = fetchStreamingInjected({
           url,
           body: JSON.stringify({ batch }),
           method: 'POST',
+          signal: abortController.signal,
         });
+
+        const handleStreamError = (error: any) => {
+          const normalizedError = normalizeError<BatchedFunctionProtocolError>(error);
+          normalizedError.code = 'STREAM';
+          for (const { future } of items) future.reject(normalizedError);
+        };
+
         stream.pipe(split('\n')).subscribe({
           next: (json: string) => {
-            const response = JSON.parse(json) as BatchResponseItem<Result, ErrorLike>;
-            if (response.error) {
-              responsesReceived++;
-              items[response.id].future.reject(response.error);
-            } else if (response.result !== undefined) {
-              responsesReceived++;
-              items[response.id].future.resolve(response.result);
+            try {
+              const response = JSON.parse(json) as BatchResponseItem<Result, ErrorLike>;
+              if (response.error) {
+                items[response.id].future.reject(response.error);
+              } else if (response.result !== undefined) {
+                items[response.id].future.resolve(response.result);
+              }
+            } catch (e) {
+              handleStreamError(e);
             }
           },
-          error: (error) => {
-            const normalizedError = normalizeError<BatchedFunctionProtocolError>(error);
-            normalizedError.code = 'STREAM';
-            for (const { future } of items) future.reject(normalizedError);
-          },
+          error: handleStreamError,
           complete: () => {
-            const streamTerminatedPrematurely = responsesReceived !== items.length;
-            if (streamTerminatedPrematurely) {
+            if (!isBatchDone) {
               const error: BatchedFunctionProtocolError = {
                 message: 'Connection terminated prematurely.',
                 code: 'CONNECTION',
diff --git a/src/plugins/bfetch/public/batching/types.ts b/src/plugins/bfetch/public/batching/types.ts
new file mode 100644
index 0000000000000..68860c5d9eedf
--- /dev/null
+++ b/src/plugins/bfetch/public/batching/types.ts
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Defer } from '../../../kibana_utils/public';
+
+export interface BatchItem<Payload, Result> {
+  payload: Payload;
+  future: Defer<Result>;
+  signal?: AbortSignal;
+}
+
+export type BatchedFunc<Payload, Result> = (
+  payload: Payload,
+  signal?: AbortSignal
+) => Promise<Result>;
diff --git a/src/plugins/bfetch/public/index.ts b/src/plugins/bfetch/public/index.ts
index 8707e5a438159..7ff110105faa0 100644
--- a/src/plugins/bfetch/public/index.ts
+++ b/src/plugins/bfetch/public/index.ts
@@ -23,6 +23,8 @@ import { BfetchPublicPlugin } from './plugin';
 export { BfetchPublicSetup, BfetchPublicStart, BfetchPublicContract } from './plugin';
 export { split } from './streaming';
 
+export { BatchedFunc } from './batching/types';
+
 export function plugin(initializerContext: PluginInitializerContext) {
   return new BfetchPublicPlugin(initializerContext);
 }
diff --git a/src/plugins/bfetch/public/plugin.ts b/src/plugins/bfetch/public/plugin.ts
index 5f01957c0908e..72aaa862b0ad2 100644
--- a/src/plugins/bfetch/public/plugin.ts
+++ b/src/plugins/bfetch/public/plugin.ts
@@ -22,9 +22,9 @@ import { fetchStreaming as fetchStreamingStatic, FetchStreamingParams } from './
 import { removeLeadingSlash } from '../common';
 import {
   createStreamingBatchedFunction,
-  BatchedFunc,
   StreamingBatchedFunctionParams,
 } from './batching/create_streaming_batched_function';
+import { BatchedFunc } from './batching/types';
 
 // eslint-disable-next-line
 export interface BfetchPublicSetupDependencies {}
diff --git a/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts b/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts
index 27adc6dc8b549..7a6827b8fee8e 100644
--- a/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts
+++ b/src/plugins/bfetch/public/streaming/fetch_streaming.test.ts
@@ -132,6 +132,33 @@ test('completes stream observable when request finishes', async () => {
   expect(spy).toHaveBeenCalledTimes(1);
 });
 
+test('completes stream observable when aborted', async () => {
+  const env = setup();
+  const abort = new AbortController();
+  const { stream } = fetchStreaming({
+    url: 'http://example.com',
+    signal: abort.signal,
+  });
+
+  const spy = jest.fn();
+  stream.subscribe({
+    complete: spy,
+  });
+
+  expect(spy).toHaveBeenCalledTimes(0);
+
+  (env.xhr as any).responseText = 'foo';
+  env.xhr.onprogress!({} as any);
+
+  abort.abort();
+
+  (env.xhr as any).readyState = 4;
+  (env.xhr as any).status = 200;
+  env.xhr.onreadystatechange!({} as any);
+
+  expect(spy).toHaveBeenCalledTimes(1);
+});
+
 test('promise throws when request errors', async () => {
   const env = setup();
   const { stream } = fetchStreaming({
diff --git a/src/plugins/bfetch/public/streaming/fetch_streaming.ts b/src/plugins/bfetch/public/streaming/fetch_streaming.ts
index 899e8a1824a41..3deee0cf66add 100644
--- a/src/plugins/bfetch/public/streaming/fetch_streaming.ts
+++ b/src/plugins/bfetch/public/streaming/fetch_streaming.ts
@@ -24,6 +24,7 @@ export interface FetchStreamingParams {
   headers?: Record<string, string>;
   method?: 'GET' | 'POST';
   body?: string;
+  signal?: AbortSignal;
 }
 
 /**
@@ -35,6 +36,7 @@ export function fetchStreaming({
   headers = {},
   method = 'POST',
   body = '',
+  signal,
 }: FetchStreamingParams) {
   const xhr = new window.XMLHttpRequest();
 
@@ -45,7 +47,7 @@ export function fetchStreaming({
   // Set the HTTP headers
   Object.entries(headers).forEach(([k, v]) => xhr.setRequestHeader(k, v));
 
-  const stream = fromStreamingXhr(xhr);
+  const stream = fromStreamingXhr(xhr, signal);
 
   // Send the payload to the server
   xhr.send(body);
diff --git a/src/plugins/bfetch/public/streaming/from_streaming_xhr.test.ts b/src/plugins/bfetch/public/streaming/from_streaming_xhr.test.ts
index 40eb3d5e2556b..b15bf9bdfbbb0 100644
--- a/src/plugins/bfetch/public/streaming/from_streaming_xhr.test.ts
+++ b/src/plugins/bfetch/public/streaming/from_streaming_xhr.test.ts
@@ -21,6 +21,7 @@ import { fromStreamingXhr } from './from_streaming_xhr';
 
 const createXhr = (): XMLHttpRequest =>
   (({
+    abort: () => {},
     onprogress: () => {},
     onreadystatechange: () => {},
     readyState: 0,
@@ -100,6 +101,39 @@ test('completes observable when request reaches end state', () => {
   expect(complete).toHaveBeenCalledTimes(1);
 });
 
+test('completes observable when aborted', () => {
+  const xhr = createXhr();
+  const abortController = new AbortController();
+  const observable = fromStreamingXhr(xhr, abortController.signal);
+
+  const next = jest.fn();
+  const complete = jest.fn();
+  observable.subscribe({
+    next,
+    complete,
+  });
+
+  (xhr as any).responseText = '1';
+  xhr.onprogress!({} as any);
+
+  (xhr as any).responseText = '2';
+  xhr.onprogress!({} as any);
+
+  expect(complete).toHaveBeenCalledTimes(0);
+
+  (xhr as any).readyState = 2;
+  abortController.abort();
+
+  expect(complete).toHaveBeenCalledTimes(1);
+
+  // Shouldn't trigger additional events
+  (xhr as any).readyState = 4;
+  (xhr as any).status = 200;
+  xhr.onreadystatechange!({} as any);
+
+  expect(complete).toHaveBeenCalledTimes(1);
+});
+
 test('errors observable if request returns with error', () => {
   const xhr = createXhr();
   const observable = fromStreamingXhr(xhr);
diff --git a/src/plugins/bfetch/public/streaming/from_streaming_xhr.ts b/src/plugins/bfetch/public/streaming/from_streaming_xhr.ts
index bba8151958492..5df1f5258cb2d 100644
--- a/src/plugins/bfetch/public/streaming/from_streaming_xhr.ts
+++ b/src/plugins/bfetch/public/streaming/from_streaming_xhr.ts
@@ -26,13 +26,17 @@ import { Observable, Subject } from 'rxjs';
 export const fromStreamingXhr = (
   xhr: Pick<
     XMLHttpRequest,
-    'onprogress' | 'onreadystatechange' | 'readyState' | 'status' | 'responseText'
-  >
+    'onprogress' | 'onreadystatechange' | 'readyState' | 'status' | 'responseText' | 'abort'
+  >,
+  signal?: AbortSignal
 ): Observable<string> => {
   const subject = new Subject<string>();
   let index = 0;
+  let aborted = false;
 
   const processBatch = () => {
+    if (aborted) return;
+
     const { responseText } = xhr;
     if (index >= responseText.length) return;
     subject.next(responseText.substr(index));
@@ -41,7 +45,19 @@ export const fromStreamingXhr = (
 
   xhr.onprogress = processBatch;
 
+  const onBatchAbort = () => {
+    if (xhr.readyState !== 4) {
+      aborted = true;
+      xhr.abort();
+      subject.complete();
+      if (signal) signal.removeEventListener('abort', onBatchAbort);
+    }
+  };
+
+  if (signal) signal.addEventListener('abort', onBatchAbort);
+
   xhr.onreadystatechange = () => {
+    if (aborted) return;
     // Older browsers don't support onprogress, so we need
     // to call this here, too. It's safe to call this multiple
     // times even for the same progress event.
@@ -49,6 +65,8 @@ export const fromStreamingXhr = (
 
     // 4 is the magic number that means the request is done
     if (xhr.readyState === 4) {
+      if (signal) signal.removeEventListener('abort', onBatchAbort);
+
       // 0 indicates a network failure. 400+ messages are considered server errors
       if (xhr.status === 0 || xhr.status >= 400) {
         subject.error(new Error(`Batch request failed with status ${xhr.status}`));
diff --git a/src/plugins/data/kibana.json b/src/plugins/data/kibana.json
index d6f2534bd5e3b..3e4d08c8faa1b 100644
--- a/src/plugins/data/kibana.json
+++ b/src/plugins/data/kibana.json
@@ -4,6 +4,7 @@
   "server": true,
   "ui": true,
   "requiredPlugins": [
+    "bfetch",
     "expressions",
     "uiActions"
   ],
diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts
index 7e8283476ffc5..dded52310a99c 100644
--- a/src/plugins/data/public/plugin.ts
+++ b/src/plugins/data/public/plugin.ts
@@ -105,7 +105,7 @@ export class DataPublicPlugin
 
   public setup(
     core: CoreSetup<DataStartDependencies, DataPublicPluginStart>,
-    { expressions, uiActions, usageCollection }: DataSetupDependencies
+    { bfetch, expressions, uiActions, usageCollection }: DataSetupDependencies
   ): DataPublicPluginSetup {
     const startServices = createStartServicesGetter(core.getStartServices);
 
@@ -152,6 +152,7 @@ export class DataPublicPlugin
     );
 
     const searchService = this.searchService.setup(core, {
+      bfetch,
       usageCollection,
       expressions,
     });
diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md
index 7c4d3ee27cf4a..a6daaf834a424 100644
--- a/src/plugins/data/public/public.api.md
+++ b/src/plugins/data/public/public.api.md
@@ -12,6 +12,7 @@ import { ApiResponse as ApiResponse_2 } from '@elastic/elasticsearch/lib/Transpo
 import { ApplicationStart } from 'kibana/public';
 import { Assign } from '@kbn/utility-types';
 import { BehaviorSubject } from 'rxjs';
+import { BfetchPublicSetup } from 'src/plugins/bfetch/public';
 import Boom from '@hapi/boom';
 import { CoreSetup } from 'src/core/public';
 import { CoreSetup as CoreSetup_2 } from 'kibana/public';
@@ -1734,7 +1735,7 @@ export class Plugin implements Plugin_2<DataPublicPluginSetup, DataPublicPluginS
     // Warning: (ae-forgotten-export) The symbol "ConfigSchema" needs to be exported by the entry point index.d.ts
     constructor(initializerContext: PluginInitializerContext_2<ConfigSchema>);
     // (undocumented)
-    setup(core: CoreSetup<DataStartDependencies, DataPublicPluginStart>, { expressions, uiActions, usageCollection }: DataSetupDependencies): DataPublicPluginSetup;
+    setup(core: CoreSetup<DataStartDependencies, DataPublicPluginStart>, { bfetch, expressions, uiActions, usageCollection }: DataSetupDependencies): DataPublicPluginSetup;
     // (undocumented)
     start(core: CoreStart_2, { uiActions }: DataStartDependencies): DataPublicPluginStart;
     // (undocumented)
@@ -2113,6 +2114,8 @@ export class SearchInterceptor {
 //
 // @public (undocumented)
 export interface SearchInterceptorDeps {
+    // (undocumented)
+    bfetch: BfetchPublicSetup;
     // (undocumented)
     http: CoreSetup_2['http'];
     // (undocumented)
diff --git a/src/plugins/data/public/search/search_interceptor.test.ts b/src/plugins/data/public/search/search_interceptor.test.ts
index 6e75f6e5eef9e..6dc52d7016797 100644
--- a/src/plugins/data/public/search/search_interceptor.test.ts
+++ b/src/plugins/data/public/search/search_interceptor.test.ts
@@ -25,9 +25,13 @@ import { AbortError } from '../../../kibana_utils/public';
 import { SearchTimeoutError, PainlessError, TimeoutErrorMode } from './errors';
 import { searchServiceMock } from './mocks';
 import { ISearchStart } from '.';
+import { bfetchPluginMock } from '../../../bfetch/public/mocks';
+import { BfetchPublicSetup } from 'src/plugins/bfetch/public';
 
 let searchInterceptor: SearchInterceptor;
 let mockCoreSetup: MockedKeys<CoreSetup>;
+let bfetchSetup: jest.Mocked<BfetchPublicSetup>;
+let fetchMock: jest.Mock<any>;
 
 const flushPromises = () => new Promise((resolve) => setImmediate(resolve));
 jest.useFakeTimers();
@@ -39,7 +43,11 @@ describe('SearchInterceptor', () => {
     mockCoreSetup = coreMock.createSetup();
     mockCoreStart = coreMock.createStart();
     searchMock = searchServiceMock.createStartContract();
+    fetchMock = jest.fn();
+    bfetchSetup = bfetchPluginMock.createSetupContract();
+    bfetchSetup.batchedFunction.mockReturnValue(fetchMock);
     searchInterceptor = new SearchInterceptor({
+      bfetch: bfetchSetup,
       toasts: mockCoreSetup.notifications.toasts,
       startServices: new Promise((resolve) => {
         resolve([mockCoreStart, {}, {}]);
@@ -91,7 +99,7 @@ describe('SearchInterceptor', () => {
   describe('search', () => {
     test('Observable should resolve if fetch is successful', async () => {
       const mockResponse: any = { result: 200 };
-      mockCoreSetup.http.fetch.mockResolvedValueOnce(mockResponse);
+      fetchMock.mockResolvedValueOnce(mockResponse);
       const mockRequest: IEsSearchRequest = {
         params: {},
       };
@@ -102,7 +110,7 @@ describe('SearchInterceptor', () => {
     describe('Should throw typed errors', () => {
       test('Observable should fail if fetch has an internal error', async () => {
         const mockResponse: any = new Error('Internal Error');
-        mockCoreSetup.http.fetch.mockRejectedValue(mockResponse);
+        fetchMock.mockRejectedValue(mockResponse);
         const mockRequest: IEsSearchRequest = {
           params: {},
         };
@@ -118,7 +126,7 @@ describe('SearchInterceptor', () => {
               message: 'Request timed out',
             },
           };
-          mockCoreSetup.http.fetch.mockRejectedValueOnce(mockResponse);
+          fetchMock.mockRejectedValueOnce(mockResponse);
           const mockRequest: IEsSearchRequest = {
             params: {},
           };
@@ -134,7 +142,7 @@ describe('SearchInterceptor', () => {
               message: 'Request timed out',
             },
           };
-          mockCoreSetup.http.fetch.mockRejectedValue(mockResponse);
+          fetchMock.mockRejectedValue(mockResponse);
           const mockRequest: IEsSearchRequest = {
             params: {},
           };
@@ -155,7 +163,7 @@ describe('SearchInterceptor', () => {
               message: 'Request timed out',
             },
           };
-          mockCoreSetup.http.fetch.mockRejectedValue(mockResponse);
+          fetchMock.mockRejectedValue(mockResponse);
           const mockRequest: IEsSearchRequest = {
             params: {},
           };
@@ -176,7 +184,7 @@ describe('SearchInterceptor', () => {
               message: 'Request timed out',
             },
           };
-          mockCoreSetup.http.fetch.mockRejectedValue(mockResponse);
+          fetchMock.mockRejectedValue(mockResponse);
           const mockRequest: IEsSearchRequest = {
             params: {},
           };
@@ -209,7 +217,7 @@ describe('SearchInterceptor', () => {
             },
           },
         };
-        mockCoreSetup.http.fetch.mockRejectedValueOnce(mockResponse);
+        fetchMock.mockRejectedValueOnce(mockResponse);
         const mockRequest: IEsSearchRequest = {
           params: {},
         };
@@ -219,7 +227,7 @@ describe('SearchInterceptor', () => {
 
       test('Observable should fail if user aborts (test merged signal)', async () => {
         const abortController = new AbortController();
-        mockCoreSetup.http.fetch.mockImplementationOnce((options: any) => {
+        fetchMock.mockImplementationOnce((options: any) => {
           return new Promise((resolve, reject) => {
             options.signal.addEventListener('abort', () => {
               reject(new AbortError());
@@ -257,7 +265,7 @@ describe('SearchInterceptor', () => {
 
         const error = (e: any) => {
           expect(e).toBeInstanceOf(AbortError);
-          expect(mockCoreSetup.http.fetch).not.toBeCalled();
+          expect(fetchMock).not.toBeCalled();
           done();
         };
         response.subscribe({ error });
diff --git a/src/plugins/data/public/search/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor.ts
index e5abac0d48fef..055b3a71705bf 100644
--- a/src/plugins/data/public/search/search_interceptor.ts
+++ b/src/plugins/data/public/search/search_interceptor.ts
@@ -17,17 +17,17 @@
  * under the License.
  */
 
-import { get, memoize, trimEnd } from 'lodash';
+import { get, memoize } from 'lodash';
 import { BehaviorSubject, throwError, timer, defer, from, Observable, NEVER } from 'rxjs';
 import { catchError, finalize } from 'rxjs/operators';
 import { PublicMethodsOf } from '@kbn/utility-types';
 import { CoreStart, CoreSetup, ToastsSetup } from 'kibana/public';
 import { i18n } from '@kbn/i18n';
+import { BatchedFunc, BfetchPublicSetup } from 'src/plugins/bfetch/public';
 import {
   IKibanaSearchRequest,
   IKibanaSearchResponse,
   ISearchOptions,
-  ES_SEARCH_STRATEGY,
   ISessionService,
 } from '../../common';
 import { SearchUsageCollector } from './collectors';
@@ -44,6 +44,7 @@ import { toMountPoint } from '../../../kibana_react/public';
 import { AbortError, getCombinedAbortSignal } from '../../../kibana_utils/public';
 
 export interface SearchInterceptorDeps {
+  bfetch: BfetchPublicSetup;
   http: CoreSetup['http'];
   uiSettings: CoreSetup['uiSettings'];
   startServices: Promise<[CoreStart, any, unknown]>;
@@ -69,6 +70,10 @@ export class SearchInterceptor {
    * @internal
    */
   protected application!: CoreStart['application'];
+  private batchedFetch!: BatchedFunc<
+    { request: IKibanaSearchRequest; options: ISearchOptions },
+    IKibanaSearchResponse
+  >;
 
   /*
    * @internal
@@ -79,6 +84,10 @@ export class SearchInterceptor {
     this.deps.startServices.then(([coreStart]) => {
       this.application = coreStart.application;
     });
+
+    this.batchedFetch = deps.bfetch.batchedFunction({
+      url: '/internal/bsearch',
+    });
   }
 
   /*
@@ -123,24 +132,14 @@ export class SearchInterceptor {
     request: IKibanaSearchRequest,
     options?: ISearchOptions
   ): Promise<IKibanaSearchResponse> {
-    const { id, ...searchRequest } = request;
-    const path = trimEnd(
-      `/internal/search/${options?.strategy ?? ES_SEARCH_STRATEGY}/${id ?? ''}`,
-      '/'
+    const { abortSignal, ...requestOptions } = options || {};
+    return this.batchedFetch(
+      {
+        request,
+        options: requestOptions,
+      },
+      abortSignal
     );
-    const body = JSON.stringify({
-      sessionId: options?.sessionId,
-      isStored: options?.isStored,
-      isRestore: options?.isRestore,
-      ...searchRequest,
-    });
-
-    return this.deps.http.fetch({
-      method: 'POST',
-      path,
-      body,
-      signal: options?.abortSignal,
-    });
   }
 
   /**
diff --git a/src/plugins/data/public/search/search_service.test.ts b/src/plugins/data/public/search/search_service.test.ts
index 20041a02067d9..3179da4d03a1a 100644
--- a/src/plugins/data/public/search/search_service.test.ts
+++ b/src/plugins/data/public/search/search_service.test.ts
@@ -21,6 +21,7 @@ import { coreMock } from '../../../../core/public/mocks';
 import { CoreSetup, CoreStart } from '../../../../core/public';
 
 import { SearchService, SearchServiceSetupDependencies } from './search_service';
+import { bfetchPluginMock } from '../../../bfetch/public/mocks';
 
 describe('Search service', () => {
   let searchService: SearchService;
@@ -39,8 +40,10 @@ describe('Search service', () => {
 
   describe('setup()', () => {
     it('exposes proper contract', async () => {
+      const bfetch = bfetchPluginMock.createSetupContract();
       const setup = searchService.setup(mockCoreSetup, ({
         packageInfo: { version: '8' },
+        bfetch,
         expressions: { registerFunction: jest.fn(), registerType: jest.fn() },
       } as unknown) as SearchServiceSetupDependencies);
       expect(setup).toHaveProperty('aggs');
diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts
index 96fb3f91ea85f..b76b5846d3d93 100644
--- a/src/plugins/data/public/search/search_service.ts
+++ b/src/plugins/data/public/search/search_service.ts
@@ -19,6 +19,7 @@
 
 import { Plugin, CoreSetup, CoreStart, PluginInitializerContext } from 'src/core/public';
 import { BehaviorSubject } from 'rxjs';
+import { BfetchPublicSetup } from 'src/plugins/bfetch/public';
 import { ISearchSetup, ISearchStart, SearchEnhancements } from './types';
 
 import { handleResponse } from './fetch';
@@ -49,6 +50,7 @@ import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn';
 
 /** @internal */
 export interface SearchServiceSetupDependencies {
+  bfetch: BfetchPublicSetup;
   expressions: ExpressionsSetup;
   usageCollection?: UsageCollectionSetup;
 }
@@ -70,7 +72,7 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
 
   public setup(
     { http, getStartServices, notifications, uiSettings }: CoreSetup,
-    { expressions, usageCollection }: SearchServiceSetupDependencies
+    { bfetch, expressions, usageCollection }: SearchServiceSetupDependencies
   ): ISearchSetup {
     this.usageCollector = createUsageCollector(getStartServices, usageCollection);
 
@@ -80,6 +82,7 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
      * all pending search requests, as well as getting the number of pending search requests.
      */
     this.searchInterceptor = new SearchInterceptor({
+      bfetch,
       toasts: notifications.toasts,
       http,
       uiSettings,
diff --git a/src/plugins/data/public/types.ts b/src/plugins/data/public/types.ts
index 21a03a49fe058..4082fbe55094c 100644
--- a/src/plugins/data/public/types.ts
+++ b/src/plugins/data/public/types.ts
@@ -19,6 +19,7 @@
 
 import React from 'react';
 import { CoreStart } from 'src/core/public';
+import { BfetchPublicSetup } from 'src/plugins/bfetch/public';
 import { IStorageWrapper } from 'src/plugins/kibana_utils/public';
 import { ExpressionsSetup } from 'src/plugins/expressions/public';
 import { UiActionsSetup, UiActionsStart } from 'src/plugins/ui_actions/public';
@@ -36,6 +37,7 @@ export interface DataPublicPluginEnhancements {
 }
 
 export interface DataSetupDependencies {
+  bfetch: BfetchPublicSetup;
   expressions: ExpressionsSetup;
   uiActions: UiActionsSetup;
   usageCollection?: UsageCollectionSetup;
diff --git a/src/plugins/data/server/plugin.ts b/src/plugins/data/server/plugin.ts
index 3ec4e7e64e382..bba2c368ff7d1 100644
--- a/src/plugins/data/server/plugin.ts
+++ b/src/plugins/data/server/plugin.ts
@@ -19,6 +19,7 @@
 
 import { PluginInitializerContext, CoreSetup, CoreStart, Plugin, Logger } from 'src/core/server';
 import { ExpressionsServerSetup } from 'src/plugins/expressions/server';
+import { BfetchServerSetup } from 'src/plugins/bfetch/server';
 import { ConfigSchema } from '../config';
 import { IndexPatternsService, IndexPatternsServiceStart } from './index_patterns';
 import { ISearchSetup, ISearchStart, SearchEnhancements } from './search';
@@ -51,6 +52,7 @@ export interface DataPluginStart {
 }
 
 export interface DataPluginSetupDependencies {
+  bfetch: BfetchServerSetup;
   expressions: ExpressionsServerSetup;
   usageCollection?: UsageCollectionSetup;
 }
@@ -85,7 +87,7 @@ export class DataServerPlugin
 
   public setup(
     core: CoreSetup<DataPluginStartDependencies, DataPluginStart>,
-    { expressions, usageCollection }: DataPluginSetupDependencies
+    { bfetch, expressions, usageCollection }: DataPluginSetupDependencies
   ) {
     this.indexPatterns.setup(core);
     this.scriptsService.setup(core);
@@ -96,6 +98,7 @@ export class DataServerPlugin
     core.uiSettings.register(getUiSettings());
 
     const searchSetup = this.searchService.setup(core, {
+      bfetch,
       expressions,
       usageCollection,
     });
diff --git a/src/plugins/data/server/search/search_service.test.ts b/src/plugins/data/server/search/search_service.test.ts
index 0700afd8d6c83..8a52d1d415f9b 100644
--- a/src/plugins/data/server/search/search_service.test.ts
+++ b/src/plugins/data/server/search/search_service.test.ts
@@ -25,6 +25,8 @@ import { createFieldFormatsStartMock } from '../field_formats/mocks';
 import { createIndexPatternsStartMock } from '../index_patterns/mocks';
 
 import { SearchService, SearchServiceSetupDependencies } from './search_service';
+import { bfetchPluginMock } from '../../../bfetch/server/mocks';
+import { of } from 'rxjs';
 
 describe('Search service', () => {
   let plugin: SearchService;
@@ -35,15 +37,29 @@ describe('Search service', () => {
     const mockLogger: any = {
       debug: () => {},
     };
-    plugin = new SearchService(coreMock.createPluginInitializerContext({}), mockLogger);
+    const context = coreMock.createPluginInitializerContext({});
+    context.config.create = jest.fn().mockImplementation(() => {
+      return of({
+        search: {
+          aggs: {
+            shardDelay: {
+              enabled: true,
+            },
+          },
+        },
+      });
+    });
+    plugin = new SearchService(context, mockLogger);
     mockCoreSetup = coreMock.createSetup();
     mockCoreStart = coreMock.createStart();
   });
 
   describe('setup()', () => {
     it('exposes proper contract', async () => {
+      const bfetch = bfetchPluginMock.createSetupContract();
       const setup = plugin.setup(mockCoreSetup, ({
         packageInfo: { version: '8' },
+        bfetch,
         expressions: {
           registerFunction: jest.fn(),
           registerType: jest.fn(),
diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts
index b44980164d097..a9539a8fd3c15 100644
--- a/src/plugins/data/server/search/search_service.ts
+++ b/src/plugins/data/server/search/search_service.ts
@@ -29,7 +29,8 @@ import {
   SharedGlobalConfig,
   StartServicesAccessor,
 } from 'src/core/server';
-import { first, switchMap } from 'rxjs/operators';
+import { catchError, first, map, switchMap } from 'rxjs/operators';
+import { BfetchServerSetup } from 'src/plugins/bfetch/server';
 import { ExpressionsServerSetup } from 'src/plugins/expressions/server';
 import {
   ISearchSetup,
@@ -43,7 +44,7 @@ import { AggsService } from './aggs';
 
 import { FieldFormatsStart } from '../field_formats';
 import { IndexPatternsServiceStart } from '../index_patterns';
-import { getCallMsearch, registerMsearchRoute, registerSearchRoute } from './routes';
+import { getCallMsearch, registerMsearchRoute, registerSearchRoute, shimHitsTotal } from './routes';
 import { ES_SEARCH_STRATEGY, esSearchStrategyProvider } from './es_search';
 import { DataPluginStart } from '../plugin';
 import { UsageCollectionSetup } from '../../../usage_collection/server';
@@ -85,6 +86,7 @@ type StrategyMap = Record<string, ISearchStrategy<any, any>>;
 
 /** @internal */
 export interface SearchServiceSetupDependencies {
+  bfetch: BfetchServerSetup;
   expressions: ExpressionsServerSetup;
   usageCollection?: UsageCollectionSetup;
 }
@@ -106,6 +108,7 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
   private readonly searchSourceService = new SearchSourceService();
   private defaultSearchStrategyName: string = ES_SEARCH_STRATEGY;
   private searchStrategies: StrategyMap = {};
+  private coreStart?: CoreStart;
   private sessionService: BackgroundSessionService = new BackgroundSessionService();
 
   constructor(
@@ -115,7 +118,7 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
 
   public setup(
     core: CoreSetup<{}, DataPluginStart>,
-    { expressions, usageCollection }: SearchServiceSetupDependencies
+    { bfetch, expressions, usageCollection }: SearchServiceSetupDependencies
   ): ISearchSetup {
     const usage = usageCollection ? usageProvider(core) : undefined;
 
@@ -128,10 +131,13 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
     registerMsearchRoute(router, routeDependencies);
     registerSessionRoutes(router);
 
+    core.getStartServices().then(([coreStart]) => {
+      this.coreStart = coreStart;
+    });
+
     core.http.registerRouteHandlerContext('search', async (context, request) => {
-      const [coreStart] = await core.getStartServices();
-      const search = this.asScopedProvider(coreStart)(request);
-      const session = this.sessionService.asScopedProvider(coreStart)(request);
+      const search = this.asScopedProvider(this.coreStart!)(request);
+      const session = this.sessionService.asScopedProvider(this.coreStart!)(request);
       return { ...search, session };
     });
 
@@ -146,6 +152,44 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
       )
     );
 
+    bfetch.addBatchProcessingRoute<
+      { request: IKibanaSearchResponse; options?: ISearchOptions },
+      any
+    >('/internal/bsearch', (request) => {
+      const search = this.asScopedProvider(this.coreStart!)(request);
+
+      return {
+        onBatchItem: async ({ request: requestData, options }) => {
+          return search
+            .search(requestData, options)
+            .pipe(
+              first(),
+              map((response) => {
+                return {
+                  ...response,
+                  ...{
+                    rawResponse: shimHitsTotal(response.rawResponse),
+                  },
+                };
+              }),
+              catchError((err) => {
+                // eslint-disable-next-line no-throw-literal
+                throw {
+                  statusCode: err.statusCode || 500,
+                  body: {
+                    message: err.message,
+                    attributes: {
+                      error: err.body?.error || err.message,
+                    },
+                  },
+                };
+              })
+            )
+            .toPromise();
+        },
+      };
+    });
+
     core.savedObjects.registerType(searchTelemetry);
     if (usageCollection) {
       registerUsageCollector(usageCollection, this.initializerContext);
diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md
index 73e2a68cb9667..86ec784834ace 100644
--- a/src/plugins/data/server/server.api.md
+++ b/src/plugins/data/server/server.api.md
@@ -9,6 +9,7 @@ import { Adapters } from 'src/plugins/inspector/common';
 import { ApiResponse } from '@elastic/elasticsearch';
 import { Assign } from '@kbn/utility-types';
 import { BehaviorSubject } from 'rxjs';
+import { BfetchServerSetup } from 'src/plugins/bfetch/server';
 import { ConfigDeprecationProvider } from '@kbn/config';
 import { CoreSetup } from 'src/core/server';
 import { CoreSetup as CoreSetup_2 } from 'kibana/server';
@@ -948,7 +949,7 @@ export function parseInterval(interval: string): moment.Duration | null;
 export class Plugin implements Plugin_2<PluginSetup, PluginStart, DataPluginSetupDependencies, DataPluginStartDependencies> {
     constructor(initializerContext: PluginInitializerContext_2<ConfigSchema>);
     // (undocumented)
-    setup(core: CoreSetup<DataPluginStartDependencies, PluginStart>, { expressions, usageCollection }: DataPluginSetupDependencies): {
+    setup(core: CoreSetup<DataPluginStartDependencies, PluginStart>, { bfetch, expressions, usageCollection }: DataPluginSetupDependencies): {
         __enhance: (enhancements: DataEnhancements) => void;
         search: ISearchSetup;
         fieldFormats: {
@@ -1250,7 +1251,7 @@ export function usageProvider(core: CoreSetup_2): SearchUsage;
 // src/plugins/data/server/index.ts:274:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/index.ts:275:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/index_patterns/index_patterns_service.ts:58:14 - (ae-forgotten-export) The symbol "IndexPatternsService" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/plugin.ts:88:66 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/plugin.ts:90:74 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/search/types.ts:104:5 - (ae-forgotten-export) The symbol "ISearchStartSearchSource" needs to be exported by the entry point index.d.ts
 
 // (No @packageDocumentation comment for this package)
diff --git a/src/plugins/data/server/ui_settings.ts b/src/plugins/data/server/ui_settings.ts
index 9393700a0e771..f5360f626ac66 100644
--- a/src/plugins/data/server/ui_settings.ts
+++ b/src/plugins/data/server/ui_settings.ts
@@ -267,14 +267,13 @@ export function getUiSettings(): Record<string, UiSettingsParams<unknown>> {
     },
     [UI_SETTINGS.COURIER_BATCH_SEARCHES]: {
       name: i18n.translate('data.advancedSettings.courier.batchSearchesTitle', {
-        defaultMessage: 'Batch concurrent searches',
+        defaultMessage: 'Use legacy search',
       }),
       value: false,
       type: 'boolean',
       description: i18n.translate('data.advancedSettings.courier.batchSearchesText', {
-        defaultMessage: `When disabled, dashboard panels will load individually, and search requests will terminate when users navigate
-           away or update the query. When enabled, dashboard panels will load together when all of the data is loaded, and
-           searches will not terminate.`,
+        defaultMessage: `Kibana uses a new search and batching infrastructure.
+           Enable this option if you prefer to fallback to the legacy synchronous behavior`,
       }),
       deprecation: {
         message: i18n.translate('data.advancedSettings.courier.batchSearchesTextDeprecation', {
diff --git a/src/plugins/embeddable/public/public.api.md b/src/plugins/embeddable/public/public.api.md
index f3f3682404e32..023cb3d19b632 100644
--- a/src/plugins/embeddable/public/public.api.md
+++ b/src/plugins/embeddable/public/public.api.md
@@ -12,6 +12,7 @@ import { ApiResponse as ApiResponse_2 } from '@elastic/elasticsearch';
 import { ApplicationStart as ApplicationStart_2 } from 'kibana/public';
 import { Assign } from '@kbn/utility-types';
 import { BehaviorSubject } from 'rxjs';
+import { BfetchPublicSetup } from 'src/plugins/bfetch/public';
 import Boom from '@hapi/boom';
 import { CoreSetup as CoreSetup_2 } from 'src/core/public';
 import { CoreSetup as CoreSetup_3 } from 'kibana/public';
diff --git a/x-pack/plugins/data_enhanced/kibana.json b/x-pack/plugins/data_enhanced/kibana.json
index bc7c8410d3df1..eea0101ec4ed7 100644
--- a/x-pack/plugins/data_enhanced/kibana.json
+++ b/x-pack/plugins/data_enhanced/kibana.json
@@ -6,6 +6,7 @@
     "xpack", "data_enhanced"
   ],
   "requiredPlugins": [
+    "bfetch",
     "data",
     "features"
   ],
diff --git a/x-pack/plugins/data_enhanced/public/plugin.ts b/x-pack/plugins/data_enhanced/public/plugin.ts
index 948858a5ed4c1..fa3206446f9fc 100644
--- a/x-pack/plugins/data_enhanced/public/plugin.ts
+++ b/x-pack/plugins/data_enhanced/public/plugin.ts
@@ -7,6 +7,7 @@
 import React from 'react';
 import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core/public';
 import { DataPublicPluginSetup, DataPublicPluginStart } from '../../../../src/plugins/data/public';
+import { BfetchPublicSetup } from '../../../../src/plugins/bfetch/public';
 
 import { setAutocompleteService } from './services';
 import { setupKqlQuerySuggestionProvider, KUERY_LANGUAGE_NAME } from './autocomplete';
@@ -16,6 +17,7 @@ import { createConnectedBackgroundSessionIndicator } from './search';
 import { ConfigSchema } from '../config';
 
 export interface DataEnhancedSetupDependencies {
+  bfetch: BfetchPublicSetup;
   data: DataPublicPluginSetup;
 }
 export interface DataEnhancedStartDependencies {
@@ -33,7 +35,7 @@ export class DataEnhancedPlugin
 
   public setup(
     core: CoreSetup<DataEnhancedStartDependencies>,
-    { data }: DataEnhancedSetupDependencies
+    { bfetch, data }: DataEnhancedSetupDependencies
   ) {
     data.autocomplete.addQuerySuggestionProvider(
       KUERY_LANGUAGE_NAME,
@@ -41,6 +43,7 @@ export class DataEnhancedPlugin
     );
 
     this.enhancedSearchInterceptor = new EnhancedSearchInterceptor({
+      bfetch,
       toasts: core.notifications.toasts,
       http: core.http,
       uiSettings: core.uiSettings,
diff --git a/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts b/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts
index 3f1cfc7a010c7..f4d7422d1c7e2 100644
--- a/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts
+++ b/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts
@@ -11,6 +11,7 @@ import { UI_SETTINGS } from '../../../../../src/plugins/data/common';
 import { AbortError } from '../../../../../src/plugins/kibana_utils/public';
 import { SearchTimeoutError } from 'src/plugins/data/public';
 import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks';
+import { bfetchPluginMock } from '../../../../../src/plugins/bfetch/public/mocks';
 
 const timeTravel = (msToRun = 0) => {
   jest.advanceTimersByTime(msToRun);
@@ -24,12 +25,13 @@ const complete = jest.fn();
 let searchInterceptor: EnhancedSearchInterceptor;
 let mockCoreSetup: MockedKeys<CoreSetup>;
 let mockCoreStart: MockedKeys<CoreStart>;
+let fetchMock: jest.Mock<any>;
 
 jest.useFakeTimers();
 
 function mockFetchImplementation(responses: any[]) {
   let i = 0;
-  mockCoreSetup.http.fetch.mockImplementation(() => {
+  fetchMock.mockImplementation(() => {
     const { time = 0, value = {}, isError = false } = responses[i++];
     return new Promise((resolve, reject) =>
       setTimeout(() => {
@@ -46,6 +48,7 @@ describe('EnhancedSearchInterceptor', () => {
     mockCoreSetup = coreMock.createSetup();
     mockCoreStart = coreMock.createStart();
     const dataPluginMockStart = dataPluginMock.createStartContract();
+    fetchMock = jest.fn();
 
     mockCoreSetup.uiSettings.get.mockImplementation((name: string) => {
       switch (name) {
@@ -74,7 +77,11 @@ describe('EnhancedSearchInterceptor', () => {
       ]);
     });
 
+    const bfetchMock = bfetchPluginMock.createSetupContract();
+    bfetchMock.batchedFunction.mockReturnValue(fetchMock);
+
     searchInterceptor = new EnhancedSearchInterceptor({
+      bfetch: bfetchMock,
       toasts: mockCoreSetup.notifications.toasts,
       startServices: mockPromise as any,
       http: mockCoreSetup.http,
@@ -245,7 +252,7 @@ describe('EnhancedSearchInterceptor', () => {
       expect(error).toHaveBeenCalled();
       expect(error.mock.calls[0][0]).toBeInstanceOf(AbortError);
 
-      expect(mockCoreSetup.http.fetch).toHaveBeenCalledTimes(2);
+      expect(fetchMock).toHaveBeenCalledTimes(2);
       expect(mockCoreSetup.http.delete).toHaveBeenCalled();
     });
 
@@ -269,7 +276,7 @@ describe('EnhancedSearchInterceptor', () => {
 
       expect(error).toHaveBeenCalled();
       expect(error.mock.calls[0][0]).toBeInstanceOf(SearchTimeoutError);
-      expect(mockCoreSetup.http.fetch).toHaveBeenCalled();
+      expect(fetchMock).toHaveBeenCalled();
       expect(mockCoreSetup.http.delete).not.toHaveBeenCalled();
     });
 
@@ -301,7 +308,7 @@ describe('EnhancedSearchInterceptor', () => {
 
       expect(next).toHaveBeenCalled();
       expect(error).not.toHaveBeenCalled();
-      expect(mockCoreSetup.http.fetch).toHaveBeenCalled();
+      expect(fetchMock).toHaveBeenCalled();
       expect(mockCoreSetup.http.delete).not.toHaveBeenCalled();
 
       // Long enough to reach the timeout but not long enough to reach the next response
@@ -309,7 +316,7 @@ describe('EnhancedSearchInterceptor', () => {
 
       expect(error).toHaveBeenCalled();
       expect(error.mock.calls[0][0]).toBeInstanceOf(SearchTimeoutError);
-      expect(mockCoreSetup.http.fetch).toHaveBeenCalledTimes(2);
+      expect(fetchMock).toHaveBeenCalledTimes(2);
       expect(mockCoreSetup.http.delete).toHaveBeenCalled();
     });
 
@@ -343,7 +350,7 @@ describe('EnhancedSearchInterceptor', () => {
 
       expect(next).toHaveBeenCalled();
       expect(error).not.toHaveBeenCalled();
-      expect(mockCoreSetup.http.fetch).toHaveBeenCalled();
+      expect(fetchMock).toHaveBeenCalled();
       expect(mockCoreSetup.http.delete).not.toHaveBeenCalled();
 
       // Long enough to reach the timeout but not long enough to reach the next response
@@ -351,7 +358,7 @@ describe('EnhancedSearchInterceptor', () => {
 
       expect(error).toHaveBeenCalled();
       expect(error.mock.calls[0][0]).toBe(responses[1].value);
-      expect(mockCoreSetup.http.fetch).toHaveBeenCalledTimes(2);
+      expect(fetchMock).toHaveBeenCalledTimes(2);
       expect(mockCoreSetup.http.delete).toHaveBeenCalled();
     });
   });
@@ -383,9 +390,7 @@ describe('EnhancedSearchInterceptor', () => {
 
       await timeTravel();
 
-      const areAllRequestsAborted = mockCoreSetup.http.fetch.mock.calls.every(
-        ([{ signal }]) => signal?.aborted
-      );
+      const areAllRequestsAborted = fetchMock.mock.calls.every(([_, signal]) => signal?.aborted);
       expect(areAllRequestsAborted).toBe(true);
       expect(mockUsageCollector.trackQueriesCancelled).toBeCalledTimes(1);
     });
diff --git a/x-pack/plugins/security_solution/cypress/cypress.json b/x-pack/plugins/security_solution/cypress/cypress.json
index 173514565c8bb..364db54b4b5d9 100644
--- a/x-pack/plugins/security_solution/cypress/cypress.json
+++ b/x-pack/plugins/security_solution/cypress/cypress.json
@@ -1,6 +1,7 @@
 {
   "baseUrl": "http://localhost:5601",
   "defaultCommandTimeout": 120000,
+  "experimentalNetworkStubbing": true,
   "retries": {
     "runMode": 2
   },
diff --git a/x-pack/plugins/security_solution/cypress/fixtures/overview_search_strategy.json b/x-pack/plugins/security_solution/cypress/fixtures/overview_search_strategy.json
index d0c7517015091..7a6d9d8ae294e 100644
--- a/x-pack/plugins/security_solution/cypress/fixtures/overview_search_strategy.json
+++ b/x-pack/plugins/security_solution/cypress/fixtures/overview_search_strategy.json
@@ -8,8 +8,7 @@
     "filebeatZeek": 71129,
     "packetbeatDNS": 1090,
     "packetbeatFlow": 722153,
-    "packetbeatTLS": 340,
-    "__typename": "OverviewNetworkData"
+    "packetbeatTLS": 340
   },
   "overviewHost": {
     "auditbeatAuditd": 123,
@@ -27,7 +26,6 @@
     "endgameSecurity": 397,
     "filebeatSystemModule": 890,
     "winlogbeatSecurity": 70,
-    "winlogbeatMWSysmonOperational": 30,
-    "__typename": "OverviewHostData"
+    "winlogbeatMWSysmonOperational": 30
   }
 }
diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts.spec.ts
index db841d2a732c4..8e3b30cddd121 100644
--- a/x-pack/plugins/security_solution/cypress/integration/alerts.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/alerts.spec.ts
@@ -25,7 +25,7 @@ import {
   markInProgressFirstAlert,
   goToInProgressAlerts,
 } from '../tasks/alerts';
-import { esArchiverLoad } from '../tasks/es_archiver';
+import { esArchiverLoad, esArchiverUnload } from '../tasks/es_archiver';
 import { loginAndWaitForPage } from '../tasks/login';
 
 import { DETECTIONS_URL } from '../urls/navigation';
@@ -37,6 +37,10 @@ describe('Alerts', () => {
       loginAndWaitForPage(DETECTIONS_URL);
     });
 
+    afterEach(() => {
+      esArchiverUnload('alerts');
+    });
+
     it('Closes and opens alerts', () => {
       waitForAlertsPanelToBeLoaded();
       waitForAlertsToBeLoaded();
@@ -165,6 +169,10 @@ describe('Alerts', () => {
       loginAndWaitForPage(DETECTIONS_URL);
     });
 
+    afterEach(() => {
+      esArchiverUnload('closed_alerts');
+    });
+
     it('Open one alert when more than one closed alerts are selected', () => {
       waitForAlerts();
       goToClosedAlerts();
@@ -212,6 +220,10 @@ describe('Alerts', () => {
       loginAndWaitForPage(DETECTIONS_URL);
     });
 
+    afterEach(() => {
+      esArchiverUnload('alerts');
+    });
+
     it('Mark one alert in progress when more than one open alerts are selected', () => {
       waitForAlerts();
       waitForAlertsToBeLoaded();
diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts
index 6a62caecfaa67..2d21e3d333c07 100644
--- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts
@@ -83,7 +83,8 @@ describe('Alerts detection rules', () => {
       });
   });
 
-  it('Auto refreshes rules', () => {
+  // FIXME: UI hangs on loading
+  it.skip('Auto refreshes rules', () => {
     cy.clock(Date.now());
 
     loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts
index dfe984cba3816..5fee3c0bce13c 100644
--- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts
@@ -225,7 +225,7 @@ describe('Custom detection rules deletion and edition', () => {
     goToManageAlertsDetectionRules();
   });
 
-  after(() => {
+  afterEach(() => {
     esArchiverUnload('custom_rules');
   });
 
diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts
index c2be6b2883c88..eb8448233c624 100644
--- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts
@@ -17,8 +17,7 @@ import { DETECTIONS_URL } from '../urls/navigation';
 
 const EXPECTED_EXPORTED_RULE_FILE_PATH = 'cypress/test_files/expected_rules_export.ndjson';
 
-// FLAKY: https://github.com/elastic/kibana/issues/69849
-describe.skip('Export rules', () => {
+describe('Export rules', () => {
   before(() => {
     esArchiverLoad('export_rule');
     cy.server();
diff --git a/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts b/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts
index 69094cad7456e..0d12019adbc99 100644
--- a/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts
+++ b/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts
@@ -11,9 +11,12 @@ import { loginAndWaitForPage } from '../tasks/login';
 
 import { OVERVIEW_URL } from '../urls/navigation';
 
+import overviewFixture from '../fixtures/overview_search_strategy.json';
+import emptyInstance from '../fixtures/empty_instance.json';
+
 describe('Overview Page', () => {
   it('Host stats render with correct values', () => {
-    cy.stubSearchStrategyApi('overview_search_strategy');
+    cy.stubSearchStrategyApi(overviewFixture, 'overviewHost');
     loginAndWaitForPage(OVERVIEW_URL);
     expandHostStats();
 
@@ -23,7 +26,7 @@ describe('Overview Page', () => {
   });
 
   it('Network stats render with correct values', () => {
-    cy.stubSearchStrategyApi('overview_search_strategy');
+    cy.stubSearchStrategyApi(overviewFixture, 'overviewNetwork');
     loginAndWaitForPage(OVERVIEW_URL);
     expandNetworkStats();
 
@@ -33,14 +36,9 @@ describe('Overview Page', () => {
   });
 
   describe('with no data', () => {
-    before(() => {
-      cy.server();
-      cy.fixture('empty_instance').as('emptyInstance');
-      loginAndWaitForPage(OVERVIEW_URL);
-      cy.route('POST', '**/internal/search/securitySolutionIndexFields', '@emptyInstance');
-    });
-
     it('Splash screen should be here', () => {
+      cy.stubSearchStrategyApi(emptyInstance, undefined, 'securitySolutionIndexFields');
+      loginAndWaitForPage(OVERVIEW_URL);
       cy.get(OVERVIEW_EMPTY_PAGE).should('be.visible');
     });
   });
diff --git a/x-pack/plugins/security_solution/cypress/support/commands.js b/x-pack/plugins/security_solution/cypress/support/commands.js
index c249e0a77690c..95a52794628b1 100644
--- a/x-pack/plugins/security_solution/cypress/support/commands.js
+++ b/x-pack/plugins/security_solution/cypress/support/commands.js
@@ -30,24 +30,66 @@
 // -- This is will overwrite an existing command --
 // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... })
 
-Cypress.Commands.add('stubSecurityApi', function (dataFileName) {
-  cy.on('window:before:load', (win) => {
-    win.fetch = null;
-  });
-  cy.server();
-  cy.fixture(dataFileName).as(`${dataFileName}JSON`);
-  cy.route('POST', 'api/solutions/security/graphql', `@${dataFileName}JSON`);
-});
+import { findIndex } from 'lodash/fp';
+
+const getFindRequestConfig = (searchStrategyName, factoryQueryType) => {
+  if (!factoryQueryType) {
+    return {
+      options: { strategy: searchStrategyName },
+    };
+  }
+
+  return {
+    options: { strategy: searchStrategyName },
+    request: { factoryQueryType },
+  };
+};
 
 Cypress.Commands.add(
   'stubSearchStrategyApi',
-  function (dataFileName, searchStrategyName = 'securitySolutionSearchStrategy') {
-    cy.on('window:before:load', (win) => {
-      win.fetch = null;
+  function (stubObject, factoryQueryType, searchStrategyName = 'securitySolutionSearchStrategy') {
+    cy.route2('POST', '/internal/bsearch', (req) => {
+      const bodyObj = JSON.parse(req.body);
+      const findRequestConfig = getFindRequestConfig(searchStrategyName, factoryQueryType);
+
+      const requestIndex = findIndex(findRequestConfig, bodyObj.batch);
+
+      if (requestIndex > -1) {
+        return req.reply((res) => {
+          const responseObjectsArray = res.body.split('\n').map((responseString) => {
+            try {
+              return JSON.parse(responseString);
+            } catch {
+              return responseString;
+            }
+          });
+          const responseIndex = findIndex({ id: requestIndex }, responseObjectsArray);
+
+          const stubbedResponseObjectsArray = [...responseObjectsArray];
+          stubbedResponseObjectsArray[responseIndex] = {
+            ...stubbedResponseObjectsArray[responseIndex],
+            result: {
+              ...stubbedResponseObjectsArray[responseIndex].result,
+              ...stubObject,
+            },
+          };
+
+          const stubbedResponse = stubbedResponseObjectsArray
+            .map((object) => {
+              try {
+                return JSON.stringify(object);
+              } catch {
+                return object;
+              }
+            })
+            .join('\n');
+
+          res.send(stubbedResponse);
+        });
+      }
+
+      req.reply();
     });
-    cy.server();
-    cy.fixture(dataFileName).as(`${dataFileName}JSON`);
-    cy.route('POST', `internal/search/${searchStrategyName}`, `@${dataFileName}JSON`);
   }
 );
 
diff --git a/x-pack/plugins/security_solution/cypress/support/index.d.ts b/x-pack/plugins/security_solution/cypress/support/index.d.ts
index fb55a2890c8b7..06285abba6531 100644
--- a/x-pack/plugins/security_solution/cypress/support/index.d.ts
+++ b/x-pack/plugins/security_solution/cypress/support/index.d.ts
@@ -7,8 +7,11 @@
 declare namespace Cypress {
   interface Chainable<Subject> {
     promisify(): Promise<Subject>;
-    stubSecurityApi(dataFileName: string): Chainable<Subject>;
-    stubSearchStrategyApi(dataFileName: string, searchStrategyName?: string): Chainable<Subject>;
+    stubSearchStrategyApi(
+      stubObject: Record<string, unknown>,
+      factoryQueryType?: string,
+      searchStrategyName?: string
+    ): Chainable<Subject>;
     attachFile(fileName: string, fileType?: string): Chainable<JQuery>;
     waitUntil(
       fn: (subject: Subject) => boolean | Chainable<boolean>,
diff --git a/x-pack/test/security_solution_cypress/es_archives/timeline_alerts/mappings.json b/x-pack/test/security_solution_cypress/es_archives/timeline_alerts/mappings.json
index a1a9e7bfeae7f..abdec252471b7 100644
--- a/x-pack/test/security_solution_cypress/es_archives/timeline_alerts/mappings.json
+++ b/x-pack/test/security_solution_cypress/es_archives/timeline_alerts/mappings.json
@@ -157,6 +157,9 @@
             "throttle": {
               "type": "keyword"
             },
+            "updatedAt": {
+              "type": "date"
+            },
             "updatedBy": {
               "type": "keyword"
             }
@@ -9060,4 +9063,4 @@
       }
     }
   }
-}
\ No newline at end of file
+}

From 58297fa131255158975449915b469ef45bbba86b Mon Sep 17 00:00:00 2001
From: Brandon Kobel <brandon.kobel@elastic.co>
Date: Wed, 25 Nov 2020 06:36:24 -0800
Subject: [PATCH 08/37] Deprecate `xpack.task_manager.index` setting (#84155)

* Deprecate `xpack.task_manager.index` setting

* Updating developer docs about configuring task manager settings
---
 x-pack/plugins/task_manager/README.md         |  2 +-
 .../plugins/task_manager/server/index.test.ts | 43 +++++++++++++++++++
 x-pack/plugins/task_manager/server/index.ts   | 18 ++++++--
 3 files changed, 59 insertions(+), 4 deletions(-)
 create mode 100644 x-pack/plugins/task_manager/server/index.test.ts

diff --git a/x-pack/plugins/task_manager/README.md b/x-pack/plugins/task_manager/README.md
index 744d657bcd790..d3c8ecb6c4505 100644
--- a/x-pack/plugins/task_manager/README.md
+++ b/x-pack/plugins/task_manager/README.md
@@ -43,7 +43,7 @@ The task_manager can be configured via `taskManager` config options (e.g. `taskM
 - `max_attempts` - The maximum number of times a task will be attempted before being abandoned as failed
 - `poll_interval` - How often the background worker should check the task_manager index for more work
 - `max_poll_inactivity_cycles` - How many poll intervals is work allowed to block polling for before it's timed out. This does not include task execution, as task execution does not block the polling, but rather includes work needed to manage Task Manager's state.
-- `index` - The name of the index that the task_manager
+- `index` - **deprecated** The name of the index that the task_manager will use. This is deprecated, and will be removed starting in 8.0
 - `max_workers` - The maximum number of tasks a Kibana will run concurrently (defaults to 10)
 - `credentials` - Encrypted user credentials. All tasks will run in the security context of this user. See [this issue](https://github.com/elastic/dev/issues/1045) for a discussion on task scheduler security.
 - `override_num_workers`: An object of `taskType: number` that overrides the `num_workers` for tasks
diff --git a/x-pack/plugins/task_manager/server/index.test.ts b/x-pack/plugins/task_manager/server/index.test.ts
new file mode 100644
index 0000000000000..3f25f4403d358
--- /dev/null
+++ b/x-pack/plugins/task_manager/server/index.test.ts
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { config } from './index';
+import { applyDeprecations, configDeprecationFactory } from '@kbn/config';
+
+const CONFIG_PATH = 'xpack.task_manager';
+
+const applyTaskManagerDeprecations = (settings: Record<string, unknown> = {}) => {
+  const deprecations = config.deprecations!(configDeprecationFactory);
+  const deprecationMessages: string[] = [];
+  const _config = {
+    [CONFIG_PATH]: settings,
+  };
+  const migrated = applyDeprecations(
+    _config,
+    deprecations.map((deprecation) => ({
+      deprecation,
+      path: CONFIG_PATH,
+    })),
+    (msg) => deprecationMessages.push(msg)
+  );
+  return {
+    messages: deprecationMessages,
+    migrated,
+  };
+};
+
+describe('deprecations', () => {
+  ['.foo', '.kibana_task_manager'].forEach((index) => {
+    it('logs a warning if index is set', () => {
+      const { messages } = applyTaskManagerDeprecations({ index });
+      expect(messages).toMatchInlineSnapshot(`
+        Array [
+          "\\"xpack.task_manager.index\\" is deprecated. Multitenancy by changing \\"kibana.index\\" will not be supported starting in 8.0. See https://ela.st/kbn-remove-legacy-multitenancy for more details",
+        ]
+      `);
+    });
+  });
+});
diff --git a/x-pack/plugins/task_manager/server/index.ts b/x-pack/plugins/task_manager/server/index.ts
index cf70e68437cc6..8f96e10430b39 100644
--- a/x-pack/plugins/task_manager/server/index.ts
+++ b/x-pack/plugins/task_manager/server/index.ts
@@ -4,9 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { PluginInitializerContext } from 'src/core/server';
+import { get } from 'lodash';
+import { PluginConfigDescriptor, PluginInitializerContext } from 'src/core/server';
 import { TaskManagerPlugin } from './plugin';
-import { configSchema } from './config';
+import { configSchema, TaskManagerConfig } from './config';
 
 export const plugin = (initContext: PluginInitializerContext) => new TaskManagerPlugin(initContext);
 
@@ -26,6 +27,17 @@ export {
   TaskManagerStartContract,
 } from './plugin';
 
-export const config = {
+export const config: PluginConfigDescriptor<TaskManagerConfig> = {
   schema: configSchema,
+  deprecations: () => [
+    (settings, fromPath, log) => {
+      const taskManager = get(settings, fromPath);
+      if (taskManager?.index) {
+        log(
+          `"${fromPath}.index" is deprecated. Multitenancy by changing "kibana.index" will not be supported starting in 8.0. See https://ela.st/kbn-remove-legacy-multitenancy for more details`
+        );
+      }
+      return settings;
+    },
+  ],
 };

From 0fe8a65a23c3de3381b58219a1e4f169b1354133 Mon Sep 17 00:00:00 2001
From: Scotty Bollinger <scotty.bollinger@elastic.co>
Date: Wed, 25 Nov 2020 09:00:10 -0600
Subject: [PATCH 09/37] [Workplace Search] Migrate DisplaySettings tree
 (#84283)

* Initial copy/paste of components

Changes for pre-commit hooks were:

- Linting
- Lodash imports
- Fixed warnings for `jsx-a11y/mouse-events-have-key-events` with stubbed onFocus and onBlue events with FIXME comments

* Add server routes

* Remove reference to shared lib

This one-liner appears only once in ent-search so adding it here in the logic file`

* Fix paths

* Add types and fix TypeScript issues

* Replace FlashMessages with global component

* More explicit Result type

* Remove routes/http in favor of HttpLogic

* Fix server routes

`urlFieldIsLinkable` was missing and `detailFields` can either be an object or an array of objects

* Add base styles

These were ported from ent-search. Decided to use spacers where some global styles were missing.

* Kibana prefers underscores in URLs

Was only going to do display-settings and result-details but decided to YOLO all of them
---
 .../applications/workplace_search/routes.ts   |  36 +-
 .../applications/workplace_search/types.ts    |  23 ++
 .../display_settings/custom_source_icon.tsx   |  36 ++
 .../display_settings/display_settings.scss    | 206 +++++++++++
 .../display_settings/display_settings.tsx     | 141 +++++++
 .../display_settings_logic.ts                 | 350 ++++++++++++++++++
 .../display_settings_router.tsx               |  31 +-
 .../example_result_detail_card.tsx            |  75 ++++
 .../example_search_result_group.tsx           |  74 ++++
 .../example_standout_result.tsx               |  66 ++++
 .../display_settings/field_editor_modal.tsx   | 101 +++++
 .../display_settings/result_detail.tsx        | 146 ++++++++
 .../display_settings/search_results.tsx       | 164 ++++++++
 .../display_settings/subtitle_field.tsx       |  35 ++
 .../display_settings/title_field.tsx          |  35 ++
 .../routes/workplace_search/sources.test.ts   | 152 ++++++++
 .../server/routes/workplace_search/sources.ts |  95 +++++
 17 files changed, 1747 insertions(+), 19 deletions(-)
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/custom_source_icon.tsx
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.scss
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.tsx
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_logic.ts
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_result_detail_card.tsx
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_search_result_group.tsx
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_standout_result.tsx
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/field_editor_modal.tsx
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/result_detail.tsx
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/search_results.tsx
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/subtitle_field.tsx
 create mode 100644 x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/title_field.tsx

diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts
index 3ec22ede888ab..14c288de5a0c8 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/routes.ts
@@ -46,7 +46,7 @@ export const ENT_SEARCH_LICENSE_MANAGEMENT = `${ENT_SEARCH_DOCS_PREFIX}/license-
 
 export const PERSONAL_PATH = '/p';
 
-export const ROLE_MAPPINGS_PATH = '/role-mappings';
+export const ROLE_MAPPINGS_PATH = '/role_mappings';
 export const ROLE_MAPPING_PATH = `${ROLE_MAPPINGS_PATH}/:roleId`;
 export const ROLE_MAPPING_NEW_PATH = `${ROLE_MAPPINGS_PATH}/new`;
 
@@ -63,18 +63,18 @@ export const PERSONAL_SOURCES_PATH = `${PERSONAL_PATH}${SOURCES_PATH}`;
 export const SOURCE_ADDED_PATH = `${SOURCES_PATH}/added`;
 export const ADD_SOURCE_PATH = `${SOURCES_PATH}/add`;
 export const ADD_BOX_PATH = `${SOURCES_PATH}/add/box`;
-export const ADD_CONFLUENCE_PATH = `${SOURCES_PATH}/add/confluence-cloud`;
-export const ADD_CONFLUENCE_SERVER_PATH = `${SOURCES_PATH}/add/confluence-server`;
+export const ADD_CONFLUENCE_PATH = `${SOURCES_PATH}/add/confluence_cloud`;
+export const ADD_CONFLUENCE_SERVER_PATH = `${SOURCES_PATH}/add/confluence_server`;
 export const ADD_DROPBOX_PATH = `${SOURCES_PATH}/add/dropbox`;
-export const ADD_GITHUB_ENTERPRISE_PATH = `${SOURCES_PATH}/add/github-enterprise-server`;
+export const ADD_GITHUB_ENTERPRISE_PATH = `${SOURCES_PATH}/add/github_enterprise_server`;
 export const ADD_GITHUB_PATH = `${SOURCES_PATH}/add/github`;
 export const ADD_GMAIL_PATH = `${SOURCES_PATH}/add/gmail`;
-export const ADD_GOOGLE_DRIVE_PATH = `${SOURCES_PATH}/add/google-drive`;
-export const ADD_JIRA_PATH = `${SOURCES_PATH}/add/jira-cloud`;
-export const ADD_JIRA_SERVER_PATH = `${SOURCES_PATH}/add/jira-server`;
+export const ADD_GOOGLE_DRIVE_PATH = `${SOURCES_PATH}/add/google_drive`;
+export const ADD_JIRA_PATH = `${SOURCES_PATH}/add/jira_cloud`;
+export const ADD_JIRA_SERVER_PATH = `${SOURCES_PATH}/add/jira_server`;
 export const ADD_ONEDRIVE_PATH = `${SOURCES_PATH}/add/onedrive`;
 export const ADD_SALESFORCE_PATH = `${SOURCES_PATH}/add/salesforce`;
-export const ADD_SALESFORCE_SANDBOX_PATH = `${SOURCES_PATH}/add/salesforce-sandbox`;
+export const ADD_SALESFORCE_SANDBOX_PATH = `${SOURCES_PATH}/add/salesforce_sandbox`;
 export const ADD_SERVICENOW_PATH = `${SOURCES_PATH}/add/servicenow`;
 export const ADD_SHAREPOINT_PATH = `${SOURCES_PATH}/add/sharepoint`;
 export const ADD_SLACK_PATH = `${SOURCES_PATH}/add/slack`;
@@ -86,30 +86,30 @@ export const PERSONAL_SETTINGS_PATH = `${PERSONAL_PATH}/settings`;
 export const SOURCE_DETAILS_PATH = `${SOURCES_PATH}/:sourceId`;
 export const SOURCE_CONTENT_PATH = `${SOURCES_PATH}/:sourceId/content`;
 export const SOURCE_SCHEMAS_PATH = `${SOURCES_PATH}/:sourceId/schemas`;
-export const SOURCE_DISPLAY_SETTINGS_PATH = `${SOURCES_PATH}/:sourceId/display-settings`;
+export const SOURCE_DISPLAY_SETTINGS_PATH = `${SOURCES_PATH}/:sourceId/display_settings`;
 export const SOURCE_SETTINGS_PATH = `${SOURCES_PATH}/:sourceId/settings`;
-export const REINDEX_JOB_PATH = `${SOURCES_PATH}/:sourceId/schema-errors/:activeReindexJobId`;
+export const REINDEX_JOB_PATH = `${SOURCES_PATH}/:sourceId/schema_errors/:activeReindexJobId`;
 
 export const DISPLAY_SETTINGS_SEARCH_RESULT_PATH = `${SOURCE_DISPLAY_SETTINGS_PATH}/`;
-export const DISPLAY_SETTINGS_RESULT_DETAIL_PATH = `${SOURCE_DISPLAY_SETTINGS_PATH}/result-detail`;
+export const DISPLAY_SETTINGS_RESULT_DETAIL_PATH = `${SOURCE_DISPLAY_SETTINGS_PATH}/result_detail`;
 
 export const ORG_SETTINGS_PATH = '/settings';
 export const ORG_SETTINGS_CUSTOMIZE_PATH = `${ORG_SETTINGS_PATH}/customize`;
 export const ORG_SETTINGS_CONNECTORS_PATH = `${ORG_SETTINGS_PATH}/connectors`;
 export const ORG_SETTINGS_OAUTH_APPLICATION_PATH = `${ORG_SETTINGS_PATH}/oauth`;
 export const EDIT_BOX_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/box/edit`;
-export const EDIT_CONFLUENCE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/confluence-cloud/edit`;
-export const EDIT_CONFLUENCE_SERVER_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/confluence-server/edit`;
+export const EDIT_CONFLUENCE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/confluence_cloud/edit`;
+export const EDIT_CONFLUENCE_SERVER_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/confluence_server/edit`;
 export const EDIT_DROPBOX_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/dropbox/edit`;
-export const EDIT_GITHUB_ENTERPRISE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/github-enterprise-server/edit`;
+export const EDIT_GITHUB_ENTERPRISE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/github_enterprise_server/edit`;
 export const EDIT_GITHUB_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/github/edit`;
 export const EDIT_GMAIL_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/gmail/edit`;
-export const EDIT_GOOGLE_DRIVE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/google-drive/edit`;
-export const EDIT_JIRA_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/jira-cloud/edit`;
-export const EDIT_JIRA_SERVER_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/jira-server/edit`;
+export const EDIT_GOOGLE_DRIVE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/google_drive/edit`;
+export const EDIT_JIRA_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/jira_cloud/edit`;
+export const EDIT_JIRA_SERVER_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/jira_server/edit`;
 export const EDIT_ONEDRIVE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/onedrive/edit`;
 export const EDIT_SALESFORCE_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/salesforce/edit`;
-export const EDIT_SALESFORCE_SANDBOX_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/salesforce-sandbox/edit`;
+export const EDIT_SALESFORCE_SANDBOX_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/salesforce_sandbox/edit`;
 export const EDIT_SERVICENOW_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/servicenow/edit`;
 export const EDIT_SHAREPOINT_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/sharepoint/edit`;
 export const EDIT_SLACK_PATH = `${ORG_SETTINGS_CONNECTORS_PATH}/slack/edit`;
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts
index 73e7f7ed701d8..9bda686ebbf00 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/types.ts
@@ -181,3 +181,26 @@ export interface CustomSource {
   name: string;
   id: string;
 }
+
+export interface Result {
+  [key: string]: string;
+}
+
+export interface OptionValue {
+  value: string;
+  text: string;
+}
+
+export interface DetailField {
+  fieldName: string;
+  label: string;
+}
+
+export interface SearchResultConfig {
+  titleField: string | null;
+  subtitleField: string | null;
+  descriptionField: string | null;
+  urlField: string | null;
+  color: string;
+  detailFields: DetailField[];
+}
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/custom_source_icon.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/custom_source_icon.tsx
new file mode 100644
index 0000000000000..16129324b56d1
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/custom_source_icon.tsx
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+const BLACK_RGB = '#000';
+
+interface CustomSourceIconProps {
+  color?: string;
+}
+
+export const CustomSourceIcon: React.FC<CustomSourceIconProps> = ({ color = BLACK_RGB }) => (
+  <svg
+    className="euiIcon euiIcon--medium"
+    width="52"
+    height="52"
+    viewBox="0 0 52 52"
+    fill="none"
+    xmlns="http://www.w3.org/2000/svg"
+    style={{ fill: color }}
+  >
+    <path
+      fillRule="evenodd"
+      clipRule="evenodd"
+      d="M18 34H20.222C21.988 31.744 23 28.935 23 26C23 23.065 21.988 20.256 20.222 18H18C13.582 18 10 21.582 10 26C10 30.418 13.582 34 18 34Z"
+    />
+    <path
+      fillRule="evenodd"
+      clipRule="evenodd"
+      d="M18 12.158C20.355 10.794 23.082 10 26 10C34.837 10 42 17.163 42 26C42 34.837 34.837 42 26 42C23.082 42 20.355 41.206 18 39.842C20.4228 38.4381 22.4459 36.4202 23.8515 34H23.848C25.1274 31.7962 25.898 29.2625 25.9906 26.5526C25.9936 26.4634 25.996 26.374 25.9975 26.2843C25.9992 26.1898 26 26.095 26 26C26 25.8613 25.9982 25.723 25.9947 25.5851C25.9937 25.5449 25.9925 25.5047 25.9912 25.4646C25.9013 22.7482 25.1301 20.2085 23.848 18H23.8515C22.4459 15.5798 20.4228 13.5619 18 12.158Z"
+    />
+  </svg>
+);
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.scss b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.scss
new file mode 100644
index 0000000000000..27935104f4ef6
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.scss
@@ -0,0 +1,206 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// --------------------------------------------------
+// Custom Source display settings
+// --------------------------------------------------
+
+@mixin source_name {
+  font-size: .6875em;
+  text-transform: uppercase;
+  font-weight: 600;
+  letter-spacing: 0.06em;
+}
+
+@mixin example_result_box_shadow {
+  box-shadow:
+    0 1px 3px rgba(black, 0.1),
+    0 0 20px $euiColorLightestShade;
+}
+
+// Wrapper
+.custom-source-display-settings {
+  font-size: 16px;
+}
+
+// Example result content
+.example-result-content {
+  & > * {
+    line-height: 1.5em;
+  }
+
+  &__title {
+    font-size: 1em;
+    font-weight: 600;
+    color: $euiColorPrimary;
+
+    .example-result-detail-card & {
+      font-size: 20px;
+    }
+  }
+
+  &__subtitle,
+  &__description {
+    font-size: .875;
+  }
+
+  &__subtitle {
+    color: $euiColorDarkestShade;
+  }
+
+  &__description {
+    padding: .1rem 0 .125rem .35rem;
+    border-left: 3px solid $euiColorLightShade;
+    color: $euiColorDarkShade;
+    line-height: 1.8;
+    word-break: break-word;
+
+    @supports (display: -webkit-box) {
+      display: -webkit-box;
+      -webkit-box-orient: vertical;
+      -webkit-line-clamp: 3;
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
+  }
+
+  &__url {
+    .example-result-detail-card & {
+      color: $euiColorDarkShade;
+    }
+  }
+}
+
+.example-result-content-placeholder {
+  color: $euiColorMediumShade;
+}
+
+// Example standout result
+.example-standout-result {
+  border-radius: 4px;
+  overflow: hidden;
+  @include example_result_box_shadow;
+
+  &__header,
+  &__content {
+    padding-left: 1em;
+    padding-right: 1em;
+  }
+
+  &__content {
+    padding-top: 1em;
+    padding-bottom: 1em;
+  }
+
+  &__source-name {
+    line-height: 34px;
+    @include source_name;
+  }
+}
+
+// Example result group
+.example-result-group {
+  &__header {
+    padding: 0 .5em;
+    border-radius: 4px;
+    display: inline-flex;
+    align-items: center;
+
+    .euiIcon {
+      margin-right: .25rem;
+    }
+  }
+
+  &__source-name {
+    line-height: 1.75em;
+    @include source_name;
+  }
+
+  &__content {
+    display: flex;
+    align-items: stretch;
+    padding: .75em 0;
+  }
+
+  &__border {
+    width: 4px;
+    border-radius: 2px;
+    flex-shrink: 0;
+    margin-left: .875rem;
+  }
+
+  &__results {
+    flex: 1;
+    max-width: 100%;
+  }
+}
+
+.example-grouped-result {
+  padding: 1em;
+}
+
+.example-result-field-hover {
+  background: lighten($euiColorVis1_behindText, 30%);
+  position: relative;
+
+  &:before,
+  &:after {
+    content: '';
+    position: absolute;
+    height: 100%;
+    width: 4px;
+    background: lighten($euiColorVis1_behindText, 30%);
+  }
+
+  &:before {
+    right: 100%;
+    border-radius: 2px 0 0 2px;
+  }
+
+  &:after {
+    left: 100%;
+    border-radius: 0 2px 2px 0;
+  }
+
+  .example-result-content-placeholder {
+    color: $euiColorFullShade;
+  }
+}
+
+.example-result-detail-card {
+  @include example_result_box_shadow;
+
+  &__header {
+    position: relative;
+    padding: 1.25em 1em 0;
+  }
+
+  &__border {
+    height: 4px;
+    position: absolute;
+    top: 0;
+    right: 0;
+    left: 0;
+  }
+
+  &__source-name {
+    margin-bottom: 1em;
+    font-weight: 500;
+  }
+
+  &__field {
+    padding: 1em;
+
+    & + & {
+      border-top: 1px solid $euiColorLightShade;
+    }
+  }
+}
+
+.visible-fields-container {
+  background: $euiColorLightestShade;
+  border-color: transparent;
+}
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.tsx
new file mode 100644
index 0000000000000..e34728beef5e5
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings.tsx
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FormEvent, useEffect } from 'react';
+
+import { History } from 'history';
+import { useActions, useValues } from 'kea';
+import { useHistory } from 'react-router-dom';
+
+import './display_settings.scss';
+
+import {
+  EuiButton,
+  EuiEmptyPrompt,
+  EuiTabbedContent,
+  EuiPanel,
+  EuiTabbedContentTab,
+} from '@elastic/eui';
+
+import {
+  DISPLAY_SETTINGS_RESULT_DETAIL_PATH,
+  DISPLAY_SETTINGS_SEARCH_RESULT_PATH,
+  getContentSourcePath,
+} from '../../../../routes';
+
+import { AppLogic } from '../../../../app_logic';
+
+import { Loading } from '../../../../../shared/loading';
+import { ViewContentHeader } from '../../../../components/shared/view_content_header';
+
+import { DisplaySettingsLogic } from './display_settings_logic';
+
+import { FieldEditorModal } from './field_editor_modal';
+import { ResultDetail } from './result_detail';
+import { SearchResults } from './search_results';
+
+const UNSAVED_MESSAGE =
+  'Your display settings have not been saved. Are you sure you want to leave?';
+
+interface DisplaySettingsProps {
+  tabId: number;
+}
+
+export const DisplaySettings: React.FC<DisplaySettingsProps> = ({ tabId }) => {
+  const history = useHistory() as History;
+  const { initializeDisplaySettings, setServerData, resetDisplaySettingsState } = useActions(
+    DisplaySettingsLogic
+  );
+
+  const {
+    dataLoading,
+    sourceId,
+    addFieldModalVisible,
+    unsavedChanges,
+    exampleDocuments,
+  } = useValues(DisplaySettingsLogic);
+
+  const { isOrganization } = useValues(AppLogic);
+
+  const hasDocuments = exampleDocuments.length > 0;
+
+  useEffect(() => {
+    initializeDisplaySettings();
+    return resetDisplaySettingsState;
+  }, []);
+
+  useEffect(() => {
+    window.onbeforeunload = hasDocuments && unsavedChanges ? () => UNSAVED_MESSAGE : null;
+    return () => {
+      window.onbeforeunload = null;
+    };
+  }, [unsavedChanges]);
+
+  if (dataLoading) return <Loading />;
+
+  const tabs = [
+    {
+      id: 'search_results',
+      name: 'Search Results',
+      content: <SearchResults />,
+    },
+    {
+      id: 'result_detail',
+      name: 'Result Detail',
+      content: <ResultDetail />,
+    },
+  ] as EuiTabbedContentTab[];
+
+  const onSelectedTabChanged = (tab: EuiTabbedContentTab) => {
+    const path =
+      tab.id === tabs[1].id
+        ? getContentSourcePath(DISPLAY_SETTINGS_RESULT_DETAIL_PATH, sourceId, isOrganization)
+        : getContentSourcePath(DISPLAY_SETTINGS_SEARCH_RESULT_PATH, sourceId, isOrganization);
+
+    history.push(path);
+  };
+
+  const handleFormSubmit = (e: FormEvent) => {
+    e.preventDefault();
+    setServerData();
+  };
+
+  return (
+    <>
+      <form onSubmit={handleFormSubmit}>
+        <ViewContentHeader
+          title="Display Settings"
+          description="Customize the content and appearance of your Custom API Source search results."
+          action={
+            hasDocuments ? (
+              <EuiButton type="submit" disabled={!unsavedChanges} fill={true}>
+                Save
+              </EuiButton>
+            ) : null
+          }
+        />
+        {hasDocuments ? (
+          <EuiTabbedContent
+            tabs={tabs}
+            selectedTab={tabs[tabId]}
+            onTabClick={onSelectedTabChanged}
+          />
+        ) : (
+          <EuiPanel className="euiPanel--inset">
+            <EuiEmptyPrompt
+              iconType="indexRollupApp"
+              title={<h2>You have no content yet</h2>}
+              body={
+                <p>You need some content to display in order to configure the display settings.</p>
+              }
+            />
+          </EuiPanel>
+        )}
+      </form>
+      {addFieldModalVisible && <FieldEditorModal />}
+    </>
+  );
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_logic.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_logic.ts
new file mode 100644
index 0000000000000..c52665524f566
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_logic.ts
@@ -0,0 +1,350 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { cloneDeep, isEqual, differenceBy } from 'lodash';
+import { DropResult } from 'react-beautiful-dnd';
+
+import { kea, MakeLogicType } from 'kea';
+
+import { HttpLogic } from '../../../../../shared/http';
+
+import {
+  setSuccessMessage,
+  FlashMessagesLogic,
+  flashAPIErrors,
+} from '../../../../../shared/flash_messages';
+
+import { AppLogic } from '../../../../app_logic';
+import { SourceLogic } from '../../source_logic';
+
+const SUCCESS_MESSAGE = 'Display Settings have been successfuly updated.';
+
+import { DetailField, SearchResultConfig, OptionValue, Result } from '../../../../types';
+
+export interface DisplaySettingsResponseProps {
+  sourceName: string;
+  searchResultConfig: SearchResultConfig;
+  schemaFields: object;
+  exampleDocuments: Result[];
+}
+
+export interface DisplaySettingsInitialData extends DisplaySettingsResponseProps {
+  sourceId: string;
+  serverRoute: string;
+}
+
+interface DisplaySettingsActions {
+  initializeDisplaySettings(): void;
+  setServerData(): void;
+  onInitializeDisplaySettings(
+    displaySettingsProps: DisplaySettingsInitialData
+  ): DisplaySettingsInitialData;
+  setServerResponseData(
+    displaySettingsProps: DisplaySettingsResponseProps
+  ): DisplaySettingsResponseProps;
+  setTitleField(titleField: string | null): string | null;
+  setUrlField(urlField: string): string;
+  setSubtitleField(subtitleField: string | null): string | null;
+  setDescriptionField(descriptionField: string | null): string | null;
+  setColorField(hex: string): string;
+  setDetailFields(result: DropResult): { result: DropResult };
+  openEditDetailField(editFieldIndex: number | null): number | null;
+  removeDetailField(index: number): number;
+  addDetailField(newField: DetailField): DetailField;
+  updateDetailField(
+    updatedField: DetailField,
+    index: number | null
+  ): { updatedField: DetailField; index: number };
+  toggleFieldEditorModal(): void;
+  toggleTitleFieldHover(): void;
+  toggleSubtitleFieldHover(): void;
+  toggleDescriptionFieldHover(): void;
+  toggleUrlFieldHover(): void;
+  resetDisplaySettingsState(): void;
+}
+
+interface DisplaySettingsValues {
+  sourceName: string;
+  sourceId: string;
+  schemaFields: object;
+  exampleDocuments: Result[];
+  serverSearchResultConfig: SearchResultConfig;
+  searchResultConfig: SearchResultConfig;
+  serverRoute: string;
+  editFieldIndex: number | null;
+  dataLoading: boolean;
+  addFieldModalVisible: boolean;
+  titleFieldHover: boolean;
+  urlFieldHover: boolean;
+  subtitleFieldHover: boolean;
+  descriptionFieldHover: boolean;
+  fieldOptions: OptionValue[];
+  optionalFieldOptions: OptionValue[];
+  availableFieldOptions: OptionValue[];
+  unsavedChanges: boolean;
+}
+
+const defaultSearchResultConfig = {
+  titleField: '',
+  subtitleField: '',
+  descriptionField: '',
+  urlField: '',
+  color: '#000000',
+  detailFields: [],
+};
+
+export const DisplaySettingsLogic = kea<
+  MakeLogicType<DisplaySettingsValues, DisplaySettingsActions>
+>({
+  actions: {
+    onInitializeDisplaySettings: (displaySettingsProps: DisplaySettingsInitialData) =>
+      displaySettingsProps,
+    setServerResponseData: (displaySettingsProps: DisplaySettingsResponseProps) =>
+      displaySettingsProps,
+    setTitleField: (titleField: string) => titleField,
+    setUrlField: (urlField: string) => urlField,
+    setSubtitleField: (subtitleField: string | null) => subtitleField,
+    setDescriptionField: (descriptionField: string) => descriptionField,
+    setColorField: (hex: string) => hex,
+    setDetailFields: (result: DropResult) => ({ result }),
+    openEditDetailField: (editFieldIndex: number | null) => editFieldIndex,
+    removeDetailField: (index: number) => index,
+    addDetailField: (newField: DetailField) => newField,
+    updateDetailField: (updatedField: DetailField, index: number) => ({ updatedField, index }),
+    toggleFieldEditorModal: () => true,
+    toggleTitleFieldHover: () => true,
+    toggleSubtitleFieldHover: () => true,
+    toggleDescriptionFieldHover: () => true,
+    toggleUrlFieldHover: () => true,
+    resetDisplaySettingsState: () => true,
+    initializeDisplaySettings: () => true,
+    setServerData: () => true,
+  },
+  reducers: {
+    sourceName: [
+      '',
+      {
+        onInitializeDisplaySettings: (_, { sourceName }) => sourceName,
+      },
+    ],
+    sourceId: [
+      '',
+      {
+        onInitializeDisplaySettings: (_, { sourceId }) => sourceId,
+      },
+    ],
+    schemaFields: [
+      {},
+      {
+        onInitializeDisplaySettings: (_, { schemaFields }) => schemaFields,
+      },
+    ],
+    exampleDocuments: [
+      [],
+      {
+        onInitializeDisplaySettings: (_, { exampleDocuments }) => exampleDocuments,
+      },
+    ],
+    serverSearchResultConfig: [
+      defaultSearchResultConfig,
+      {
+        onInitializeDisplaySettings: (_, { searchResultConfig }) =>
+          setDefaultColor(searchResultConfig),
+        setServerResponseData: (_, { searchResultConfig }) => searchResultConfig,
+      },
+    ],
+    searchResultConfig: [
+      defaultSearchResultConfig,
+      {
+        onInitializeDisplaySettings: (_, { searchResultConfig }) =>
+          setDefaultColor(searchResultConfig),
+        setServerResponseData: (_, { searchResultConfig }) => searchResultConfig,
+        setTitleField: (searchResultConfig, titleField) => ({ ...searchResultConfig, titleField }),
+        setSubtitleField: (searchResultConfig, subtitleField) => ({
+          ...searchResultConfig,
+          subtitleField,
+        }),
+        setUrlField: (searchResultConfig, urlField) => ({ ...searchResultConfig, urlField }),
+        setDescriptionField: (searchResultConfig, descriptionField) => ({
+          ...searchResultConfig,
+          descriptionField,
+        }),
+        setColorField: (searchResultConfig, color) => ({ ...searchResultConfig, color }),
+        setDetailFields: (searchResultConfig, { result: { destination, source } }) => {
+          const detailFields = cloneDeep(searchResultConfig.detailFields);
+          const element = detailFields[source.index];
+          detailFields.splice(source.index, 1);
+          detailFields.splice(destination!.index, 0, element);
+          return {
+            ...searchResultConfig,
+            detailFields,
+          };
+        },
+        addDetailField: (searchResultConfig, newfield) => {
+          const detailFields = cloneDeep(searchResultConfig.detailFields);
+          detailFields.push(newfield);
+          return {
+            ...searchResultConfig,
+            detailFields,
+          };
+        },
+        removeDetailField: (searchResultConfig, index) => {
+          const detailFields = cloneDeep(searchResultConfig.detailFields);
+          detailFields.splice(index, 1);
+          return {
+            ...searchResultConfig,
+            detailFields,
+          };
+        },
+        updateDetailField: (searchResultConfig, { updatedField, index }) => {
+          const detailFields = cloneDeep(searchResultConfig.detailFields);
+          detailFields[index] = updatedField;
+          return {
+            ...searchResultConfig,
+            detailFields,
+          };
+        },
+      },
+    ],
+    serverRoute: [
+      '',
+      {
+        onInitializeDisplaySettings: (_, { serverRoute }) => serverRoute,
+      },
+    ],
+    editFieldIndex: [
+      null,
+      {
+        openEditDetailField: (_, openEditDetailField) => openEditDetailField,
+        toggleFieldEditorModal: () => null,
+      },
+    ],
+    dataLoading: [
+      true,
+      {
+        onInitializeDisplaySettings: () => false,
+      },
+    ],
+    addFieldModalVisible: [
+      false,
+      {
+        toggleFieldEditorModal: (addFieldModalVisible) => !addFieldModalVisible,
+        openEditDetailField: () => true,
+        updateDetailField: () => false,
+        addDetailField: () => false,
+      },
+    ],
+    titleFieldHover: [
+      false,
+      {
+        toggleTitleFieldHover: (titleFieldHover) => !titleFieldHover,
+      },
+    ],
+    urlFieldHover: [
+      false,
+      {
+        toggleUrlFieldHover: (urlFieldHover) => !urlFieldHover,
+      },
+    ],
+    subtitleFieldHover: [
+      false,
+      {
+        toggleSubtitleFieldHover: (subtitleFieldHover) => !subtitleFieldHover,
+      },
+    ],
+    descriptionFieldHover: [
+      false,
+      {
+        toggleDescriptionFieldHover: (addFieldModalVisible) => !addFieldModalVisible,
+      },
+    ],
+  },
+  selectors: ({ selectors }) => ({
+    fieldOptions: [
+      () => [selectors.schemaFields],
+      (schemaFields) => Object.keys(schemaFields).map(euiSelectObjectFromValue),
+    ],
+    optionalFieldOptions: [
+      () => [selectors.fieldOptions],
+      (fieldOptions) => {
+        const optionalFieldOptions = cloneDeep(fieldOptions);
+        optionalFieldOptions.unshift({ value: '', text: '' });
+        return optionalFieldOptions;
+      },
+    ],
+    // We don't want to let the user add a duplicate detailField.
+    availableFieldOptions: [
+      () => [selectors.fieldOptions, selectors.searchResultConfig],
+      (fieldOptions, { detailFields }) => {
+        const usedFields = detailFields.map((usedField: DetailField) =>
+          euiSelectObjectFromValue(usedField.fieldName)
+        );
+        return differenceBy(fieldOptions, usedFields, 'value');
+      },
+    ],
+    unsavedChanges: [
+      () => [selectors.searchResultConfig, selectors.serverSearchResultConfig],
+      (uiConfig, serverConfig) => !isEqual(uiConfig, serverConfig),
+    ],
+  }),
+  listeners: ({ actions, values }) => ({
+    initializeDisplaySettings: async () => {
+      const { isOrganization } = AppLogic.values;
+      const {
+        contentSource: { id: sourceId },
+      } = SourceLogic.values;
+
+      const route = isOrganization
+        ? `/api/workplace_search/org/sources/${sourceId}/display_settings/config`
+        : `/api/workplace_search/account/sources/${sourceId}/display_settings/config`;
+
+      try {
+        const response = await HttpLogic.values.http.get(route);
+        actions.onInitializeDisplaySettings({
+          isOrganization,
+          sourceId,
+          serverRoute: route,
+          ...response,
+        });
+      } catch (e) {
+        flashAPIErrors(e);
+      }
+    },
+    setServerData: async () => {
+      const { searchResultConfig, serverRoute } = values;
+
+      try {
+        const response = await HttpLogic.values.http.post(serverRoute, {
+          body: JSON.stringify({ ...searchResultConfig }),
+        });
+        actions.setServerResponseData(response);
+      } catch (e) {
+        flashAPIErrors(e);
+      }
+    },
+    setServerResponseData: () => {
+      setSuccessMessage(SUCCESS_MESSAGE);
+    },
+    toggleFieldEditorModal: () => {
+      FlashMessagesLogic.actions.clearFlashMessages();
+    },
+    resetDisplaySettingsState: () => {
+      FlashMessagesLogic.actions.clearFlashMessages();
+    },
+  }),
+});
+
+const euiSelectObjectFromValue = (value: string) => ({ text: value, value });
+
+// By default, the color is `null` on the server. The color is a required field and the
+// EuiColorPicker components doesn't allow the field to be required so the form can be
+// submitted with no color and this results in a server error. The default should be black
+// and this allows the `searchResultConfig` and the `serverSearchResultConfig` reducers to
+// stay synced on initialization.
+const setDefaultColor = (searchResultConfig: SearchResultConfig) => ({
+  ...searchResultConfig,
+  color: searchResultConfig.color || '#000000',
+});
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_router.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_router.tsx
index 5cebaad95e3a8..01ac93735b8a8 100644
--- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_router.tsx
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/display_settings_router.tsx
@@ -6,4 +6,33 @@
 
 import React from 'react';
 
-export const DisplaySettingsRouter: React.FC = () => <>Display Settings Placeholder</>;
+import { useValues } from 'kea';
+import { Route, Switch } from 'react-router-dom';
+
+import { AppLogic } from '../../../../app_logic';
+
+import {
+  DISPLAY_SETTINGS_RESULT_DETAIL_PATH,
+  DISPLAY_SETTINGS_SEARCH_RESULT_PATH,
+  getSourcesPath,
+} from '../../../../routes';
+
+import { DisplaySettings } from './display_settings';
+
+export const DisplaySettingsRouter: React.FC = () => {
+  const { isOrganization } = useValues(AppLogic);
+  return (
+    <Switch>
+      <Route
+        exact
+        path={getSourcesPath(DISPLAY_SETTINGS_SEARCH_RESULT_PATH, isOrganization)}
+        render={() => <DisplaySettings tabId={0} />}
+      />
+      <Route
+        exact
+        path={getSourcesPath(DISPLAY_SETTINGS_RESULT_DETAIL_PATH, isOrganization)}
+        render={() => <DisplaySettings tabId={1} />}
+      />
+    </Switch>
+  );
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_result_detail_card.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_result_detail_card.tsx
new file mode 100644
index 0000000000000..468f7d2f7ad05
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_result_detail_card.tsx
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import classNames from 'classnames';
+import { useValues } from 'kea';
+
+import { EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText, EuiTitle } from '@elastic/eui';
+
+import { DisplaySettingsLogic } from './display_settings_logic';
+
+import { CustomSourceIcon } from './custom_source_icon';
+import { TitleField } from './title_field';
+
+export const ExampleResultDetailCard: React.FC = () => {
+  const {
+    sourceName,
+    searchResultConfig: { titleField, urlField, color, detailFields },
+    titleFieldHover,
+    urlFieldHover,
+    exampleDocuments,
+  } = useValues(DisplaySettingsLogic);
+
+  const result = exampleDocuments[0];
+
+  return (
+    <div className="example-result-detail-card">
+      <div className="example-result-detail-card__header">
+        <div className="example-result-detail-card__border" style={{ backgroundColor: color }} />
+        <div className="example-result-detail-card__source-name">
+          <EuiFlexGroup alignItems="center" gutterSize="s">
+            <EuiFlexItem grow={false}>
+              <CustomSourceIcon color={color} />
+            </EuiFlexItem>
+            <EuiFlexItem>{sourceName}</EuiFlexItem>
+          </EuiFlexGroup>
+        </div>
+        <div className="example-result-content">
+          <TitleField titleFieldHover={titleFieldHover} titleField={titleField} result={result} />
+          <div
+            className={classNames('example-result-content__url', {
+              'example-result-field-hover': urlFieldHover,
+            })}
+          >
+            {urlField ? (
+              <div className="eui-textTruncate">{result[urlField]}</div>
+            ) : (
+              <span className="example-result-content-placeholder">URL</span>
+            )}
+          </div>
+        </div>
+      </div>
+      <div className="example-result-detail-card__content">
+        {detailFields.length > 0 ? (
+          detailFields.map(({ fieldName, label }, index) => (
+            <div className="example-result-detail-card__field" key={index}>
+              <EuiTitle size="xs">
+                <h4>{label}</h4>
+              </EuiTitle>
+              <EuiText size="s" color="subdued">
+                <div className="eui-textBreakWord">{result[fieldName]}</div>
+              </EuiText>
+            </div>
+          ))
+        ) : (
+          <EuiSpacer size="m" />
+        )}
+      </div>
+    </div>
+  );
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_search_result_group.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_search_result_group.tsx
new file mode 100644
index 0000000000000..14239b1654308
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_search_result_group.tsx
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import { isColorDark, hexToRgb } from '@elastic/eui';
+import classNames from 'classnames';
+import { useValues } from 'kea';
+
+import { DisplaySettingsLogic } from './display_settings_logic';
+
+import { CustomSourceIcon } from './custom_source_icon';
+import { SubtitleField } from './subtitle_field';
+import { TitleField } from './title_field';
+
+export const ExampleSearchResultGroup: React.FC = () => {
+  const {
+    sourceName,
+    searchResultConfig: { titleField, subtitleField, descriptionField, color },
+    titleFieldHover,
+    subtitleFieldHover,
+    descriptionFieldHover,
+    exampleDocuments,
+  } = useValues(DisplaySettingsLogic);
+
+  return (
+    <div className="example-result-group">
+      <div className="example-result-group__header" style={{ backgroundColor: color }}>
+        <CustomSourceIcon color={isColorDark.apply(null, hexToRgb(color)) ? 'white' : 'black'} />
+        <span
+          className="example-result-group__source-name"
+          style={{ color: isColorDark.apply(null, hexToRgb(color)) ? 'white' : 'black' }}
+        >
+          {sourceName}
+        </span>
+      </div>
+      <div className="example-result-group__content">
+        <div className="example-result-group__border" style={{ backgroundColor: color }} />
+        <div className="example-result-group__results">
+          {exampleDocuments.map((result, id) => (
+            <div key={id} className="example-grouped-result">
+              <div className="example-result-content">
+                <TitleField
+                  titleFieldHover={titleFieldHover}
+                  titleField={titleField}
+                  result={result}
+                />
+                <SubtitleField
+                  subtitleFieldHover={subtitleFieldHover}
+                  subtitleField={subtitleField}
+                  result={result}
+                />
+                <div
+                  className={classNames('example-result-content__description', {
+                    'example-result-field-hover': descriptionFieldHover,
+                  })}
+                >
+                  {descriptionField ? (
+                    <div>{result[descriptionField]}</div>
+                  ) : (
+                    <span className="example-result-content-placeholder">Description</span>
+                  )}
+                </div>
+              </div>
+            </div>
+          ))}
+        </div>
+      </div>
+    </div>
+  );
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_standout_result.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_standout_result.tsx
new file mode 100644
index 0000000000000..4ef3b1fe14b93
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/example_standout_result.tsx
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import classNames from 'classnames';
+import { useValues } from 'kea';
+
+import { isColorDark, hexToRgb } from '@elastic/eui';
+
+import { DisplaySettingsLogic } from './display_settings_logic';
+
+import { CustomSourceIcon } from './custom_source_icon';
+import { SubtitleField } from './subtitle_field';
+import { TitleField } from './title_field';
+
+export const ExampleStandoutResult: React.FC = () => {
+  const {
+    sourceName,
+    searchResultConfig: { titleField, subtitleField, descriptionField, color },
+    titleFieldHover,
+    subtitleFieldHover,
+    descriptionFieldHover,
+    exampleDocuments,
+  } = useValues(DisplaySettingsLogic);
+
+  const result = exampleDocuments[0];
+
+  return (
+    <div className="example-standout-result">
+      <div className="example-standout-result__header" style={{ backgroundColor: color }}>
+        <CustomSourceIcon color={isColorDark.apply(null, hexToRgb(color)) ? 'white' : 'black'} />
+        <span
+          className="example-standout-result__source-name"
+          style={{ color: isColorDark.apply(null, hexToRgb(color)) ? 'white' : 'black' }}
+        >
+          {sourceName}
+        </span>
+      </div>
+      <div className="example-standout-result__content">
+        <div className="example-result-content">
+          <TitleField titleFieldHover={titleFieldHover} titleField={titleField} result={result} />
+          <SubtitleField
+            subtitleFieldHover={subtitleFieldHover}
+            subtitleField={subtitleField}
+            result={result}
+          />
+          <div
+            className={classNames('example-result-content__description', {
+              'example-result-field-hover': descriptionFieldHover,
+            })}
+          >
+            {descriptionField ? (
+              <span>{result[descriptionField]}</span>
+            ) : (
+              <span className="example-result-content-placeholder">Description</span>
+            )}
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/field_editor_modal.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/field_editor_modal.tsx
new file mode 100644
index 0000000000000..587916a741d66
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/field_editor_modal.tsx
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FormEvent, useState } from 'react';
+
+import { useActions, useValues } from 'kea';
+
+import {
+  EuiButton,
+  EuiButtonEmpty,
+  EuiFieldText,
+  EuiForm,
+  EuiFormRow,
+  EuiModal,
+  EuiModalBody,
+  EuiModalFooter,
+  EuiModalHeader,
+  EuiModalHeaderTitle,
+  EuiOverlayMask,
+  EuiSelect,
+} from '@elastic/eui';
+
+import { DisplaySettingsLogic } from './display_settings_logic';
+
+const emptyField = { fieldName: '', label: '' };
+
+export const FieldEditorModal: React.FC = () => {
+  const { toggleFieldEditorModal, addDetailField, updateDetailField } = useActions(
+    DisplaySettingsLogic
+  );
+
+  const {
+    searchResultConfig: { detailFields },
+    availableFieldOptions,
+    fieldOptions,
+    editFieldIndex,
+  } = useValues(DisplaySettingsLogic);
+
+  const isEditing = editFieldIndex || editFieldIndex === 0;
+  const field = isEditing ? detailFields[editFieldIndex || 0] : emptyField;
+  const [fieldName, setName] = useState(field.fieldName || '');
+  const [label, setLabel] = useState(field.label || '');
+
+  const handleSubmit = (e: FormEvent) => {
+    e.preventDefault();
+    if (isEditing) {
+      updateDetailField({ fieldName, label }, editFieldIndex);
+    } else {
+      addDetailField({ fieldName, label });
+    }
+  };
+
+  const ACTION_LABEL = isEditing ? 'Update' : 'Add';
+
+  return (
+    <EuiOverlayMask>
+      <form onSubmit={handleSubmit}>
+        <EuiModal onClose={toggleFieldEditorModal} maxWidth={475}>
+          <EuiModalHeader>
+            <EuiModalHeaderTitle>{ACTION_LABEL} Field</EuiModalHeaderTitle>
+          </EuiModalHeader>
+          <EuiModalBody>
+            <EuiForm>
+              <EuiFormRow label="Field">
+                <EuiSelect
+                  options={isEditing ? fieldOptions : availableFieldOptions}
+                  name="field"
+                  required={true}
+                  className="field-selector"
+                  hasNoInitialSelection={true}
+                  data-test-subj="AvailableFieldOptions"
+                  value={fieldName}
+                  disabled={!!isEditing}
+                  onChange={(e) => setName(e.target.value)}
+                />
+              </EuiFormRow>
+              <EuiFormRow label="Label">
+                <EuiFieldText
+                  name="label"
+                  required={true}
+                  data-test-subj="VisibleFieldName"
+                  value={label}
+                  onChange={(e) => setLabel(e.target.value)}
+                />
+              </EuiFormRow>
+            </EuiForm>
+          </EuiModalBody>
+          <EuiModalFooter>
+            <EuiButtonEmpty onClick={toggleFieldEditorModal}>Cancel</EuiButtonEmpty>
+            <EuiButton data-test-subj="FieldSubmitButton" color="primary" fill={true} type="submit">
+              {ACTION_LABEL} Field
+            </EuiButton>
+          </EuiModalFooter>
+        </EuiModal>
+      </form>
+    </EuiOverlayMask>
+  );
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/result_detail.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/result_detail.tsx
new file mode 100644
index 0000000000000..cb65d8ef671e6
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/result_detail.tsx
@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import { useActions, useValues } from 'kea';
+
+import {
+  EuiButtonEmpty,
+  EuiButtonIcon,
+  EuiDragDropContext,
+  EuiDraggable,
+  EuiDroppable,
+  EuiFlexGrid,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiForm,
+  EuiFormRow,
+  EuiIcon,
+  EuiPanel,
+  EuiSpacer,
+  EuiTextColor,
+  EuiTitle,
+} from '@elastic/eui';
+
+import { DisplaySettingsLogic } from './display_settings_logic';
+
+import { ExampleResultDetailCard } from './example_result_detail_card';
+
+export const ResultDetail: React.FC = () => {
+  const {
+    toggleFieldEditorModal,
+    setDetailFields,
+    openEditDetailField,
+    removeDetailField,
+  } = useActions(DisplaySettingsLogic);
+
+  const {
+    searchResultConfig: { detailFields },
+    availableFieldOptions,
+  } = useValues(DisplaySettingsLogic);
+
+  return (
+    <>
+      <EuiSpacer />
+      <EuiFlexGrid columns={2}>
+        <EuiFlexItem>
+          <EuiSpacer size="m" />
+          <EuiForm>
+            <EuiFormRow>
+              <>
+                <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
+                  <EuiFlexItem>
+                    <EuiTitle size="s">
+                      <h3>Visible Fields</h3>
+                    </EuiTitle>
+                  </EuiFlexItem>
+                  <EuiFlexItem grow={false}>
+                    <EuiButtonEmpty
+                      color="primary"
+                      onClick={toggleFieldEditorModal}
+                      disabled={availableFieldOptions.length < 1}
+                      data-test-subj="AddFieldButton"
+                    >
+                      Add Field
+                    </EuiButtonEmpty>
+                  </EuiFlexItem>
+                </EuiFlexGroup>
+                <EuiSpacer size="s" />
+                {detailFields.length > 0 ? (
+                  <EuiDragDropContext onDragEnd={setDetailFields}>
+                    <EuiDroppable
+                      droppableId="visible_fields"
+                      className="visible-fields-container"
+                      withPanel={true}
+                    >
+                      <>
+                        {detailFields.map(({ fieldName, label }, index) => (
+                          <EuiDraggable
+                            key={index}
+                            index={index}
+                            draggableId={`${fieldName}-${index}`}
+                            customDragHandle={true}
+                            spacing="m"
+                          >
+                            {(provided) => (
+                              <EuiPanel>
+                                <EuiFlexGroup alignItems="center">
+                                  <EuiFlexItem grow={false}>
+                                    <div {...provided.dragHandleProps}>
+                                      <EuiIcon color="subdued" type="grab" />
+                                    </div>
+                                  </EuiFlexItem>
+                                  <EuiFlexItem>
+                                    <EuiTitle size="xs">
+                                      <h4>{fieldName}</h4>
+                                    </EuiTitle>
+                                    <EuiTextColor color="subdued">
+                                      <div>&ldquo;{label || ''}&rdquo;</div>
+                                    </EuiTextColor>
+                                  </EuiFlexItem>
+                                  <EuiFlexItem grow={false}>
+                                    <div>
+                                      <EuiButtonIcon
+                                        aria-label="Edit Field"
+                                        iconType="pencil"
+                                        onClick={() => openEditDetailField(index)}
+                                      />
+                                      <EuiButtonIcon
+                                        aria-label="Remove Field"
+                                        iconType="cross"
+                                        onClick={() => removeDetailField(index)}
+                                      />
+                                    </div>
+                                  </EuiFlexItem>
+                                </EuiFlexGroup>
+                              </EuiPanel>
+                            )}
+                          </EuiDraggable>
+                        ))}
+                      </>
+                    </EuiDroppable>
+                  </EuiDragDropContext>
+                ) : (
+                  <p>Add fields and move them into the order you want them to appear.</p>
+                )}
+              </>
+            </EuiFormRow>
+          </EuiForm>
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiPanel>
+            <EuiTitle size="s">
+              <h3>Preview</h3>
+            </EuiTitle>
+            <EuiSpacer />
+            <ExampleResultDetailCard />
+          </EuiPanel>
+        </EuiFlexItem>
+      </EuiFlexGrid>
+    </>
+  );
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/search_results.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/search_results.tsx
new file mode 100644
index 0000000000000..cfe0ddb1533ec
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/search_results.tsx
@@ -0,0 +1,164 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import { useActions, useValues } from 'kea';
+
+import {
+  EuiColorPicker,
+  EuiFlexGrid,
+  EuiFlexItem,
+  EuiForm,
+  EuiFormRow,
+  EuiPanel,
+  EuiSelect,
+  EuiSpacer,
+  EuiTitle,
+} from '@elastic/eui';
+
+import { DisplaySettingsLogic } from './display_settings_logic';
+
+import { ExampleSearchResultGroup } from './example_search_result_group';
+import { ExampleStandoutResult } from './example_standout_result';
+
+export const SearchResults: React.FC = () => {
+  const {
+    toggleTitleFieldHover,
+    toggleSubtitleFieldHover,
+    toggleDescriptionFieldHover,
+    setTitleField,
+    setSubtitleField,
+    setDescriptionField,
+    setUrlField,
+    setColorField,
+  } = useActions(DisplaySettingsLogic);
+
+  const {
+    searchResultConfig: { titleField, descriptionField, subtitleField, urlField, color },
+    fieldOptions,
+    optionalFieldOptions,
+  } = useValues(DisplaySettingsLogic);
+
+  return (
+    <>
+      <EuiSpacer />
+      <EuiFlexGrid columns={2}>
+        <EuiFlexItem>
+          <EuiSpacer size="m" />
+          <EuiTitle size="s">
+            <h3>Search Result Settings</h3>
+          </EuiTitle>
+          <EuiSpacer size="s" />
+          <EuiForm>
+            <EuiFormRow
+              label="Title"
+              onMouseOver={toggleTitleFieldHover}
+              onMouseOut={toggleTitleFieldHover}
+              onFocus={() => null} // FIXME
+              onBlur={() => null} // FIXME
+            >
+              <EuiSelect
+                options={fieldOptions}
+                required={true}
+                name="titleField"
+                className="field-selector"
+                hasNoInitialSelection={true}
+                data-test-subj="TitleFieldSelect"
+                value={titleField || ''}
+                onChange={(e) => setTitleField(e.target.value)}
+              />
+            </EuiFormRow>
+            <EuiFormRow label="URL">
+              <EuiSelect
+                options={fieldOptions}
+                required={true}
+                className="field-selector"
+                hasNoInitialSelection={true}
+                data-test-subj="UrlFieldSelect"
+                value={urlField || ''}
+                onChange={(e) => setUrlField(e.target.value)}
+              />
+            </EuiFormRow>
+            <EuiFormRow label="Color">
+              <EuiColorPicker
+                color={color}
+                onChange={setColorField}
+                onFocus={() => null} // FIXME
+                onBlur={() => null} // FIXME
+              />
+            </EuiFormRow>
+            <EuiFormRow
+              label="Subtitle"
+              helpText="This area is optional"
+              onMouseOver={toggleSubtitleFieldHover}
+              onMouseOut={toggleSubtitleFieldHover}
+              onFocus={() => null} // FIXME
+              onBlur={() => null} // FIXME
+            >
+              <EuiSelect
+                options={optionalFieldOptions}
+                className="field-selector"
+                hasNoInitialSelection={true}
+                data-test-subj="SubtitleFieldSelect"
+                value={subtitleField || ''}
+                onChange={({ target: { value } }) => setSubtitleField(value === '' ? null : value)}
+              />
+            </EuiFormRow>
+            <EuiFormRow
+              label="Description"
+              helpText="This area is optional"
+              onMouseOver={toggleDescriptionFieldHover}
+              onMouseOut={toggleDescriptionFieldHover}
+              onFocus={() => null} // FIXME
+              onBlur={() => null} // FIXME
+            >
+              <EuiSelect
+                options={optionalFieldOptions}
+                className="field-selector"
+                hasNoInitialSelection={true}
+                data-test-subj="DescriptionFieldSelect"
+                value={descriptionField || ''}
+                onChange={({ target: { value } }) =>
+                  setDescriptionField(value === '' ? null : value)
+                }
+              />
+            </EuiFormRow>
+          </EuiForm>
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiPanel>
+            <EuiTitle size="s">
+              <h3>Preview</h3>
+            </EuiTitle>
+            <EuiSpacer />
+            <div className="section-header">
+              <EuiTitle size="xs">
+                <h4>Featured Results</h4>
+              </EuiTitle>
+              <p className="section-header__description">
+                A matching document will appear as a single bold card.
+              </p>
+            </div>
+            <EuiSpacer />
+            <ExampleStandoutResult />
+            <EuiSpacer />
+            <div className="section-header">
+              <EuiTitle size="xs">
+                <h4>Standard Results</h4>
+              </EuiTitle>
+              <p className="section-header__description">
+                Somewhat matching documents will appear as a set.
+              </p>
+            </div>
+            <EuiSpacer />
+            <ExampleSearchResultGroup />
+          </EuiPanel>
+        </EuiFlexItem>
+      </EuiFlexGrid>
+    </>
+  );
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/subtitle_field.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/subtitle_field.tsx
new file mode 100644
index 0000000000000..e27052ddffc04
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/subtitle_field.tsx
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import classNames from 'classnames';
+
+import { Result } from '../../../../types';
+
+interface SubtitleFieldProps {
+  result: Result;
+  subtitleField: string | null;
+  subtitleFieldHover: boolean;
+}
+
+export const SubtitleField: React.FC<SubtitleFieldProps> = ({
+  result,
+  subtitleField,
+  subtitleFieldHover,
+}) => (
+  <div
+    className={classNames('example-result-content__subtitle', {
+      'example-result-field-hover': subtitleFieldHover,
+    })}
+  >
+    {subtitleField ? (
+      <div className="eui-textTruncate">{result[subtitleField]}</div>
+    ) : (
+      <span className="example-result-content-placeholder">Subtitle</span>
+    )}
+  </div>
+);
diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/title_field.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/title_field.tsx
new file mode 100644
index 0000000000000..a54c0977b464f
--- /dev/null
+++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/display_settings/title_field.tsx
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import classNames from 'classnames';
+
+import { Result } from '../../../../types';
+
+interface TitleFieldProps {
+  result: Result;
+  titleField: string | null;
+  titleFieldHover: boolean;
+}
+
+export const TitleField: React.FC<TitleFieldProps> = ({ result, titleField, titleFieldHover }) => {
+  const title = titleField ? result[titleField] : '';
+  const titleDisplay = Array.isArray(title) ? title.join(', ') : title;
+  return (
+    <div
+      className={classNames('example-result-content__title', {
+        'example-result-field-hover': titleFieldHover,
+      })}
+    >
+      {titleField ? (
+        <div className="eui-textTruncate">{titleDisplay}</div>
+      ) : (
+        <span className="example-result-content-placeholder">Title</span>
+      )}
+    </div>
+  );
+};
diff --git a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts
index 22e2deaace1dc..62f4dceeac363 100644
--- a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts
+++ b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.test.ts
@@ -18,6 +18,7 @@ import {
   registerAccountPreSourceRoute,
   registerAccountPrepareSourcesRoute,
   registerAccountSourceSearchableRoute,
+  registerAccountSourceDisplaySettingsConfig,
   registerOrgSourcesRoute,
   registerOrgSourcesStatusRoute,
   registerOrgSourceRoute,
@@ -29,6 +30,7 @@ import {
   registerOrgPreSourceRoute,
   registerOrgPrepareSourcesRoute,
   registerOrgSourceSearchableRoute,
+  registerOrgSourceDisplaySettingsConfig,
   registerOrgSourceOauthConfigurationsRoute,
   registerOrgSourceOauthConfigurationRoute,
 } from './sources';
@@ -446,6 +448,81 @@ describe('sources routes', () => {
     });
   });
 
+  describe('GET /api/workplace_search/account/sources/{id}/display_settings/config', () => {
+    let mockRouter: MockRouter;
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('creates a request handler', () => {
+      mockRouter = new MockRouter({
+        method: 'get',
+        path: '/api/workplace_search/account/sources/{id}/display_settings/config',
+        payload: 'params',
+      });
+
+      registerAccountSourceDisplaySettingsConfig({
+        ...mockDependencies,
+        router: mockRouter.router,
+      });
+
+      const mockRequest = {
+        params: {
+          id: '123',
+        },
+      };
+
+      mockRouter.callRoute(mockRequest);
+
+      expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({
+        path: '/ws/sources/123/display_settings/config',
+      });
+    });
+  });
+
+  describe('POST /api/workplace_search/account/sources/{id}/display_settings/config', () => {
+    let mockRouter: MockRouter;
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+      mockRouter = new MockRouter({
+        method: 'post',
+        path: '/api/workplace_search/account/sources/{id}/display_settings/config',
+        payload: 'body',
+      });
+
+      registerAccountSourceDisplaySettingsConfig({
+        ...mockDependencies,
+        router: mockRouter.router,
+      });
+    });
+
+    it('creates a request handler', () => {
+      const mockRequest = {
+        params: { id: '123' },
+        body: {
+          titleField: 'foo',
+          subtitleField: 'bar',
+          descriptionField: 'this is a thing',
+          urlField: 'http://youknowfor.search',
+          color: '#aaa',
+          detailFields: {
+            fieldName: 'myField',
+            label: 'My Field',
+          },
+        },
+      };
+
+      mockRouter.callRoute(mockRequest);
+
+      expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({
+        path: '/ws/sources/123/display_settings/config',
+        body: mockRequest.body,
+      });
+    });
+  });
+
   describe('GET /api/workplace_search/org/sources', () => {
     let mockRouter: MockRouter;
 
@@ -848,6 +925,81 @@ describe('sources routes', () => {
     });
   });
 
+  describe('GET /api/workplace_search/org/sources/{id}/display_settings/config', () => {
+    let mockRouter: MockRouter;
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('creates a request handler', () => {
+      mockRouter = new MockRouter({
+        method: 'get',
+        path: '/api/workplace_search/org/sources/{id}/display_settings/config',
+        payload: 'params',
+      });
+
+      registerOrgSourceDisplaySettingsConfig({
+        ...mockDependencies,
+        router: mockRouter.router,
+      });
+
+      const mockRequest = {
+        params: {
+          id: '123',
+        },
+      };
+
+      mockRouter.callRoute(mockRequest);
+
+      expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({
+        path: '/ws/org/sources/123/display_settings/config',
+      });
+    });
+  });
+
+  describe('POST /api/workplace_search/org/sources/{id}/display_settings/config', () => {
+    let mockRouter: MockRouter;
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+      mockRouter = new MockRouter({
+        method: 'post',
+        path: '/api/workplace_search/org/sources/{id}/display_settings/config',
+        payload: 'body',
+      });
+
+      registerOrgSourceDisplaySettingsConfig({
+        ...mockDependencies,
+        router: mockRouter.router,
+      });
+    });
+
+    it('creates a request handler', () => {
+      const mockRequest = {
+        params: { id: '123' },
+        body: {
+          titleField: 'foo',
+          subtitleField: 'bar',
+          descriptionField: 'this is a thing',
+          urlField: 'http://youknowfor.search',
+          color: '#aaa',
+          detailFields: {
+            fieldName: 'myField',
+            label: 'My Field',
+          },
+        },
+      };
+
+      mockRouter.callRoute(mockRequest);
+
+      expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({
+        path: '/ws/org/sources/123/display_settings/config',
+        body: mockRequest.body,
+      });
+    });
+  });
+
   describe('GET /api/workplace_search/org/settings/connectors', () => {
     let mockRouter: MockRouter;
 
diff --git a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts
index 24473388c03b1..d43a4252c7e1f 100644
--- a/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts
+++ b/x-pack/plugins/enterprise_search/server/routes/workplace_search/sources.ts
@@ -25,6 +25,21 @@ const oAuthConfigSchema = schema.object({
   consumer_key: schema.string(),
 });
 
+const displayFieldSchema = schema.object({
+  fieldName: schema.string(),
+  label: schema.string(),
+});
+
+const displaySettingsSchema = schema.object({
+  titleField: schema.maybe(schema.string()),
+  subtitleField: schema.maybe(schema.string()),
+  descriptionField: schema.maybe(schema.string()),
+  urlField: schema.maybe(schema.string()),
+  color: schema.string(),
+  urlFieldIsLinkable: schema.boolean(),
+  detailFields: schema.oneOf([schema.arrayOf(displayFieldSchema), displayFieldSchema]),
+});
+
 export function registerAccountSourcesRoute({
   router,
   enterpriseSearchRequestHandler,
@@ -285,6 +300,45 @@ export function registerAccountSourceSearchableRoute({
   );
 }
 
+export function registerAccountSourceDisplaySettingsConfig({
+  router,
+  enterpriseSearchRequestHandler,
+}: RouteDependencies) {
+  router.get(
+    {
+      path: '/api/workplace_search/account/sources/{id}/display_settings/config',
+      validate: {
+        params: schema.object({
+          id: schema.string(),
+        }),
+      },
+    },
+    async (context, request, response) => {
+      return enterpriseSearchRequestHandler.createRequest({
+        path: `/ws/sources/${request.params.id}/display_settings/config`,
+      })(context, request, response);
+    }
+  );
+
+  router.post(
+    {
+      path: '/api/workplace_search/account/sources/{id}/display_settings/config',
+      validate: {
+        body: displaySettingsSchema,
+        params: schema.object({
+          id: schema.string(),
+        }),
+      },
+    },
+    async (context, request, response) => {
+      return enterpriseSearchRequestHandler.createRequest({
+        path: `/ws/sources/${request.params.id}/display_settings/config`,
+        body: request.body,
+      })(context, request, response);
+    }
+  );
+}
+
 export function registerOrgSourcesRoute({
   router,
   enterpriseSearchRequestHandler,
@@ -545,6 +599,45 @@ export function registerOrgSourceSearchableRoute({
   );
 }
 
+export function registerOrgSourceDisplaySettingsConfig({
+  router,
+  enterpriseSearchRequestHandler,
+}: RouteDependencies) {
+  router.get(
+    {
+      path: '/api/workplace_search/org/sources/{id}/display_settings/config',
+      validate: {
+        params: schema.object({
+          id: schema.string(),
+        }),
+      },
+    },
+    async (context, request, response) => {
+      return enterpriseSearchRequestHandler.createRequest({
+        path: `/ws/org/sources/${request.params.id}/display_settings/config`,
+      })(context, request, response);
+    }
+  );
+
+  router.post(
+    {
+      path: '/api/workplace_search/org/sources/{id}/display_settings/config',
+      validate: {
+        body: displaySettingsSchema,
+        params: schema.object({
+          id: schema.string(),
+        }),
+      },
+    },
+    async (context, request, response) => {
+      return enterpriseSearchRequestHandler.createRequest({
+        path: `/ws/org/sources/${request.params.id}/display_settings/config`,
+        body: request.body,
+      })(context, request, response);
+    }
+  );
+}
+
 export function registerOrgSourceOauthConfigurationsRoute({
   router,
   enterpriseSearchRequestHandler,
@@ -647,6 +740,7 @@ export const registerSourcesRoutes = (dependencies: RouteDependencies) => {
   registerAccountPreSourceRoute(dependencies);
   registerAccountPrepareSourcesRoute(dependencies);
   registerAccountSourceSearchableRoute(dependencies);
+  registerAccountSourceDisplaySettingsConfig(dependencies);
   registerOrgSourcesRoute(dependencies);
   registerOrgSourcesStatusRoute(dependencies);
   registerOrgSourceRoute(dependencies);
@@ -658,6 +752,7 @@ export const registerSourcesRoutes = (dependencies: RouteDependencies) => {
   registerOrgPreSourceRoute(dependencies);
   registerOrgPrepareSourcesRoute(dependencies);
   registerOrgSourceSearchableRoute(dependencies);
+  registerOrgSourceDisplaySettingsConfig(dependencies);
   registerOrgSourceOauthConfigurationsRoute(dependencies);
   registerOrgSourceOauthConfigurationRoute(dependencies);
 };

From ea8ea4e4e0c62839ccf3d6c7315b9166b4a3a607 Mon Sep 17 00:00:00 2001
From: Spencer <email@spalger.com>
Date: Wed, 25 Nov 2020 08:03:47 -0700
Subject: [PATCH 10/37] [dev/cli] detect worker type using env, not cluster
 module (#83977)

* [dev/cli] detect worker type using env, not cluster module

* remove unused property

* assume that if process.send is undefined we are not a child

* update comment

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 packages/kbn-config/src/__mocks__/env.ts            |  3 +--
 .../kbn-config/src/__snapshots__/env.test.ts.snap   | 12 ++++++------
 packages/kbn-config/src/env.test.ts                 |  2 +-
 packages/kbn-config/src/env.ts                      |  8 ++++----
 .../kbn-legacy-logging/src/log_format_string.ts     |  4 ++--
 packages/kbn-legacy-logging/src/rotate/index.ts     |  7 -------
 .../kbn-legacy-logging/src/rotate/log_rotator.ts    | 13 ++-----------
 src/apm.js                                          |  4 ----
 src/cli/cluster/cluster_manager.ts                  |  2 --
 src/cli/cluster/worker.ts                           |  4 +---
 src/cli/serve/serve.js                              |  6 +++---
 src/core/server/bootstrap.ts                        | 11 +++++------
 src/core/server/http/http_service.test.ts           |  2 +-
 src/core/server/http/http_service.ts                |  2 +-
 src/core/server/legacy/legacy_service.test.ts       |  4 ++--
 src/core/server/legacy/legacy_service.ts            |  8 +++-----
 src/core/server/plugins/plugins_service.test.ts     |  8 ++++----
 src/core/server/plugins/plugins_service.ts          |  4 ++--
 src/core/server/server.test.ts                      |  4 ++--
 src/core/server/server.ts                           |  2 +-
 src/core/test_helpers/kbn_server.ts                 |  2 +-
 src/legacy/server/kbn_server.js                     |  3 +--
 22 files changed, 43 insertions(+), 72 deletions(-)

diff --git a/packages/kbn-config/src/__mocks__/env.ts b/packages/kbn-config/src/__mocks__/env.ts
index e0f6f6add1686..8b7475680ecf5 100644
--- a/packages/kbn-config/src/__mocks__/env.ts
+++ b/packages/kbn-config/src/__mocks__/env.ts
@@ -42,7 +42,6 @@ export function getEnvOptions(options: DeepPartial<EnvOptions> = {}): EnvOptions
       runExamples: false,
       ...(options.cliArgs || {}),
     },
-    isDevClusterMaster:
-      options.isDevClusterMaster !== undefined ? options.isDevClusterMaster : false,
+    isDevCliParent: options.isDevCliParent !== undefined ? options.isDevCliParent : false,
   };
 }
diff --git a/packages/kbn-config/src/__snapshots__/env.test.ts.snap b/packages/kbn-config/src/__snapshots__/env.test.ts.snap
index 884896266344c..9236c83f9c921 100644
--- a/packages/kbn-config/src/__snapshots__/env.test.ts.snap
+++ b/packages/kbn-config/src/__snapshots__/env.test.ts.snap
@@ -22,7 +22,7 @@ Env {
     "/some/other/path/some-kibana.yml",
   ],
   "homeDir": "/test/kibanaRoot",
-  "isDevClusterMaster": false,
+  "isDevCliParent": false,
   "logDir": "/test/kibanaRoot/log",
   "mode": Object {
     "dev": true,
@@ -67,7 +67,7 @@ Env {
     "/some/other/path/some-kibana.yml",
   ],
   "homeDir": "/test/kibanaRoot",
-  "isDevClusterMaster": false,
+  "isDevCliParent": false,
   "logDir": "/test/kibanaRoot/log",
   "mode": Object {
     "dev": false,
@@ -111,7 +111,7 @@ Env {
     "/test/cwd/config/kibana.yml",
   ],
   "homeDir": "/test/kibanaRoot",
-  "isDevClusterMaster": true,
+  "isDevCliParent": true,
   "logDir": "/test/kibanaRoot/log",
   "mode": Object {
     "dev": true,
@@ -155,7 +155,7 @@ Env {
     "/some/other/path/some-kibana.yml",
   ],
   "homeDir": "/test/kibanaRoot",
-  "isDevClusterMaster": false,
+  "isDevCliParent": false,
   "logDir": "/test/kibanaRoot/log",
   "mode": Object {
     "dev": false,
@@ -199,7 +199,7 @@ Env {
     "/some/other/path/some-kibana.yml",
   ],
   "homeDir": "/test/kibanaRoot",
-  "isDevClusterMaster": false,
+  "isDevCliParent": false,
   "logDir": "/test/kibanaRoot/log",
   "mode": Object {
     "dev": false,
@@ -243,7 +243,7 @@ Env {
     "/some/other/path/some-kibana.yml",
   ],
   "homeDir": "/some/home/dir",
-  "isDevClusterMaster": false,
+  "isDevCliParent": false,
   "logDir": "/some/home/dir/log",
   "mode": Object {
     "dev": false,
diff --git a/packages/kbn-config/src/env.test.ts b/packages/kbn-config/src/env.test.ts
index 1613a90951d40..5aee33e6fda5e 100644
--- a/packages/kbn-config/src/env.test.ts
+++ b/packages/kbn-config/src/env.test.ts
@@ -47,7 +47,7 @@ test('correctly creates default environment in dev mode.', () => {
     REPO_ROOT,
     getEnvOptions({
       configs: ['/test/cwd/config/kibana.yml'],
-      isDevClusterMaster: true,
+      isDevCliParent: true,
     })
   );
 
diff --git a/packages/kbn-config/src/env.ts b/packages/kbn-config/src/env.ts
index 7edb4b1c8dad9..4ae8d7b7f9aa5 100644
--- a/packages/kbn-config/src/env.ts
+++ b/packages/kbn-config/src/env.ts
@@ -26,7 +26,7 @@ import { PackageInfo, EnvironmentMode } from './types';
 export interface EnvOptions {
   configs: string[];
   cliArgs: CliArgs;
-  isDevClusterMaster: boolean;
+  isDevCliParent: boolean;
 }
 
 /** @internal */
@@ -101,10 +101,10 @@ export class Env {
   public readonly configs: readonly string[];
 
   /**
-   * Indicates that this Kibana instance is run as development Node Cluster master.
+   * Indicates that this Kibana instance is running in the parent process of the dev cli.
    * @internal
    */
-  public readonly isDevClusterMaster: boolean;
+  public readonly isDevCliParent: boolean;
 
   /**
    * @internal
@@ -122,7 +122,7 @@ export class Env {
 
     this.cliArgs = Object.freeze(options.cliArgs);
     this.configs = Object.freeze(options.configs);
-    this.isDevClusterMaster = options.isDevClusterMaster;
+    this.isDevCliParent = options.isDevCliParent;
 
     const isDevMode = this.cliArgs.dev || this.cliArgs.envName === 'development';
     this.mode = Object.freeze<EnvironmentMode>({
diff --git a/packages/kbn-legacy-logging/src/log_format_string.ts b/packages/kbn-legacy-logging/src/log_format_string.ts
index 3f024fac55119..b4217c37b960e 100644
--- a/packages/kbn-legacy-logging/src/log_format_string.ts
+++ b/packages/kbn-legacy-logging/src/log_format_string.ts
@@ -54,7 +54,7 @@ const type = _.memoize((t: string) => {
   return color(t)(_.pad(t, 7).slice(0, 7));
 });
 
-const workerType = process.env.kbnWorkerType ? `${type(process.env.kbnWorkerType)} ` : '';
+const prefix = process.env.isDevCliChild ? `${type('server')} ` : '';
 
 export class KbnLoggerStringFormat extends BaseLogFormat {
   format(data: Record<string, any>) {
@@ -71,6 +71,6 @@ export class KbnLoggerStringFormat extends BaseLogFormat {
         return s + `[${color(t)(t)}]`;
       }, '');
 
-    return `${workerType}${type(data.type)} [${time}] ${tags} ${msg}`;
+    return `${prefix}${type(data.type)} [${time}] ${tags} ${msg}`;
   }
 }
diff --git a/packages/kbn-legacy-logging/src/rotate/index.ts b/packages/kbn-legacy-logging/src/rotate/index.ts
index 2387fc530e58b..9a83c625b9431 100644
--- a/packages/kbn-legacy-logging/src/rotate/index.ts
+++ b/packages/kbn-legacy-logging/src/rotate/index.ts
@@ -17,7 +17,6 @@
  * under the License.
  */
 
-import { isMaster, isWorker } from 'cluster';
 import { Server } from '@hapi/hapi';
 import { LogRotator } from './log_rotator';
 import { LegacyLoggingConfig } from '../schema';
@@ -30,12 +29,6 @@ export async function setupLoggingRotate(server: Server, config: LegacyLoggingCo
     return;
   }
 
-  // We just want to start the logging rotate service once
-  // and we choose to use the master (prod) or the worker server (dev)
-  if (!isMaster && isWorker && process.env.kbnWorkerType !== 'server') {
-    return;
-  }
-
   // We don't want to run logging rotate server if
   // we are not logging to a file
   if (config.dest === 'stdout') {
diff --git a/packages/kbn-legacy-logging/src/rotate/log_rotator.ts b/packages/kbn-legacy-logging/src/rotate/log_rotator.ts
index 54181e30d6007..fc2c088f01dde 100644
--- a/packages/kbn-legacy-logging/src/rotate/log_rotator.ts
+++ b/packages/kbn-legacy-logging/src/rotate/log_rotator.ts
@@ -18,7 +18,6 @@
  */
 
 import * as chokidar from 'chokidar';
-import { isMaster } from 'cluster';
 import fs from 'fs';
 import { Server } from '@hapi/hapi';
 import { throttle } from 'lodash';
@@ -351,22 +350,14 @@ export class LogRotator {
   }
 
   _sendReloadLogConfigSignal() {
-    if (isMaster) {
-      (process as NodeJS.EventEmitter).emit('SIGHUP');
+    if (!process.env.isDevCliChild || !process.send) {
+      process.emit('SIGHUP', 'SIGHUP');
       return;
     }
 
     // Send a special message to the cluster manager
     // so it can forward it correctly
     // It will only run when we are under cluster mode (not under a production environment)
-    if (!process.send) {
-      this.log(
-        ['error', 'logging:rotate'],
-        'For some unknown reason process.send is not defined, the rotation was not successful'
-      );
-      return;
-    }
-
     process.send(['RELOAD_LOGGING_CONFIG_FROM_SERVER_WORKER']);
   }
 }
diff --git a/src/apm.js b/src/apm.js
index bde37fa006c61..4f5fe29cbb5fa 100644
--- a/src/apm.js
+++ b/src/apm.js
@@ -30,10 +30,6 @@ let apmConfig;
 const isKibanaDistributable = Boolean(build && build.distributable === true);
 
 module.exports = function (serviceName = name) {
-  if (process.env.kbnWorkerType === 'optmzr') {
-    return;
-  }
-
   apmConfig = loadConfiguration(process.argv, ROOT_DIR, isKibanaDistributable);
   const conf = apmConfig.getConfig(serviceName);
   const apm = require('elastic-apm-node');
diff --git a/src/cli/cluster/cluster_manager.ts b/src/cli/cluster/cluster_manager.ts
index f427c8750912b..7a14f617b5d5a 100644
--- a/src/cli/cluster/cluster_manager.ts
+++ b/src/cli/cluster/cluster_manager.ts
@@ -33,8 +33,6 @@ import { BasePathProxyServer } from '../../core/server/http';
 import { Log } from './log';
 import { Worker } from './worker';
 
-process.env.kbnWorkerType = 'managr';
-
 export type SomeCliArgs = Pick<
   CliArgs,
   | 'quiet'
diff --git a/src/cli/cluster/worker.ts b/src/cli/cluster/worker.ts
index d28065765070b..26b2a643e5373 100644
--- a/src/cli/cluster/worker.ts
+++ b/src/cli/cluster/worker.ts
@@ -56,7 +56,6 @@ export class Worker extends EventEmitter {
   private readonly clusterBinder: BinderFor;
   private readonly processBinder: BinderFor;
 
-  private type: string;
   private title: string;
   private log: any;
   private forkBinder: BinderFor | null = null;
@@ -76,7 +75,6 @@ export class Worker extends EventEmitter {
     super();
 
     this.log = opts.log;
-    this.type = opts.type;
     this.title = opts.title || opts.type;
     this.watch = opts.watch !== false;
     this.startCount = 0;
@@ -88,7 +86,7 @@ export class Worker extends EventEmitter {
 
     this.env = {
       NODE_OPTIONS: process.env.NODE_OPTIONS || '',
-      kbnWorkerType: this.type,
+      isDevCliChild: 'true',
       kbnWorkerArgv: JSON.stringify([...(opts.baseArgv || baseArgv), ...(opts.argv || [])]),
       ELASTIC_APM_SERVICE_NAME: opts.apmServiceName || '',
     };
diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js
index 2fa24cc7f3791..61f880d80633d 100644
--- a/src/cli/serve/serve.js
+++ b/src/cli/serve/serve.js
@@ -43,7 +43,7 @@ function canRequire(path) {
 }
 
 const CLUSTER_MANAGER_PATH = resolve(__dirname, '../cluster/cluster_manager');
-const CAN_CLUSTER = canRequire(CLUSTER_MANAGER_PATH);
+const DEV_MODE_SUPPORTED = canRequire(CLUSTER_MANAGER_PATH);
 
 const REPL_PATH = resolve(__dirname, '../repl');
 const CAN_REPL = canRequire(REPL_PATH);
@@ -189,7 +189,7 @@ export default function (program) {
       );
   }
 
-  if (CAN_CLUSTER) {
+  if (DEV_MODE_SUPPORTED) {
     command
       .option('--dev', 'Run the server with development mode defaults')
       .option('--ssl', 'Run the dev server using HTTPS')
@@ -240,7 +240,7 @@ export default function (program) {
         dist: !!opts.dist,
       },
       features: {
-        isClusterModeSupported: CAN_CLUSTER,
+        isCliDevModeSupported: DEV_MODE_SUPPORTED,
         isReplModeSupported: CAN_REPL,
       },
       applyConfigOverrides: (rawConfig) => applyConfigOverrides(rawConfig, opts, unknownOptions),
diff --git a/src/core/server/bootstrap.ts b/src/core/server/bootstrap.ts
index ff1a5c0340c46..6711a8b8987e5 100644
--- a/src/core/server/bootstrap.ts
+++ b/src/core/server/bootstrap.ts
@@ -18,16 +18,15 @@
  */
 
 import chalk from 'chalk';
-import { isMaster } from 'cluster';
 import { CliArgs, Env, RawConfigService } from './config';
 import { Root } from './root';
 import { CriticalError } from './errors';
 
 interface KibanaFeatures {
-  // Indicates whether we can run Kibana in a so called cluster mode in which
-  // Kibana is run as a "worker" process together with optimizer "worker" process
-  // that are orchestrated by the "master" process (dev mode only feature).
-  isClusterModeSupported: boolean;
+  // Indicates whether we can run Kibana in dev mode in which Kibana is run as
+  // a child process together with optimizer "worker" processes that are
+  // orchestrated by a parent process (dev mode only feature).
+  isCliDevModeSupported: boolean;
 
   // Indicates whether we can run Kibana in REPL mode (dev mode only feature).
   isReplModeSupported: boolean;
@@ -71,7 +70,7 @@ export async function bootstrap({
   const env = Env.createDefault(REPO_ROOT, {
     configs,
     cliArgs,
-    isDevClusterMaster: isMaster && cliArgs.dev && features.isClusterModeSupported,
+    isDevCliParent: cliArgs.dev && features.isCliDevModeSupported && !process.env.isDevCliChild,
   });
 
   const rawConfigService = new RawConfigService(env.configs, applyConfigOverrides);
diff --git a/src/core/server/http/http_service.test.ts b/src/core/server/http/http_service.test.ts
index 11cea88fa0dd2..3d55322461288 100644
--- a/src/core/server/http/http_service.test.ts
+++ b/src/core/server/http/http_service.test.ts
@@ -264,7 +264,7 @@ test('does not start http server if process is dev cluster master', async () =>
   const service = new HttpService({
     coreId,
     configService,
-    env: Env.createDefault(REPO_ROOT, getEnvOptions({ isDevClusterMaster: true })),
+    env: Env.createDefault(REPO_ROOT, getEnvOptions({ isDevCliParent: true })),
     logger,
   });
 
diff --git a/src/core/server/http/http_service.ts b/src/core/server/http/http_service.ts
index 0127a6493e7fd..171a20160d26d 100644
--- a/src/core/server/http/http_service.ts
+++ b/src/core/server/http/http_service.ts
@@ -158,7 +158,7 @@ export class HttpService
    * @internal
    * */
   private shouldListen(config: HttpConfig) {
-    return !this.coreContext.env.isDevClusterMaster && config.autoListen;
+    return !this.coreContext.env.isDevCliParent && config.autoListen;
   }
 
   public async stop() {
diff --git a/src/core/server/legacy/legacy_service.test.ts b/src/core/server/legacy/legacy_service.test.ts
index 5cc6fcb133507..fe19ef9d0a774 100644
--- a/src/core/server/legacy/legacy_service.test.ts
+++ b/src/core/server/legacy/legacy_service.test.ts
@@ -362,7 +362,7 @@ describe('once LegacyService is set up in `devClusterMaster` mode', () => {
         REPO_ROOT,
         getEnvOptions({
           cliArgs: { silent: true, basePath: false },
-          isDevClusterMaster: true,
+          isDevCliParent: true,
         })
       ),
       logger,
@@ -391,7 +391,7 @@ describe('once LegacyService is set up in `devClusterMaster` mode', () => {
         REPO_ROOT,
         getEnvOptions({
           cliArgs: { quiet: true, basePath: true },
-          isDevClusterMaster: true,
+          isDevCliParent: true,
         })
       ),
       logger,
diff --git a/src/core/server/legacy/legacy_service.ts b/src/core/server/legacy/legacy_service.ts
index 3111c8daf7981..4ae6c9d437576 100644
--- a/src/core/server/legacy/legacy_service.ts
+++ b/src/core/server/legacy/legacy_service.ts
@@ -144,7 +144,7 @@ export class LegacyService implements CoreService {
     this.log.debug('starting legacy service');
 
     // Receive initial config and create kbnServer/ClusterManager.
-    if (this.coreContext.env.isDevClusterMaster) {
+    if (this.coreContext.env.isDevCliParent) {
       await this.createClusterManager(this.legacyRawConfig!);
     } else {
       this.kbnServer = await this.createKbnServer(
@@ -310,10 +310,8 @@ export class LegacyService implements CoreService {
       logger: this.coreContext.logger,
     });
 
-    // The kbnWorkerType check is necessary to prevent the repl
-    // from being started multiple times in different processes.
-    // We only want one REPL.
-    if (this.coreContext.env.cliArgs.repl && process.env.kbnWorkerType === 'server') {
+    // Prevent the repl from being started multiple times in different processes.
+    if (this.coreContext.env.cliArgs.repl && process.env.isDevCliChild) {
       // eslint-disable-next-line @typescript-eslint/no-var-requires
       require('./cli').startRepl(kbnServer);
     }
diff --git a/src/core/server/plugins/plugins_service.test.ts b/src/core/server/plugins/plugins_service.test.ts
index 02b82c17ed4fc..601e0038b0cf0 100644
--- a/src/core/server/plugins/plugins_service.test.ts
+++ b/src/core/server/plugins/plugins_service.test.ts
@@ -102,7 +102,7 @@ const createPlugin = (
   });
 };
 
-async function testSetup(options: { isDevClusterMaster?: boolean } = {}) {
+async function testSetup(options: { isDevCliParent?: boolean } = {}) {
   mockPackage.raw = {
     branch: 'feature-v1',
     version: 'v1',
@@ -116,7 +116,7 @@ async function testSetup(options: { isDevClusterMaster?: boolean } = {}) {
   coreId = Symbol('core');
   env = Env.createDefault(REPO_ROOT, {
     ...getEnvOptions(),
-    isDevClusterMaster: options.isDevClusterMaster ?? false,
+    isDevCliParent: options.isDevCliParent ?? false,
   });
 
   config$ = new BehaviorSubject<Record<string, any>>({ plugins: { initialize: true } });
@@ -638,10 +638,10 @@ describe('PluginsService', () => {
   });
 });
 
-describe('PluginService when isDevClusterMaster is true', () => {
+describe('PluginService when isDevCliParent is true', () => {
   beforeEach(async () => {
     await testSetup({
-      isDevClusterMaster: true,
+      isDevCliParent: true,
     });
   });
 
diff --git a/src/core/server/plugins/plugins_service.ts b/src/core/server/plugins/plugins_service.ts
index 5967e6d5358de..e1622b1e19231 100644
--- a/src/core/server/plugins/plugins_service.ts
+++ b/src/core/server/plugins/plugins_service.ts
@@ -90,7 +90,7 @@ export class PluginsService implements CoreService<PluginsServiceSetup, PluginsS
 
   constructor(private readonly coreContext: CoreContext) {
     this.log = coreContext.logger.get('plugins-service');
-    this.discoveryDisabled = coreContext.env.isDevClusterMaster;
+    this.discoveryDisabled = coreContext.env.isDevCliParent;
     this.pluginsSystem = new PluginsSystem(coreContext);
     this.configService = coreContext.configService;
     this.config$ = coreContext.configService
@@ -133,7 +133,7 @@ export class PluginsService implements CoreService<PluginsServiceSetup, PluginsS
     const config = await this.config$.pipe(first()).toPromise();
 
     let contracts = new Map<PluginName, unknown>();
-    const initialize = config.initialize && !this.coreContext.env.isDevClusterMaster;
+    const initialize = config.initialize && !this.coreContext.env.isDevCliParent;
     if (initialize) {
       contracts = await this.pluginsSystem.setupPlugins(deps);
       this.registerPluginStaticDirs(deps);
diff --git a/src/core/server/server.test.ts b/src/core/server/server.test.ts
index 0c7ebbcb527ec..f377bfc321735 100644
--- a/src/core/server/server.test.ts
+++ b/src/core/server/server.test.ts
@@ -216,10 +216,10 @@ test(`doesn't setup core services if legacy config validation fails`, async () =
   expect(mockI18nService.setup).not.toHaveBeenCalled();
 });
 
-test(`doesn't validate config if env.isDevClusterMaster is true`, async () => {
+test(`doesn't validate config if env.isDevCliParent is true`, async () => {
   const devParentEnv = Env.createDefault(REPO_ROOT, {
     ...getEnvOptions(),
-    isDevClusterMaster: true,
+    isDevCliParent: true,
   });
 
   const server = new Server(rawConfigService, devParentEnv, logger);
diff --git a/src/core/server/server.ts b/src/core/server/server.ts
index 0f7e8cced999c..e253663d8dc8d 100644
--- a/src/core/server/server.ts
+++ b/src/core/server/server.ts
@@ -124,7 +124,7 @@ export class Server {
     const legacyConfigSetup = await this.legacy.setupLegacyConfig();
 
     // rely on dev server to validate config, don't validate in the parent process
-    if (!this.env.isDevClusterMaster) {
+    if (!this.env.isDevCliParent) {
       // Immediately terminate in case of invalid configuration
       // This needs to be done after plugin discovery
       await this.configService.validate();
diff --git a/src/core/test_helpers/kbn_server.ts b/src/core/test_helpers/kbn_server.ts
index f95ea66d3cbc1..3161420b94d22 100644
--- a/src/core/test_helpers/kbn_server.ts
+++ b/src/core/test_helpers/kbn_server.ts
@@ -82,7 +82,7 @@ export function createRootWithSettings(
       dist: false,
       ...cliArgs,
     },
-    isDevClusterMaster: false,
+    isDevCliParent: false,
   });
 
   return new Root(
diff --git a/src/legacy/server/kbn_server.js b/src/legacy/server/kbn_server.js
index b61a86326ca1a..85d75b4e18772 100644
--- a/src/legacy/server/kbn_server.js
+++ b/src/legacy/server/kbn_server.js
@@ -20,7 +20,6 @@
 import { constant, once, compact, flatten } from 'lodash';
 import { reconfigureLogging } from '@kbn/legacy-logging';
 
-import { isWorker } from 'cluster';
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
 import { fromRoot, pkg } from '../../core/server/utils';
 import { Config } from './config';
@@ -121,7 +120,7 @@ export default class KbnServer {
 
     const { server, config } = this;
 
-    if (isWorker) {
+    if (process.env.isDevCliChild) {
       // help parent process know when we are ready
       process.send(['WORKER_LISTENING']);
     }

From 280ce7e5fa9256b79e7900c3b3b898d3048d5ddd Mon Sep 17 00:00:00 2001
From: Dima Arnautov <dmitrii.arnautov@elastic.co>
Date: Wed, 25 Nov 2020 16:49:45 +0100
Subject: [PATCH 11/37] [ML] Persisted URL state for Anomalies table (#84314)

* [ML] Persisted URL state for Anomalies table

* [ML] adjust cell selection according to the time range
---
 .../anomalies_table/anomalies_table.js        | 66 +++++++++++++++----
 .../explorer/hooks/use_selected_cells.ts      | 53 +++++++++++++--
 .../reducers/explorer_reducer/state.ts        |  3 +-
 .../application/routing/routes/explorer.tsx   |  7 +-
 .../ml/public/application/util/url_state.tsx  |  2 +-
 5 files changed, 111 insertions(+), 20 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table.js b/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table.js
index 0a2c67a3b0dcb..ebc782fe4625b 100644
--- a/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table.js
+++ b/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table.js
@@ -25,8 +25,9 @@ import { mlTableService } from '../../services/table_service';
 import { RuleEditorFlyout } from '../rule_editor';
 import { ml } from '../../services/ml_api_service';
 import { INFLUENCERS_LIMIT, ANOMALIES_TABLE_TABS, MAX_CHARS } from './anomalies_table_constants';
+import { usePageUrlState } from '../../util/url_state';
 
-class AnomaliesTable extends Component {
+export class AnomaliesTableInternal extends Component {
   constructor(props) {
     super(props);
 
@@ -145,8 +146,20 @@ class AnomaliesTable extends Component {
     });
   };
 
+  onTableChange = ({ page, sort }) => {
+    const { tableState, updateTableState } = this.props;
+    const result = {
+      pageIndex: page && page.index !== undefined ? page.index : tableState.pageIndex,
+      pageSize: page && page.size !== undefined ? page.size : tableState.pageSize,
+      sortField: sort && sort.field !== undefined ? sort.field : tableState.sortField,
+      sortDirection:
+        sort && sort.direction !== undefined ? sort.direction : tableState.sortDirection,
+    };
+    updateTableState(result);
+  };
+
   render() {
-    const { bounds, tableData, filter, influencerFilter } = this.props;
+    const { bounds, tableData, filter, influencerFilter, tableState } = this.props;
 
     if (
       tableData === undefined ||
@@ -186,8 +199,8 @@ class AnomaliesTable extends Component {
 
     const sorting = {
       sort: {
-        field: 'severity',
-        direction: 'desc',
+        field: tableState.sortField,
+        direction: tableState.sortDirection,
       },
     };
 
@@ -199,8 +212,15 @@ class AnomaliesTable extends Component {
       };
     };
 
+    const pagination = {
+      pageIndex: tableState.pageIndex,
+      pageSize: tableState.pageSize,
+      totalItemCount: tableData.anomalies.length,
+      pageSizeOptions: [10, 25, 100],
+    };
+
     return (
-      <React.Fragment>
+      <>
         <RuleEditorFlyout
           setShowFunction={this.setShowRuleEditorFlyoutFunction}
           unsetShowFunction={this.unsetShowRuleEditorFlyoutFunction}
@@ -209,26 +229,46 @@ class AnomaliesTable extends Component {
           className="ml-anomalies-table eui-textOverflowWrap"
           items={tableData.anomalies}
           columns={columns}
-          pagination={{
-            pageSizeOptions: [10, 25, 100],
-            initialPageSize: 25,
-          }}
+          pagination={pagination}
           sorting={sorting}
           itemId="rowId"
           itemIdToExpandedRowMap={this.state.itemIdToExpandedRowMap}
           compressed={true}
           rowProps={getRowProps}
           data-test-subj="mlAnomaliesTable"
+          onTableChange={this.onTableChange}
         />
-      </React.Fragment>
+      </>
     );
   }
 }
-AnomaliesTable.propTypes = {
+
+export const getDefaultAnomaliesTableState = () => ({
+  pageIndex: 0,
+  pageSize: 25,
+  sortField: 'severity',
+  sortDirection: 'desc',
+});
+
+export const AnomaliesTable = (props) => {
+  const [tableState, updateTableState] = usePageUrlState(
+    'mlAnomaliesTable',
+    getDefaultAnomaliesTableState()
+  );
+  return (
+    <AnomaliesTableInternal
+      {...props}
+      tableState={tableState}
+      updateTableState={updateTableState}
+    />
+  );
+};
+
+AnomaliesTableInternal.propTypes = {
   bounds: PropTypes.object.isRequired,
   tableData: PropTypes.object,
   filter: PropTypes.func,
   influencerFilter: PropTypes.func,
+  tableState: PropTypes.object.isRequired,
+  updateTableState: PropTypes.func.isRequired,
 };
-
-export { AnomaliesTable };
diff --git a/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts b/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts
index 7602954b4c8c3..f940fdc2387e2 100644
--- a/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts
+++ b/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts
@@ -4,14 +4,17 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { useCallback, useMemo } from 'react';
+import { useCallback, useEffect, useMemo } from 'react';
+import { Duration } from 'moment';
 import { SWIMLANE_TYPE } from '../explorer_constants';
-import { AppStateSelectedCells } from '../explorer_utils';
+import { AppStateSelectedCells, TimeRangeBounds } from '../explorer_utils';
 import { ExplorerAppState } from '../../../../common/types/ml_url_generator';
 
 export const useSelectedCells = (
   appState: ExplorerAppState,
-  setAppState: (update: Partial<ExplorerAppState>) => void
+  setAppState: (update: Partial<ExplorerAppState>) => void,
+  timeBounds: TimeRangeBounds | undefined,
+  bucketInterval: Duration | undefined
 ): [AppStateSelectedCells | undefined, (swimlaneSelectedCells: AppStateSelectedCells) => void] => {
   // keep swimlane selection, restore selectedCells from AppState
   const selectedCells = useMemo(() => {
@@ -28,7 +31,7 @@ export const useSelectedCells = (
   }, [JSON.stringify(appState?.mlExplorerSwimlane)]);
 
   const setSelectedCells = useCallback(
-    (swimlaneSelectedCells: AppStateSelectedCells) => {
+    (swimlaneSelectedCells?: AppStateSelectedCells) => {
       const mlExplorerSwimlane = {
         ...appState.mlExplorerSwimlane,
       } as ExplorerAppState['mlExplorerSwimlane'];
@@ -65,5 +68,47 @@ export const useSelectedCells = (
     [appState?.mlExplorerSwimlane, selectedCells, setAppState]
   );
 
+  /**
+   * Adjust cell selection with respect to the time boundaries.
+   * Reset it entirely when it out of range.
+   */
+  useEffect(() => {
+    if (
+      timeBounds === undefined ||
+      selectedCells?.times === undefined ||
+      bucketInterval === undefined
+    )
+      return;
+
+    let [selectedFrom, selectedTo] = selectedCells.times;
+
+    const rangeFrom = timeBounds.min!.unix();
+    /**
+     * Because each cell on the swim lane represent the fixed bucket interval,
+     * the selection range could be outside of the time boundaries with
+     * correction within the bucket interval.
+     */
+    const rangeTo = timeBounds.max!.unix() + bucketInterval.asSeconds();
+
+    selectedFrom = Math.max(selectedFrom, rangeFrom);
+
+    selectedTo = Math.min(selectedTo, rangeTo);
+
+    const isSelectionOutOfRange = rangeFrom > selectedTo || rangeTo < selectedFrom;
+
+    if (isSelectionOutOfRange) {
+      // reset selection
+      setSelectedCells();
+      return;
+    }
+
+    if (selectedFrom !== rangeFrom || selectedTo !== rangeTo) {
+      setSelectedCells({
+        ...selectedCells,
+        times: [selectedFrom, selectedTo],
+      });
+    }
+  }, [timeBounds, selectedCells, bucketInterval]);
+
   return [selectedCells, setSelectedCells];
 };
diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts
index ea9a8b5c18054..14b0a6033999c 100644
--- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts
+++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts
@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { Duration } from 'moment';
 import { ML_RESULTS_INDEX_PATTERN } from '../../../../../common/constants/index_patterns';
 import { Dictionary } from '../../../../../common/types/common';
 
@@ -43,7 +44,7 @@ export interface ExplorerState {
   queryString: string;
   selectedCells: AppStateSelectedCells | undefined;
   selectedJobs: ExplorerJob[] | null;
-  swimlaneBucketInterval: any;
+  swimlaneBucketInterval: Duration | undefined;
   swimlaneContainerWidth: number;
   tableData: AnomaliesTableData;
   tableQueryString: string;
diff --git a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx
index f8b4de6903ad2..2126cbceae6b1 100644
--- a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx
+++ b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx
@@ -205,7 +205,12 @@ const ExplorerUrlStateManager: FC<ExplorerUrlStateManagerProps> = ({ jobsWithTim
   const [tableInterval] = useTableInterval();
   const [tableSeverity] = useTableSeverity();
 
-  const [selectedCells, setSelectedCells] = useSelectedCells(explorerUrlState, setExplorerUrlState);
+  const [selectedCells, setSelectedCells] = useSelectedCells(
+    explorerUrlState,
+    setExplorerUrlState,
+    explorerState?.bounds,
+    explorerState?.swimlaneBucketInterval
+  );
 
   useEffect(() => {
     explorerService.setSelectedCells(selectedCells);
diff --git a/x-pack/plugins/ml/public/application/util/url_state.tsx b/x-pack/plugins/ml/public/application/util/url_state.tsx
index fdc6dd135cd69..6cdc069096dcc 100644
--- a/x-pack/plugins/ml/public/application/util/url_state.tsx
+++ b/x-pack/plugins/ml/public/application/util/url_state.tsx
@@ -162,7 +162,7 @@ export const useUrlState = (accessor: Accessor) => {
   return [urlState, setUrlState];
 };
 
-type AppStateKey = 'mlSelectSeverity' | 'mlSelectInterval' | MlPages;
+type AppStateKey = 'mlSelectSeverity' | 'mlSelectInterval' | 'mlAnomaliesTable' | MlPages;
 
 /**
  * Hook for managing the URL state of the page.

From 5b2e119356c33543a20283a64e1f7ae8555a1fc3 Mon Sep 17 00:00:00 2001
From: Joe Reuter <johannes.reuter@elastic.co>
Date: Wed, 25 Nov 2020 16:57:03 +0100
Subject: [PATCH 12/37] add live region for field search (#84310)

---
 .../datapanel.test.tsx                        | 19 +++++++++++++++++++
 .../indexpattern_datasource/datapanel.tsx     | 19 +++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
index ac82caf9d5227..3d55494fd260c 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
@@ -718,6 +718,25 @@ describe('IndexPattern Data Panel', () => {
       ]);
     });
 
+    it('should announce filter in live region', () => {
+      const wrapper = mountWithIntl(<InnerIndexPatternDataPanel {...props} />);
+      act(() => {
+        wrapper.find('[data-test-subj="lnsIndexPatternFieldSearch"]').prop('onChange')!({
+          target: { value: 'me' },
+        } as ChangeEvent<HTMLInputElement>);
+      });
+
+      wrapper
+        .find('[data-test-subj="lnsIndexPatternEmptyFields"]')
+        .find('button')
+        .first()
+        .simulate('click');
+
+      expect(wrapper.find('[aria-live="polite"]').text()).toEqual(
+        '1 available field. 1 empty field. 0 meta fields.'
+      );
+    });
+
     it('should filter down by type', () => {
       const wrapper = mountWithIntl(<InnerIndexPatternDataPanel {...props} />);
 
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx
index f2c7d7fc20926..ad5509dd88bc9 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx
@@ -18,10 +18,12 @@ import {
   EuiSpacer,
   EuiFilterGroup,
   EuiFilterButton,
+  EuiScreenReaderOnly,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { DataPublicPluginStart, EsQueryConfig, Query, Filter } from 'src/plugins/data/public';
+import { htmlIdGenerator } from '@elastic/eui';
 import { DatasourceDataPanelProps, DataType, StateSetter } from '../types';
 import { ChildDragDropProvider, DragContextState } from '../drag_drop';
 import {
@@ -222,6 +224,9 @@ const fieldFiltersLabel = i18n.translate('xpack.lens.indexPatterns.fieldFiltersL
   defaultMessage: 'Field filters',
 });
 
+const htmlId = htmlIdGenerator('datapanel');
+const fieldSearchDescriptionId = htmlId();
+
 export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
   currentIndexPatternId,
   indexPatternRefs,
@@ -489,6 +494,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
               aria-label={i18n.translate('xpack.lens.indexPatterns.filterByNameAriaLabel', {
                 defaultMessage: 'Search fields',
               })}
+              aria-describedby={fieldSearchDescriptionId}
             />
           </EuiFormControlLayout>
 
@@ -550,6 +556,19 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
             </EuiPopover>
           </EuiFilterGroup>
         </EuiFlexItem>
+        <EuiScreenReaderOnly>
+          <div aria-live="polite" id={fieldSearchDescriptionId}>
+            {i18n.translate('xpack.lens.indexPatterns.fieldSearchLiveRegion', {
+              defaultMessage:
+                '{availableFields} available {availableFields, plural, one {field} other {fields}}. {emptyFields} empty {emptyFields, plural, one {field} other {fields}}. {metaFields} meta {metaFields, plural, one {field} other {fields}}.',
+              values: {
+                availableFields: fieldGroups.AvailableFields.fields.length,
+                emptyFields: fieldGroups.EmptyFields.fields.length,
+                metaFields: fieldGroups.MetaFields.fields.length,
+              },
+            })}
+          </div>
+        </EuiScreenReaderOnly>
         <EuiFlexItem>
           <FieldList
             exists={(field) =>

From 9bef42b2a8078d070b206c6860c65df638d3b323 Mon Sep 17 00:00:00 2001
From: Devon Thomson <devon.thomson@hotmail.com>
Date: Wed, 25 Nov 2020 10:59:52 -0500
Subject: [PATCH 13/37] redirect to visualize listing page when by value
 visualization editor doesn't have a value input (#84287)

---
 .../application/components/visualize_byvalue_editor.tsx       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx b/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx
index a63f597f10135..1c1eb9956a329 100644
--- a/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx
+++ b/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx
@@ -33,6 +33,7 @@ import {
 import { VisualizeServices } from '../types';
 import { VisualizeEditorCommon } from './visualize_editor_common';
 import { VisualizeAppProps } from '../app';
+import { VisualizeConstants } from '../..';
 
 export const VisualizeByValueEditor = ({ onAppLeave }: VisualizeAppProps) => {
   const [originatingApp, setOriginatingApp] = useState<string>();
@@ -52,7 +53,8 @@ export const VisualizeByValueEditor = ({ onAppLeave }: VisualizeAppProps) => {
     setValueInput(valueInputValue);
     setEmbeddableId(embeddableIdValue);
     if (!valueInputValue) {
-      history.back();
+      // if there is no value input to load, redirect to the visualize listing page.
+      services.history.replace(VisualizeConstants.LANDING_PAGE_PATH);
     }
   }, [services]);
 

From 351cd6d19d0367ee606f4dd32aac4ee4190844ae Mon Sep 17 00:00:00 2001
From: Devon Thomson <devon.thomson@hotmail.com>
Date: Wed, 25 Nov 2020 11:03:00 -0500
Subject: [PATCH 14/37] [Time to Visualize] Fix Unlink Action via Rollback of
 ReplacePanel (#83873)

* Fixed unlink action via rollback of replacePanel changes
---
 .../actions/add_to_library_action.test.tsx    | 18 +++++++---
 .../actions/add_to_library_action.tsx         |  5 ++-
 .../unlink_from_library_action.test.tsx       | 18 +++++++---
 .../actions/unlink_from_library_action.tsx    |  5 ++-
 .../embeddable/dashboard_container.tsx        | 33 ++++++++++++++++---
 5 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx b/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx
index feb30b248c066..5f3945e733527 100644
--- a/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx
+++ b/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx
@@ -137,12 +137,17 @@ test('Add to library is not compatible when embeddable is not in a dashboard con
 test('Add to library replaces embeddableId and retains panel count', async () => {
   const dashboard = embeddable.getRoot() as IContainer;
   const originalPanelCount = Object.keys(dashboard.getInput().panels).length;
+  const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels));
 
   const action = new AddToLibraryAction({ toasts: coreStart.notifications.toasts });
   await action.execute({ embeddable });
   expect(Object.keys(container.getInput().panels).length).toEqual(originalPanelCount);
-  expect(Object.keys(container.getInput().panels)).toContain(embeddable.id);
-  const newPanel = container.getInput().panels[embeddable.id!];
+
+  const newPanelId = Object.keys(container.getInput().panels).find(
+    (key) => !originalPanelKeySet.has(key)
+  );
+  expect(newPanelId).toBeDefined();
+  const newPanel = container.getInput().panels[newPanelId!];
   expect(newPanel.type).toEqual(embeddable.type);
 });
 
@@ -158,10 +163,15 @@ test('Add to library returns reference type input', async () => {
     mockedByReferenceInput: { savedObjectId: 'testSavedObjectId', id: embeddable.id },
     mockedByValueInput: { attributes: complicatedAttributes, id: embeddable.id } as EmbeddableInput,
   });
+  const dashboard = embeddable.getRoot() as IContainer;
+  const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels));
   const action = new AddToLibraryAction({ toasts: coreStart.notifications.toasts });
   await action.execute({ embeddable });
-  expect(Object.keys(container.getInput().panels)).toContain(embeddable.id);
-  const newPanel = container.getInput().panels[embeddable.id!];
+  const newPanelId = Object.keys(container.getInput().panels).find(
+    (key) => !originalPanelKeySet.has(key)
+  );
+  expect(newPanelId).toBeDefined();
+  const newPanel = container.getInput().panels[newPanelId!];
   expect(newPanel.type).toEqual(embeddable.type);
   expect(newPanel.explicitInput.attributes).toBeUndefined();
   expect(newPanel.explicitInput.savedObjectId).toBe('testSavedObjectId');
diff --git a/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx b/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx
index 179e5d522a2b3..08cd0c7a15381 100644
--- a/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx
+++ b/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx
@@ -19,7 +19,6 @@
 
 import { i18n } from '@kbn/i18n';
 import _ from 'lodash';
-import uuid from 'uuid';
 import { ActionByType, IncompatibleActionError } from '../../ui_actions_plugin';
 import { ViewMode, PanelState, IEmbeddable } from '../../embeddable_plugin';
 import {
@@ -89,9 +88,9 @@ export class AddToLibraryAction implements ActionByType<typeof ACTION_ADD_TO_LIB
 
     const newPanel: PanelState<EmbeddableInput> = {
       type: embeddable.type,
-      explicitInput: { ...newInput, id: uuid.v4() },
+      explicitInput: { ...newInput },
     };
-    dashboard.replacePanel(panelToReplace, newPanel);
+    dashboard.replacePanel(panelToReplace, newPanel, true);
 
     const title = i18n.translate('dashboard.panel.addToLibrary.successMessage', {
       defaultMessage: `Panel '{panelTitle}' was added to the visualize library`,
diff --git a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx
index f191be6f7baad..6a9769b0c8d16 100644
--- a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx
+++ b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx
@@ -135,11 +135,16 @@ test('Unlink is not compatible when embeddable is not in a dashboard container',
 test('Unlink replaces embeddableId and retains panel count', async () => {
   const dashboard = embeddable.getRoot() as IContainer;
   const originalPanelCount = Object.keys(dashboard.getInput().panels).length;
+  const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels));
   const action = new UnlinkFromLibraryAction({ toasts: coreStart.notifications.toasts });
   await action.execute({ embeddable });
   expect(Object.keys(container.getInput().panels).length).toEqual(originalPanelCount);
-  expect(Object.keys(container.getInput().panels)).toContain(embeddable.id);
-  const newPanel = container.getInput().panels[embeddable.id!];
+
+  const newPanelId = Object.keys(container.getInput().panels).find(
+    (key) => !originalPanelKeySet.has(key)
+  );
+  expect(newPanelId).toBeDefined();
+  const newPanel = container.getInput().panels[newPanelId!];
   expect(newPanel.type).toEqual(embeddable.type);
 });
 
@@ -159,10 +164,15 @@ test('Unlink unwraps all attributes from savedObject', async () => {
     mockedByReferenceInput: { savedObjectId: 'testSavedObjectId', id: embeddable.id },
     mockedByValueInput: { attributes: complicatedAttributes, id: embeddable.id },
   });
+  const dashboard = embeddable.getRoot() as IContainer;
+  const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels));
   const action = new UnlinkFromLibraryAction({ toasts: coreStart.notifications.toasts });
   await action.execute({ embeddable });
-  expect(Object.keys(container.getInput().panels)).toContain(embeddable.id);
-  const newPanel = container.getInput().panels[embeddable.id!];
+  const newPanelId = Object.keys(container.getInput().panels).find(
+    (key) => !originalPanelKeySet.has(key)
+  );
+  expect(newPanelId).toBeDefined();
+  const newPanel = container.getInput().panels[newPanelId!];
   expect(newPanel.type).toEqual(embeddable.type);
   expect(newPanel.explicitInput.attributes).toEqual(complicatedAttributes);
 });
diff --git a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx
index 5e16145364712..b20bbc6350aaa 100644
--- a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx
+++ b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx
@@ -19,7 +19,6 @@
 
 import { i18n } from '@kbn/i18n';
 import _ from 'lodash';
-import uuid from 'uuid';
 import { ActionByType, IncompatibleActionError } from '../../ui_actions_plugin';
 import { ViewMode, PanelState, IEmbeddable } from '../../embeddable_plugin';
 import {
@@ -88,9 +87,9 @@ export class UnlinkFromLibraryAction implements ActionByType<typeof ACTION_UNLIN
 
     const newPanel: PanelState<EmbeddableInput> = {
       type: embeddable.type,
-      explicitInput: { ...newInput, id: uuid.v4() },
+      explicitInput: { ...newInput },
     };
-    dashboard.replacePanel(panelToReplace, newPanel);
+    dashboard.replacePanel(panelToReplace, newPanel, true);
 
     const title = embeddable.getTitle()
       ? i18n.translate('dashboard.panel.unlinkFromLibrary.successMessageWithTitle', {
diff --git a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx
index 051a7ef8bfb92..e80d387fa3066 100644
--- a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx
+++ b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx
@@ -173,11 +173,30 @@ export class DashboardContainer extends Container<InheritedChildInput, Dashboard
 
   public replacePanel(
     previousPanelState: DashboardPanelState<EmbeddableInput>,
-    newPanelState: Partial<PanelState>
+    newPanelState: Partial<PanelState>,
+    generateNewId?: boolean
   ) {
-    // Because the embeddable type can change, we have to operate at the container level here
-    return this.updateInput({
-      panels: {
+    let panels;
+    if (generateNewId) {
+      // replace panel can be called with generateNewId in order to totally destroy and recreate the embeddable
+      panels = { ...this.input.panels };
+      delete panels[previousPanelState.explicitInput.id];
+      const newId = uuid.v4();
+      panels[newId] = {
+        ...previousPanelState,
+        ...newPanelState,
+        gridData: {
+          ...previousPanelState.gridData,
+          i: newId,
+        },
+        explicitInput: {
+          ...newPanelState.explicitInput,
+          id: newId,
+        },
+      };
+    } else {
+      // Because the embeddable type can change, we have to operate at the container level here
+      panels = {
         ...this.input.panels,
         [previousPanelState.explicitInput.id]: {
           ...previousPanelState,
@@ -190,7 +209,11 @@ export class DashboardContainer extends Container<InheritedChildInput, Dashboard
             id: previousPanelState.explicitInput.id,
           },
         },
-      },
+      };
+    }
+
+    return this.updateInput({
+      panels,
       lastReloadRequestTime: new Date().getTime(),
     });
   }

From 8804b504a0689254b3d376bbafe4dc245202b08c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cau=C3=AA=20Marcondes?=
 <55978943+cauemarcondes@users.noreply.github.com>
Date: Wed, 25 Nov 2020 17:08:15 +0100
Subject: [PATCH 15/37] [APM] Elastic chart issues (#84238)

* fixing charts

* addressing pr comments
---
 .../ErrorGroupDetails/Distribution/index.tsx  |   7 +-
 .../TransactionDetails/Distribution/index.tsx |   2 +-
 .../shared/charts/annotations/index.tsx       |  45 -------
 .../shared/charts/timeseries_chart.tsx        |  38 +++++-
 .../transaction_breakdown_chart_contents.tsx  |  36 +++++-
 .../charts/transaction_charts/index.tsx       | 119 +++++++++---------
 .../transaction_error_rate_chart/index.tsx    |   1 +
 .../public/context/annotations_context.tsx    |  49 ++++++++
 .../apm/public/hooks/use_annotations.ts       |  34 ++---
 9 files changed, 194 insertions(+), 137 deletions(-)
 delete mode 100644 x-pack/plugins/apm/public/components/shared/charts/annotations/index.tsx
 create mode 100644 x-pack/plugins/apm/public/context/annotations_context.tsx

diff --git a/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx b/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx
index 99316e3520a76..159f111bee04c 100644
--- a/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx
@@ -90,7 +90,12 @@ export function ErrorDistribution({ distribution, title }: Props) {
             showOverlappingTicks
             tickFormat={xFormatter}
           />
-          <Axis id="y-axis" position={Position.Left} ticks={2} showGridLines />
+          <Axis
+            id="y-axis"
+            position={Position.Left}
+            ticks={2}
+            gridLine={{ visible: true }}
+          />
           <HistogramBarSeries
             minBarHeight={2}
             id="errorOccurrences"
diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx
index bbc99fb122fc6..003f2ed05b09e 100644
--- a/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx
@@ -224,7 +224,7 @@ export function TransactionDistribution({
             id="y-axis"
             position={Position.Left}
             ticks={3}
-            showGridLines
+            gridLine={{ visible: true }}
             tickFormat={(value: number) => formatYShort(value)}
           />
           <HistogramBarSeries
diff --git a/x-pack/plugins/apm/public/components/shared/charts/annotations/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/annotations/index.tsx
deleted file mode 100644
index 683c66b2a96fe..0000000000000
--- a/x-pack/plugins/apm/public/components/shared/charts/annotations/index.tsx
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import {
-  AnnotationDomainTypes,
-  LineAnnotation,
-  Position,
-} from '@elastic/charts';
-import { EuiIcon } from '@elastic/eui';
-import { i18n } from '@kbn/i18n';
-import React from 'react';
-import { asAbsoluteDateTime } from '../../../../../common/utils/formatters';
-import { useTheme } from '../../../../hooks/useTheme';
-import { useAnnotations } from '../../../../hooks/use_annotations';
-
-export function Annotations() {
-  const { annotations } = useAnnotations();
-  const theme = useTheme();
-
-  if (!annotations.length) {
-    return null;
-  }
-
-  const color = theme.eui.euiColorSecondary;
-
-  return (
-    <LineAnnotation
-      id="annotations"
-      domainType={AnnotationDomainTypes.XDomain}
-      dataValues={annotations.map((annotation) => ({
-        dataValue: annotation['@timestamp'],
-        header: asAbsoluteDateTime(annotation['@timestamp']),
-        details: `${i18n.translate('xpack.apm.chart.annotation.version', {
-          defaultMessage: 'Version',
-        })} ${annotation.text}`,
-      }))}
-      style={{ line: { strokeWidth: 1, stroke: color, opacity: 1 } }}
-      marker={<EuiIcon type="dot" color={color} />}
-      markerPosition={Position.Top}
-    />
-  );
-}
diff --git a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx
index c4f5abe104aa9..ea6f2a4a233e5 100644
--- a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx
+++ b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx
@@ -5,28 +5,35 @@
  */
 
 import {
+  AnnotationDomainTypes,
   AreaSeries,
   Axis,
   Chart,
   CurveType,
   LegendItemListener,
+  LineAnnotation,
   LineSeries,
   niceTimeFormatter,
   Placement,
   Position,
   ScaleType,
   Settings,
+  YDomainRange,
 } from '@elastic/charts';
+import { EuiIcon } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
 import moment from 'moment';
 import React, { useEffect } from 'react';
 import { useHistory } from 'react-router-dom';
 import { useChartTheme } from '../../../../../observability/public';
+import { asAbsoluteDateTime } from '../../../../common/utils/formatters';
 import { TimeSeries } from '../../../../typings/timeseries';
 import { FETCH_STATUS } from '../../../hooks/useFetcher';
+import { useTheme } from '../../../hooks/useTheme';
 import { useUrlParams } from '../../../hooks/useUrlParams';
+import { useAnnotations } from '../../../hooks/use_annotations';
 import { useChartPointerEvent } from '../../../hooks/use_chart_pointer_event';
 import { unit } from '../../../style/variables';
-import { Annotations } from './annotations';
 import { ChartContainer } from './chart_container';
 import { onBrushEnd } from './helper/helper';
 
@@ -45,6 +52,7 @@ interface Props {
    */
   yTickFormat?: (y: number) => string;
   showAnnotations?: boolean;
+  yDomain?: YDomainRange;
 }
 
 export function TimeseriesChart({
@@ -56,12 +64,16 @@ export function TimeseriesChart({
   yLabelFormat,
   yTickFormat,
   showAnnotations = true,
+  yDomain,
 }: Props) {
   const history = useHistory();
   const chartRef = React.createRef<Chart>();
+  const { annotations } = useAnnotations();
   const chartTheme = useChartTheme();
   const { pointerEvent, setPointerEvent } = useChartPointerEvent();
   const { urlParams } = useUrlParams();
+  const theme = useTheme();
+
   const { start, end } = urlParams;
 
   useEffect(() => {
@@ -83,6 +95,8 @@ export function TimeseriesChart({
         y === null || y === undefined
     );
 
+  const annotationColor = theme.eui.euiColorSecondary;
+
   return (
     <ChartContainer hasData={!isEmpty} height={height} status={fetchStatus}>
       <Chart ref={chartRef} id={id}>
@@ -108,17 +122,35 @@ export function TimeseriesChart({
           position={Position.Bottom}
           showOverlappingTicks
           tickFormat={xFormatter}
+          gridLine={{ visible: false }}
         />
         <Axis
+          domain={yDomain}
           id="y-axis"
           ticks={3}
           position={Position.Left}
           tickFormat={yTickFormat ? yTickFormat : yLabelFormat}
           labelFormat={yLabelFormat}
-          showGridLines
         />
 
-        {showAnnotations && <Annotations />}
+        {showAnnotations && (
+          <LineAnnotation
+            id="annotations"
+            domainType={AnnotationDomainTypes.XDomain}
+            dataValues={annotations.map((annotation) => ({
+              dataValue: annotation['@timestamp'],
+              header: asAbsoluteDateTime(annotation['@timestamp']),
+              details: `${i18n.translate('xpack.apm.chart.annotation.version', {
+                defaultMessage: 'Version',
+              })} ${annotation.text}`,
+            }))}
+            style={{
+              line: { strokeWidth: 1, stroke: annotationColor, opacity: 1 },
+            }}
+            marker={<EuiIcon type="dot" color={annotationColor} />}
+            markerPosition={Position.Top}
+          />
+        )}
 
         {timeseries.map((serie) => {
           const Series = serie.type === 'area' ? AreaSeries : LineSeries;
diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx
index 04c07c01442a4..20056a6831adf 100644
--- a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx
+++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx
@@ -5,27 +5,35 @@
  */
 
 import {
+  AnnotationDomainTypes,
   AreaSeries,
   Axis,
   Chart,
   CurveType,
+  LineAnnotation,
   niceTimeFormatter,
   Placement,
   Position,
   ScaleType,
   Settings,
 } from '@elastic/charts';
+import { EuiIcon } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
 import moment from 'moment';
 import React, { useEffect } from 'react';
 import { useHistory } from 'react-router-dom';
 import { useChartTheme } from '../../../../../../observability/public';
-import { asPercent } from '../../../../../common/utils/formatters';
+import {
+  asAbsoluteDateTime,
+  asPercent,
+} from '../../../../../common/utils/formatters';
 import { TimeSeries } from '../../../../../typings/timeseries';
 import { FETCH_STATUS } from '../../../../hooks/useFetcher';
+import { useTheme } from '../../../../hooks/useTheme';
 import { useUrlParams } from '../../../../hooks/useUrlParams';
+import { useAnnotations } from '../../../../hooks/use_annotations';
 import { useChartPointerEvent } from '../../../../hooks/use_chart_pointer_event';
 import { unit } from '../../../../style/variables';
-import { Annotations } from '../../charts/annotations';
 import { ChartContainer } from '../../charts/chart_container';
 import { onBrushEnd } from '../../charts/helper/helper';
 
@@ -44,9 +52,11 @@ export function TransactionBreakdownChartContents({
 }: Props) {
   const history = useHistory();
   const chartRef = React.createRef<Chart>();
+  const { annotations } = useAnnotations();
   const chartTheme = useChartTheme();
   const { pointerEvent, setPointerEvent } = useChartPointerEvent();
   const { urlParams } = useUrlParams();
+  const theme = useTheme();
   const { start, end } = urlParams;
 
   useEffect(() => {
@@ -64,6 +74,8 @@ export function TransactionBreakdownChartContents({
 
   const xFormatter = niceTimeFormatter([min, max]);
 
+  const annotationColor = theme.eui.euiColorSecondary;
+
   return (
     <ChartContainer height={height} hasData={!!timeseries} status={fetchStatus}>
       <Chart ref={chartRef} id="timeSpentBySpan">
@@ -85,6 +97,7 @@ export function TransactionBreakdownChartContents({
           position={Position.Bottom}
           showOverlappingTicks
           tickFormat={xFormatter}
+          gridLine={{ visible: false }}
         />
         <Axis
           id="y-axis"
@@ -93,7 +106,24 @@ export function TransactionBreakdownChartContents({
           tickFormat={(y: number) => asPercent(y ?? 0, 1)}
         />
 
-        {showAnnotations && <Annotations />}
+        {showAnnotations && (
+          <LineAnnotation
+            id="annotations"
+            domainType={AnnotationDomainTypes.XDomain}
+            dataValues={annotations.map((annotation) => ({
+              dataValue: annotation['@timestamp'],
+              header: asAbsoluteDateTime(annotation['@timestamp']),
+              details: `${i18n.translate('xpack.apm.chart.annotation.version', {
+                defaultMessage: 'Version',
+              })} ${annotation.text}`,
+            }))}
+            style={{
+              line: { strokeWidth: 1, stroke: annotationColor, opacity: 1 },
+            }}
+            marker={<EuiIcon type="dot" color={annotationColor} />}
+            markerPosition={Position.Top}
+          />
+        )}
 
         {timeseries?.length ? (
           timeseries.map((serie) => {
diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx
index 221f17bb9e1d5..3f8071ec39f0f 100644
--- a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx
+++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx
@@ -20,13 +20,14 @@ import {
   TRANSACTION_ROUTE_CHANGE,
 } from '../../../../../common/transaction_types';
 import { asTransactionRate } from '../../../../../common/utils/formatters';
+import { AnnotationsContextProvider } from '../../../../context/annotations_context';
 import { ChartPointerEventContextProvider } from '../../../../context/chart_pointer_event_context';
 import { LicenseContext } from '../../../../context/LicenseContext';
 import { IUrlParams } from '../../../../context/UrlParamsContext/types';
 import { FETCH_STATUS } from '../../../../hooks/useFetcher';
 import { ITransactionChartData } from '../../../../selectors/chart_selectors';
-import { TransactionBreakdownChart } from '../transaction_breakdown_chart';
 import { TimeseriesChart } from '../timeseries_chart';
+import { TransactionBreakdownChart } from '../transaction_breakdown_chart';
 import { TransactionErrorRateChart } from '../transaction_error_rate_chart/';
 import { getResponseTimeTickFormatter } from './helper';
 import { MLHeader } from './ml_header';
@@ -51,65 +52,69 @@ export function TransactionCharts({
 
   return (
     <>
-      <ChartPointerEventContextProvider>
-        <EuiFlexGrid columns={2} gutterSize="s">
-          <EuiFlexItem data-cy={`transaction-duration-charts`}>
-            <EuiPanel>
-              <EuiFlexGroup justifyContent="spaceBetween">
-                <EuiFlexItem>
-                  <EuiTitle size="xs">
-                    <span>{responseTimeLabel(transactionType)}</span>
-                  </EuiTitle>
-                </EuiFlexItem>
-                <LicenseContext.Consumer>
-                  {(license) => (
-                    <MLHeader
-                      hasValidMlLicense={license?.getFeature('ml').isAvailable}
-                      mlJobId={charts.mlJobId}
-                    />
-                  )}
-                </LicenseContext.Consumer>
-              </EuiFlexGroup>
-              <TimeseriesChart
-                fetchStatus={fetchStatus}
-                id="transactionDuration"
-                timeseries={responseTimeSeries || []}
-                yLabelFormat={getResponseTimeTickFormatter(formatter)}
-                onToggleLegend={(serie) => {
-                  if (serie) {
-                    toggleSerie(serie);
-                  }
-                }}
-              />
-            </EuiPanel>
-          </EuiFlexItem>
+      <AnnotationsContextProvider>
+        <ChartPointerEventContextProvider>
+          <EuiFlexGrid columns={2} gutterSize="s">
+            <EuiFlexItem data-cy={`transaction-duration-charts`}>
+              <EuiPanel>
+                <EuiFlexGroup justifyContent="spaceBetween">
+                  <EuiFlexItem>
+                    <EuiTitle size="xs">
+                      <span>{responseTimeLabel(transactionType)}</span>
+                    </EuiTitle>
+                  </EuiFlexItem>
+                  <LicenseContext.Consumer>
+                    {(license) => (
+                      <MLHeader
+                        hasValidMlLicense={
+                          license?.getFeature('ml').isAvailable
+                        }
+                        mlJobId={charts.mlJobId}
+                      />
+                    )}
+                  </LicenseContext.Consumer>
+                </EuiFlexGroup>
+                <TimeseriesChart
+                  fetchStatus={fetchStatus}
+                  id="transactionDuration"
+                  timeseries={responseTimeSeries || []}
+                  yLabelFormat={getResponseTimeTickFormatter(formatter)}
+                  onToggleLegend={(serie) => {
+                    if (serie) {
+                      toggleSerie(serie);
+                    }
+                  }}
+                />
+              </EuiPanel>
+            </EuiFlexItem>
 
-          <EuiFlexItem style={{ flexShrink: 1 }}>
-            <EuiPanel>
-              <EuiTitle size="xs">
-                <span>{tpmLabel(transactionType)}</span>
-              </EuiTitle>
-              <TimeseriesChart
-                fetchStatus={fetchStatus}
-                id="requestPerMinutes"
-                timeseries={tpmSeries || []}
-                yLabelFormat={asTransactionRate}
-              />
-            </EuiPanel>
-          </EuiFlexItem>
-        </EuiFlexGrid>
+            <EuiFlexItem style={{ flexShrink: 1 }}>
+              <EuiPanel>
+                <EuiTitle size="xs">
+                  <span>{tpmLabel(transactionType)}</span>
+                </EuiTitle>
+                <TimeseriesChart
+                  fetchStatus={fetchStatus}
+                  id="requestPerMinutes"
+                  timeseries={tpmSeries || []}
+                  yLabelFormat={asTransactionRate}
+                />
+              </EuiPanel>
+            </EuiFlexItem>
+          </EuiFlexGrid>
 
-        <EuiSpacer size="s" />
+          <EuiSpacer size="s" />
 
-        <EuiFlexGrid columns={2} gutterSize="s">
-          <EuiFlexItem>
-            <TransactionErrorRateChart />
-          </EuiFlexItem>
-          <EuiFlexItem>
-            <TransactionBreakdownChart />
-          </EuiFlexItem>
-        </EuiFlexGrid>
-      </ChartPointerEventContextProvider>
+          <EuiFlexGrid columns={2} gutterSize="s">
+            <EuiFlexItem>
+              <TransactionErrorRateChart />
+            </EuiFlexItem>
+            <EuiFlexItem>
+              <TransactionBreakdownChart />
+            </EuiFlexItem>
+          </EuiFlexGrid>
+        </ChartPointerEventContextProvider>
+      </AnnotationsContextProvider>
     </>
   );
 }
diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx
index b9028ff2e9e8c..00472df95c4b1 100644
--- a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx
+++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx
@@ -91,6 +91,7 @@ export function TransactionErrorRateChart({
         ]}
         yLabelFormat={yLabelFormat}
         yTickFormat={yTickFormat}
+        yDomain={{ min: 0, max: 1 }}
       />
     </EuiPanel>
   );
diff --git a/x-pack/plugins/apm/public/context/annotations_context.tsx b/x-pack/plugins/apm/public/context/annotations_context.tsx
new file mode 100644
index 0000000000000..4e09a3d227b11
--- /dev/null
+++ b/x-pack/plugins/apm/public/context/annotations_context.tsx
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { createContext } from 'react';
+import { useParams } from 'react-router-dom';
+import { Annotation } from '../../common/annotations';
+import { useFetcher } from '../hooks/useFetcher';
+import { useUrlParams } from '../hooks/useUrlParams';
+import { callApmApi } from '../services/rest/createCallApmApi';
+
+export const AnnotationsContext = createContext({ annotations: [] } as {
+  annotations: Annotation[];
+});
+
+const INITIAL_STATE = { annotations: [] };
+
+export function AnnotationsContextProvider({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  const { serviceName } = useParams<{ serviceName?: string }>();
+  const { urlParams, uiFilters } = useUrlParams();
+  const { start, end } = urlParams;
+  const { environment } = uiFilters;
+
+  const { data = INITIAL_STATE } = useFetcher(() => {
+    if (start && end && serviceName) {
+      return callApmApi({
+        endpoint: 'GET /api/apm/services/{serviceName}/annotation/search',
+        params: {
+          path: {
+            serviceName,
+          },
+          query: {
+            start,
+            end,
+            environment,
+          },
+        },
+      });
+    }
+  }, [start, end, environment, serviceName]);
+
+  return <AnnotationsContext.Provider value={data} children={children} />;
+}
diff --git a/x-pack/plugins/apm/public/hooks/use_annotations.ts b/x-pack/plugins/apm/public/hooks/use_annotations.ts
index e8f6785706a91..1cd9a7e65dda2 100644
--- a/x-pack/plugins/apm/public/hooks/use_annotations.ts
+++ b/x-pack/plugins/apm/public/hooks/use_annotations.ts
@@ -3,36 +3,16 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { useParams } from 'react-router-dom';
-import { callApmApi } from '../services/rest/createCallApmApi';
-import { useFetcher } from './useFetcher';
-import { useUrlParams } from './useUrlParams';
 
-const INITIAL_STATE = { annotations: [] };
+import { useContext } from 'react';
+import { AnnotationsContext } from '../context/annotations_context';
 
 export function useAnnotations() {
-  const { serviceName } = useParams<{ serviceName?: string }>();
-  const { urlParams, uiFilters } = useUrlParams();
-  const { start, end } = urlParams;
-  const { environment } = uiFilters;
+  const context = useContext(AnnotationsContext);
 
-  const { data = INITIAL_STATE } = useFetcher(() => {
-    if (start && end && serviceName) {
-      return callApmApi({
-        endpoint: 'GET /api/apm/services/{serviceName}/annotation/search',
-        params: {
-          path: {
-            serviceName,
-          },
-          query: {
-            start,
-            end,
-            environment,
-          },
-        },
-      });
-    }
-  }, [start, end, environment, serviceName]);
+  if (!context) {
+    throw new Error('Missing Annotations context provider');
+  }
 
-  return data;
+  return context;
 }

From 284b1046df6981f0da1e743680bf99247523627e Mon Sep 17 00:00:00 2001
From: Jen Huang <its.jenetic@gmail.com>
Date: Wed, 25 Nov 2020 10:13:36 -0800
Subject: [PATCH 16/37] [Fleet] Support input-level vars & templates (#83878)

* Fix bug creating new policy on the fly

* Adjust UI for input with vars but no streams

* Revert "Fix bug creating new policy on the fly"

This reverts commit 34f7014d6977cf73b792a64bcd9bf7c54b72cf42.

* Add `compiled_input` field and compile input template, if any. Make compilation method names more generic (instead of only for streams). Add testts

* Add compiled input to generated agent yaml

* Don't return empty streams in agent yaml when there aren't any

* Update missed assertion

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../package_policies_to_agent_inputs.test.ts  |  42 +++++-
 .../package_policies_to_agent_inputs.ts       |  33 +++--
 .../fleet/common/types/models/agent_policy.ts |   2 +-
 .../plugins/fleet/common/types/models/epm.ts  |   1 +
 .../common/types/models/package_policy.ts     |   1 +
 .../package_policy_input_config.tsx           |  24 ++--
 .../components/package_policy_input_panel.tsx |  32 +++--
 x-pack/plugins/fleet/server/mocks.ts          |   2 +-
 .../routes/package_policy/handlers.test.ts    |   2 +-
 .../fleet/server/saved_objects/index.ts       |   1 +
 .../server/services/epm/agent/agent.test.ts   |  14 +-
 .../fleet/server/services/epm/agent/agent.ts  |  35 ++---
 .../server/services/package_policy.test.ts    | 128 +++++++++++++++++-
 .../fleet/server/services/package_policy.ts   |  71 +++++++---
 .../apps/endpoint/policy_details.ts           |   1 -
 15 files changed, 306 insertions(+), 83 deletions(-)

diff --git a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts
index f721afb639141..a370f92e97fe1 100644
--- a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts
+++ b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts
@@ -100,7 +100,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => {
     ).toEqual([]);
   });
 
-  it('returns agent inputs', () => {
+  it('returns agent inputs with streams', () => {
     expect(
       storedPackagePoliciesToAgentInputs([
         {
@@ -143,6 +143,46 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => {
     ]);
   });
 
+  it('returns agent inputs without streams', () => {
+    expect(
+      storedPackagePoliciesToAgentInputs([
+        {
+          ...mockPackagePolicy,
+          package: {
+            name: 'mock-package',
+            title: 'Mock package',
+            version: '0.0.0',
+          },
+          inputs: [
+            {
+              ...mockInput,
+              compiled_input: {
+                inputVar: 'input-value',
+              },
+              streams: [],
+            },
+          ],
+        },
+      ])
+    ).toEqual([
+      {
+        id: 'some-uuid',
+        name: 'mock-package-policy',
+        revision: 1,
+        type: 'test-logs',
+        data_stream: { namespace: 'default' },
+        use_output: 'default',
+        meta: {
+          package: {
+            name: 'mock-package',
+            version: '0.0.0',
+          },
+        },
+        inputVar: 'input-value',
+      },
+    ]);
+  });
+
   it('returns agent inputs without disabled streams', () => {
     expect(
       storedPackagePoliciesToAgentInputs([
diff --git a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts
index e74256ce732a6..d780fb791aa8e 100644
--- a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts
+++ b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts
@@ -33,20 +33,25 @@ export const storedPackagePoliciesToAgentInputs = (
           acc[key] = value;
           return acc;
         }, {} as { [k: string]: any }),
-        streams: input.streams
-          .filter((stream) => stream.enabled)
-          .map((stream) => {
-            const fullStream: FullAgentPolicyInputStream = {
-              id: stream.id,
-              data_stream: stream.data_stream,
-              ...stream.compiled_stream,
-              ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
-                acc[key] = value;
-                return acc;
-              }, {} as { [k: string]: any }),
-            };
-            return fullStream;
-          }),
+        ...(input.compiled_input || {}),
+        ...(input.streams.length
+          ? {
+              streams: input.streams
+                .filter((stream) => stream.enabled)
+                .map((stream) => {
+                  const fullStream: FullAgentPolicyInputStream = {
+                    id: stream.id,
+                    data_stream: stream.data_stream,
+                    ...stream.compiled_stream,
+                    ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
+                      acc[key] = value;
+                      return acc;
+                    }, {} as { [k: string]: any }),
+                  };
+                  return fullStream;
+                }),
+            }
+          : {}),
       };
 
       if (packagePolicy.package) {
diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts
index f43f65fb317f3..75bb2998f2d92 100644
--- a/x-pack/plugins/fleet/common/types/models/agent_policy.ts
+++ b/x-pack/plugins/fleet/common/types/models/agent_policy.ts
@@ -49,7 +49,7 @@ export interface FullAgentPolicyInput {
     package?: Pick<PackagePolicyPackage, 'name' | 'version'>;
     [key: string]: unknown;
   };
-  streams: FullAgentPolicyInputStream[];
+  streams?: FullAgentPolicyInputStream[];
   [key: string]: any;
 }
 
diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
index 7a6f6232b2d4f..53e507f6fb494 100644
--- a/x-pack/plugins/fleet/common/types/models/epm.ts
+++ b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -121,6 +121,7 @@ export interface RegistryInput {
   title: string;
   description?: string;
   vars?: RegistryVarsEntry[];
+  template_path?: string;
 }
 
 export interface RegistryStream {
diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts
index ae16899a4b6f9..6da98a51ef1ff 100644
--- a/x-pack/plugins/fleet/common/types/models/package_policy.ts
+++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts
@@ -42,6 +42,7 @@ export interface NewPackagePolicyInput {
 
 export interface PackagePolicyInput extends Omit<NewPackagePolicyInput, 'streams'> {
   streams: PackagePolicyInputStream[];
+  compiled_input?: any;
 }
 
 export interface NewPackagePolicy {
diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx
index 75000ad7e1d3b..9015cd09f78a3 100644
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx
@@ -27,6 +27,7 @@ const FlexItemWithMaxWidth = styled(EuiFlexItem)`
 `;
 
 export const PackagePolicyInputConfig: React.FunctionComponent<{
+  hasInputStreams: boolean;
   packageInputVars?: RegistryVarsEntry[];
   packagePolicyInput: NewPackagePolicyInput;
   updatePackagePolicyInput: (updatedInput: Partial<NewPackagePolicyInput>) => void;
@@ -34,6 +35,7 @@ export const PackagePolicyInputConfig: React.FunctionComponent<{
   forceShowErrors?: boolean;
 }> = memo(
   ({
+    hasInputStreams,
     packageInputVars,
     packagePolicyInput,
     updatePackagePolicyInput,
@@ -82,15 +84,19 @@ export const PackagePolicyInputConfig: React.FunctionComponent<{
                   />
                 </h4>
               </EuiText>
-              <EuiSpacer size="s" />
-              <EuiText color="subdued" size="s">
-                <p>
-                  <FormattedMessage
-                    id="xpack.fleet.createPackagePolicy.stepConfigure.inputSettingsDescription"
-                    defaultMessage="The following settings are applicable to all inputs below."
-                  />
-                </p>
-              </EuiText>
+              {hasInputStreams ? (
+                <>
+                  <EuiSpacer size="s" />
+                  <EuiText color="subdued" size="s">
+                    <p>
+                      <FormattedMessage
+                        id="xpack.fleet.createPackagePolicy.stepConfigure.inputSettingsDescription"
+                        defaultMessage="The following settings are applicable to all inputs below."
+                      />
+                    </p>
+                  </EuiText>
+                </>
+              ) : null}
             </EuiFlexItem>
           </EuiFlexGroup>
         </EuiFlexItem>
diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx
index 79ff0cc29850c..8e242980ce807 100644
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import React, { useState, Fragment, memo } from 'react';
+import React, { useState, Fragment, memo, useMemo } from 'react';
 import styled from 'styled-components';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
@@ -85,16 +85,23 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{
     const errorCount = countValidationErrors(inputValidationResults);
     const hasErrors = forceShowErrors && errorCount;
 
-    const inputStreams = packageInputStreams
-      .map((packageInputStream) => {
-        return {
-          packageInputStream,
-          packagePolicyInputStream: packagePolicyInput.streams.find(
-            (stream) => stream.data_stream.dataset === packageInputStream.data_stream.dataset
-          ),
-        };
-      })
-      .filter((stream) => Boolean(stream.packagePolicyInputStream));
+    const hasInputStreams = useMemo(() => packageInputStreams.length > 0, [
+      packageInputStreams.length,
+    ]);
+    const inputStreams = useMemo(
+      () =>
+        packageInputStreams
+          .map((packageInputStream) => {
+            return {
+              packageInputStream,
+              packagePolicyInputStream: packagePolicyInput.streams.find(
+                (stream) => stream.data_stream.dataset === packageInputStream.data_stream.dataset
+              ),
+            };
+          })
+          .filter((stream) => Boolean(stream.packagePolicyInputStream)),
+      [packageInputStreams, packagePolicyInput.streams]
+    );
 
     return (
       <>
@@ -179,13 +186,14 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{
         {isShowingStreams && packageInput.vars && packageInput.vars.length ? (
           <Fragment>
             <PackagePolicyInputConfig
+              hasInputStreams={hasInputStreams}
               packageInputVars={packageInput.vars}
               packagePolicyInput={packagePolicyInput}
               updatePackagePolicyInput={updatePackagePolicyInput}
               inputVarsValidationResults={{ vars: inputValidationResults.vars }}
               forceShowErrors={forceShowErrors}
             />
-            <ShortenedHorizontalRule margin="m" />
+            {hasInputStreams ? <ShortenedHorizontalRule margin="m" /> : <EuiSpacer size="l" />}
           </Fragment>
         ) : null}
 
diff --git a/x-pack/plugins/fleet/server/mocks.ts b/x-pack/plugins/fleet/server/mocks.ts
index 91098c87c312a..bc3e89ef6d3ce 100644
--- a/x-pack/plugins/fleet/server/mocks.ts
+++ b/x-pack/plugins/fleet/server/mocks.ts
@@ -25,7 +25,7 @@ export const createAppContextStartContractMock = (): FleetAppContext => {
 
 export const createPackagePolicyServiceMock = () => {
   return {
-    assignPackageStream: jest.fn(),
+    compilePackagePolicyInputs: jest.fn(),
     buildPackagePolicyFromPackage: jest.fn(),
     bulkCreate: jest.fn(),
     create: jest.fn(),
diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts
index f47b8499a1b69..fee74e39c833a 100644
--- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts
+++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts
@@ -22,7 +22,7 @@ jest.mock('../../services/package_policy', (): {
 } => {
   return {
     packagePolicyService: {
-      assignPackageStream: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)),
+      compilePackagePolicyInputs: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)),
       buildPackagePolicyFromPackage: jest.fn(),
       bulkCreate: jest.fn(),
       create: jest.fn((soClient, callCluster, newData) =>
diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
index 9d85a151efbbf..201ca1c7a97bc 100644
--- a/x-pack/plugins/fleet/server/saved_objects/index.ts
+++ b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -242,6 +242,7 @@ const getSavedObjectTypes = (
             enabled: { type: 'boolean' },
             vars: { type: 'flattened' },
             config: { type: 'flattened' },
+            compiled_input: { type: 'flattened' },
             streams: {
               type: 'nested',
               properties: {
diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts
index 54b40400bb4e7..dba6f442d76e2 100644
--- a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts
+++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts
@@ -4,9 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { createStream } from './agent';
+import { compileTemplate } from './agent';
 
-describe('createStream', () => {
+describe('compileTemplate', () => {
   it('should work', () => {
     const streamTemplate = `
 input: log
@@ -27,7 +27,7 @@ hidden_password: {{password}}
       password: { type: 'password', value: '' },
     };
 
-    const output = createStream(vars, streamTemplate);
+    const output = compileTemplate(vars, streamTemplate);
     expect(output).toEqual({
       input: 'log',
       paths: ['/usr/local/var/log/nginx/access.log'],
@@ -67,7 +67,7 @@ foo: bar
       password: { type: 'password', value: '' },
     };
 
-    const output = createStream(vars, streamTemplate);
+    const output = compileTemplate(vars, streamTemplate);
     expect(output).toEqual({
       input: 'redis/metrics',
       metricsets: ['key'],
@@ -114,7 +114,7 @@ hidden_password: {{password}}
         tags: { value: ['foo', 'bar', 'forwarded'] },
       };
 
-      const output = createStream(vars, streamTemplate);
+      const output = compileTemplate(vars, streamTemplate);
       expect(output).toEqual({
         input: 'log',
         paths: ['/usr/local/var/log/nginx/access.log'],
@@ -133,7 +133,7 @@ hidden_password: {{password}}
         tags: { value: ['foo', 'bar'] },
       };
 
-      const output = createStream(vars, streamTemplate);
+      const output = compileTemplate(vars, streamTemplate);
       expect(output).toEqual({
         input: 'log',
         paths: ['/usr/local/var/log/nginx/access.log'],
@@ -157,7 +157,7 @@ input: logs
       },
     };
 
-    const output = createStream(vars, streamTemplate);
+    const output = compileTemplate(vars, streamTemplate);
     expect(output).toEqual({
       input: 'logs',
     });
diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts
index eeadac6e168b1..400a688722f99 100644
--- a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts
+++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts
@@ -10,27 +10,30 @@ import { PackagePolicyConfigRecord } from '../../../../common';
 
 const handlebars = Handlebars.create();
 
-export function createStream(variables: PackagePolicyConfigRecord, streamTemplate: string) {
-  const { vars, yamlValues } = buildTemplateVariables(variables, streamTemplate);
+export function compileTemplate(variables: PackagePolicyConfigRecord, templateStr: string) {
+  const { vars, yamlValues } = buildTemplateVariables(variables, templateStr);
 
-  const template = handlebars.compile(streamTemplate, { noEscape: true });
-  let stream = template(vars);
-  stream = replaceRootLevelYamlVariables(yamlValues, stream);
+  const template = handlebars.compile(templateStr, { noEscape: true });
+  let compiledTemplate = template(vars);
+  compiledTemplate = replaceRootLevelYamlVariables(yamlValues, compiledTemplate);
 
-  const yamlFromStream = safeLoad(stream, {});
+  const yamlFromCompiledTemplate = safeLoad(compiledTemplate, {});
 
   // Hack to keep empty string ('') values around in the end yaml because
   // `safeLoad` replaces empty strings with null
-  const patchedYamlFromStream = Object.entries(yamlFromStream).reduce((acc, [key, value]) => {
-    if (value === null && typeof vars[key] === 'string' && vars[key].trim() === '') {
-      acc[key] = '';
-    } else {
-      acc[key] = value;
-    }
-    return acc;
-  }, {} as { [k: string]: any });
+  const patchedYamlFromCompiledTemplate = Object.entries(yamlFromCompiledTemplate).reduce(
+    (acc, [key, value]) => {
+      if (value === null && typeof vars[key] === 'string' && vars[key].trim() === '') {
+        acc[key] = '';
+      } else {
+        acc[key] = value;
+      }
+      return acc;
+    },
+    {} as { [k: string]: any }
+  );
 
-  return replaceVariablesInYaml(yamlValues, patchedYamlFromStream);
+  return replaceVariablesInYaml(yamlValues, patchedYamlFromCompiledTemplate);
 }
 
 function isValidKey(key: string) {
@@ -54,7 +57,7 @@ function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any)
   return yaml;
 }
 
-function buildTemplateVariables(variables: PackagePolicyConfigRecord, streamTemplate: string) {
+function buildTemplateVariables(variables: PackagePolicyConfigRecord, templateStr: string) {
   const yamlValues: { [k: string]: any } = {};
   const vars = Object.entries(variables).reduce((acc, [key, recordEntry]) => {
     // support variables with . like key.patterns
diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts
index 6ae76c56436d5..30a980ab07f70 100644
--- a/x-pack/plugins/fleet/server/services/package_policy.test.ts
+++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts
@@ -25,7 +25,16 @@ paths:
       },
     ];
   }
-  return [];
+  return [
+    {
+      buffer: Buffer.from(`
+hosts:
+{{#each hosts}}
+- {{this}}
+{{/each}}
+`),
+    },
+  ];
 }
 
 jest.mock('./epm/packages/assets', () => {
@@ -47,9 +56,9 @@ jest.mock('./epm/registry', () => {
 });
 
 describe('Package policy service', () => {
-  describe('assignPackageStream', () => {
+  describe('compilePackagePolicyInputs', () => {
     it('should work with config variables from the stream', async () => {
-      const inputs = await packagePolicyService.assignPackageStream(
+      const inputs = await packagePolicyService.compilePackagePolicyInputs(
         ({
           data_streams: [
             {
@@ -110,7 +119,7 @@ describe('Package policy service', () => {
     });
 
     it('should work with config variables at the input level', async () => {
-      const inputs = await packagePolicyService.assignPackageStream(
+      const inputs = await packagePolicyService.compilePackagePolicyInputs(
         ({
           data_streams: [
             {
@@ -169,6 +178,117 @@ describe('Package policy service', () => {
         },
       ]);
     });
+
+    it('should work with an input with a template and no streams', async () => {
+      const inputs = await packagePolicyService.compilePackagePolicyInputs(
+        ({
+          data_streams: [],
+          policy_templates: [
+            {
+              inputs: [{ type: 'log', template_path: 'some_template_path.yml' }],
+            },
+          ],
+        } as unknown) as PackageInfo,
+        [
+          {
+            type: 'log',
+            enabled: true,
+            vars: {
+              hosts: {
+                value: ['localhost'],
+              },
+            },
+            streams: [],
+          },
+        ]
+      );
+
+      expect(inputs).toEqual([
+        {
+          type: 'log',
+          enabled: true,
+          vars: {
+            hosts: {
+              value: ['localhost'],
+            },
+          },
+          compiled_input: {
+            hosts: ['localhost'],
+          },
+          streams: [],
+        },
+      ]);
+    });
+
+    it('should work with an input with a template and streams', async () => {
+      const inputs = await packagePolicyService.compilePackagePolicyInputs(
+        ({
+          data_streams: [
+            {
+              dataset: 'package.dataset1',
+              type: 'logs',
+              streams: [{ input: 'log', template_path: 'some_template_path.yml' }],
+            },
+          ],
+          policy_templates: [
+            {
+              inputs: [{ type: 'log', template_path: 'some_template_path.yml' }],
+            },
+          ],
+        } as unknown) as PackageInfo,
+        [
+          {
+            type: 'log',
+            enabled: true,
+            vars: {
+              hosts: {
+                value: ['localhost'],
+              },
+              paths: {
+                value: ['/var/log/set.log'],
+              },
+            },
+            streams: [
+              {
+                id: 'datastream01',
+                data_stream: { dataset: 'package.dataset1', type: 'logs' },
+                enabled: true,
+              },
+            ],
+          },
+        ]
+      );
+
+      expect(inputs).toEqual([
+        {
+          type: 'log',
+          enabled: true,
+          vars: {
+            hosts: {
+              value: ['localhost'],
+            },
+            paths: {
+              value: ['/var/log/set.log'],
+            },
+          },
+          compiled_input: {
+            hosts: ['localhost'],
+          },
+          streams: [
+            {
+              id: 'datastream01',
+              data_stream: { dataset: 'package.dataset1', type: 'logs' },
+              enabled: true,
+              compiled_stream: {
+                metricset: ['dataset1'],
+                paths: ['/var/log/set.log'],
+                type: 'log',
+              },
+            },
+          ],
+        },
+      ]);
+    });
   });
 
   describe('update', () => {
diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts
index 0f78c97a6f2bd..7b8952bdea2cd 100644
--- a/x-pack/plugins/fleet/server/services/package_policy.ts
+++ b/x-pack/plugins/fleet/server/services/package_policy.ts
@@ -31,7 +31,7 @@ import { outputService } from './output';
 import * as Registry from './epm/registry';
 import { getPackageInfo, getInstallation, ensureInstalledPackage } from './epm/packages';
 import { getAssetsData } from './epm/packages/assets';
-import { createStream } from './epm/agent/agent';
+import { compileTemplate } from './epm/agent/agent';
 import { normalizeKuery } from './saved_object';
 
 const SAVED_OBJECT_TYPE = PACKAGE_POLICY_SAVED_OBJECT_TYPE;
@@ -92,7 +92,7 @@ class PackagePolicyService {
         }
       }
 
-      inputs = await this.assignPackageStream(pkgInfo, inputs);
+      inputs = await this.compilePackagePolicyInputs(pkgInfo, inputs);
     }
 
     const isoDate = new Date().toISOString();
@@ -285,7 +285,7 @@ class PackagePolicyService {
         pkgVersion: packagePolicy.package.version,
       });
 
-      inputs = await this.assignPackageStream(pkgInfo, inputs);
+      inputs = await this.compilePackagePolicyInputs(pkgInfo, inputs);
     }
 
     await soClient.update<PackagePolicySOAttributes>(
@@ -374,14 +374,20 @@ class PackagePolicyService {
     }
   }
 
-  public async assignPackageStream(
+  public async compilePackagePolicyInputs(
     pkgInfo: PackageInfo,
     inputs: PackagePolicyInput[]
   ): Promise<PackagePolicyInput[]> {
     const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version);
-    const inputsPromises = inputs.map((input) =>
-      _assignPackageStreamToInput(registryPkgInfo, pkgInfo, input)
-    );
+    const inputsPromises = inputs.map(async (input) => {
+      const compiledInput = await _compilePackagePolicyInput(registryPkgInfo, pkgInfo, input);
+      const compiledStreams = await _compilePackageStreams(registryPkgInfo, pkgInfo, input);
+      return {
+        ...input,
+        compiled_input: compiledInput,
+        streams: compiledStreams,
+      };
+    });
 
     return Promise.all(inputsPromises);
   }
@@ -396,20 +402,53 @@ function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyI
   };
 }
 
-async function _assignPackageStreamToInput(
+async function _compilePackagePolicyInput(
+  registryPkgInfo: RegistryPackage,
+  pkgInfo: PackageInfo,
+  input: PackagePolicyInput
+) {
+  if (!input.enabled || !pkgInfo.policy_templates?.[0].inputs) {
+    return undefined;
+  }
+
+  const packageInputs = pkgInfo.policy_templates[0].inputs;
+  const packageInput = packageInputs.find((pkgInput) => pkgInput.type === input.type);
+  if (!packageInput) {
+    throw new Error(`Input template not found, unable to find input type ${input.type}`);
+  }
+
+  if (!packageInput.template_path) {
+    return undefined;
+  }
+
+  const [pkgInputTemplate] = await getAssetsData(registryPkgInfo, (path: string) =>
+    path.endsWith(`/agent/input/${packageInput.template_path!}`)
+  );
+
+  if (!pkgInputTemplate || !pkgInputTemplate.buffer) {
+    throw new Error(`Unable to load input template at /agent/input/${packageInput.template_path!}`);
+  }
+
+  return compileTemplate(
+    // Populate template variables from input vars
+    Object.assign({}, input.vars),
+    pkgInputTemplate.buffer.toString()
+  );
+}
+
+async function _compilePackageStreams(
   registryPkgInfo: RegistryPackage,
   pkgInfo: PackageInfo,
   input: PackagePolicyInput
 ) {
   const streamsPromises = input.streams.map((stream) =>
-    _assignPackageStreamToStream(registryPkgInfo, pkgInfo, input, stream)
+    _compilePackageStream(registryPkgInfo, pkgInfo, input, stream)
   );
 
-  const streams = await Promise.all(streamsPromises);
-  return { ...input, streams };
+  return await Promise.all(streamsPromises);
 }
 
-async function _assignPackageStreamToStream(
+async function _compilePackageStream(
   registryPkgInfo: RegistryPackage,
   pkgInfo: PackageInfo,
   input: PackagePolicyInput,
@@ -442,22 +481,22 @@ async function _assignPackageStreamToStream(
     throw new Error(`Stream template path not found for dataset ${datasetPath}`);
   }
 
-  const [pkgStream] = await getAssetsData(
+  const [pkgStreamTemplate] = await getAssetsData(
     registryPkgInfo,
     (path: string) => path.endsWith(streamFromPkg.template_path),
     datasetPath
   );
 
-  if (!pkgStream || !pkgStream.buffer) {
+  if (!pkgStreamTemplate || !pkgStreamTemplate.buffer) {
     throw new Error(
       `Unable to load stream template ${streamFromPkg.template_path} for dataset ${datasetPath}`
     );
   }
 
-  const yaml = createStream(
+  const yaml = compileTemplate(
     // Populate template variables from input vars and stream vars
     Object.assign({}, input.vars, stream.vars),
-    pkgStream.buffer.toString()
+    pkgStreamTemplate.buffer.toString()
   );
 
   stream.compiled_stream = yaml;
diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts
index b3c130ea1e5dc..46085b0db3063 100644
--- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts
+++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts
@@ -249,7 +249,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
                 },
               },
             },
-            streams: [],
             type: 'endpoint',
             use_output: 'default',
           },

From 497511272308d2aae3f81e26121588aa73dfdd24 Mon Sep 17 00:00:00 2001
From: Spencer <email@spalger.com>
Date: Wed, 25 Nov 2020 12:12:28 -0700
Subject: [PATCH 17/37] [cli/dev] log a warning when --no-base-path is used
 with --dev (#84354)

Co-authored-by: spalger <spalger@users.noreply.github.com>
---
 src/cli/cluster/cluster_manager.ts | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/cli/cluster/cluster_manager.ts b/src/cli/cluster/cluster_manager.ts
index 7a14f617b5d5a..b0f7cded938dd 100644
--- a/src/cli/cluster/cluster_manager.ts
+++ b/src/cli/cluster/cluster_manager.ts
@@ -73,6 +73,19 @@ export class ClusterManager {
     this.inReplMode = !!opts.repl;
     this.basePathProxy = basePathProxy;
 
+    if (!this.basePathProxy) {
+      this.log.warn(
+        '===================================================================================================='
+      );
+      this.log.warn(
+        'no-base-path',
+        'Running Kibana in dev mode with --no-base-path disables several useful features and is not recommended'
+      );
+      this.log.warn(
+        '===================================================================================================='
+      );
+    }
+
     // run @kbn/optimizer and write it's state to kbnOptimizerReady$
     if (opts.disableOptimizer) {
       this.kbnOptimizerReady$.next(true);

From 5fda30001f69d536bfa7f0f84ecd5f6e7df1dcc8 Mon Sep 17 00:00:00 2001
From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com>
Date: Wed, 25 Nov 2020 14:47:11 -0500
Subject: [PATCH 18/37] [Security Solution][Resolver] Add support for
 predefined schemas for endpoint and winlogbeat (#84103)

* Refactoring entity route to return schema

* Refactoring frontend middleware to pick off id field from entity route

* Refactoring schema and adding name and comments

* Adding name to schema mocks

* Fixing type issue
---
 .../common/endpoint/types/index.ts            |  39 ++++-
 .../mocks/no_ancestors_two_children.ts        |  13 +-
 ..._children_in_index_called_awesome_index.ts |  13 +-
 ...ith_related_events_and_cursor_on_origin.ts |  13 +-
 ..._children_with_related_events_on_origin.ts |  13 +-
 .../one_node_with_paginated_related_events.ts |  13 +-
 .../store/middleware/resolver_tree_fetcher.ts |   2 +-
 .../server/endpoint/routes/resolver/entity.ts | 147 ++++++++++++------
 .../resolver/tree/queries/descendants.ts      |   8 +-
 .../routes/resolver/tree/queries/lifecycle.ts |   8 +-
 .../routes/resolver/tree/queries/stats.ts     |   8 +-
 .../routes/resolver/tree/utils/fetch.test.ts  |   9 +-
 .../routes/resolver/tree/utils/fetch.ts       |  25 +--
 .../routes/resolver/tree/utils/index.ts       |  27 +---
 .../apis/resolver/common.ts                   |  25 +--
 .../apis/resolver/entity.ts                   |   9 +-
 .../apis/resolver/tree.ts                     |  12 +-
 17 files changed, 259 insertions(+), 125 deletions(-)

diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts
index cd5c60e2698cb..d6be83d7cbbe3 100644
--- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts
+++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts
@@ -870,10 +870,47 @@ export interface SafeLegacyEndpointEvent {
   }>;
 }
 
+/**
+ * The fields to use to identify nodes within a resolver tree.
+ */
+export interface ResolverSchema {
+  /**
+   * the ancestry field should be set to a field that contains an order array representing
+   * the ancestors of a node.
+   */
+  ancestry?: string;
+  /**
+   * id represents the field to use as the unique ID for a node.
+   */
+  id: string;
+  /**
+   * field to use for the name of the node
+   */
+  name?: string;
+  /**
+   * parent represents the field that is the edge between two nodes.
+   */
+  parent: string;
+}
+
 /**
  * The response body for the resolver '/entity' index API
  */
-export type ResolverEntityIndex = Array<{ entity_id: string }>;
+export type ResolverEntityIndex = Array<{
+  /**
+   * A name for the schema that is being used (e.g. endpoint, winlogbeat, etc)
+   */
+  name: string;
+  /**
+   * The schema to pass to the /tree api and other backend requests, based on the contents of the document found using
+   * the _id
+   */
+  schema: ResolverSchema;
+  /**
+   * Unique ID value for the requested document using the `_id` field passed to the /entity route
+   */
+  id: string;
+}>;
 
 /**
  * Takes a @kbn/config-schema 'schema' type and returns a type that represents valid inputs.
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts
index 09625e5726b1d..472fdc79d1f02 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts
@@ -99,7 +99,18 @@ export function noAncestorsTwoChildren(): { dataAccessLayer: DataAccessLayer; me
        * Get entities matching a document.
        */
       entities(): Promise<ResolverEntityIndex> {
-        return Promise.resolve([{ entity_id: metadata.entityIDs.origin }]);
+        return Promise.resolve([
+          {
+            name: 'endpoint',
+            schema: {
+              id: 'process.entity_id',
+              parent: 'process.parent.entity_id',
+              ancestry: 'process.Ext.ancestry',
+              name: 'process.name',
+            },
+            id: metadata.entityIDs.origin,
+          },
+        ]);
       },
     },
   };
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts
index 3bbe4bcf51060..b085738d3fd2e 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts
@@ -115,7 +115,18 @@ export function noAncestorsTwoChildenInIndexCalledAwesomeIndex(): {
       entities({ indices }): Promise<ResolverEntityIndex> {
         // Only return values if the `indices` array contains exactly `'awesome_index'`
         if (indices.length === 1 && indices[0] === 'awesome_index') {
-          return Promise.resolve([{ entity_id: metadata.entityIDs.origin }]);
+          return Promise.resolve([
+            {
+              name: 'endpoint',
+              schema: {
+                id: 'process.entity_id',
+                parent: 'process.parent.entity_id',
+                ancestry: 'process.Ext.ancestry',
+                name: 'process.name',
+              },
+              id: metadata.entityIDs.origin,
+            },
+          ]);
         }
         return Promise.resolve([]);
       },
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts
index 7682165ac5e94..43704db358d7e 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts
@@ -140,7 +140,18 @@ export function noAncestorsTwoChildrenWithRelatedEventsOnOriginWithOneAfterCurso
        * Get entities matching a document.
        */
       async entities(): Promise<ResolverEntityIndex> {
-        return [{ entity_id: metadata.entityIDs.origin }];
+        return [
+          {
+            name: 'endpoint',
+            schema: {
+              id: 'process.entity_id',
+              parent: 'process.parent.entity_id',
+              ancestry: 'process.Ext.ancestry',
+              name: 'process.name',
+            },
+            id: metadata.entityIDs.origin,
+          },
+        ];
       },
     },
   };
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts
index 837d824db8748..c4d538d2eed94 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts
@@ -112,7 +112,18 @@ export function noAncestorsTwoChildrenWithRelatedEventsOnOrigin(): {
        * Get entities matching a document.
        */
       async entities(): Promise<ResolverEntityIndex> {
-        return [{ entity_id: metadata.entityIDs.origin }];
+        return [
+          {
+            name: 'endpoint',
+            schema: {
+              id: 'process.entity_id',
+              parent: 'process.parent.entity_id',
+              ancestry: 'process.Ext.ancestry',
+              name: 'process.name',
+            },
+            id: metadata.entityIDs.origin,
+          },
+        ];
       },
     },
   };
diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts
index 01477ff16868e..7849776ed1378 100644
--- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts
+++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts
@@ -103,7 +103,18 @@ export function oneNodeWithPaginatedEvents(): {
        * Get entities matching a document.
        */
       async entities(): Promise<ResolverEntityIndex> {
-        return [{ entity_id: metadata.entityIDs.origin }];
+        return [
+          {
+            name: 'endpoint',
+            schema: {
+              id: 'process.entity_id',
+              parent: 'process.parent.entity_id',
+              ancestry: 'process.Ext.ancestry',
+              name: 'process.name',
+            },
+            id: metadata.entityIDs.origin,
+          },
+        ];
       },
     },
   };
diff --git a/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts b/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts
index ef4ca2380ebf4..aecdd6b92a463 100644
--- a/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts
+++ b/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts
@@ -56,7 +56,7 @@ export function ResolverTreeFetcher(
           });
           return;
         }
-        const entityIDToFetch = matchingEntities[0].entity_id;
+        const entityIDToFetch = matchingEntities[0].id;
         result = await dataAccessLayer.resolverTree(
           entityIDToFetch,
           lastRequestAbortController.signal
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts
index 510bb6c545558..c731692e6fb89 100644
--- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts
+++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts
@@ -3,10 +3,70 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { RequestHandler } from 'kibana/server';
+import _ from 'lodash';
+import { RequestHandler, SearchResponse } from 'kibana/server';
 import { TypeOf } from '@kbn/config-schema';
+import { ApiResponse } from '@elastic/elasticsearch';
 import { validateEntities } from '../../../../common/endpoint/schema/resolver';
-import { ResolverEntityIndex } from '../../../../common/endpoint/types';
+import { ResolverEntityIndex, ResolverSchema } from '../../../../common/endpoint/types';
+
+interface SupportedSchema {
+  /**
+   * A name for the schema being used
+   */
+  name: string;
+
+  /**
+   * A constraint to search for in the documented returned by Elasticsearch
+   */
+  constraint: { field: string; value: string };
+
+  /**
+   * Schema to return to the frontend so that it can be passed in to call to the /tree API
+   */
+  schema: ResolverSchema;
+}
+
+/**
+ * This structure defines the preset supported schemas for a resolver graph. We'll probably want convert this
+ * implementation to something similar to how row renderers is implemented.
+ */
+const supportedSchemas: SupportedSchema[] = [
+  {
+    name: 'endpoint',
+    constraint: {
+      field: 'agent.type',
+      value: 'endpoint',
+    },
+    schema: {
+      id: 'process.entity_id',
+      parent: 'process.parent.entity_id',
+      ancestry: 'process.Ext.ancestry',
+      name: 'process.name',
+    },
+  },
+  {
+    name: 'winlogbeat',
+    constraint: {
+      field: 'agent.type',
+      value: 'winlogbeat',
+    },
+    schema: {
+      id: 'process.entity_id',
+      parent: 'process.parent.entity_id',
+      name: 'process.name',
+    },
+  },
+];
+
+function getFieldAsString(doc: unknown, field: string): string | undefined {
+  const value = _.get(doc, field);
+  if (value === undefined) {
+    return undefined;
+  }
+
+  return String(value);
+}
 
 /**
  * This is used to get an 'entity_id' which is an internal-to-Resolver concept, from an `_id`, which
@@ -18,61 +78,46 @@ export function handleEntities(): RequestHandler<unknown, TypeOf<typeof validate
       query: { _id, indices },
     } = request;
 
-    /**
-     * A safe type for the response based on the semantics of the query.
-     * We specify _source, asking for `process.entity_id` and we only
-     * accept documents that have it.
-     * Also, we only request 1 document.
-     */
-    interface ExpectedQueryResponse {
-      hits: {
-        hits:
-          | []
-          | [
+    const queryResponse: ApiResponse<
+      SearchResponse<unknown>
+    > = await context.core.elasticsearch.client.asCurrentUser.search({
+      ignore_unavailable: true,
+      index: indices,
+      body: {
+        // only return 1 match at most
+        size: 1,
+        query: {
+          bool: {
+            filter: [
               {
-                _source: {
-                  process?: {
-                    entity_id?: string;
-                  };
-                };
-              }
-            ];
-      };
-    }
-
-    const queryResponse: ExpectedQueryResponse = await context.core.elasticsearch.legacy.client.callAsCurrentUser(
-      'search',
-      {
-        ignoreUnavailable: true,
-        index: indices,
-        body: {
-          // only return process.entity_id
-          _source: 'process.entity_id',
-          // only return 1 match at most
-          size: 1,
-          query: {
-            bool: {
-              filter: [
-                {
-                  // only return documents with the matching _id
-                  ids: {
-                    values: _id,
-                  },
+                // only return documents with the matching _id
+                ids: {
+                  values: _id,
                 },
-              ],
-            },
+              },
+            ],
           },
         },
-      }
-    );
+      },
+    });
 
     const responseBody: ResolverEntityIndex = [];
-    for (const hit of queryResponse.hits.hits) {
-      // check that the field is defined and that is not an empty string
-      if (hit._source.process?.entity_id) {
-        responseBody.push({
-          entity_id: hit._source.process.entity_id,
-        });
+    for (const hit of queryResponse.body.hits.hits) {
+      for (const supportedSchema of supportedSchemas) {
+        const fieldValue = getFieldAsString(hit._source, supportedSchema.constraint.field);
+        const id = getFieldAsString(hit._source, supportedSchema.schema.id);
+        // check that the constraint and id fields are defined and that the id field is not an empty string
+        if (
+          fieldValue?.toLowerCase() === supportedSchema.constraint.value.toLowerCase() &&
+          id !== undefined &&
+          id !== ''
+        ) {
+          responseBody.push({
+            name: supportedSchema.name,
+            schema: supportedSchema.schema,
+            id,
+          });
+        }
       }
     }
     return response.ok({ body: responseBody });
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts
index 405429cc24191..3baf3a8667529 100644
--- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts
+++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts
@@ -6,12 +6,12 @@
 import { SearchResponse } from 'elasticsearch';
 import { ApiResponse } from '@elastic/elasticsearch';
 import { IScopedClusterClient } from 'src/core/server';
-import { FieldsObject } from '../../../../../../common/endpoint/types';
+import { FieldsObject, ResolverSchema } from '../../../../../../common/endpoint/types';
 import { JsonObject, JsonValue } from '../../../../../../../../../src/plugins/kibana_utils/common';
-import { NodeID, Schema, Timerange, docValueFields } from '../utils/index';
+import { NodeID, Timerange, docValueFields } from '../utils/index';
 
 interface DescendantsParams {
-  schema: Schema;
+  schema: ResolverSchema;
   indexPatterns: string | string[];
   timerange: Timerange;
 }
@@ -20,7 +20,7 @@ interface DescendantsParams {
  * Builds a query for retrieving descendants of a node.
  */
 export class DescendantsQuery {
-  private readonly schema: Schema;
+  private readonly schema: ResolverSchema;
   private readonly indexPatterns: string | string[];
   private readonly timerange: Timerange;
   private readonly docValueFields: JsonValue[];
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts
index 606a4538ba88c..5253806be66ba 100644
--- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts
+++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts
@@ -6,12 +6,12 @@
 import { SearchResponse } from 'elasticsearch';
 import { ApiResponse } from '@elastic/elasticsearch';
 import { IScopedClusterClient } from 'src/core/server';
-import { FieldsObject } from '../../../../../../common/endpoint/types';
+import { FieldsObject, ResolverSchema } from '../../../../../../common/endpoint/types';
 import { JsonObject, JsonValue } from '../../../../../../../../../src/plugins/kibana_utils/common';
-import { NodeID, Schema, Timerange, docValueFields } from '../utils/index';
+import { NodeID, Timerange, docValueFields } from '../utils/index';
 
 interface LifecycleParams {
-  schema: Schema;
+  schema: ResolverSchema;
   indexPatterns: string | string[];
   timerange: Timerange;
 }
@@ -20,7 +20,7 @@ interface LifecycleParams {
  * Builds a query for retrieving descendants of a node.
  */
 export class LifecycleQuery {
-  private readonly schema: Schema;
+  private readonly schema: ResolverSchema;
   private readonly indexPatterns: string | string[];
   private readonly timerange: Timerange;
   private readonly docValueFields: JsonValue[];
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts
index 33dcdce8987f5..117cc3647dd0e 100644
--- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts
+++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts
@@ -7,8 +7,8 @@ import { SearchResponse } from 'elasticsearch';
 import { ApiResponse } from '@elastic/elasticsearch';
 import { IScopedClusterClient } from 'src/core/server';
 import { JsonObject } from '../../../../../../../../../src/plugins/kibana_utils/common';
-import { EventStats } from '../../../../../../common/endpoint/types';
-import { NodeID, Schema, Timerange } from '../utils/index';
+import { EventStats, ResolverSchema } from '../../../../../../common/endpoint/types';
+import { NodeID, Timerange } from '../utils/index';
 
 interface AggBucket {
   key: string;
@@ -26,7 +26,7 @@ interface CategoriesAgg extends AggBucket {
 }
 
 interface StatsParams {
-  schema: Schema;
+  schema: ResolverSchema;
   indexPatterns: string | string[];
   timerange: Timerange;
 }
@@ -35,7 +35,7 @@ interface StatsParams {
  * Builds a query for retrieving descendants of a node.
  */
 export class StatsQuery {
-  private readonly schema: Schema;
+  private readonly schema: ResolverSchema;
   private readonly indexPatterns: string | string[];
   private readonly timerange: Timerange;
   constructor({ schema, indexPatterns, timerange }: StatsParams) {
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts
index 8105f1125d01d..d5e0af9dea239 100644
--- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts
+++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts
@@ -18,14 +18,17 @@ import { DescendantsQuery } from '../queries/descendants';
 import { StatsQuery } from '../queries/stats';
 import { IScopedClusterClient } from 'src/core/server';
 import { elasticsearchServiceMock } from 'src/core/server/mocks';
-import { FieldsObject, ResolverNode } from '../../../../../../common/endpoint/types';
-import { Schema } from './index';
+import {
+  FieldsObject,
+  ResolverNode,
+  ResolverSchema,
+} from '../../../../../../common/endpoint/types';
 
 jest.mock('../queries/descendants');
 jest.mock('../queries/lifecycle');
 jest.mock('../queries/stats');
 
-function formatResponse(results: FieldsObject[], schema: Schema): ResolverNode[] {
+function formatResponse(results: FieldsObject[], schema: ResolverSchema): ResolverNode[] {
   return results.map((node) => {
     return {
       id: getIDField(node, schema) ?? '',
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts
index eaecad6c47970..356357082d6ee 100644
--- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts
+++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts
@@ -8,9 +8,14 @@ import {
   firstNonNullValue,
   values,
 } from '../../../../../../common/endpoint/models/ecs_safety_helpers';
-import { ECSField, ResolverNode, FieldsObject } from '../../../../../../common/endpoint/types';
+import {
+  ECSField,
+  ResolverNode,
+  FieldsObject,
+  ResolverSchema,
+} from '../../../../../../common/endpoint/types';
 import { DescendantsQuery } from '../queries/descendants';
-import { Schema, NodeID } from './index';
+import { NodeID } from './index';
 import { LifecycleQuery } from '../queries/lifecycle';
 import { StatsQuery } from '../queries/stats';
 
@@ -26,7 +31,7 @@ export interface TreeOptions {
     from: string;
     to: string;
   };
-  schema: Schema;
+  schema: ResolverSchema;
   nodes: NodeID[];
   indexPatterns: string[];
 }
@@ -98,7 +103,7 @@ export class Fetcher {
 
   private static getNextAncestorsToFind(
     results: FieldsObject[],
-    schema: Schema,
+    schema: ResolverSchema,
     levelsLeft: number
   ): NodeID[] {
     const nodesByID = results.reduce((accMap: Map<NodeID, FieldsObject>, result: FieldsObject) => {
@@ -216,7 +221,7 @@ export class Fetcher {
 export function getLeafNodes(
   results: FieldsObject[],
   nodes: Array<string | number>,
-  schema: Schema
+  schema: ResolverSchema
 ): NodeID[] {
   let largestAncestryArray = 0;
   const nodesToQueryNext: Map<number, Set<NodeID>> = new Map();
@@ -269,7 +274,7 @@ export function getLeafNodes(
  * @param obj the doc value fields retrieved from a document returned by Elasticsearch
  * @param schema the schema used for identifying connections between documents
  */
-export function getIDField(obj: FieldsObject, schema: Schema): NodeID | undefined {
+export function getIDField(obj: FieldsObject, schema: ResolverSchema): NodeID | undefined {
   const id: ECSField<NodeID> = obj[schema.id];
   return firstNonNullValue(id);
 }
@@ -281,7 +286,7 @@ export function getIDField(obj: FieldsObject, schema: Schema): NodeID | undefine
  * @param obj the doc value fields retrieved from a document returned by Elasticsearch
  * @param schema the schema used for identifying connections between documents
  */
-export function getNameField(obj: FieldsObject, schema: Schema): string | undefined {
+export function getNameField(obj: FieldsObject, schema: ResolverSchema): string | undefined {
   if (!schema.name) {
     return undefined;
   }
@@ -297,12 +302,12 @@ export function getNameField(obj: FieldsObject, schema: Schema): string | undefi
  * @param obj the doc value fields retrieved from a document returned by Elasticsearch
  * @param schema the schema used for identifying connections between documents
  */
-export function getParentField(obj: FieldsObject, schema: Schema): NodeID | undefined {
+export function getParentField(obj: FieldsObject, schema: ResolverSchema): NodeID | undefined {
   const parent: ECSField<NodeID> = obj[schema.parent];
   return firstNonNullValue(parent);
 }
 
-function getAncestryField(obj: FieldsObject, schema: Schema): NodeID[] | undefined {
+function getAncestryField(obj: FieldsObject, schema: ResolverSchema): NodeID[] | undefined {
   if (!schema.ancestry) {
     return undefined;
   }
@@ -324,7 +329,7 @@ function getAncestryField(obj: FieldsObject, schema: Schema): NodeID[] | undefin
  * @param obj the doc value fields retrieved from a document returned by Elasticsearch
  * @param schema the schema used for identifying connections between documents
  */
-export function getAncestryAsArray(obj: FieldsObject, schema: Schema): NodeID[] {
+export function getAncestryAsArray(obj: FieldsObject, schema: ResolverSchema): NodeID[] {
   const ancestry = getAncestryField(obj, schema);
   if (!ancestry || ancestry.length <= 0) {
     const parentField = getParentField(obj, schema);
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts
index 21a49e268310b..be08b4390a69c 100644
--- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts
+++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts
@@ -4,6 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { ResolverSchema } from '../../../../../../common/endpoint/types';
+
 /**
  * Represents a time range filter
  */
@@ -17,29 +19,6 @@ export interface Timerange {
  */
 export type NodeID = string | number;
 
-/**
- * The fields to use to identify nodes within a resolver tree.
- */
-export interface Schema {
-  /**
-   * the ancestry field should be set to a field that contains an order array representing
-   * the ancestors of a node.
-   */
-  ancestry?: string;
-  /**
-   * id represents the field to use as the unique ID for a node.
-   */
-  id: string;
-  /**
-   * field to use for the name of the node
-   */
-  name?: string;
-  /**
-   * parent represents the field that is the edge between two nodes.
-   */
-  parent: string;
-}
-
 /**
  * Returns the doc value fields filter to use in queries to limit the number of fields returned in the
  * query response.
@@ -49,7 +28,7 @@ export interface Schema {
  * @param schema is the node schema information describing how relationships are formed between nodes
  *  in the resolver graph.
  */
-export function docValueFields(schema: Schema): Array<{ field: string }> {
+export function docValueFields(schema: ResolverSchema): Array<{ field: string }> {
   const filter = [{ field: '@timestamp' }, { field: schema.id }, { field: schema.parent }];
   if (schema.ancestry) {
     filter.push({ field: schema.ancestry });
diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts
index b4e98d7d4b95e..3cc833c6a2475 100644
--- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts
+++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts
@@ -6,16 +6,14 @@
 import _ from 'lodash';
 import expect from '@kbn/expect';
 import { firstNonNullValue } from '../../../../plugins/security_solution/common/endpoint/models/ecs_safety_helpers';
-import {
-  NodeID,
-  Schema,
-} from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils';
+import { NodeID } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils';
 import {
   SafeResolverChildNode,
   SafeResolverLifecycleNode,
   SafeResolverEvent,
   ResolverNodeStats,
   ResolverNode,
+  ResolverSchema,
 } from '../../../../plugins/security_solution/common/endpoint/types';
 import {
   parentEntityIDSafeVersion,
@@ -41,7 +39,7 @@ const createLevels = ({
   descendantsByParent: Map<NodeID, Map<NodeID, ResolverNode>>;
   levels: Array<Map<NodeID, ResolverNode>>;
   currentNodes: Map<NodeID, ResolverNode> | undefined;
-  schema: Schema;
+  schema: ResolverSchema;
 }): Array<Map<NodeID, ResolverNode>> => {
   if (!currentNodes || currentNodes.size === 0) {
     return levels;
@@ -98,7 +96,7 @@ export interface APIResponse {
  * @param node a resolver node
  * @param schema the schema that was used to retrieve this resolver node
  */
-export const getID = (node: ResolverNode | undefined, schema: Schema): NodeID => {
+export const getID = (node: ResolverNode | undefined, schema: ResolverSchema): NodeID => {
   const id = firstNonNullValue(node?.data[schema.id]);
   if (!id) {
     throw new Error(`Unable to find id ${schema.id} in node: ${JSON.stringify(node)}`);
@@ -106,7 +104,10 @@ export const getID = (node: ResolverNode | undefined, schema: Schema): NodeID =>
   return id;
 };
 
-const getParentInternal = (node: ResolverNode | undefined, schema: Schema): NodeID | undefined => {
+const getParentInternal = (
+  node: ResolverNode | undefined,
+  schema: ResolverSchema
+): NodeID | undefined => {
   if (node) {
     return firstNonNullValue(node?.data[schema.parent]);
   }
@@ -119,7 +120,7 @@ const getParentInternal = (node: ResolverNode | undefined, schema: Schema): Node
  * @param node a resolver node
  * @param schema the schema that was used to retrieve this resolver node
  */
-export const getParent = (node: ResolverNode | undefined, schema: Schema): NodeID => {
+export const getParent = (node: ResolverNode | undefined, schema: ResolverSchema): NodeID => {
   const parent = getParentInternal(node, schema);
   if (!parent) {
     throw new Error(`Unable to find parent ${schema.parent} in node: ${JSON.stringify(node)}`);
@@ -138,7 +139,7 @@ export const getParent = (node: ResolverNode | undefined, schema: Schema): NodeI
 const createTreeFromResponse = (
   treeExpectations: TreeExpectation[],
   nodes: ResolverNode[],
-  schema: Schema
+  schema: ResolverSchema
 ) => {
   const nodesByID = new Map<NodeID, ResolverNode>();
   const nodesByParent = new Map<NodeID, Map<NodeID, ResolverNode>>();
@@ -206,7 +207,7 @@ const verifyAncestry = ({
   genTree,
 }: {
   responseTrees: APIResponse;
-  schema: Schema;
+  schema: ResolverSchema;
   genTree: Tree;
 }) => {
   const allGenNodes = new Map<string, TreeNode>([
@@ -277,7 +278,7 @@ const verifyChildren = ({
   genTree,
 }: {
   responseTrees: APIResponse;
-  schema: Schema;
+  schema: ResolverSchema;
   genTree: Tree;
 }) => {
   const allGenNodes = new Map<string, TreeNode>([
@@ -358,7 +359,7 @@ export const verifyTree = ({
 }: {
   expectations: TreeExpectation[];
   response: ResolverNode[];
-  schema: Schema;
+  schema: ResolverSchema;
   genTree: Tree;
   relatedEventsCategories?: RelatedEventInfo[];
 }) => {
diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts
index 7fbba4e04798d..2607b934e7df2 100644
--- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts
+++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts
@@ -29,8 +29,15 @@ export default function ({ getService }: FtrProviderContext) {
       );
       expect(body).eql([
         {
+          name: 'endpoint',
+          schema: {
+            id: 'process.entity_id',
+            parent: 'process.parent.entity_id',
+            ancestry: 'process.Ext.ancestry',
+            name: 'process.name',
+          },
           // this value is from the es archive
-          entity_id:
+          id:
             'MTIwNWY1NWQtODRkYS00MzkxLWIyNWQtYTNkNGJmNDBmY2E1LTc1NTItMTMyNDM1NDY1MTQuNjI0MjgxMDA=',
         },
       ]);
diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts
index 646a666629ac9..7a1210c6b762f 100644
--- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts
+++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts
@@ -5,8 +5,10 @@
  */
 import expect from '@kbn/expect';
 import { getNameField } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch';
-import { Schema } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils';
-import { ResolverNode } from '../../../../plugins/security_solution/common/endpoint/types';
+import {
+  ResolverNode,
+  ResolverSchema,
+} from '../../../../plugins/security_solution/common/endpoint/types';
 import {
   parentEntityIDSafeVersion,
   timestampSafeVersion,
@@ -44,18 +46,18 @@ export default function ({ getService }: FtrProviderContext) {
     ancestryArraySize: 2,
   };
 
-  const schemaWithAncestry: Schema = {
+  const schemaWithAncestry: ResolverSchema = {
     ancestry: 'process.Ext.ancestry',
     id: 'process.entity_id',
     parent: 'process.parent.entity_id',
   };
 
-  const schemaWithoutAncestry: Schema = {
+  const schemaWithoutAncestry: ResolverSchema = {
     id: 'process.entity_id',
     parent: 'process.parent.entity_id',
   };
 
-  const schemaWithName: Schema = {
+  const schemaWithName: ResolverSchema = {
     id: 'process.entity_id',
     parent: 'process.parent.entity_id',
     name: 'process.name',

From dfa9c75021f1872b31f770c4009cc2e0d556dbfe Mon Sep 17 00:00:00 2001
From: Chris Roberson <chrisronline@gmail.com>
Date: Wed, 25 Nov 2020 15:15:46 -0500
Subject: [PATCH 19/37] Fix issues with show_license_expiration (#84361)

---
 .../cluster/overview/elasticsearch_panel.js   | 71 +++++++++++--------
 .../alerts/license_expiration_alert.test.ts   | 28 ++++++++
 .../server/alerts/license_expiration_alert.ts |  9 ++-
 3 files changed, 77 insertions(+), 31 deletions(-)

diff --git a/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js b/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js
index 7e85d62c4bbd6..ded309ce64e2e 100644
--- a/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js
+++ b/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js
@@ -230,6 +230,46 @@ export function ElasticsearchPanel(props) {
     return null;
   };
 
+  const showLicense = () => {
+    if (!props.showLicenseExpiration) {
+      return null;
+    }
+    return (
+      <Fragment>
+        <EuiDescriptionListTitle className="eui-textBreakWord">
+          <FormattedMessage
+            id="xpack.monitoring.cluster.overview.esPanel.licenseLabel"
+            defaultMessage="License"
+          />
+        </EuiDescriptionListTitle>
+        <EuiDescriptionListDescription data-test-subj="esLicenseType">
+          <EuiFlexGroup direction="column" gutterSize="xs">
+            <EuiFlexItem grow={false}>
+              <EuiLink href={getSafeForExternalLink('#/license')}>
+                {capitalize(props.license.type)}
+              </EuiLink>
+            </EuiFlexItem>
+            <EuiFlexItem grow={false}>
+              <EuiText size="s">
+                {props.license.expiry_date_in_millis === undefined ? (
+                  ''
+                ) : (
+                  <FormattedMessage
+                    id="xpack.monitoring.cluster.overview.esPanel.expireDateText"
+                    defaultMessage="expires on {expiryDate}"
+                    values={{
+                      expiryDate: formatDateLocal(props.license.expiry_date_in_millis),
+                    }}
+                  />
+                )}
+              </EuiText>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiDescriptionListDescription>
+      </Fragment>
+    );
+  };
+
   const statusColorMap = {
     green: 'success',
     yellow: 'warning',
@@ -325,36 +365,7 @@ export function ElasticsearchPanel(props) {
                 {formatNumber(get(nodes, 'jvm.max_uptime_in_millis'), 'time_since')}
               </EuiDescriptionListDescription>
               {showMlJobs()}
-              <EuiDescriptionListTitle className="eui-textBreakWord">
-                <FormattedMessage
-                  id="xpack.monitoring.cluster.overview.esPanel.licenseLabel"
-                  defaultMessage="License"
-                />
-              </EuiDescriptionListTitle>
-              <EuiDescriptionListDescription data-test-subj="esLicenseType">
-                <EuiFlexGroup direction="column" gutterSize="xs">
-                  <EuiFlexItem grow={false}>
-                    <EuiLink href={getSafeForExternalLink('#/license')}>
-                      {capitalize(props.license.type)}
-                    </EuiLink>
-                  </EuiFlexItem>
-                  <EuiFlexItem grow={false}>
-                    <EuiText size="s">
-                      {props.license.expiry_date_in_millis === undefined ? (
-                        ''
-                      ) : (
-                        <FormattedMessage
-                          id="xpack.monitoring.cluster.overview.esPanel.expireDateText"
-                          defaultMessage="expires on {expiryDate}"
-                          values={{
-                            expiryDate: formatDateLocal(props.license.expiry_date_in_millis),
-                          }}
-                        />
-                      )}
-                    </EuiText>
-                  </EuiFlexItem>
-                </EuiFlexGroup>
-              </EuiDescriptionListDescription>
+              {showLicense()}
             </EuiDescriptionList>
           </EuiPanel>
         </EuiFlexItem>
diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts
index 74c300d971898..b82b4c235acba 100644
--- a/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts
+++ b/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts
@@ -76,6 +76,7 @@ describe('LicenseExpirationAlert', () => {
     const monitoringCluster = null;
     const config = {
       ui: {
+        show_license_expiration: true,
         ccs: { enabled: true },
         container: { elasticsearch: { enabled: false } },
         metricbeat: { index: 'metricbeat-*' },
@@ -282,5 +283,32 @@ describe('LicenseExpirationAlert', () => {
         state: 'resolved',
       });
     });
+
+    it('should not fire actions if we are not showing license expiration', async () => {
+      const alert = new LicenseExpirationAlert();
+      const customConfig = {
+        ...config,
+        ui: {
+          ...config.ui,
+          show_license_expiration: false,
+        },
+      };
+      alert.initializeAlertType(
+        getUiSettingsService as any,
+        monitoringCluster as any,
+        getLogger as any,
+        customConfig as any,
+        kibanaUrl,
+        false
+      );
+      const type = alert.getAlertType();
+      await type.executor({
+        ...executorOptions,
+        // @ts-ignore
+        params: alert.defaultParams,
+      } as any);
+      expect(replaceState).not.toHaveBeenCalledWith({});
+      expect(scheduleActions).not.toHaveBeenCalled();
+    });
   });
 });
diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.ts
index 00846e9cf7599..9692d95bfc6fe 100644
--- a/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.ts
+++ b/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.ts
@@ -18,7 +18,7 @@ import {
   LegacyAlert,
   CommonAlertParams,
 } from '../../common/types/alerts';
-import { AlertInstance } from '../../../alerts/server';
+import { AlertExecutorOptions, AlertInstance } from '../../../alerts/server';
 import {
   INDEX_ALERTS,
   ALERT_LICENSE_EXPIRATION,
@@ -64,6 +64,13 @@ export class LicenseExpirationAlert extends BaseAlert {
     AlertingDefaults.ALERT_TYPE.context.actionPlain,
   ];
 
+  protected async execute(options: AlertExecutorOptions): Promise<any> {
+    if (!this.config.ui.show_license_expiration) {
+      return;
+    }
+    return await super.execute(options);
+  }
+
   protected async fetchData(
     params: CommonAlertParams,
     callCluster: any,

From 71f77862e7a6fa00c248c854a7814a2331d22841 Mon Sep 17 00:00:00 2001
From: Dan Panzarella <pzl@users.noreply.github.com>
Date: Wed, 25 Nov 2020 15:29:23 -0500
Subject: [PATCH 20/37] [Security Solution] Add Endpoint policy feature checks
 (#83972)

---
 .../common/endpoint/models/policy_config.ts   |   5 +
 .../common/license/license.ts                 |  33 +++---
 .../common/license/policy_config.test.ts      | 110 ++++++++++++++++++
 .../common/license/policy_config.ts           |  66 +++++++++++
 .../policy/store/policy_details/middleware.ts |   7 +-
 5 files changed, 198 insertions(+), 23 deletions(-)
 create mode 100644 x-pack/plugins/security_solution/common/license/policy_config.test.ts
 create mode 100644 x-pack/plugins/security_solution/common/license/policy_config.ts

diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts
index 890def5b63d4a..22037c021701f 100644
--- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts
+++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts
@@ -68,3 +68,8 @@ export const factory = (): PolicyConfig => {
     },
   };
 };
+
+/**
+ * Reflects what string the Endpoint will use when message field is default/empty
+ */
+export const DefaultMalwareMessage = 'Elastic Security { action } { filename }';
diff --git a/x-pack/plugins/security_solution/common/license/license.ts b/x-pack/plugins/security_solution/common/license/license.ts
index 96c1a14ceb1f4..2d424ab9c960a 100644
--- a/x-pack/plugins/security_solution/common/license/license.ts
+++ b/x-pack/plugins/security_solution/common/license/license.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { Observable, Subscription } from 'rxjs';
-import { ILicense } from '../../../licensing/common/types';
+import { ILicense, LicenseType } from '../../../licensing/common/types';
 
 // Generic license service class that works with the license observable
 // Both server and client plugins instancates a singleton version of this class
@@ -36,25 +36,20 @@ export class LicenseService {
     return this.observable;
   }
 
-  public isGoldPlus() {
-    return (
-      this.licenseInformation?.isAvailable &&
-      this.licenseInformation?.isActive &&
-      this.licenseInformation?.hasAtLeast('gold')
-    );
+  public isAtLeast(level: LicenseType): boolean {
+    return isAtLeast(this.licenseInformation, level);
   }
-  public isPlatinumPlus() {
-    return (
-      this.licenseInformation?.isAvailable &&
-      this.licenseInformation?.isActive &&
-      this.licenseInformation?.hasAtLeast('platinum')
-    );
+  public isGoldPlus(): boolean {
+    return this.isAtLeast('gold');
   }
-  public isEnterprise() {
-    return (
-      this.licenseInformation?.isAvailable &&
-      this.licenseInformation?.isActive &&
-      this.licenseInformation?.hasAtLeast('enterprise')
-    );
+  public isPlatinumPlus(): boolean {
+    return this.isAtLeast('platinum');
+  }
+  public isEnterprise(): boolean {
+    return this.isAtLeast('enterprise');
   }
 }
+
+export const isAtLeast = (license: ILicense | null, level: LicenseType): boolean => {
+  return license !== null && license.isAvailable && license.isActive && license.hasAtLeast(level);
+};
diff --git a/x-pack/plugins/security_solution/common/license/policy_config.test.ts b/x-pack/plugins/security_solution/common/license/policy_config.test.ts
new file mode 100644
index 0000000000000..6923bf00055f6
--- /dev/null
+++ b/x-pack/plugins/security_solution/common/license/policy_config.test.ts
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  isEndpointPolicyValidForLicense,
+  unsetPolicyFeaturesAboveLicenseLevel,
+} from './policy_config';
+import { DefaultMalwareMessage, factory } from '../endpoint/models/policy_config';
+import { licenseMock } from '../../../licensing/common/licensing.mock';
+
+describe('policy_config and licenses', () => {
+  const Platinum = licenseMock.createLicense({ license: { type: 'platinum', mode: 'platinum' } });
+  const Gold = licenseMock.createLicense({ license: { type: 'gold', mode: 'gold' } });
+  const Basic = licenseMock.createLicense({ license: { type: 'basic', mode: 'basic' } });
+
+  describe('isEndpointPolicyValidForLicense', () => {
+    it('allows malware notification to be disabled with a Platinum license', () => {
+      const policy = factory();
+      policy.windows.popup.malware.enabled = false; // make policy change
+      const valid = isEndpointPolicyValidForLicense(policy, Platinum);
+      expect(valid).toBeTruthy();
+    });
+    it('blocks windows malware notification changes below Platinum licenses', () => {
+      const policy = factory();
+      policy.windows.popup.malware.enabled = false; // make policy change
+      let valid = isEndpointPolicyValidForLicense(policy, Gold);
+      expect(valid).toBeFalsy();
+
+      valid = isEndpointPolicyValidForLicense(policy, Basic);
+      expect(valid).toBeFalsy();
+    });
+
+    it('blocks mac malware notification changes below Platinum licenses', () => {
+      const policy = factory();
+      policy.mac.popup.malware.enabled = false; // make policy change
+      let valid = isEndpointPolicyValidForLicense(policy, Gold);
+      expect(valid).toBeFalsy();
+
+      valid = isEndpointPolicyValidForLicense(policy, Basic);
+      expect(valid).toBeFalsy();
+    });
+
+    it('allows malware notification message changes with a Platinum license', () => {
+      const policy = factory();
+      policy.windows.popup.malware.message = 'BOOM'; // make policy change
+      const valid = isEndpointPolicyValidForLicense(policy, Platinum);
+      expect(valid).toBeTruthy();
+    });
+    it('blocks windows malware notification message changes below Platinum licenses', () => {
+      const policy = factory();
+      policy.windows.popup.malware.message = 'BOOM'; // make policy change
+      let valid = isEndpointPolicyValidForLicense(policy, Gold);
+      expect(valid).toBeFalsy();
+
+      valid = isEndpointPolicyValidForLicense(policy, Basic);
+      expect(valid).toBeFalsy();
+    });
+    it('blocks mac malware notification message changes below Platinum licenses', () => {
+      const policy = factory();
+      policy.mac.popup.malware.message = 'BOOM'; // make policy change
+      let valid = isEndpointPolicyValidForLicense(policy, Gold);
+      expect(valid).toBeFalsy();
+
+      valid = isEndpointPolicyValidForLicense(policy, Basic);
+      expect(valid).toBeFalsy();
+    });
+
+    it('allows default policyConfig with Basic', () => {
+      const policy = factory();
+      const valid = isEndpointPolicyValidForLicense(policy, Basic);
+      expect(valid).toBeTruthy();
+    });
+  });
+
+  describe('unsetPolicyFeaturesAboveLicenseLevel', () => {
+    it('does not change any fields with a Platinum license', () => {
+      const policy = factory();
+      const popupMessage = 'WOOP WOOP';
+      policy.windows.popup.malware.message = popupMessage;
+      policy.mac.popup.malware.message = popupMessage;
+      policy.windows.popup.malware.enabled = false;
+
+      const retPolicy = unsetPolicyFeaturesAboveLicenseLevel(policy, Platinum);
+      expect(retPolicy.windows.popup.malware.enabled).toBeFalsy();
+      expect(retPolicy.windows.popup.malware.message).toEqual(popupMessage);
+      expect(retPolicy.mac.popup.malware.message).toEqual(popupMessage);
+    });
+    it('resets Platinum-paid fields for lower license tiers', () => {
+      const defaults = factory(); // reference
+      const policy = factory(); // what we will modify, and should be reset
+      const popupMessage = 'WOOP WOOP';
+      policy.windows.popup.malware.message = popupMessage;
+      policy.mac.popup.malware.message = popupMessage;
+      policy.windows.popup.malware.enabled = false;
+
+      const retPolicy = unsetPolicyFeaturesAboveLicenseLevel(policy, Gold);
+      expect(retPolicy.windows.popup.malware.enabled).toEqual(
+        defaults.windows.popup.malware.enabled
+      );
+      expect(retPolicy.windows.popup.malware.message).not.toEqual(popupMessage);
+      expect(retPolicy.mac.popup.malware.message).not.toEqual(popupMessage);
+
+      // need to invert the test, since it could be either value
+      expect(['', DefaultMalwareMessage]).toContain(retPolicy.windows.popup.malware.message);
+    });
+  });
+});
diff --git a/x-pack/plugins/security_solution/common/license/policy_config.ts b/x-pack/plugins/security_solution/common/license/policy_config.ts
new file mode 100644
index 0000000000000..da2260ad55e8b
--- /dev/null
+++ b/x-pack/plugins/security_solution/common/license/policy_config.ts
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ILicense } from '../../../licensing/common/types';
+import { isAtLeast } from './license';
+import { PolicyConfig } from '../endpoint/types';
+import { DefaultMalwareMessage, factory } from '../endpoint/models/policy_config';
+
+/**
+ * Given an endpoint package policy, verifies that all enabled features that
+ * require a certain license level have a valid license for them.
+ */
+export const isEndpointPolicyValidForLicense = (
+  policy: PolicyConfig,
+  license: ILicense | null
+): boolean => {
+  if (isAtLeast(license, 'platinum')) {
+    return true; // currently, platinum allows all features
+  }
+
+  const defaults = factory();
+
+  // only platinum or higher may disable malware notification
+  if (
+    policy.windows.popup.malware.enabled !== defaults.windows.popup.malware.enabled ||
+    policy.mac.popup.malware.enabled !== defaults.mac.popup.malware.enabled
+  ) {
+    return false;
+  }
+
+  // Only Platinum or higher may change the malware message (which can be blank or what Endpoint defaults)
+  if (
+    [policy.windows, policy.mac].some(
+      (p) => p.popup.malware.message !== '' && p.popup.malware.message !== DefaultMalwareMessage
+    )
+  ) {
+    return false;
+  }
+
+  return true;
+};
+
+/**
+ * Resets paid features in a PolicyConfig back to default values
+ * when unsupported by the given license level.
+ */
+export const unsetPolicyFeaturesAboveLicenseLevel = (
+  policy: PolicyConfig,
+  license: ILicense | null
+): PolicyConfig => {
+  if (isAtLeast(license, 'platinum')) {
+    return policy;
+  }
+
+  const defaults = factory();
+  // set any license-gated features back to the defaults
+  policy.windows.popup.malware.enabled = defaults.windows.popup.malware.enabled;
+  policy.mac.popup.malware.enabled = defaults.mac.popup.malware.enabled;
+  policy.windows.popup.malware.message = defaults.windows.popup.malware.message;
+  policy.mac.popup.malware.message = defaults.mac.popup.malware.message;
+
+  return policy;
+};
diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts
index 36649d22f730c..f039324b3af64 100644
--- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts
+++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts
@@ -5,6 +5,7 @@
  */
 
 import { IHttpFetchError } from 'kibana/public';
+import { DefaultMalwareMessage } from '../../../../../../common/endpoint/models/policy_config';
 import { PolicyDetailsState, UpdatePolicyResponse } from '../../types';
 import {
   policyIdFromParams,
@@ -38,10 +39,8 @@ export const policyDetailsMiddlewareFactory: ImmutableMiddlewareFactory<PolicyDe
         policyItem = (await sendGetPackagePolicy(http, id)).item;
         // sets default user notification message if policy config message is empty
         if (policyItem.inputs[0].config.policy.value.windows.popup.malware.message === '') {
-          policyItem.inputs[0].config.policy.value.windows.popup.malware.message =
-            'Elastic Security { action } { filename }';
-          policyItem.inputs[0].config.policy.value.mac.popup.malware.message =
-            'Elastic Security { action } { filename }';
+          policyItem.inputs[0].config.policy.value.windows.popup.malware.message = DefaultMalwareMessage;
+          policyItem.inputs[0].config.policy.value.mac.popup.malware.message = DefaultMalwareMessage;
         }
       } catch (error) {
         dispatch({

From 5c57f3db362a52ed2891f68535d026a3152166da Mon Sep 17 00:00:00 2001
From: Spencer <email@spalger.com>
Date: Wed, 25 Nov 2020 13:34:59 -0700
Subject: [PATCH 21/37] [basePathProxy] include query in redirect (#84356)

Co-authored-by: spalger <spalger@users.noreply.github.com>
---
 src/core/server/http/base_path_proxy_server.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/core/server/http/base_path_proxy_server.ts b/src/core/server/http/base_path_proxy_server.ts
index 42841377e7369..737aab00cff0e 100644
--- a/src/core/server/http/base_path_proxy_server.ts
+++ b/src/core/server/http/base_path_proxy_server.ts
@@ -199,8 +199,13 @@ export class BasePathProxyServer {
         const isGet = request.method === 'get';
         const isBasepathLike = oldBasePath.length === 3;
 
+        const newUrl = Url.format({
+          pathname: `${this.httpConfig.basePath}/${kbnPath}`,
+          query: request.query,
+        });
+
         return isGet && isBasepathLike && shouldRedirectFromOldBasePath(kbnPath)
-          ? responseToolkit.redirect(`${this.httpConfig.basePath}/${kbnPath}`)
+          ? responseToolkit.redirect(newUrl)
           : responseToolkit.response('Not Found').code(404);
       },
       method: '*',

From 459263fd596b91f35c71f025ad9990dde794a9b5 Mon Sep 17 00:00:00 2001
From: Jen Huang <its.jenetic@gmail.com>
Date: Wed, 25 Nov 2020 13:29:57 -0800
Subject: [PATCH 22/37] [Fleet] Support URL query state in agent logs UI
 (#84298)

* Initial attempt at URL state

* Break into smaller files

* Handle invalid date range expressions
---
 .../components/agent_logs/agent_logs.tsx      | 278 ++++++++++++++++++
 .../components/agent_logs/constants.tsx       |   9 +
 .../components/agent_logs/index.tsx           | 277 ++++-------------
 3 files changed, 339 insertions(+), 225 deletions(-)
 create mode 100644 x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx

diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx
new file mode 100644
index 0000000000000..00deeff89503f
--- /dev/null
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx
@@ -0,0 +1,278 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React, { memo, useMemo, useState, useCallback, useEffect } from 'react';
+import styled from 'styled-components';
+import url from 'url';
+import { encode } from 'rison-node';
+import { stringify } from 'query-string';
+import {
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiSuperDatePicker,
+  EuiFilterGroup,
+  EuiPanel,
+  EuiButtonEmpty,
+} from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+import semverGte from 'semver/functions/gte';
+import semverCoerce from 'semver/functions/coerce';
+import { createStateContainerReactHelpers } from '../../../../../../../../../../../src/plugins/kibana_utils/public';
+import { RedirectAppLinks } from '../../../../../../../../../../../src/plugins/kibana_react/public';
+import { TimeRange, esKuery } from '../../../../../../../../../../../src/plugins/data/public';
+import { LogStream } from '../../../../../../../../../infra/public';
+import { Agent } from '../../../../../types';
+import { useStartServices } from '../../../../../hooks';
+import { DEFAULT_DATE_RANGE } from './constants';
+import { DatasetFilter } from './filter_dataset';
+import { LogLevelFilter } from './filter_log_level';
+import { LogQueryBar } from './query_bar';
+import { buildQuery } from './build_query';
+import { SelectLogLevel } from './select_log_level';
+
+const WrapperFlexGroup = styled(EuiFlexGroup)`
+  height: 100%;
+`;
+
+const DatePickerFlexItem = styled(EuiFlexItem)`
+  max-width: 312px;
+`;
+
+export interface AgentLogsProps {
+  agent: Agent;
+  state: AgentLogsState;
+}
+
+export interface AgentLogsState {
+  start: string;
+  end: string;
+  logLevels: string[];
+  datasets: string[];
+  query: string;
+}
+
+export const AgentLogsUrlStateHelper = createStateContainerReactHelpers();
+
+export const AgentLogsUI: React.FunctionComponent<AgentLogsProps> = memo(({ agent, state }) => {
+  const { data, application, http } = useStartServices();
+  const { update: updateState } = AgentLogsUrlStateHelper.useTransitions();
+
+  // Util to convert date expressions (returned by datepicker) to timestamps (used by LogStream)
+  const getDateRangeTimestamps = useCallback(
+    (timeRange: TimeRange) => {
+      const { min, max } = data.query.timefilter.timefilter.calculateBounds(timeRange);
+      return min && max
+        ? {
+            start: min.valueOf(),
+            end: max.valueOf(),
+          }
+        : undefined;
+    },
+    [data.query.timefilter.timefilter]
+  );
+
+  const tryUpdateDateRange = useCallback(
+    (timeRange: TimeRange) => {
+      const timestamps = getDateRangeTimestamps(timeRange);
+      if (timestamps) {
+        updateState({
+          start: timeRange.from,
+          end: timeRange.to,
+        });
+      }
+    },
+    [getDateRangeTimestamps, updateState]
+  );
+
+  const [dateRangeTimestamps, setDateRangeTimestamps] = useState<{ start: number; end: number }>(
+    getDateRangeTimestamps({
+      from: state.start,
+      to: state.end,
+    }) ||
+      getDateRangeTimestamps({
+        from: DEFAULT_DATE_RANGE.start,
+        to: DEFAULT_DATE_RANGE.end,
+      })!
+  );
+
+  // Attempts to parse for timestamps when start/end date expressions change
+  // If invalid date expressions, set expressions back to default
+  // Otherwise set the new timestamps
+  useEffect(() => {
+    const timestampsFromDateRange = getDateRangeTimestamps({
+      from: state.start,
+      to: state.end,
+    });
+    if (!timestampsFromDateRange) {
+      tryUpdateDateRange({
+        from: DEFAULT_DATE_RANGE.start,
+        to: DEFAULT_DATE_RANGE.end,
+      });
+    } else {
+      setDateRangeTimestamps(timestampsFromDateRange);
+    }
+  }, [state.start, state.end, getDateRangeTimestamps, tryUpdateDateRange]);
+
+  // Query validation helper
+  const isQueryValid = useCallback((testQuery: string) => {
+    try {
+      esKuery.fromKueryExpression(testQuery);
+      return true;
+    } catch (err) {
+      return false;
+    }
+  }, []);
+
+  // User query state
+  const [draftQuery, setDraftQuery] = useState<string>(state.query);
+  const [isDraftQueryValid, setIsDraftQueryValid] = useState<boolean>(isQueryValid(state.query));
+  const onUpdateDraftQuery = useCallback(
+    (newDraftQuery: string, runQuery?: boolean) => {
+      setDraftQuery(newDraftQuery);
+      if (isQueryValid(newDraftQuery)) {
+        setIsDraftQueryValid(true);
+        if (runQuery) {
+          updateState({ query: newDraftQuery });
+        }
+      } else {
+        setIsDraftQueryValid(false);
+      }
+    },
+    [isQueryValid, updateState]
+  );
+
+  // Build final log stream query from agent id, datasets, log levels, and user input
+  const logStreamQuery = useMemo(
+    () =>
+      buildQuery({
+        agentId: agent.id,
+        datasets: state.datasets,
+        logLevels: state.logLevels,
+        userQuery: state.query,
+      }),
+    [agent.id, state.datasets, state.logLevels, state.query]
+  );
+
+  // Generate URL to pass page state to Logs UI
+  const viewInLogsUrl = useMemo(
+    () =>
+      http.basePath.prepend(
+        url.format({
+          pathname: '/app/logs/stream',
+          search: stringify(
+            {
+              logPosition: encode({
+                start: state.start,
+                end: state.end,
+                streamLive: false,
+              }),
+              logFilter: encode({
+                expression: logStreamQuery,
+                kind: 'kuery',
+              }),
+            },
+            { sort: false, encode: false }
+          ),
+        })
+      ),
+    [http.basePath, state.start, state.end, logStreamQuery]
+  );
+
+  const agentVersion = agent.local_metadata?.elastic?.agent?.version;
+  const isLogLevelSelectionAvailable = useMemo(() => {
+    if (!agentVersion) {
+      return false;
+    }
+    const agentVersionWithPrerelease = semverCoerce(agentVersion)?.version;
+    if (!agentVersionWithPrerelease) {
+      return false;
+    }
+    return semverGte(agentVersionWithPrerelease, '7.11.0');
+  }, [agentVersion]);
+
+  return (
+    <WrapperFlexGroup direction="column" gutterSize="m">
+      <EuiFlexItem grow={false}>
+        <EuiFlexGroup gutterSize="m">
+          <EuiFlexItem>
+            <LogQueryBar
+              query={draftQuery}
+              onUpdateQuery={onUpdateDraftQuery}
+              isQueryValid={isDraftQueryValid}
+            />
+          </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            <EuiFilterGroup>
+              <DatasetFilter
+                selectedDatasets={state.datasets}
+                onToggleDataset={(dataset: string) => {
+                  const currentDatasets = [...state.datasets];
+                  const datasetPosition = currentDatasets.indexOf(dataset);
+                  if (datasetPosition >= 0) {
+                    currentDatasets.splice(datasetPosition, 1);
+                    updateState({ datasets: currentDatasets });
+                  } else {
+                    updateState({ datasets: [...state.datasets, dataset] });
+                  }
+                }}
+              />
+              <LogLevelFilter
+                selectedLevels={state.logLevels}
+                onToggleLevel={(level: string) => {
+                  const currentLevels = [...state.logLevels];
+                  const levelPosition = currentLevels.indexOf(level);
+                  if (levelPosition >= 0) {
+                    currentLevels.splice(levelPosition, 1);
+                    updateState({ logLevels: currentLevels });
+                  } else {
+                    updateState({ logLevels: [...state.logLevels, level] });
+                  }
+                }}
+              />
+            </EuiFilterGroup>
+          </EuiFlexItem>
+          <DatePickerFlexItem grow={false}>
+            <EuiSuperDatePicker
+              showUpdateButton={false}
+              start={state.start}
+              end={state.end}
+              onTimeChange={({ start, end }) => {
+                tryUpdateDateRange({
+                  from: start,
+                  to: end,
+                });
+              }}
+            />
+          </DatePickerFlexItem>
+          <EuiFlexItem grow={false}>
+            <RedirectAppLinks application={application}>
+              <EuiButtonEmpty href={viewInLogsUrl} iconType="popout" flush="both">
+                <FormattedMessage
+                  id="xpack.fleet.agentLogs.openInLogsUiLinkText"
+                  defaultMessage="Open in Logs"
+                />
+              </EuiButtonEmpty>
+            </RedirectAppLinks>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </EuiFlexItem>
+      <EuiFlexItem>
+        <EuiPanel paddingSize="none">
+          <LogStream
+            height="100%"
+            startTimestamp={dateRangeTimestamps.start}
+            endTimestamp={dateRangeTimestamps.end}
+            query={logStreamQuery}
+          />
+        </EuiPanel>
+      </EuiFlexItem>
+      {isLogLevelSelectionAvailable && (
+        <EuiFlexItem grow={false}>
+          <SelectLogLevel agent={agent} />
+        </EuiFlexItem>
+      )}
+    </WrapperFlexGroup>
+  );
+});
diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx
index ea98de3560246..89fe1a916605d 100644
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx
@@ -3,6 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { AgentLogsState } from './agent_logs';
+
 export const AGENT_LOG_INDEX_PATTERN = 'logs-elastic_agent-*,logs-elastic_agent.*-*';
 export const AGENT_DATASET = 'elastic_agent';
 export const AGENT_DATASET_PATTERN = 'elastic_agent.*';
@@ -24,6 +26,13 @@ export const DEFAULT_DATE_RANGE = {
   start: 'now-1d',
   end: 'now',
 };
+export const DEFAULT_LOGS_STATE: AgentLogsState = {
+  start: DEFAULT_DATE_RANGE.start,
+  end: DEFAULT_DATE_RANGE.end,
+  logLevels: [],
+  datasets: [AGENT_DATASET],
+  query: '',
+};
 
 export const AGENT_LOG_LEVELS = {
   ERROR: 'error',
diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/index.tsx
index bed857c073099..0d888a88ec2cb 100644
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/index.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/index.tsx
@@ -3,236 +3,63 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import React, { memo, useMemo, useState, useCallback } from 'react';
-import styled from 'styled-components';
-import url from 'url';
-import { encode } from 'rison-node';
-import { stringify } from 'query-string';
+import React, { memo, useEffect, useState } from 'react';
 import {
-  EuiFlexGroup,
-  EuiFlexItem,
-  EuiSuperDatePicker,
-  EuiFilterGroup,
-  EuiPanel,
-  EuiButtonEmpty,
-} from '@elastic/eui';
-import { FormattedMessage } from '@kbn/i18n/react';
-import semverGte from 'semver/functions/gte';
-import semverCoerce from 'semver/functions/coerce';
-import { RedirectAppLinks } from '../../../../../../../../../../../src/plugins/kibana_react/public';
-import { TimeRange, esKuery } from '../../../../../../../../../../../src/plugins/data/public';
-import { LogStream } from '../../../../../../../../../infra/public';
-import { Agent } from '../../../../../types';
-import { useStartServices } from '../../../../../hooks';
-import { AGENT_DATASET, DEFAULT_DATE_RANGE } from './constants';
-import { DatasetFilter } from './filter_dataset';
-import { LogLevelFilter } from './filter_log_level';
-import { LogQueryBar } from './query_bar';
-import { buildQuery } from './build_query';
-import { SelectLogLevel } from './select_log_level';
+  createStateContainer,
+  syncState,
+  createKbnUrlStateStorage,
+  INullableBaseStateContainer,
+  PureTransition,
+  getStateFromKbnUrl,
+} from '../../../../../../../../../../../src/plugins/kibana_utils/public';
+import { DEFAULT_LOGS_STATE } from './constants';
+import { AgentLogsUI, AgentLogsProps, AgentLogsState, AgentLogsUrlStateHelper } from './agent_logs';
 
-const WrapperFlexGroup = styled(EuiFlexGroup)`
-  height: 100%;
-`;
+const stateStorageKey = '_q';
 
-const DatePickerFlexItem = styled(EuiFlexItem)`
-  max-width: 312px;
-`;
+const stateContainer = createStateContainer<
+  AgentLogsState,
+  {
+    update: PureTransition<AgentLogsState, [Partial<AgentLogsState>]>;
+  }
+>(
+  {
+    ...DEFAULT_LOGS_STATE,
+    ...getStateFromKbnUrl<AgentLogsState>(stateStorageKey, window.location.href),
+  },
+  {
+    update: (state) => (updatedState) => ({ ...state, ...updatedState }),
+  }
+);
 
-export const AgentLogs: React.FunctionComponent<{ agent: Agent }> = memo(({ agent }) => {
-  const { data, application, http } = useStartServices();
+const AgentLogsConnected = AgentLogsUrlStateHelper.connect<AgentLogsProps, 'state'>((state) => ({
+  state: state || DEFAULT_LOGS_STATE,
+}))(AgentLogsUI);
 
-  // Util to convert date expressions (returned by datepicker) to timestamps (used by LogStream)
-  const getDateRangeTimestamps = useCallback(
-    (timeRange: TimeRange) => {
-      const { min, max } = data.query.timefilter.timefilter.calculateBounds(timeRange);
-      return min && max
-        ? {
-            startTimestamp: min.valueOf(),
-            endTimestamp: max.valueOf(),
-          }
-        : undefined;
-    },
-    [data.query.timefilter.timefilter]
-  );
+export const AgentLogs: React.FunctionComponent<Pick<AgentLogsProps, 'agent'>> = memo(
+  ({ agent }) => {
+    const [isSyncReady, setIsSyncReady] = useState<boolean>(false);
 
-  // Initial time range filter
-  const [dateRange, setDateRange] = useState<{
-    startExpression: string;
-    endExpression: string;
-    startTimestamp: number;
-    endTimestamp: number;
-  }>({
-    startExpression: DEFAULT_DATE_RANGE.start,
-    endExpression: DEFAULT_DATE_RANGE.end,
-    ...getDateRangeTimestamps({ from: DEFAULT_DATE_RANGE.start, to: DEFAULT_DATE_RANGE.end })!,
-  });
+    useEffect(() => {
+      const stateStorage = createKbnUrlStateStorage();
+      const { start, stop } = syncState({
+        storageKey: stateStorageKey,
+        stateContainer: stateContainer as INullableBaseStateContainer<AgentLogsState>,
+        stateStorage,
+      });
+      start();
+      setIsSyncReady(true);
 
-  const tryUpdateDateRange = useCallback(
-    (timeRange: TimeRange) => {
-      const timestamps = getDateRangeTimestamps(timeRange);
-      if (timestamps) {
-        setDateRange({
-          startExpression: timeRange.from,
-          endExpression: timeRange.to,
-          ...timestamps,
-        });
-      }
-    },
-    [getDateRangeTimestamps]
-  );
+      return () => {
+        stop();
+        stateContainer.set(DEFAULT_LOGS_STATE);
+      };
+    }, []);
 
-  // Filters
-  const [selectedLogLevels, setSelectedLogLevels] = useState<string[]>([]);
-  const [selectedDatasets, setSelectedDatasets] = useState<string[]>([AGENT_DATASET]);
-
-  // User query state
-  const [query, setQuery] = useState<string>('');
-  const [draftQuery, setDraftQuery] = useState<string>('');
-  const [isDraftQueryValid, setIsDraftQueryValid] = useState<boolean>(true);
-  const onUpdateDraftQuery = useCallback((newDraftQuery: string, runQuery?: boolean) => {
-    setDraftQuery(newDraftQuery);
-    try {
-      esKuery.fromKueryExpression(newDraftQuery);
-      setIsDraftQueryValid(true);
-      if (runQuery) {
-        setQuery(newDraftQuery);
-      }
-    } catch (err) {
-      setIsDraftQueryValid(false);
-    }
-  }, []);
-
-  // Build final log stream query from agent id, datasets, log levels, and user input
-  const logStreamQuery = useMemo(
-    () =>
-      buildQuery({
-        agentId: agent.id,
-        datasets: selectedDatasets,
-        logLevels: selectedLogLevels,
-        userQuery: query,
-      }),
-    [agent.id, query, selectedDatasets, selectedLogLevels]
-  );
-
-  // Generate URL to pass page state to Logs UI
-  const viewInLogsUrl = useMemo(
-    () =>
-      http.basePath.prepend(
-        url.format({
-          pathname: '/app/logs/stream',
-          search: stringify(
-            {
-              logPosition: encode({
-                start: dateRange.startExpression,
-                end: dateRange.endExpression,
-                streamLive: false,
-              }),
-              logFilter: encode({
-                expression: logStreamQuery,
-                kind: 'kuery',
-              }),
-            },
-            { sort: false, encode: false }
-          ),
-        })
-      ),
-    [logStreamQuery, dateRange.endExpression, dateRange.startExpression, http.basePath]
-  );
-
-  const agentVersion = agent.local_metadata?.elastic?.agent?.version;
-  const isLogLevelSelectionAvailable = useMemo(() => {
-    if (!agentVersion) {
-      return false;
-    }
-    const agentVersionWithPrerelease = semverCoerce(agentVersion)?.version;
-    if (!agentVersionWithPrerelease) {
-      return false;
-    }
-    return semverGte(agentVersionWithPrerelease, '7.11.0');
-  }, [agentVersion]);
-
-  return (
-    <WrapperFlexGroup direction="column" gutterSize="m">
-      <EuiFlexItem grow={false}>
-        <EuiFlexGroup gutterSize="m">
-          <EuiFlexItem>
-            <LogQueryBar
-              query={draftQuery}
-              onUpdateQuery={onUpdateDraftQuery}
-              isQueryValid={isDraftQueryValid}
-            />
-          </EuiFlexItem>
-          <EuiFlexItem grow={false}>
-            <EuiFilterGroup>
-              <DatasetFilter
-                selectedDatasets={selectedDatasets}
-                onToggleDataset={(level: string) => {
-                  const currentLevels = [...selectedDatasets];
-                  const levelPosition = currentLevels.indexOf(level);
-                  if (levelPosition >= 0) {
-                    currentLevels.splice(levelPosition, 1);
-                    setSelectedDatasets(currentLevels);
-                  } else {
-                    setSelectedDatasets([...selectedDatasets, level]);
-                  }
-                }}
-              />
-              <LogLevelFilter
-                selectedLevels={selectedLogLevels}
-                onToggleLevel={(level: string) => {
-                  const currentLevels = [...selectedLogLevels];
-                  const levelPosition = currentLevels.indexOf(level);
-                  if (levelPosition >= 0) {
-                    currentLevels.splice(levelPosition, 1);
-                    setSelectedLogLevels(currentLevels);
-                  } else {
-                    setSelectedLogLevels([...selectedLogLevels, level]);
-                  }
-                }}
-              />
-            </EuiFilterGroup>
-          </EuiFlexItem>
-          <DatePickerFlexItem grow={false}>
-            <EuiSuperDatePicker
-              showUpdateButton={false}
-              start={dateRange.startExpression}
-              end={dateRange.endExpression}
-              onTimeChange={({ start, end }) => {
-                tryUpdateDateRange({
-                  from: start,
-                  to: end,
-                });
-              }}
-            />
-          </DatePickerFlexItem>
-          <EuiFlexItem grow={false}>
-            <RedirectAppLinks application={application}>
-              <EuiButtonEmpty href={viewInLogsUrl} iconType="popout" flush="both">
-                <FormattedMessage
-                  id="xpack.fleet.agentLogs.openInLogsUiLinkText"
-                  defaultMessage="Open in Logs"
-                />
-              </EuiButtonEmpty>
-            </RedirectAppLinks>
-          </EuiFlexItem>
-        </EuiFlexGroup>
-      </EuiFlexItem>
-      <EuiFlexItem>
-        <EuiPanel paddingSize="none">
-          <LogStream
-            height="100%"
-            startTimestamp={dateRange.startTimestamp}
-            endTimestamp={dateRange.endTimestamp}
-            query={logStreamQuery}
-          />
-        </EuiPanel>
-      </EuiFlexItem>
-      {isLogLevelSelectionAvailable && (
-        <EuiFlexItem grow={false}>
-          <SelectLogLevel agent={agent} />
-        </EuiFlexItem>
-      )}
-    </WrapperFlexGroup>
-  );
-});
+    return (
+      <AgentLogsUrlStateHelper.Provider value={stateContainer}>
+        {isSyncReady ? <AgentLogsConnected agent={agent} /> : null}
+      </AgentLogsUrlStateHelper.Provider>
+    );
+  }
+);

From fac792778e5dada6c672c9ef082db837629a847e Mon Sep 17 00:00:00 2001
From: Silvia Mitter <silvia.mitter@elastic.co>
Date: Thu, 26 Nov 2020 09:32:04 +0100
Subject: [PATCH 23/37] [fleet] Add config options to accepted docker env vars
 (#84338)

---
 .../os_packages/docker_generator/resources/bin/kibana-docker   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker
index 4c833f5be6c5b..3e440c89b82d8 100755
--- a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker
+++ b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker
@@ -166,6 +166,9 @@ kibana_vars=(
     xpack.code.security.gitProtocolWhitelist
     xpack.encryptedSavedObjects.encryptionKey
     xpack.encryptedSavedObjects.keyRotation.decryptionOnlyKeys
+    xpack.fleet.agents.elasticsearch.host
+    xpack.fleet.agents.kibana.host
+    xpack.fleet.agents.tlsCheckDisabled
     xpack.graph.enabled
     xpack.graph.canEditDrillDownUrls
     xpack.graph.savePolicy

From a4c8dca02147dc3e3f9fef86c4e8f26320ed2058 Mon Sep 17 00:00:00 2001
From: Shahzad <shahzad.muhammad@elastic.co>
Date: Thu, 26 Nov 2020 10:08:45 +0100
Subject: [PATCH 24/37] [Uptime] Fix headers io-ts type (#84089)

---
 x-pack/plugins/uptime/common/runtime_types/ping/ping.ts     | 6 +++++-
 .../uptime/public/components/monitor/ping_list/headers.tsx  | 3 ++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts b/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts
index 9e5cd7641b65d..f9dde011b25fe 100644
--- a/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts
+++ b/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts
@@ -86,6 +86,10 @@ export const MonitorType = t.intersection([
 
 export type Monitor = t.TypeOf<typeof MonitorType>;
 
+export const PingHeadersType = t.record(t.string, t.union([t.string, t.array(t.string)]));
+
+export type PingHeaders = t.TypeOf<typeof PingHeadersType>;
+
 export const PingType = t.intersection([
   t.type({
     timestamp: t.string,
@@ -135,7 +139,7 @@ export const PingType = t.intersection([
         bytes: t.number,
         redirects: t.array(t.string),
         status_code: t.number,
-        headers: t.record(t.string, t.string),
+        headers: PingHeadersType,
       }),
       version: t.string,
     }),
diff --git a/x-pack/plugins/uptime/public/components/monitor/ping_list/headers.tsx b/x-pack/plugins/uptime/public/components/monitor/ping_list/headers.tsx
index 52fe26a7e08ca..a8cd8e2a26f98 100644
--- a/x-pack/plugins/uptime/public/components/monitor/ping_list/headers.tsx
+++ b/x-pack/plugins/uptime/public/components/monitor/ping_list/headers.tsx
@@ -7,9 +7,10 @@
 import React from 'react';
 import { EuiAccordion, EuiDescriptionList, EuiSpacer, EuiText } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
+import { PingHeaders as HeadersProp } from '../../../../common/runtime_types';
 
 interface Props {
-  headers: Record<string, string>;
+  headers: HeadersProp;
 }
 
 export const PingHeaders = ({ headers }: Props) => {

From d2552c426d6b0385ab075284c06232df8d25ab68 Mon Sep 17 00:00:00 2001
From: Mikhail Shustov <restrry@gmail.com>
Date: Thu, 26 Nov 2020 14:08:47 +0300
Subject: [PATCH 25/37] fix identation in list (#84301)

---
 .../plugin/migrating-legacy-plugins-examples.asciidoc        | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc b/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc
index 469f7a4f3adb1..8a0e487971b20 100644
--- a/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc
+++ b/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc
@@ -902,8 +902,9 @@ The most significant changes on the Kibana side for the consumers are the follow
 ===== User client accessor
 Internal /current user client accessors has been renamed and are now
 properties instead of functions:
-** `callAsInternalUser('ping')` -> `asInternalUser.ping()`
-** `callAsCurrentUser('ping')` -> `asCurrentUser.ping()`
+
+* `callAsInternalUser('ping')` -> `asInternalUser.ping()`
+* `callAsCurrentUser('ping')` -> `asCurrentUser.ping()`
 * the API now reflects the `Client`’s instead of leveraging the
 string-based endpoint names the `LegacyAPICaller` was using.
 

From 4004dd4012b569a72f53d09a482958f62de64383 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?=
 <alejandro.haro@elastic.co>
Date: Thu, 26 Nov 2020 12:25:05 +0100
Subject: [PATCH 26/37] [Application Usage] Update `schema` with new `fleet`
 rename (#84327)

---
 .../server/collectors/application_usage/schema.ts               | 2 +-
 src/plugins/telemetry/schema/oss_plugins.json                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
index 2e79cdaa7fc6b..76141f098a065 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
@@ -73,7 +73,7 @@ export const applicationUsageSchema = {
   logs: commonSchema,
   metrics: commonSchema,
   infra: commonSchema, // It's a forward app so we'll likely never report it
-  ingestManager: commonSchema,
+  fleet: commonSchema,
   lens: commonSchema,
   maps: commonSchema,
   ml: commonSchema,
diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json
index a1eae69ffaed0..3d79d7c6cf0e1 100644
--- a/src/plugins/telemetry/schema/oss_plugins.json
+++ b/src/plugins/telemetry/schema/oss_plugins.json
@@ -616,7 +616,7 @@
             }
           }
         },
-        "ingestManager": {
+        "fleet": {
           "properties": {
             "clicks_total": {
               "type": "long"

From 36ab99e546ead4f0a3e688d2575512172f94470b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?=
 <alejandro.fernandez@elastic.co>
Date: Thu, 26 Nov 2020 14:01:53 +0100
Subject: [PATCH 27/37] [Logs UI] Limit the height of the "view in context"
 container (#83178)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../infra/public/pages/logs/stream/page_view_log_in_context.tsx  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx b/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx
index 4ac3d15a82222..8a4081288b283 100644
--- a/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx
+++ b/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx
@@ -84,6 +84,7 @@ const LogInContextWrapper = euiStyled.div<{ width: number | string; height: numb
   padding: 16px;
   width: ${(props) => (typeof props.width === 'number' ? `${props.width}px` : props.width)};
   height: ${(props) => (typeof props.height === 'number' ? `${props.height}px` : props.height)};
+  max-height: 75vh; // Same as EuiModal
 `;
 
 const LogEntryContext: React.FC<{ context: LogEntry['context'] }> = ({ context }) => {

From b246701d39a582b4ea3c32f9a3ca8ed8fcdb2480 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?=
 <alejandro.fernandez@elastic.co>
Date: Thu, 26 Nov 2020 14:18:42 +0100
Subject: [PATCH 28/37] [Logs UI] Polish the UI for the log entry examples in
 the anomaly table (#82139)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../log_entry_examples/log_entry_examples.tsx        |  1 +
 .../log_entry_examples_empty_indicator.tsx           |  2 +-
 .../log_entry_examples_failure_indicator.tsx         |  2 +-
 .../sections/anomalies/expanded_row.tsx              | 12 +++++++-----
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx
index 2ec9922d94555..2d15068e51da5 100644
--- a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx
+++ b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx
@@ -46,4 +46,5 @@ const Wrapper = euiStyled.div`
   flex-direction: column;
   flex: 1 0 0%;
   overflow: hidden;
+  padding-top: 1px; // Buffer for the "Reload" buttons' hover state
 `;
diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx
index 1d6028ed032a2..d3d309948529f 100644
--- a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx
+++ b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx
@@ -10,7 +10,7 @@ import React from 'react';
 export const LogEntryExampleMessagesEmptyIndicator: React.FunctionComponent<{
   onReload: () => void;
 }> = ({ onReload }) => (
-  <EuiFlexGroup alignItems="center" justifyContent="center">
+  <EuiFlexGroup alignItems="center">
     <EuiFlexItem grow={false} className="eui-textNoWrap">
       <FormattedMessage
         id="xpack.infra.logs.logEntryExamples.exampleEmptyDescription"
diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_failure_indicator.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_failure_indicator.tsx
index dca786bce3b71..2a81265f043df 100644
--- a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_failure_indicator.tsx
+++ b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_failure_indicator.tsx
@@ -10,7 +10,7 @@ import React from 'react';
 export const LogEntryExampleMessagesFailureIndicator: React.FunctionComponent<{
   onRetry: () => void;
 }> = ({ onRetry }) => (
-  <EuiFlexGroup alignItems="center" justifyContent="center">
+  <EuiFlexGroup alignItems="center">
     <EuiFlexItem grow={false} className="eui-textNoWrap">
       <EuiTextColor color="danger">
         <FormattedMessage
diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx
index 9c78c421af752..e6489417f2d07 100644
--- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx
+++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx
@@ -51,9 +51,11 @@ export const AnomaliesTableExpandedRow: React.FunctionComponent<{
     <>
       <ExpandedContentWrapper direction="column">
         <EuiFlexItem>
-          <EuiTitle size="s">
+          <EuiTitle size="xs">
             <h3>{examplesTitle}</h3>
           </EuiTitle>
+        </EuiFlexItem>
+        <EuiFlexItem>
           <LogEntryExampleMessages
             isLoading={isLoadingLogEntryExamples}
             hasFailedLoading={hasFailedLoadingLogEntryExamples}
@@ -84,8 +86,8 @@ export const AnomaliesTableExpandedRow: React.FunctionComponent<{
           <EuiFlexGroup>
             <EuiFlexItem grow={false}>
               <EuiStat
-                titleSize="s"
-                title={`${numeral(anomaly.typical).format('0.00a')} ${i18n.translate(
+                titleSize="xs"
+                title={`${numeral(anomaly.typical).format('0.[00]a')} ${i18n.translate(
                   'xpack.infra.logs.analysis.anomaliesExpandedRowTypicalRateTitle',
                   {
                     defaultMessage: '{typicalCount, plural, one {message} other {messages}}',
@@ -102,8 +104,8 @@ export const AnomaliesTableExpandedRow: React.FunctionComponent<{
             </EuiFlexItem>
             <EuiFlexItem grow={false}>
               <EuiStat
-                titleSize="s"
-                title={`${numeral(anomaly.actual).format('0.00a')} ${i18n.translate(
+                titleSize="xs"
+                title={`${numeral(anomaly.actual).format('0.[00]a')} ${i18n.translate(
                   'xpack.infra.logs.analysis.anomaliesExpandedRowActualRateTitle',
                   {
                     defaultMessage: '{actualCount, plural, one {message} other {messages}}',

From f0192c1c03e95807a8bc66829551a010e2bb33a5 Mon Sep 17 00:00:00 2001
From: Matthias Wilhelm <matthias.wilhelm@elastic.co>
Date: Thu, 26 Nov 2020 15:45:21 +0100
Subject: [PATCH 29/37] [Discover] Fix navigating back when changing index
 pattern (#84061)

---
 .../public/application/angular/discover.js      |  6 +++---
 .../discover/_indexpattern_without_timefield.ts | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js
index 272c2f2ca6187..7059593c0c4e7 100644
--- a/src/plugins/discover/public/application/angular/discover.js
+++ b/src/plugins/discover/public/application/angular/discover.js
@@ -252,7 +252,8 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
     if (!_.isEqual(newStatePartial, oldStatePartial)) {
       $scope.$evalAsync(async () => {
         if (oldStatePartial.index !== newStatePartial.index) {
-          //in case of index switch the route has currently to be reloaded, legacy
+          //in case of index pattern switch the route has currently to be reloaded, legacy
+          $route.reload();
           return;
         }
 
@@ -289,8 +290,7 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise
         $scope.state.sort,
         config.get(MODIFY_COLUMNS_ON_SWITCH)
       );
-      await replaceUrlAppState(nextAppState);
-      $route.reload();
+      await setAppState(nextAppState);
     }
   };
 
diff --git a/test/functional/apps/discover/_indexpattern_without_timefield.ts b/test/functional/apps/discover/_indexpattern_without_timefield.ts
index 677b27c31bd86..20d783690277f 100644
--- a/test/functional/apps/discover/_indexpattern_without_timefield.ts
+++ b/test/functional/apps/discover/_indexpattern_without_timefield.ts
@@ -20,6 +20,7 @@
 import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const browser = getService('browser');
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
   const security = getService('security');
@@ -50,5 +51,21 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         throw new Error('Expected timepicker to exist');
       }
     });
+    it('should switch between with and without timefield using the browser back button', async () => {
+      await PageObjects.discover.selectIndexPattern('without-timefield');
+      if (await PageObjects.timePicker.timePickerExists()) {
+        throw new Error('Expected timepicker not to exist');
+      }
+
+      await PageObjects.discover.selectIndexPattern('with-timefield');
+      if (!(await PageObjects.timePicker.timePickerExists())) {
+        throw new Error('Expected timepicker to exist');
+      }
+      // Navigating back to discover
+      await browser.goBack();
+      if (await PageObjects.timePicker.timePickerExists()) {
+        throw new Error('Expected timepicker not to exist');
+      }
+    });
   });
 }

From 93b0273ccddb89aed744ef1d64eafd390dcf9090 Mon Sep 17 00:00:00 2001
From: Alexey Antonov <alexwizp@gmail.com>
Date: Thu, 26 Nov 2020 18:04:27 +0300
Subject: [PATCH 30/37] TSVB offsets (#83051)

Closes: #40299

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../response_processors/series/time_shift.js         | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js
index 14de6aa18f872..c00b0894073d2 100644
--- a/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js
+++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import _ from 'lodash';
+import { startsWith } from 'lodash';
 import moment from 'moment';
 
 export function timeShift(resp, panel, series) {
@@ -26,13 +26,15 @@ export function timeShift(resp, panel, series) {
       const matches = series.offset_time.match(/^([+-]?[\d]+)([shmdwMy]|ms)$/);
 
       if (matches) {
-        const offsetValue = Number(matches[1]);
+        const offsetValue = matches[1];
         const offsetUnit = matches[2];
-        const offset = moment.duration(offsetValue, offsetUnit).valueOf();
 
         results.forEach((item) => {
-          if (_.startsWith(item.id, series.id)) {
-            item.data = item.data.map(([time, value]) => [time + offset, value]);
+          if (startsWith(item.id, series.id)) {
+            item.data = item.data.map((row) => [
+              moment.utc(row[0]).add(offsetValue, offsetUnit).valueOf(),
+              row[1],
+            ]);
           }
         });
       }

From 4d8b7f9084f46bc4694ea8d78aee2b13a049a80c Mon Sep 17 00:00:00 2001
From: Joe Portner <5295965+jportner@users.noreply.github.com>
Date: Thu, 26 Nov 2020 10:51:10 -0500
Subject: [PATCH 31/37] Improve short-url redirect validation (#84366)

---
 .../routes/lib/short_url_assert_valid.test.ts | 45 ++++++++++---------
 .../routes/lib/short_url_assert_valid.ts      | 12 +++--
 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts b/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts
index 02a5e123b6481..232429ac4ec33 100644
--- a/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts
+++ b/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts
@@ -19,36 +19,39 @@
 
 import { shortUrlAssertValid } from './short_url_assert_valid';
 
+const PROTOCOL_ERROR = /^Short url targets cannot have a protocol/;
+const HOSTNAME_ERROR = /^Short url targets cannot have a hostname/;
+const PATH_ERROR = /^Short url target path must be in the format/;
+
 describe('shortUrlAssertValid()', () => {
   const invalid = [
-    ['protocol', 'http://localhost:5601/app/kibana'],
-    ['protocol', 'https://localhost:5601/app/kibana'],
-    ['protocol', 'mailto:foo@bar.net'],
-    ['protocol', 'javascript:alert("hi")'], // eslint-disable-line no-script-url
-    ['hostname', 'localhost/app/kibana'],
-    ['hostname and port', 'local.host:5601/app/kibana'],
-    ['hostname and auth', 'user:pass@localhost.net/app/kibana'],
-    ['path traversal', '/app/../../not-kibana'],
-    ['deep path', '/app/kibana/foo'],
-    ['deep path', '/app/kibana/foo/bar'],
-    ['base path', '/base/app/kibana'],
+    ['protocol', 'http://localhost:5601/app/kibana', PROTOCOL_ERROR],
+    ['protocol', 'https://localhost:5601/app/kibana', PROTOCOL_ERROR],
+    ['protocol', 'mailto:foo@bar.net', PROTOCOL_ERROR],
+    ['protocol', 'javascript:alert("hi")', PROTOCOL_ERROR], // eslint-disable-line no-script-url
+    ['hostname', 'localhost/app/kibana', PATH_ERROR], // according to spec, this is not a valid URL -- you cannot specify a hostname without a protocol
+    ['hostname and port', 'local.host:5601/app/kibana', PROTOCOL_ERROR], // parser detects 'local.host' as the protocol
+    ['hostname and auth', 'user:pass@localhost.net/app/kibana', PROTOCOL_ERROR], // parser detects 'user' as the protocol
+    ['path traversal', '/app/../../not-kibana', PATH_ERROR], // fails because there are >2 path parts
+    ['path traversal', '/../not-kibana', PATH_ERROR], // fails because first path part is not 'app'
+    ['deep path', '/app/kibana/foo', PATH_ERROR], // fails because there are >2 path parts
+    ['deeper path', '/app/kibana/foo/bar', PATH_ERROR], // fails because there are >2 path parts
+    ['base path', '/base/app/kibana', PATH_ERROR], // fails because there are >2 path parts
+    ['path with an extra leading slash', '//foo/app/kibana', HOSTNAME_ERROR], // parser detects 'foo' as the hostname
+    ['path with an extra leading slash', '///app/kibana', HOSTNAME_ERROR], // parser detects '' as the hostname
+    ['path without app', '/foo/kibana', PATH_ERROR], // fails because first path part is not 'app'
+    ['path without appId', '/app/', PATH_ERROR], // fails because there is only one path part (leading and trailing slashes are trimmed)
   ];
 
-  invalid.forEach(([desc, url]) => {
-    it(`fails when url has ${desc}`, () => {
-      try {
-        shortUrlAssertValid(url);
-        throw new Error(`expected assertion to throw`);
-      } catch (err) {
-        if (!err || !err.isBoom) {
-          throw err;
-        }
-      }
+  invalid.forEach(([desc, url, error]) => {
+    it(`fails when url has ${desc as string}`, () => {
+      expect(() => shortUrlAssertValid(url as string)).toThrowError(error);
     });
   });
 
   const valid = [
     '/app/kibana',
+    '/app/kibana/', // leading and trailing slashes are trimmed
     '/app/monitoring#angular/route',
     '/app/text#document-id',
     '/app/some?with=query',
diff --git a/src/plugins/share/server/routes/lib/short_url_assert_valid.ts b/src/plugins/share/server/routes/lib/short_url_assert_valid.ts
index 581410359322f..773e3acdcb902 100644
--- a/src/plugins/share/server/routes/lib/short_url_assert_valid.ts
+++ b/src/plugins/share/server/routes/lib/short_url_assert_valid.ts
@@ -22,18 +22,22 @@ import { trim } from 'lodash';
 import Boom from '@hapi/boom';
 
 export function shortUrlAssertValid(url: string) {
-  const { protocol, hostname, pathname } = parse(url);
+  const { protocol, hostname, pathname } = parse(
+    url,
+    false /* parseQueryString */,
+    true /* slashesDenoteHost */
+  );
 
-  if (protocol) {
+  if (protocol !== null) {
     throw Boom.notAcceptable(`Short url targets cannot have a protocol, found "${protocol}"`);
   }
 
-  if (hostname) {
+  if (hostname !== null) {
     throw Boom.notAcceptable(`Short url targets cannot have a hostname, found "${hostname}"`);
   }
 
   const pathnameParts = trim(pathname === null ? undefined : pathname, '/').split('/');
-  if (pathnameParts.length !== 2) {
+  if (pathnameParts.length !== 2 || pathnameParts[0] !== 'app' || !pathnameParts[1]) {
     throw Boom.notAcceptable(
       `Short url target path must be in the format "/app/{{appId}}", found "${pathname}"`
     );

From 51f75a5655224d77c97b944f8b603ff34ec92714 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yulia=20=C4=8Cech?=
 <6585477+yuliacech@users.noreply.github.com>
Date: Thu, 26 Nov 2020 17:04:24 +0100
Subject: [PATCH 32/37] Added data streams privileges to better control delete
 actions in UI (#83573)

* Added data streams privileges to better control delete actions in UI

* Fix type check issues

* Change data streams privileges request

* Fixed type check issue

* Fixed api integration test

* Cleaned up not needed code

* Renamed some data streams and added a default value for stats find

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../helpers/test_subjects.ts                  |   2 +
 .../home/data_streams_tab.helpers.ts          |  12 +
 .../home/data_streams_tab.test.ts             |  73 ++++++
 .../common/lib/data_stream_serialization.ts   |   2 +
 .../common/types/data_streams.ts              |  16 +-
 .../data_stream_detail_panel.tsx              |   2 +-
 .../data_stream_list/data_stream_list.tsx     |   9 +-
 .../data_stream_table/data_stream_table.tsx   |   6 +-
 .../server/client/elasticsearch.ts            |  22 --
 .../plugins/index_management/server/plugin.ts |   3 +-
 .../component_templates/privileges.test.ts    |   2 +
 .../api/data_streams/register_get_route.ts    | 221 ++++++++++++------
 .../index_management/server/shared_imports.ts |   2 +-
 .../plugins/index_management/server/types.ts  |   3 +-
 .../index_management/data_streams.ts          |   9 +
 15 files changed, 271 insertions(+), 113 deletions(-)

diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts b/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts
index 04843cae6a57e..e8105ac2937c0 100644
--- a/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts
+++ b/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts
@@ -13,6 +13,8 @@ export type TestSubjects =
   | 'createTemplateButton'
   | 'dataStreamsEmptyPromptTemplateLink'
   | 'dataStreamTable'
+  | 'deleteDataStreamsButton'
+  | 'deleteDataStreamButton'
   | 'deleteSystemTemplateCallOut'
   | 'deleteTemplateButton'
   | 'deleteTemplatesConfirmation'
diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts
index 4e0486e55720d..9c92af30097a2 100644
--- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts
+++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts
@@ -24,6 +24,7 @@ export interface DataStreamsTabTestBed extends TestBed<TestSubjects> {
     clickNameAt: (index: number) => void;
     clickIndicesAt: (index: number) => void;
     clickDeleteActionAt: (index: number) => void;
+    selectDataStream: (name: string, selected: boolean) => void;
     clickConfirmDelete: () => void;
     clickDeleteDataStreamButton: () => void;
     clickDetailPanelIndexTemplateLink: () => void;
@@ -125,6 +126,13 @@ export const setup = async (overridingDependencies: any = {}): Promise<DataStrea
     findDeleteActionAt(index).simulate('click');
   };
 
+  const selectDataStream = (name: string, selected: boolean) => {
+    const {
+      form: { selectCheckBox },
+    } = testBed;
+    selectCheckBox(`checkboxSelectRow-${name}`, selected);
+  };
+
   const findDeleteConfirmationModal = () => {
     const { find } = testBed;
     return find('deleteDataStreamsConfirmation');
@@ -194,6 +202,7 @@ export const setup = async (overridingDependencies: any = {}): Promise<DataStrea
       clickNameAt,
       clickIndicesAt,
       clickDeleteActionAt,
+      selectDataStream,
       clickConfirmDelete,
       clickDeleteDataStreamButton,
       clickDetailPanelIndexTemplateLink,
@@ -223,6 +232,9 @@ export const createDataStreamPayload = (dataStream: Partial<DataStream>): DataSt
   indexTemplateName: 'indexTemplate',
   storageSize: '1b',
   maxTimeStamp: 420,
+  privileges: {
+    delete_index: true,
+  },
   ...dataStream,
 });
 
diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts
index 8ce307c103f4c..91502621d50c5 100644
--- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts
+++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts
@@ -449,4 +449,77 @@ describe('Data Streams tab', () => {
       expect(tableCellsValues).toEqual([['', 'non-managed-data-stream', 'green', '1', 'Delete']]);
     });
   });
+
+  describe('data stream privileges', () => {
+    describe('delete', () => {
+      const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers;
+
+      const dataStreamWithDelete = createDataStreamPayload({
+        name: 'dataStreamWithDelete',
+        privileges: { delete_index: true },
+      });
+      const dataStreamNoDelete = createDataStreamPayload({
+        name: 'dataStreamNoDelete',
+        privileges: { delete_index: false },
+      });
+
+      beforeEach(async () => {
+        setLoadDataStreamsResponse([dataStreamWithDelete, dataStreamNoDelete]);
+
+        testBed = await setup({ history: createMemoryHistory() });
+        await act(async () => {
+          testBed.actions.goToDataStreamsList();
+        });
+        testBed.component.update();
+      });
+
+      test('displays/hides delete button depending on data streams privileges', async () => {
+        const { table } = testBed;
+        const { tableCellsValues } = table.getMetaData('dataStreamTable');
+
+        expect(tableCellsValues).toEqual([
+          ['', 'dataStreamNoDelete', 'green', '1', ''],
+          ['', 'dataStreamWithDelete', 'green', '1', 'Delete'],
+        ]);
+      });
+
+      test('displays/hides delete action depending on data streams privileges', async () => {
+        const {
+          actions: { selectDataStream },
+          find,
+        } = testBed;
+
+        selectDataStream('dataStreamNoDelete', true);
+        expect(find('deleteDataStreamsButton').exists()).toBeFalsy();
+
+        selectDataStream('dataStreamWithDelete', true);
+        expect(find('deleteDataStreamsButton').exists()).toBeFalsy();
+
+        selectDataStream('dataStreamNoDelete', false);
+        expect(find('deleteDataStreamsButton').exists()).toBeTruthy();
+      });
+
+      test('displays delete button in detail panel', async () => {
+        const {
+          actions: { clickNameAt },
+          find,
+        } = testBed;
+        setLoadDataStreamResponse(dataStreamWithDelete);
+        await clickNameAt(1);
+
+        expect(find('deleteDataStreamButton').exists()).toBeTruthy();
+      });
+
+      test('hides delete button in detail panel', async () => {
+        const {
+          actions: { clickNameAt },
+          find,
+        } = testBed;
+        setLoadDataStreamResponse(dataStreamNoDelete);
+        await clickNameAt(0);
+
+        expect(find('deleteDataStreamButton').exists()).toBeFalsy();
+      });
+    });
+  });
 });
diff --git a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts b/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts
index 2d8e038d2a60f..fe7db99c98db1 100644
--- a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts
+++ b/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts
@@ -18,6 +18,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS
     store_size: storageSize,
     maximum_timestamp: maxTimeStamp,
     _meta,
+    privileges,
   } = dataStreamFromEs;
 
   return {
@@ -37,6 +38,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS
     storageSize,
     maxTimeStamp,
     _meta,
+    privileges,
   };
 }
 
diff --git a/x-pack/plugins/index_management/common/types/data_streams.ts b/x-pack/plugins/index_management/common/types/data_streams.ts
index adb7104043fbb..fdfe6278eb985 100644
--- a/x-pack/plugins/index_management/common/types/data_streams.ts
+++ b/x-pack/plugins/index_management/common/types/data_streams.ts
@@ -10,13 +10,19 @@ interface TimestampFieldFromEs {
 
 type TimestampField = TimestampFieldFromEs;
 
-interface MetaFieldFromEs {
+interface MetaFromEs {
   managed_by: string;
   package: any;
   managed: boolean;
 }
 
-type MetaField = MetaFieldFromEs;
+type Meta = MetaFromEs;
+
+interface PrivilegesFromEs {
+  delete_index: boolean;
+}
+
+type Privileges = PrivilegesFromEs;
 
 export type HealthFromEs = 'GREEN' | 'YELLOW' | 'RED';
 
@@ -25,12 +31,13 @@ export interface DataStreamFromEs {
   timestamp_field: TimestampFieldFromEs;
   indices: DataStreamIndexFromEs[];
   generation: number;
-  _meta?: MetaFieldFromEs;
+  _meta?: MetaFromEs;
   status: HealthFromEs;
   template: string;
   ilm_policy?: string;
   store_size?: string;
   maximum_timestamp?: number;
+  privileges: PrivilegesFromEs;
 }
 
 export interface DataStreamIndexFromEs {
@@ -50,7 +57,8 @@ export interface DataStream {
   ilmPolicyName?: string;
   storageSize?: string;
   maxTimeStamp?: number;
-  _meta?: MetaField;
+  _meta?: Meta;
+  privileges: Privileges;
 }
 
 export interface DataStreamIndex {
diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx
index 05d7e97745b9e..ec47b2c062aa9 100644
--- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx
+++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx
@@ -290,7 +290,7 @@ export const DataStreamDetailPanel: React.FunctionComponent<Props> = ({
               </EuiButtonEmpty>
             </EuiFlexItem>
 
-            {!isLoading && !error ? (
+            {!isLoading && !error && dataStream?.privileges.delete_index ? (
               <EuiFlexItem grow={false}>
                 <EuiButton
                   color="danger"
diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx
index bc7df7a70196e..f43b9799082a0 100644
--- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx
+++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx
@@ -46,6 +46,7 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
   history,
 }) => {
   const { isDeepLink } = extractQueryParams(search);
+  const decodedDataStreamName = attemptToURIDecode(dataStreamName);
 
   const {
     core: { getUrlForApp },
@@ -241,8 +242,8 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
 
         <DataStreamTable
           filters={
-            isDeepLink && dataStreamName !== undefined
-              ? `name="${attemptToURIDecode(dataStreamName)}"`
+            isDeepLink && decodedDataStreamName !== undefined
+              ? `name="${decodedDataStreamName}"`
               : ''
           }
           dataStreams={filteredDataStreams}
@@ -262,9 +263,9 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
         If the user has been deep-linked, they'll expect to see the detail panel because it reflects
         the URL state, even if there are no data streams or if there was an error loading them.
       */}
-      {dataStreamName && (
+      {decodedDataStreamName && (
         <DataStreamDetailPanel
-          dataStreamName={attemptToURIDecode(dataStreamName)!}
+          dataStreamName={decodedDataStreamName}
           onClose={(shouldReload?: boolean) => {
             history.push(`/${Section.DataStreams}`);
 
diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx
index c1fd33a39569c..7a3e719d013c8 100644
--- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx
+++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx
@@ -162,6 +162,7 @@ export const DataStreamTable: React.FunctionComponent<Props> = ({
         },
         isPrimary: true,
         'data-test-subj': 'deleteDataStream',
+        available: ({ privileges: { delete_index: deleteIndex } }: DataStream) => deleteIndex,
       },
     ],
   });
@@ -188,9 +189,10 @@ export const DataStreamTable: React.FunctionComponent<Props> = ({
       incremental: true,
     },
     toolsLeft:
-      selection.length > 0 ? (
+      selection.length > 0 &&
+      selection.every((dataStream: DataStream) => dataStream.privileges.delete_index) ? (
         <EuiButton
-          data-test-subj="deletDataStreamsButton"
+          data-test-subj="deleteDataStreamsButton"
           onClick={() => setDataStreamsToDelete(selection.map(({ name }: DataStream) => name))}
           color="danger"
         >
diff --git a/x-pack/plugins/index_management/server/client/elasticsearch.ts b/x-pack/plugins/index_management/server/client/elasticsearch.ts
index ed5ede07479ca..8b7749335131e 100644
--- a/x-pack/plugins/index_management/server/client/elasticsearch.ts
+++ b/x-pack/plugins/index_management/server/client/elasticsearch.ts
@@ -11,28 +11,6 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any)
   const dataManagement = Client.prototype.dataManagement.prototype;
 
   // Data streams
-  dataManagement.getDataStreams = ca({
-    urls: [
-      {
-        fmt: '/_data_stream',
-      },
-    ],
-    method: 'GET',
-  });
-
-  dataManagement.getDataStream = ca({
-    urls: [
-      {
-        fmt: '/_data_stream/<%=name%>',
-        req: {
-          name: {
-            type: 'string',
-          },
-        },
-      },
-    ],
-    method: 'GET',
-  });
 
   // We don't allow the user to create a data stream in the UI or API. We're just adding this here
   // to enable the API integration tests.
diff --git a/x-pack/plugins/index_management/server/plugin.ts b/x-pack/plugins/index_management/server/plugin.ts
index ae9633f3e22b9..3d70140fa60b7 100644
--- a/x-pack/plugins/index_management/server/plugin.ts
+++ b/x-pack/plugins/index_management/server/plugin.ts
@@ -24,7 +24,7 @@ import { PLUGIN } from '../common';
 import { Dependencies } from './types';
 import { ApiRoutes } from './routes';
 import { License, IndexDataEnricher } from './services';
-import { isEsError } from './shared_imports';
+import { isEsError, handleEsError } from './shared_imports';
 import { elasticsearchJsPlugin } from './client/elasticsearch';
 
 export interface DataManagementContext {
@@ -110,6 +110,7 @@ export class IndexMgmtServerPlugin implements Plugin<IndexManagementPluginSetup,
       indexDataEnricher: this.indexDataEnricher,
       lib: {
         isEsError,
+        handleEsError,
       },
     });
 
diff --git a/x-pack/plugins/index_management/server/routes/api/component_templates/privileges.test.ts b/x-pack/plugins/index_management/server/routes/api/component_templates/privileges.test.ts
index 060a423350ada..b1eb430ecd2c3 100644
--- a/x-pack/plugins/index_management/server/routes/api/component_templates/privileges.test.ts
+++ b/x-pack/plugins/index_management/server/routes/api/component_templates/privileges.test.ts
@@ -54,6 +54,7 @@ describe('GET privileges', () => {
       indexDataEnricher: mockedIndexDataEnricher,
       lib: {
         isEsError: jest.fn(),
+        handleEsError: jest.fn(),
       },
     });
 
@@ -123,6 +124,7 @@ describe('GET privileges', () => {
         indexDataEnricher: mockedIndexDataEnricher,
         lib: {
           isEsError: jest.fn(),
+          handleEsError: jest.fn(),
         },
       });
 
diff --git a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts
index fa93d9bd0c563..d19383d892cbd 100644
--- a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts
+++ b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts
@@ -6,122 +6,189 @@
 
 import { schema, TypeOf } from '@kbn/config-schema';
 
+import { ElasticsearchClient } from 'kibana/server';
 import { deserializeDataStream, deserializeDataStreamList } from '../../../../common/lib';
+import { DataStreamFromEs } from '../../../../common/types';
 import { RouteDependencies } from '../../../types';
 import { addBasePath } from '../index';
 
-const querySchema = schema.object({
-  includeStats: schema.maybe(schema.oneOf([schema.literal('true'), schema.literal('false')])),
-});
+interface PrivilegesFromEs {
+  username: string;
+  has_all_requested: boolean;
+  cluster: Record<string, boolean>;
+  index: Record<string, Record<string, boolean>>;
+  application: Record<string, boolean>;
+}
+
+interface StatsFromEs {
+  data_stream: string;
+  store_size: string;
+  maximum_timestamp: number;
+}
 
-export function registerGetAllRoute({ router, license, lib: { isEsError } }: RouteDependencies) {
+const enhanceDataStreams = ({
+  dataStreams,
+  dataStreamsStats,
+  dataStreamsPrivileges,
+}: {
+  dataStreams: DataStreamFromEs[];
+  dataStreamsStats?: StatsFromEs[];
+  dataStreamsPrivileges?: PrivilegesFromEs;
+}): DataStreamFromEs[] => {
+  return dataStreams.map((dataStream: DataStreamFromEs) => {
+    let enhancedDataStream = { ...dataStream };
+
+    if (dataStreamsStats) {
+      // eslint-disable-next-line @typescript-eslint/naming-convention
+      const { store_size, maximum_timestamp } =
+        dataStreamsStats.find(
+          ({ data_stream: statsName }: { data_stream: string }) => statsName === dataStream.name
+        ) || {};
+
+      enhancedDataStream = {
+        ...enhancedDataStream,
+        store_size,
+        maximum_timestamp,
+      };
+    }
+
+    enhancedDataStream = {
+      ...enhancedDataStream,
+      privileges: {
+        delete_index: dataStreamsPrivileges
+          ? dataStreamsPrivileges.index[dataStream.name].delete_index
+          : true,
+      },
+    };
+
+    return enhancedDataStream;
+  });
+};
+
+const getDataStreamsStats = (client: ElasticsearchClient, name = '*') => {
+  return client.transport.request({
+    path: `/_data_stream/${encodeURIComponent(name)}/_stats`,
+    method: 'GET',
+    querystring: {
+      human: true,
+    },
+  });
+};
+
+const getDataStreamsPrivileges = (client: ElasticsearchClient, names: string[]) => {
+  return client.security.hasPrivileges<PrivilegesFromEs>({
+    body: {
+      index: [
+        {
+          names,
+          privileges: ['delete_index'],
+        },
+      ],
+    },
+  });
+};
+
+export function registerGetAllRoute({
+  router,
+  license,
+  lib: { handleEsError },
+  config,
+}: RouteDependencies) {
+  const querySchema = schema.object({
+    includeStats: schema.maybe(schema.oneOf([schema.literal('true'), schema.literal('false')])),
+  });
   router.get(
     { path: addBasePath('/data_streams'), validate: { query: querySchema } },
-    license.guardApiRoute(async (ctx, req, res) => {
-      const { callAsCurrentUser } = ctx.dataManagement!.client;
+    license.guardApiRoute(async (ctx, req, response) => {
+      const { asCurrentUser } = ctx.core.elasticsearch.client;
 
       const includeStats = (req.query as TypeOf<typeof querySchema>).includeStats === 'true';
 
       try {
-        const { data_streams: dataStreams } = await callAsCurrentUser(
-          'dataManagement.getDataStreams'
-        );
-
-        if (includeStats) {
-          const {
-            data_streams: dataStreamsStats,
-          } = await ctx.core.elasticsearch.legacy.client.callAsCurrentUser('transport.request', {
-            path: '/_data_stream/*/_stats',
-            method: 'GET',
-            query: {
-              human: true,
-            },
-          });
+        let {
+          body: { data_streams: dataStreams },
+        } = await asCurrentUser.indices.getDataStream();
 
-          // Merge stats into data streams.
-          for (let i = 0; i < dataStreams.length; i++) {
-            const dataStream = dataStreams[i];
+        let dataStreamsStats;
+        let dataStreamsPrivileges;
 
-            // eslint-disable-next-line @typescript-eslint/naming-convention
-            const { store_size, maximum_timestamp } = dataStreamsStats.find(
-              ({ data_stream: statsName }: { data_stream: string }) => statsName === dataStream.name
-            );
-
-            dataStreams[i] = {
-              ...dataStream,
-              store_size,
-              maximum_timestamp,
-            };
-          }
+        if (includeStats) {
+          ({
+            body: { data_streams: dataStreamsStats },
+          } = await getDataStreamsStats(asCurrentUser));
         }
 
-        return res.ok({ body: deserializeDataStreamList(dataStreams) });
-      } catch (error) {
-        if (isEsError(error)) {
-          return res.customError({
-            statusCode: error.statusCode,
-            body: error,
-          });
+        if (config.isSecurityEnabled() && dataStreams.length > 0) {
+          ({ body: dataStreamsPrivileges } = await getDataStreamsPrivileges(
+            asCurrentUser,
+            dataStreams.map((dataStream: DataStreamFromEs) => dataStream.name)
+          ));
         }
 
-        return res.internalError({ body: error });
+        dataStreams = enhanceDataStreams({
+          dataStreams,
+          dataStreamsStats,
+          dataStreamsPrivileges,
+        });
+
+        return response.ok({ body: deserializeDataStreamList(dataStreams) });
+      } catch (error) {
+        return handleEsError({ error, response });
       }
     })
   );
 }
 
-export function registerGetOneRoute({ router, license, lib: { isEsError } }: RouteDependencies) {
+export function registerGetOneRoute({
+  router,
+  license,
+  lib: { handleEsError },
+  config,
+}: RouteDependencies) {
   const paramsSchema = schema.object({
     name: schema.string(),
   });
-
   router.get(
     {
       path: addBasePath('/data_streams/{name}'),
       validate: { params: paramsSchema },
     },
-    license.guardApiRoute(async (ctx, req, res) => {
+    license.guardApiRoute(async (ctx, req, response) => {
       const { name } = req.params as TypeOf<typeof paramsSchema>;
-      const { callAsCurrentUser } = ctx.dataManagement!.client;
+      const { asCurrentUser } = ctx.core.elasticsearch.client;
       try {
         const [
-          { data_streams: dataStream },
-          { data_streams: dataStreamsStats },
+          {
+            body: { data_streams: dataStreams },
+          },
+          {
+            body: { data_streams: dataStreamsStats },
+          },
         ] = await Promise.all([
-          callAsCurrentUser('dataManagement.getDataStream', {
-            name,
-          }),
-          ctx.core.elasticsearch.legacy.client.callAsCurrentUser('transport.request', {
-            path: `/_data_stream/${encodeURIComponent(name)}/_stats`,
-            method: 'GET',
-            query: {
-              human: true,
-            },
-          }),
+          asCurrentUser.indices.getDataStream({ name }),
+          getDataStreamsStats(asCurrentUser, name),
         ]);
 
-        if (dataStream[0]) {
-          // eslint-disable-next-line @typescript-eslint/naming-convention
-          const { store_size, maximum_timestamp } = dataStreamsStats[0];
-          dataStream[0] = {
-            ...dataStream[0],
-            store_size,
-            maximum_timestamp,
-          };
-          const body = deserializeDataStream(dataStream[0]);
-          return res.ok({ body });
-        }
+        if (dataStreams[0]) {
+          let dataStreamsPrivileges;
+          if (config.isSecurityEnabled()) {
+            ({ body: dataStreamsPrivileges } = await getDataStreamsPrivileges(asCurrentUser, [
+              dataStreams[0].name,
+            ]));
+          }
 
-        return res.notFound();
-      } catch (e) {
-        if (isEsError(e)) {
-          return res.customError({
-            statusCode: e.statusCode,
-            body: e,
+          const enhancedDataStreams = enhanceDataStreams({
+            dataStreams,
+            dataStreamsStats,
+            dataStreamsPrivileges,
           });
+          const body = deserializeDataStream(enhancedDataStreams[0]);
+          return response.ok({ body });
         }
-        // Case: default
-        return res.internalError({ body: e });
+
+        return response.notFound();
+      } catch (error) {
+        return handleEsError({ error, response });
       }
     })
   );
diff --git a/x-pack/plugins/index_management/server/shared_imports.ts b/x-pack/plugins/index_management/server/shared_imports.ts
index 454beda5394c7..0606f474897b5 100644
--- a/x-pack/plugins/index_management/server/shared_imports.ts
+++ b/x-pack/plugins/index_management/server/shared_imports.ts
@@ -4,4 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export { isEsError } from '../../../../src/plugins/es_ui_shared/server';
+export { isEsError, handleEsError } from '../../../../src/plugins/es_ui_shared/server';
diff --git a/x-pack/plugins/index_management/server/types.ts b/x-pack/plugins/index_management/server/types.ts
index 7aa91629f0a47..177dedeb87bb4 100644
--- a/x-pack/plugins/index_management/server/types.ts
+++ b/x-pack/plugins/index_management/server/types.ts
@@ -8,7 +8,7 @@ import { PluginSetupContract as FeaturesPluginSetup } from '../../features/serve
 import { LicensingPluginSetup } from '../../licensing/server';
 import { SecurityPluginSetup } from '../../security/server';
 import { License, IndexDataEnricher } from './services';
-import { isEsError } from './shared_imports';
+import { isEsError, handleEsError } from './shared_imports';
 
 export interface Dependencies {
   security: SecurityPluginSetup;
@@ -25,6 +25,7 @@ export interface RouteDependencies {
   indexDataEnricher: IndexDataEnricher;
   lib: {
     isEsError: typeof isEsError;
+    handleEsError: typeof handleEsError;
   };
 }
 
diff --git a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts
index f4b947336e044..6cf1a40a4d5a1 100644
--- a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts
+++ b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts
@@ -77,6 +77,9 @@ export default function ({ getService }: FtrProviderContext) {
         expect(dataStreams).to.eql([
           {
             name: testDataStreamName,
+            privileges: {
+              delete_index: true,
+            },
             timeStampField: { name: '@timestamp' },
             indices: [
               {
@@ -105,6 +108,9 @@ export default function ({ getService }: FtrProviderContext) {
         expect(dataStreams.length).to.be(1);
         expect(dataStreamWithoutStorageSize).to.eql({
           name: testDataStreamName,
+          privileges: {
+            delete_index: true,
+          },
           timeStampField: { name: '@timestamp' },
           indices: [
             {
@@ -132,6 +138,9 @@ export default function ({ getService }: FtrProviderContext) {
 
         expect(dataStreamWithoutStorageSize).to.eql({
           name: testDataStreamName,
+          privileges: {
+            delete_index: true,
+          },
           timeStampField: { name: '@timestamp' },
           indices: [
             {

From ee5c9bceebf1b0ddd3f4da371e8009d455e7b3f6 Mon Sep 17 00:00:00 2001
From: Rudolf Meijering <skaapgif@gmail.com>
Date: Thu, 26 Nov 2020 20:34:06 +0100
Subject: [PATCH 33/37] Upgrade fp-ts to 2.8.6 (#83866)

* Upgrade fp-ts to 2.8.6

* reduce import size from io-ts

* removed unused imports

* remove usage of fpts from alerts

Co-authored-by: Gidi Meir Morris <github@gidi.io>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../plugins/alerts/public/alert_api.test.ts   | 66 +------------------
 x-pack/plugins/alerts/public/alert_api.ts     | 41 ++----------
 .../public/application/lib/alert_api.ts       |  6 +-
 yarn.lock                                     |  6 +-
 4 files changed, 14 insertions(+), 105 deletions(-)

diff --git a/x-pack/plugins/alerts/public/alert_api.test.ts b/x-pack/plugins/alerts/public/alert_api.test.ts
index 3ee67b79b7bda..0fa2e7f25323b 100644
--- a/x-pack/plugins/alerts/public/alert_api.test.ts
+++ b/x-pack/plugins/alerts/public/alert_api.test.ts
@@ -6,7 +6,7 @@
 
 import { AlertType } from '../common';
 import { httpServiceMock } from '../../../../src/core/public/mocks';
-import { loadAlert, loadAlertState, loadAlertType, loadAlertTypes } from './alert_api';
+import { loadAlert, loadAlertType, loadAlertTypes } from './alert_api';
 import uuid from 'uuid';
 
 const http = httpServiceMock.createStartContract();
@@ -114,67 +114,3 @@ describe('loadAlert', () => {
     expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}`);
   });
 });
-
-describe('loadAlertState', () => {
-  test('should call get API with base parameters', async () => {
-    const alertId = uuid.v4();
-    const resolvedValue = {
-      alertTypeState: {
-        some: 'value',
-      },
-      alertInstances: {
-        first_instance: {},
-        second_instance: {},
-      },
-    };
-    http.get.mockResolvedValueOnce(resolvedValue);
-
-    expect(await loadAlertState({ http, alertId })).toEqual(resolvedValue);
-    expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}/state`);
-  });
-
-  test('should parse AlertInstances', async () => {
-    const alertId = uuid.v4();
-    const resolvedValue = {
-      alertTypeState: {
-        some: 'value',
-      },
-      alertInstances: {
-        first_instance: {
-          state: {},
-          meta: {
-            lastScheduledActions: {
-              group: 'first_group',
-              date: '2020-02-09T23:15:41.941Z',
-            },
-          },
-        },
-      },
-    };
-    http.get.mockResolvedValueOnce(resolvedValue);
-
-    expect(await loadAlertState({ http, alertId })).toEqual({
-      ...resolvedValue,
-      alertInstances: {
-        first_instance: {
-          state: {},
-          meta: {
-            lastScheduledActions: {
-              group: 'first_group',
-              date: new Date('2020-02-09T23:15:41.941Z'),
-            },
-          },
-        },
-      },
-    });
-    expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}/state`);
-  });
-
-  test('should handle empty response from api', async () => {
-    const alertId = uuid.v4();
-    http.get.mockResolvedValueOnce('');
-
-    expect(await loadAlertState({ http, alertId })).toEqual({});
-    expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}/state`);
-  });
-});
diff --git a/x-pack/plugins/alerts/public/alert_api.ts b/x-pack/plugins/alerts/public/alert_api.ts
index 5b7cd2128f386..753158ed1ed42 100644
--- a/x-pack/plugins/alerts/public/alert_api.ts
+++ b/x-pack/plugins/alerts/public/alert_api.ts
@@ -5,15 +5,9 @@
  */
 
 import { HttpSetup } from 'kibana/public';
-import * as t from 'io-ts';
-import { pipe } from 'fp-ts/lib/pipeable';
-import { fold } from 'fp-ts/lib/Either';
-import { findFirst } from 'fp-ts/lib/Array';
-import { isNone } from 'fp-ts/lib/Option';
-
 import { i18n } from '@kbn/i18n';
-import { BASE_ALERT_API_PATH, alertStateSchema } from '../common';
-import { Alert, AlertType, AlertTaskState } from '../common';
+import { BASE_ALERT_API_PATH } from '../common';
+import type { Alert, AlertType } from '../common';
 
 export async function loadAlertTypes({ http }: { http: HttpSetup }): Promise<AlertType[]> {
   return await http.get(`${BASE_ALERT_API_PATH}/list_alert_types`);
@@ -26,10 +20,10 @@ export async function loadAlertType({
   http: HttpSetup;
   id: AlertType['id'];
 }): Promise<AlertType> {
-  const maybeAlertType = findFirst<AlertType>((type) => type.id === id)(
-    await http.get(`${BASE_ALERT_API_PATH}/list_alert_types`)
-  );
-  if (isNone(maybeAlertType)) {
+  const maybeAlertType = ((await http.get(
+    `${BASE_ALERT_API_PATH}/list_alert_types`
+  )) as AlertType[]).find((type) => type.id === id);
+  if (!maybeAlertType) {
     throw new Error(
       i18n.translate('xpack.alerts.loadAlertType.missingAlertTypeError', {
         defaultMessage: 'Alert type "{id}" is not registered.',
@@ -39,7 +33,7 @@ export async function loadAlertType({
       })
     );
   }
-  return maybeAlertType.value;
+  return maybeAlertType;
 }
 
 export async function loadAlert({
@@ -51,24 +45,3 @@ export async function loadAlert({
 }): Promise<Alert> {
   return await http.get(`${BASE_ALERT_API_PATH}/alert/${alertId}`);
 }
-
-type EmptyHttpResponse = '';
-export async function loadAlertState({
-  http,
-  alertId,
-}: {
-  http: HttpSetup;
-  alertId: string;
-}): Promise<AlertTaskState> {
-  return await http
-    .get(`${BASE_ALERT_API_PATH}/alert/${alertId}/state`)
-    .then((state: AlertTaskState | EmptyHttpResponse) => (state ? state : {}))
-    .then((state: AlertTaskState) => {
-      return pipe(
-        alertStateSchema.decode(state),
-        fold((e: t.Errors) => {
-          throw new Error(`Alert "${alertId}" has invalid state`);
-        }, t.identity)
-      );
-    });
-}
diff --git a/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts b/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts
index 7c2f50211d4af..d34481850ca4a 100644
--- a/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts
+++ b/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts
@@ -5,7 +5,7 @@
  */
 
 import { HttpSetup } from 'kibana/public';
-import * as t from 'io-ts';
+import { Errors, identity } from 'io-ts';
 import { pipe } from 'fp-ts/lib/pipeable';
 import { fold } from 'fp-ts/lib/Either';
 import { pick } from 'lodash';
@@ -48,9 +48,9 @@ export async function loadAlertState({
     .then((state: AlertTaskState) => {
       return pipe(
         alertStateSchema.decode(state),
-        fold((e: t.Errors) => {
+        fold((e: Errors) => {
           throw new Error(`Alert "${alertId}" has invalid state`);
-        }, t.identity)
+        }, identity)
       );
     });
 }
diff --git a/yarn.lock b/yarn.lock
index 1cde1266ca38f..1f5eba54fb835 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -14045,9 +14045,9 @@ fp-ts@^1.0.0:
   integrity sha512-fWwnAgVlTsV26Ruo9nx+fxNHIm6l1puE1VJ/C0XJ3nRQJJJIgRHYw6sigB3MuNFZL1o4fpGlhwFhcbxHK0RsOA==
 
 fp-ts@^2.3.1:
-  version "2.3.1"
-  resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.3.1.tgz#8068bfcca118227932941101e062134d7ecd9119"
-  integrity sha512-KevPBnYt0aaJiuUzmU9YIxjrhC9AgJ8CLtLlXmwArovlNTeYM5NtEoKd86B0wHd7FIbzeE8sNXzCoYIOr7e6Iw==
+  version "2.8.6"
+  resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.8.6.tgz#1a0e6c3f29f5b0fbfa3120f034ea266aa73c811b"
+  integrity sha512-fGGpKf/Jy3UT4s16oM+hr/8F5QXFcZ+20NAvaZXH5Y5jsiLPMDCaNqffXq0z1Kr6ZUJj0346cH9tq+cI2SoJ4w==
 
 fragment-cache@^0.2.1:
   version "0.2.1"

From f99766e7ab838c1424f7633a3c5182705abf0eaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= <sorenlouv@gmail.com>
Date: Fri, 27 Nov 2020 10:19:11 +0100
Subject: [PATCH 34/37] [APM] Fix missing `service.node.name` (#84269)

---
 .../app/service_node_metrics/index.tsx        | 25 ++++-------------
 .../public/hooks/useServiceMetricCharts.ts    |  6 ++--
 .../server/lib/metrics/by_agent/default.ts    |  4 +--
 .../by_agent/java/gc/get_gc_rate_chart.ts     | 16 +++++++----
 .../by_agent/java/gc/get_gc_time_chart.ts     | 16 +++++++----
 .../by_agent/java/heap_memory/index.ts        | 14 ++++++----
 .../server/lib/metrics/by_agent/java/index.ts | 28 +++++++++++--------
 .../by_agent/java/non_heap_memory/index.ts    | 14 ++++++----
 .../by_agent/java/thread_count/index.ts       | 14 ++++++----
 .../lib/metrics/by_agent/shared/cpu/index.ts  | 14 ++++++----
 .../metrics/by_agent/shared/memory/index.ts   | 14 ++++++----
 .../get_metrics_chart_data_by_agent.ts        |  2 +-
 .../apm/server/lib/metrics/queries.test.ts    | 10 +++----
 13 files changed, 98 insertions(+), 79 deletions(-)

diff --git a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx
index a74ff574bc0c8..a886c3f29d57c 100644
--- a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx
+++ b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx
@@ -59,7 +59,11 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) {
   const { urlParams, uiFilters } = useUrlParams();
   const { serviceName, serviceNodeName } = match.params;
   const { agentName } = useAgentName();
-  const { data } = useServiceMetricCharts(urlParams, agentName);
+  const { data } = useServiceMetricCharts(
+    urlParams,
+    agentName,
+    serviceNodeName
+  );
   const { start, end } = urlParams;
 
   const { data: { host, containerId } = INITIAL_DATA, status } = useFetcher(
@@ -177,25 +181,6 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) {
           </EuiFlexItem>
         </MetadataFlexGroup>
       )}
-      {agentName && (
-        <ChartPointerEventContextProvider>
-          <EuiFlexGrid columns={2} gutterSize="s">
-            {data.charts.map((chart) => (
-              <EuiFlexItem key={chart.key}>
-                <EuiPanel>
-                  <MetricsChart
-                    start={start}
-                    end={end}
-                    chart={chart}
-                    fetchStatus={status}
-                  />
-                </EuiPanel>
-              </EuiFlexItem>
-            ))}
-          </EuiFlexGrid>
-          <EuiSpacer size="xxl" />
-        </ChartPointerEventContextProvider>
-      )}
       <SearchBar />
       <EuiPage>
         {agentName && (
diff --git a/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts b/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts
index d264ad6069db3..7a54c6ffc6dbe 100644
--- a/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts
+++ b/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts
@@ -17,7 +17,8 @@ const INITIAL_DATA: MetricsChartsByAgentAPIResponse = {
 
 export function useServiceMetricCharts(
   urlParams: IUrlParams,
-  agentName?: string
+  agentName?: string,
+  serviceNodeName?: string
 ) {
   const { serviceName } = useParams<{ serviceName?: string }>();
   const { start, end } = urlParams;
@@ -30,6 +31,7 @@ export function useServiceMetricCharts(
           params: {
             path: { serviceName },
             query: {
+              serviceNodeName,
               start,
               end,
               agentName,
@@ -39,7 +41,7 @@ export function useServiceMetricCharts(
         });
       }
     },
-    [serviceName, start, end, agentName, uiFilters]
+    [serviceName, start, end, agentName, serviceNodeName, uiFilters]
   );
 
   return {
diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts
index fbcbc9f12791f..71bc1018b619b 100644
--- a/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts
@@ -13,8 +13,8 @@ export async function getDefaultMetricsCharts(
   serviceName: string
 ) {
   const charts = await Promise.all([
-    getCPUChartData(setup, serviceName),
-    getMemoryChartData(setup, serviceName),
+    getCPUChartData({ setup, serviceName }),
+    getMemoryChartData({ setup, serviceName }),
   ]);
 
   return { charts };
diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts
index 7cedeb828e3b7..8e68c229c555c 100644
--- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts
@@ -30,11 +30,15 @@ const chartBase: ChartBase = {
   series,
 };
 
-const getGcRateChart = (
-  setup: Setup & SetupTimeRange,
-  serviceName: string,
-  serviceNodeName?: string
-) => {
+function getGcRateChart({
+  setup,
+  serviceName,
+  serviceNodeName,
+}: {
+  setup: Setup & SetupTimeRange;
+  serviceName: string;
+  serviceNodeName?: string;
+}) {
   return fetchAndTransformGcMetrics({
     setup,
     serviceName,
@@ -42,6 +46,6 @@ const getGcRateChart = (
     chartBase,
     fieldName: METRIC_JAVA_GC_COUNT,
   });
-};
+}
 
 export { getGcRateChart };
diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts
index f21d3d8e7c056..e7a83e71a22e9 100644
--- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts
@@ -30,11 +30,15 @@ const chartBase: ChartBase = {
   series,
 };
 
-const getGcTimeChart = (
-  setup: Setup & SetupTimeRange,
-  serviceName: string,
-  serviceNodeName?: string
-) => {
+function getGcTimeChart({
+  setup,
+  serviceName,
+  serviceNodeName,
+}: {
+  setup: Setup & SetupTimeRange;
+  serviceName: string;
+  serviceNodeName?: string;
+}) {
   return fetchAndTransformGcMetrics({
     setup,
     serviceName,
@@ -42,6 +46,6 @@ const getGcTimeChart = (
     chartBase,
     fieldName: METRIC_JAVA_GC_TIME,
   });
-};
+}
 
 export { getGcTimeChart };
diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts
index eb79897f9f055..4f087594acc21 100644
--- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts
@@ -50,11 +50,15 @@ const chartBase: ChartBase = {
   series,
 };
 
-export async function getHeapMemoryChart(
-  setup: Setup & SetupTimeRange,
-  serviceName: string,
-  serviceNodeName?: string
-) {
+export async function getHeapMemoryChart({
+  setup,
+  serviceName,
+  serviceNodeName,
+}: {
+  setup: Setup & SetupTimeRange;
+  serviceName: string;
+  serviceNodeName?: string;
+}) {
   return fetchAndTransformMetrics({
     setup,
     serviceName,
diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts
index d4084701f0f49..4a1f3f572b3a5 100644
--- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts
@@ -13,19 +13,23 @@ import { getMemoryChartData } from '../shared/memory';
 import { getGcRateChart } from './gc/get_gc_rate_chart';
 import { getGcTimeChart } from './gc/get_gc_time_chart';
 
-export async function getJavaMetricsCharts(
-  setup: Setup & SetupTimeRange,
-  serviceName: string,
-  serviceNodeName?: string
-) {
+export async function getJavaMetricsCharts({
+  setup,
+  serviceName,
+  serviceNodeName,
+}: {
+  setup: Setup & SetupTimeRange;
+  serviceName: string;
+  serviceNodeName?: string;
+}) {
   const charts = await Promise.all([
-    getCPUChartData(setup, serviceName, serviceNodeName),
-    getMemoryChartData(setup, serviceName, serviceNodeName),
-    getHeapMemoryChart(setup, serviceName, serviceNodeName),
-    getNonHeapMemoryChart(setup, serviceName, serviceNodeName),
-    getThreadCountChart(setup, serviceName, serviceNodeName),
-    getGcRateChart(setup, serviceName, serviceNodeName),
-    getGcTimeChart(setup, serviceName, serviceNodeName),
+    getCPUChartData({ setup, serviceName, serviceNodeName }),
+    getMemoryChartData({ setup, serviceName, serviceNodeName }),
+    getHeapMemoryChart({ setup, serviceName, serviceNodeName }),
+    getNonHeapMemoryChart({ setup, serviceName, serviceNodeName }),
+    getThreadCountChart({ setup, serviceName, serviceNodeName }),
+    getGcRateChart({ setup, serviceName, serviceNodeName }),
+    getGcTimeChart({ setup, serviceName, serviceNodeName }),
   ]);
 
   return { charts };
diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts
index 50cc449da3c15..dc2684f707d17 100644
--- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts
@@ -47,11 +47,15 @@ const chartBase: ChartBase = {
   series,
 };
 
-export async function getNonHeapMemoryChart(
-  setup: Setup & SetupTimeRange,
-  serviceName: string,
-  serviceNodeName?: string
-) {
+export async function getNonHeapMemoryChart({
+  setup,
+  serviceName,
+  serviceNodeName,
+}: {
+  setup: Setup & SetupTimeRange;
+  serviceName: string;
+  serviceNodeName?: string;
+}) {
   return fetchAndTransformMetrics({
     setup,
     serviceName,
diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts
index 0062f0a423970..b42ab61f5c766 100644
--- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts
@@ -39,11 +39,15 @@ const chartBase: ChartBase = {
   series,
 };
 
-export async function getThreadCountChart(
-  setup: Setup & SetupTimeRange,
-  serviceName: string,
-  serviceNodeName?: string
-) {
+export async function getThreadCountChart({
+  setup,
+  serviceName,
+  serviceNodeName,
+}: {
+  setup: Setup & SetupTimeRange;
+  serviceName: string;
+  serviceNodeName?: string;
+}) {
   return fetchAndTransformMetrics({
     setup,
     serviceName,
diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts
index ca642aa12fff1..83311caf99516 100644
--- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts
@@ -51,11 +51,15 @@ const chartBase: ChartBase = {
   series,
 };
 
-export async function getCPUChartData(
-  setup: Setup & SetupTimeRange,
-  serviceName: string,
-  serviceNodeName?: string
-) {
+export async function getCPUChartData({
+  setup,
+  serviceName,
+  serviceNodeName,
+}: {
+  setup: Setup & SetupTimeRange;
+  serviceName: string;
+  serviceNodeName?: string;
+}) {
   const metricsChart = await fetchAndTransformMetrics({
     setup,
     serviceName,
diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts
index e6ee47cc815ef..8fd234903d58e 100644
--- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts
@@ -68,11 +68,15 @@ export const percentCgroupMemoryUsedScript = {
     `,
 };
 
-export async function getMemoryChartData(
-  setup: Setup & SetupTimeRange,
-  serviceName: string,
-  serviceNodeName?: string
-) {
+export async function getMemoryChartData({
+  setup,
+  serviceName,
+  serviceNodeName,
+}: {
+  setup: Setup & SetupTimeRange;
+  serviceName: string;
+  serviceNodeName?: string;
+}) {
   const cgroupResponse = await fetchAndTransformMetrics({
     setup,
     serviceName,
diff --git a/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts b/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts
index 72cd65deebff6..1fc510b77c856 100644
--- a/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts
@@ -25,7 +25,7 @@ export async function getMetricsChartDataByAgent({
 }): Promise<MetricsChartsByAgentAPIResponse> {
   switch (agentName) {
     case 'java': {
-      return getJavaMetricsCharts(setup, serviceName, serviceNodeName);
+      return getJavaMetricsCharts({ setup, serviceName, serviceNodeName });
     }
 
     default: {
diff --git a/x-pack/plugins/apm/server/lib/metrics/queries.test.ts b/x-pack/plugins/apm/server/lib/metrics/queries.test.ts
index fc24377ebf390..0bf3d3daaad66 100644
--- a/x-pack/plugins/apm/server/lib/metrics/queries.test.ts
+++ b/x-pack/plugins/apm/server/lib/metrics/queries.test.ts
@@ -21,7 +21,7 @@ describe('metrics queries', () => {
   const createTests = (serviceNodeName?: string) => {
     it('fetches cpu chart data', async () => {
       mock = await inspectSearchParams((setup) =>
-        getCPUChartData(setup, 'foo', serviceNodeName)
+        getCPUChartData({ setup, serviceName: 'foo', serviceNodeName })
       );
 
       expect(mock.params).toMatchSnapshot();
@@ -29,7 +29,7 @@ describe('metrics queries', () => {
 
     it('fetches memory chart data', async () => {
       mock = await inspectSearchParams((setup) =>
-        getMemoryChartData(setup, 'foo', serviceNodeName)
+        getMemoryChartData({ setup, serviceName: 'foo', serviceNodeName })
       );
 
       expect(mock.params).toMatchSnapshot();
@@ -37,7 +37,7 @@ describe('metrics queries', () => {
 
     it('fetches heap memory chart data', async () => {
       mock = await inspectSearchParams((setup) =>
-        getHeapMemoryChart(setup, 'foo', serviceNodeName)
+        getHeapMemoryChart({ setup, serviceName: 'foo', serviceNodeName })
       );
 
       expect(mock.params).toMatchSnapshot();
@@ -45,7 +45,7 @@ describe('metrics queries', () => {
 
     it('fetches non heap memory chart data', async () => {
       mock = await inspectSearchParams((setup) =>
-        getNonHeapMemoryChart(setup, 'foo', serviceNodeName)
+        getNonHeapMemoryChart({ setup, serviceName: 'foo', serviceNodeName })
       );
 
       expect(mock.params).toMatchSnapshot();
@@ -53,7 +53,7 @@ describe('metrics queries', () => {
 
     it('fetches thread count chart data', async () => {
       mock = await inspectSearchParams((setup) =>
-        getThreadCountChart(setup, 'foo', serviceNodeName)
+        getThreadCountChart({ setup, serviceName: 'foo', serviceNodeName })
       );
 
       expect(mock.params).toMatchSnapshot();

From 31c1ca0026ffd17db16ff46214cb367e4f2fa513 Mon Sep 17 00:00:00 2001
From: Tiago Costa <tiagoffcc@hotmail.com>
Date: Fri, 27 Nov 2020 15:14:42 +0000
Subject: [PATCH 35/37] skip flaky suite (#84445)

---
 test/api_integration/apis/saved_objects/migrations.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/api_integration/apis/saved_objects/migrations.ts b/test/api_integration/apis/saved_objects/migrations.ts
index 99a58620b17f5..b5fa16558514a 100644
--- a/test/api_integration/apis/saved_objects/migrations.ts
+++ b/test/api_integration/apis/saved_objects/migrations.ts
@@ -54,7 +54,8 @@ function getLogMock() {
 export default ({ getService }: FtrProviderContext) => {
   const esClient = getService('es');
 
-  describe('Kibana index migration', () => {
+  // FLAKY: https://github.com/elastic/kibana/issues/84445
+  describe.skip('Kibana index migration', () => {
     before(() => esClient.indices.delete({ index: '.migrate-*' }));
 
     it('Migrates an existing index that has never been migrated before', async () => {

From d21e4f5af41369289023d2ead593b7a6d0bec436 Mon Sep 17 00:00:00 2001
From: Tiago Costa <tiagoffcc@hotmail.com>
Date: Sat, 28 Nov 2020 03:49:04 +0000
Subject: [PATCH 36/37] skip flaky suite (#84440)

---
 x-pack/test/functional/apps/grok_debugger/grok_debugger.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/functional/apps/grok_debugger/grok_debugger.js b/x-pack/test/functional/apps/grok_debugger/grok_debugger.js
index 771ca2088c055..8513429639e58 100644
--- a/x-pack/test/functional/apps/grok_debugger/grok_debugger.js
+++ b/x-pack/test/functional/apps/grok_debugger/grok_debugger.js
@@ -11,7 +11,8 @@ export default function ({ getService, getPageObjects }) {
 
   const PageObjects = getPageObjects(['grokDebugger']);
 
-  describe('grok debugger app', function () {
+  // FLAKY: https://github.com/elastic/kibana/issues/84440
+  describe.skip('grok debugger app', function () {
     this.tags('includeFirefox');
     before(async () => {
       await esArchiver.load('empty_kibana');

From bdf7b88b45329b29d8affcb5a0dc5452ed97210e Mon Sep 17 00:00:00 2001
From: Madison Caldwell <madison.rey.caldwell@gmail.com>
Date: Sun, 29 Nov 2020 22:10:23 -0500
Subject: [PATCH 37/37] [Security Solution][Detections] Handle dupes when
 processing threshold rules (#83062)

* Fix threshold rule synthetic signal generation

* Use top_hits aggregation

* Find signals and aggregate over search terms

* Exclude dupes

* Fixes to algorithm

* Sync timestamps with events/signals

* Add timestampOverride

* Revert changes in signal creation

* Simplify query, return 10k buckets

* Account for when threshold.field is not supplied

* Ensure we're getting the last event when threshold.field is not provided

* Add missing import

* Handle case where threshold field not supplied

* Fix type errors

* Handle non-ECS fields

* Regorganize

* Address comments

* Fix type error

* Add unit test for buildBulkBody on threshold results

* Add threshold_count back to mapping (and deprecate)

* Timestamp fixes

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../routes/index/get_signals_template.ts      |   2 +-
 .../routes/index/signals_mapping.json         |  10 ++
 .../signals/build_bulk_body.test.ts           | 109 ++++++++++++++++++
 .../signals/build_bulk_body.ts                |   1 +
 .../detection_engine/signals/build_signal.ts  |   4 +-
 .../signals/bulk_create_threshold_signals.ts  |  10 +-
 .../find_previous_threshold_signals.ts        |  87 ++++++++++++++
 .../signals/find_threshold_signals.ts         |   1 +
 .../signals/signal_rule_alert_type.ts         |  45 +++++++-
 .../lib/detection_engine/signals/types.ts     |  14 ++-
 .../endpoint/resolver/signals/mappings.json   |  10 ++
 .../es_archives/export_rule/mappings.json     |  10 ++
 12 files changed, 296 insertions(+), 7 deletions(-)
 create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts

diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts
index d1a9b701b2c9d..8ea1faa84cfba 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts
@@ -7,7 +7,7 @@
 import signalsMapping from './signals_mapping.json';
 import ecsMapping from './ecs_mapping.json';
 
-export const SIGNALS_TEMPLATE_VERSION = 2;
+export const SIGNALS_TEMPLATE_VERSION = 3;
 export const MIN_EQL_RULE_INDEX_VERSION = 2;
 
 export const getSignalsTemplate = (index: string) => {
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json
index 4e9477f3f2f73..96868e62ea978 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json
@@ -337,6 +337,16 @@
           "threshold_count": {
             "type": "float"
           },
+          "threshold_result": {
+            "properties": {
+              "count": {
+                "type": "long"
+              },
+              "value": {
+                "type": "keyword"
+              }
+            }
+          },
           "depth": {
             "type": "integer"
           }
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts
index ad060a9304e84..cd61fbcfd0fc7 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts
@@ -123,6 +123,115 @@ describe('buildBulkBody', () => {
     expect(fakeSignalSourceHit).toEqual(expected);
   });
 
+  test('bulk body builds well-defined body with threshold results', () => {
+    const sampleParams = sampleRuleAlertParams();
+    const baseDoc = sampleDocNoSortId();
+    const doc: SignalSourceHit = {
+      ...baseDoc,
+      _source: {
+        ...baseDoc._source,
+        threshold_result: {
+          count: 5,
+          value: 'abcd',
+        },
+      },
+    };
+    delete doc._source.source;
+    const fakeSignalSourceHit = buildBulkBody({
+      doc,
+      ruleParams: sampleParams,
+      id: sampleRuleGuid,
+      name: 'rule-name',
+      actions: [],
+      createdAt: '2020-01-28T15:58:34.810Z',
+      updatedAt: '2020-01-28T15:59:14.004Z',
+      createdBy: 'elastic',
+      updatedBy: 'elastic',
+      interval: '5m',
+      enabled: true,
+      tags: ['some fake tag 1', 'some fake tag 2'],
+      throttle: 'no_actions',
+    });
+    // Timestamp will potentially always be different so remove it for the test
+    // @ts-expect-error
+    delete fakeSignalSourceHit['@timestamp'];
+    const expected: Omit<SignalHit, '@timestamp'> & { someKey: 'someValue' } = {
+      someKey: 'someValue',
+      event: {
+        kind: 'signal',
+      },
+      signal: {
+        parent: {
+          id: sampleIdGuid,
+          type: 'event',
+          index: 'myFakeSignalIndex',
+          depth: 0,
+        },
+        parents: [
+          {
+            id: sampleIdGuid,
+            type: 'event',
+            index: 'myFakeSignalIndex',
+            depth: 0,
+          },
+        ],
+        ancestors: [
+          {
+            id: sampleIdGuid,
+            type: 'event',
+            index: 'myFakeSignalIndex',
+            depth: 0,
+          },
+        ],
+        original_time: '2020-04-20T21:27:45+0000',
+        status: 'open',
+        rule: {
+          actions: [],
+          author: ['Elastic'],
+          building_block_type: 'default',
+          id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+          rule_id: 'rule-1',
+          false_positives: [],
+          max_signals: 10000,
+          risk_score: 50,
+          risk_score_mapping: [],
+          output_index: '.siem-signals',
+          description: 'Detecting root and admin users',
+          from: 'now-6m',
+          immutable: false,
+          index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+          interval: '5m',
+          language: 'kuery',
+          license: 'Elastic License',
+          name: 'rule-name',
+          query: 'user.name: root or user.name: admin',
+          references: ['http://google.com'],
+          severity: 'high',
+          severity_mapping: [],
+          tags: ['some fake tag 1', 'some fake tag 2'],
+          threat: [],
+          throttle: 'no_actions',
+          type: 'query',
+          to: 'now',
+          note: '',
+          enabled: true,
+          created_by: 'elastic',
+          updated_by: 'elastic',
+          version: 1,
+          created_at: fakeSignalSourceHit.signal.rule?.created_at,
+          updated_at: fakeSignalSourceHit.signal.rule?.updated_at,
+          exceptions_list: getListArrayMock(),
+        },
+        threshold_result: {
+          count: 5,
+          value: 'abcd',
+        },
+        depth: 1,
+      },
+    };
+    expect(fakeSignalSourceHit).toEqual(expected);
+  });
+
   test('bulk body builds original_event if it exists on the event to begin with', () => {
     const sampleParams = sampleRuleAlertParams();
     const doc = sampleDocNoSortId();
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts
index e50956e9ef752..e6e6d2e0f5fa9 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts
@@ -71,6 +71,7 @@ export const buildBulkBody = ({
     ...buildSignal([doc], rule),
     ...additionalSignalFields(doc),
   };
+  delete doc._source.threshold_result;
   const event = buildEventTypeSignal(doc);
   const signalHit: SignalHit = {
     ...doc._source,
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts
index b36a1cbb4a6b3..9cf2462526cfc 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts
@@ -96,9 +96,9 @@ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal =>
 export const additionalSignalFields = (doc: BaseSignalHit) => {
   return {
     parent: buildParent(removeClashes(doc)),
-    original_time: doc._source['@timestamp'],
+    original_time: doc._source['@timestamp'], // This field has already been replaced with timestampOverride, if provided.
     original_event: doc._source.event ?? undefined,
-    threshold_count: doc._source.threshold_count ?? undefined,
+    threshold_result: doc._source.threshold_result,
     original_signal:
       doc._source.signal != null && !isEventTypeSignal(doc) ? doc._source.signal : undefined,
   };
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts
index edaaa345d8a69..a98aae4ec8107 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts
@@ -149,7 +149,10 @@ const getTransformedHits = (
 
     const source = {
       '@timestamp': get(timestampOverride ?? '@timestamp', hit._source),
-      threshold_count: totalResults,
+      threshold_result: {
+        count: totalResults,
+        value: ruleId,
+      },
       ...getThresholdSignalQueryFields(hit, filter),
     };
 
@@ -176,7 +179,10 @@ const getTransformedHits = (
 
         const source = {
           '@timestamp': get(timestampOverride ?? '@timestamp', hit._source),
-          threshold_count: docCount,
+          threshold_result: {
+            count: docCount,
+            value: get(threshold.field, hit._source),
+          },
           ...getThresholdSignalQueryFields(hit, filter),
         };
 
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts
new file mode 100644
index 0000000000000..960693bc703d6
--- /dev/null
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts
@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas';
+import { singleSearchAfter } from './single_search_after';
+
+import { AlertServices } from '../../../../../alerts/server';
+import { Logger } from '../../../../../../../src/core/server';
+import { SignalSearchResponse } from './types';
+import { BuildRuleMessage } from './rule_messages';
+
+interface FindPreviousThresholdSignalsParams {
+  from: string;
+  to: string;
+  indexPattern: string[];
+  services: AlertServices;
+  logger: Logger;
+  ruleId: string;
+  bucketByField: string;
+  timestampOverride: TimestampOverrideOrUndefined;
+  buildRuleMessage: BuildRuleMessage;
+}
+
+export const findPreviousThresholdSignals = async ({
+  from,
+  to,
+  indexPattern,
+  services,
+  logger,
+  ruleId,
+  bucketByField,
+  timestampOverride,
+  buildRuleMessage,
+}: FindPreviousThresholdSignalsParams): Promise<{
+  searchResult: SignalSearchResponse;
+  searchDuration: string;
+  searchErrors: string[];
+}> => {
+  const aggregations = {
+    threshold: {
+      terms: {
+        field: 'signal.threshold_result.value',
+      },
+      aggs: {
+        lastSignalTimestamp: {
+          max: {
+            field: 'signal.original_time', // timestamp of last event captured by bucket
+          },
+        },
+      },
+    },
+  };
+
+  const filter = {
+    bool: {
+      must: [
+        {
+          term: {
+            'signal.rule.rule_id': ruleId,
+          },
+        },
+        {
+          term: {
+            'signal.rule.threshold.field': bucketByField,
+          },
+        },
+      ],
+    },
+  };
+
+  return singleSearchAfter({
+    aggregations,
+    searchAfterSortId: undefined,
+    timestampOverride,
+    index: indexPattern,
+    from,
+    to,
+    services,
+    logger,
+    filter,
+    pageSize: 0,
+    buildRuleMessage,
+  });
+};
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts
index 01e4812b9c8bf..7141b61a23e6e 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts
@@ -51,6 +51,7 @@ export const findThresholdSignals = async ({
             terms: {
               field: threshold.field,
               min_doc_count: threshold.value,
+              size: 10000, // max 10k buckets
             },
             aggs: {
               // Get the most recent hit per bucket
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
index 003626e319007..f0b1825c7cc99 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
@@ -8,6 +8,7 @@
 
 import { Logger, KibanaRequest } from 'src/core/server';
 
+import { Filter } from 'src/plugins/data/common';
 import {
   SIGNALS_ID,
   DEFAULT_SEARCH_AFTER_PAGE_SIZE,
@@ -29,6 +30,7 @@ import {
   RuleAlertAttributes,
   EqlSignalSearchResponse,
   BaseSignalHit,
+  ThresholdQueryBucket,
 } from './types';
 import {
   getGapBetweenRuns,
@@ -46,6 +48,7 @@ import { signalParamsSchema } from './signal_params_schema';
 import { siemRuleActionGroups } from './siem_rule_action_groups';
 import { findMlSignals } from './find_ml_signals';
 import { findThresholdSignals } from './find_threshold_signals';
+import { findPreviousThresholdSignals } from './find_previous_threshold_signals';
 import { bulkCreateMlSignals } from './bulk_create_ml_signals';
 import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals';
 import {
@@ -300,6 +303,46 @@ export const signalRulesAlertType = ({
             lists: exceptionItems ?? [],
           });
 
+          const {
+            searchResult: previousSignals,
+            searchErrors: previousSearchErrors,
+          } = await findPreviousThresholdSignals({
+            indexPattern: [outputIndex],
+            from,
+            to,
+            services,
+            logger,
+            ruleId,
+            bucketByField: threshold.field,
+            timestampOverride,
+            buildRuleMessage,
+          });
+
+          previousSignals.aggregations.threshold.buckets.forEach((bucket: ThresholdQueryBucket) => {
+            esFilter.bool.filter.push(({
+              bool: {
+                must_not: {
+                  bool: {
+                    must: [
+                      {
+                        term: {
+                          [threshold.field ?? 'signal.rule.rule_id']: bucket.key,
+                        },
+                      },
+                      {
+                        range: {
+                          [timestampOverride ?? '@timestamp']: {
+                            lte: bucket.lastSignalTimestamp.value_as_string,
+                          },
+                        },
+                      },
+                    ],
+                  },
+                },
+              },
+            } as unknown) as Filter);
+          });
+
           const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({
             inputIndexPattern: inputIndex,
             from,
@@ -349,7 +392,7 @@ export const signalRulesAlertType = ({
             }),
             createSearchAfterReturnType({
               success,
-              errors: [...errors, ...searchErrors],
+              errors: [...errors, ...previousSearchErrors, ...searchErrors],
               createdSignalsCount: createdItemsCount,
               bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [],
             }),
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
index cda3c97c08531..9e81797b14731 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
@@ -46,6 +46,11 @@ export interface SignalsStatusParams {
   status: Status;
 }
 
+export interface ThresholdResult {
+  count: number;
+  value: string;
+}
+
 export interface SignalSource {
   [key: string]: SearchTypes;
   // TODO: SignalSource is being used as the type for documents matching detection engine queries, but they may not
@@ -67,6 +72,7 @@ export interface SignalSource {
     // signal.depth doesn't exist on pre-7.10 signals
     depth?: number;
   };
+  threshold_result?: ThresholdResult;
 }
 
 export interface BulkItem {
@@ -156,7 +162,7 @@ export interface Signal {
   original_time?: string;
   original_event?: SearchTypes;
   status: Status;
-  threshold_count?: SearchTypes;
+  threshold_result?: ThresholdResult;
   original_signal?: SearchTypes;
   depth: number;
 }
@@ -239,3 +245,9 @@ export interface SearchAfterAndBulkCreateReturnType {
 export interface ThresholdAggregationBucket extends TermAggregationBucket {
   top_threshold_hits: BaseSearchResponse<SignalSource>;
 }
+
+export interface ThresholdQueryBucket extends TermAggregationBucket {
+  lastSignalTimestamp: {
+    value_as_string: string;
+  };
+}
diff --git a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json
index ad77961a41445..3c6042e8efd24 100644
--- a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json
+++ b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json
@@ -2582,6 +2582,16 @@
             },
             "threshold_count": {
               "type": "float"
+            },
+            "threshold_result": {
+              "properties": {
+                "count": {
+                  "type": "long"
+                },
+                "value": {
+                  "type": "keyword"
+                }
+              }
             }
           }
         },
diff --git a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json
index 757121df53d44..f701f811b244b 100644
--- a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json
+++ b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json
@@ -5131,6 +5131,16 @@
             },
             "threshold_count": {
               "type": "float"
+            },
+            "threshold_result": {
+              "properties": {
+                "count": {
+                  "type": "long"
+                },
+                "value": {
+                  "type": "keyword"
+                }
+              }
             }
           }
         },