From 5b2e119356c33543a20283a64e1f7ae8555a1fc3 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 25 Nov 2020 16:57:03 +0100 Subject: [PATCH 01/37] add live region for field search (#84310) --- .../datapanel.test.tsx | 19 +++++++++++++++++++ .../indexpattern_datasource/datapanel.tsx | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx index ac82caf9d5227..3d55494fd260c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx @@ -718,6 +718,25 @@ describe('IndexPattern Data Panel', () => { ]); }); + it('should announce filter in live region', () => { + const wrapper = mountWithIntl(); + act(() => { + wrapper.find('[data-test-subj="lnsIndexPatternFieldSearch"]').prop('onChange')!({ + target: { value: 'me' }, + } as ChangeEvent); + }); + + wrapper + .find('[data-test-subj="lnsIndexPatternEmptyFields"]') + .find('button') + .first() + .simulate('click'); + + expect(wrapper.find('[aria-live="polite"]').text()).toEqual( + '1 available field. 1 empty field. 0 meta fields.' + ); + }); + it('should filter down by type', () => { const wrapper = mountWithIntl(); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx index f2c7d7fc20926..ad5509dd88bc9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx @@ -18,10 +18,12 @@ import { EuiSpacer, EuiFilterGroup, EuiFilterButton, + EuiScreenReaderOnly, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { DataPublicPluginStart, EsQueryConfig, Query, Filter } from 'src/plugins/data/public'; +import { htmlIdGenerator } from '@elastic/eui'; import { DatasourceDataPanelProps, DataType, StateSetter } from '../types'; import { ChildDragDropProvider, DragContextState } from '../drag_drop'; import { @@ -222,6 +224,9 @@ const fieldFiltersLabel = i18n.translate('xpack.lens.indexPatterns.fieldFiltersL defaultMessage: 'Field filters', }); +const htmlId = htmlIdGenerator('datapanel'); +const fieldSearchDescriptionId = htmlId(); + export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ currentIndexPatternId, indexPatternRefs, @@ -489,6 +494,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ aria-label={i18n.translate('xpack.lens.indexPatterns.filterByNameAriaLabel', { defaultMessage: 'Search fields', })} + aria-describedby={fieldSearchDescriptionId} /> @@ -550,6 +556,19 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ + +
+ {i18n.translate('xpack.lens.indexPatterns.fieldSearchLiveRegion', { + defaultMessage: + '{availableFields} available {availableFields, plural, one {field} other {fields}}. {emptyFields} empty {emptyFields, plural, one {field} other {fields}}. {metaFields} meta {metaFields, plural, one {field} other {fields}}.', + values: { + availableFields: fieldGroups.AvailableFields.fields.length, + emptyFields: fieldGroups.EmptyFields.fields.length, + metaFields: fieldGroups.MetaFields.fields.length, + }, + })} +
+ From 9bef42b2a8078d070b206c6860c65df638d3b323 Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Wed, 25 Nov 2020 10:59:52 -0500 Subject: [PATCH 02/37] redirect to visualize listing page when by value visualization editor doesn't have a value input (#84287) --- .../application/components/visualize_byvalue_editor.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx b/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx index a63f597f10135..1c1eb9956a329 100644 --- a/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx +++ b/src/plugins/visualize/public/application/components/visualize_byvalue_editor.tsx @@ -33,6 +33,7 @@ import { import { VisualizeServices } from '../types'; import { VisualizeEditorCommon } from './visualize_editor_common'; import { VisualizeAppProps } from '../app'; +import { VisualizeConstants } from '../..'; export const VisualizeByValueEditor = ({ onAppLeave }: VisualizeAppProps) => { const [originatingApp, setOriginatingApp] = useState(); @@ -52,7 +53,8 @@ export const VisualizeByValueEditor = ({ onAppLeave }: VisualizeAppProps) => { setValueInput(valueInputValue); setEmbeddableId(embeddableIdValue); if (!valueInputValue) { - history.back(); + // if there is no value input to load, redirect to the visualize listing page. + services.history.replace(VisualizeConstants.LANDING_PAGE_PATH); } }, [services]); From 351cd6d19d0367ee606f4dd32aac4ee4190844ae Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Wed, 25 Nov 2020 11:03:00 -0500 Subject: [PATCH 03/37] [Time to Visualize] Fix Unlink Action via Rollback of ReplacePanel (#83873) * Fixed unlink action via rollback of replacePanel changes --- .../actions/add_to_library_action.test.tsx | 18 +++++++--- .../actions/add_to_library_action.tsx | 5 ++- .../unlink_from_library_action.test.tsx | 18 +++++++--- .../actions/unlink_from_library_action.tsx | 5 ++- .../embeddable/dashboard_container.tsx | 33 ++++++++++++++++--- 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx b/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx index feb30b248c066..5f3945e733527 100644 --- a/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx +++ b/src/plugins/dashboard/public/application/actions/add_to_library_action.test.tsx @@ -137,12 +137,17 @@ test('Add to library is not compatible when embeddable is not in a dashboard con test('Add to library replaces embeddableId and retains panel count', async () => { const dashboard = embeddable.getRoot() as IContainer; const originalPanelCount = Object.keys(dashboard.getInput().panels).length; + const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels)); const action = new AddToLibraryAction({ toasts: coreStart.notifications.toasts }); await action.execute({ embeddable }); expect(Object.keys(container.getInput().panels).length).toEqual(originalPanelCount); - expect(Object.keys(container.getInput().panels)).toContain(embeddable.id); - const newPanel = container.getInput().panels[embeddable.id!]; + + const newPanelId = Object.keys(container.getInput().panels).find( + (key) => !originalPanelKeySet.has(key) + ); + expect(newPanelId).toBeDefined(); + const newPanel = container.getInput().panels[newPanelId!]; expect(newPanel.type).toEqual(embeddable.type); }); @@ -158,10 +163,15 @@ test('Add to library returns reference type input', async () => { mockedByReferenceInput: { savedObjectId: 'testSavedObjectId', id: embeddable.id }, mockedByValueInput: { attributes: complicatedAttributes, id: embeddable.id } as EmbeddableInput, }); + const dashboard = embeddable.getRoot() as IContainer; + const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels)); const action = new AddToLibraryAction({ toasts: coreStart.notifications.toasts }); await action.execute({ embeddable }); - expect(Object.keys(container.getInput().panels)).toContain(embeddable.id); - const newPanel = container.getInput().panels[embeddable.id!]; + const newPanelId = Object.keys(container.getInput().panels).find( + (key) => !originalPanelKeySet.has(key) + ); + expect(newPanelId).toBeDefined(); + const newPanel = container.getInput().panels[newPanelId!]; expect(newPanel.type).toEqual(embeddable.type); expect(newPanel.explicitInput.attributes).toBeUndefined(); expect(newPanel.explicitInput.savedObjectId).toBe('testSavedObjectId'); diff --git a/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx b/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx index 179e5d522a2b3..08cd0c7a15381 100644 --- a/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx +++ b/src/plugins/dashboard/public/application/actions/add_to_library_action.tsx @@ -19,7 +19,6 @@ import { i18n } from '@kbn/i18n'; import _ from 'lodash'; -import uuid from 'uuid'; import { ActionByType, IncompatibleActionError } from '../../ui_actions_plugin'; import { ViewMode, PanelState, IEmbeddable } from '../../embeddable_plugin'; import { @@ -89,9 +88,9 @@ export class AddToLibraryAction implements ActionByType = { type: embeddable.type, - explicitInput: { ...newInput, id: uuid.v4() }, + explicitInput: { ...newInput }, }; - dashboard.replacePanel(panelToReplace, newPanel); + dashboard.replacePanel(panelToReplace, newPanel, true); const title = i18n.translate('dashboard.panel.addToLibrary.successMessage', { defaultMessage: `Panel '{panelTitle}' was added to the visualize library`, diff --git a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx index f191be6f7baad..6a9769b0c8d16 100644 --- a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx +++ b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.test.tsx @@ -135,11 +135,16 @@ test('Unlink is not compatible when embeddable is not in a dashboard container', test('Unlink replaces embeddableId and retains panel count', async () => { const dashboard = embeddable.getRoot() as IContainer; const originalPanelCount = Object.keys(dashboard.getInput().panels).length; + const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels)); const action = new UnlinkFromLibraryAction({ toasts: coreStart.notifications.toasts }); await action.execute({ embeddable }); expect(Object.keys(container.getInput().panels).length).toEqual(originalPanelCount); - expect(Object.keys(container.getInput().panels)).toContain(embeddable.id); - const newPanel = container.getInput().panels[embeddable.id!]; + + const newPanelId = Object.keys(container.getInput().panels).find( + (key) => !originalPanelKeySet.has(key) + ); + expect(newPanelId).toBeDefined(); + const newPanel = container.getInput().panels[newPanelId!]; expect(newPanel.type).toEqual(embeddable.type); }); @@ -159,10 +164,15 @@ test('Unlink unwraps all attributes from savedObject', async () => { mockedByReferenceInput: { savedObjectId: 'testSavedObjectId', id: embeddable.id }, mockedByValueInput: { attributes: complicatedAttributes, id: embeddable.id }, }); + const dashboard = embeddable.getRoot() as IContainer; + const originalPanelKeySet = new Set(Object.keys(dashboard.getInput().panels)); const action = new UnlinkFromLibraryAction({ toasts: coreStart.notifications.toasts }); await action.execute({ embeddable }); - expect(Object.keys(container.getInput().panels)).toContain(embeddable.id); - const newPanel = container.getInput().panels[embeddable.id!]; + const newPanelId = Object.keys(container.getInput().panels).find( + (key) => !originalPanelKeySet.has(key) + ); + expect(newPanelId).toBeDefined(); + const newPanel = container.getInput().panels[newPanelId!]; expect(newPanel.type).toEqual(embeddable.type); expect(newPanel.explicitInput.attributes).toEqual(complicatedAttributes); }); diff --git a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx index 5e16145364712..b20bbc6350aaa 100644 --- a/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx +++ b/src/plugins/dashboard/public/application/actions/unlink_from_library_action.tsx @@ -19,7 +19,6 @@ import { i18n } from '@kbn/i18n'; import _ from 'lodash'; -import uuid from 'uuid'; import { ActionByType, IncompatibleActionError } from '../../ui_actions_plugin'; import { ViewMode, PanelState, IEmbeddable } from '../../embeddable_plugin'; import { @@ -88,9 +87,9 @@ export class UnlinkFromLibraryAction implements ActionByType = { type: embeddable.type, - explicitInput: { ...newInput, id: uuid.v4() }, + explicitInput: { ...newInput }, }; - dashboard.replacePanel(panelToReplace, newPanel); + dashboard.replacePanel(panelToReplace, newPanel, true); const title = embeddable.getTitle() ? i18n.translate('dashboard.panel.unlinkFromLibrary.successMessageWithTitle', { diff --git a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx index 051a7ef8bfb92..e80d387fa3066 100644 --- a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx +++ b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx @@ -173,11 +173,30 @@ export class DashboardContainer extends Container, - newPanelState: Partial + newPanelState: Partial, + generateNewId?: boolean ) { - // Because the embeddable type can change, we have to operate at the container level here - return this.updateInput({ - panels: { + let panels; + if (generateNewId) { + // replace panel can be called with generateNewId in order to totally destroy and recreate the embeddable + panels = { ...this.input.panels }; + delete panels[previousPanelState.explicitInput.id]; + const newId = uuid.v4(); + panels[newId] = { + ...previousPanelState, + ...newPanelState, + gridData: { + ...previousPanelState.gridData, + i: newId, + }, + explicitInput: { + ...newPanelState.explicitInput, + id: newId, + }, + }; + } else { + // Because the embeddable type can change, we have to operate at the container level here + panels = { ...this.input.panels, [previousPanelState.explicitInput.id]: { ...previousPanelState, @@ -190,7 +209,11 @@ export class DashboardContainer extends Container Date: Wed, 25 Nov 2020 17:08:15 +0100 Subject: [PATCH 04/37] [APM] Elastic chart issues (#84238) * fixing charts * addressing pr comments --- .../ErrorGroupDetails/Distribution/index.tsx | 7 +- .../TransactionDetails/Distribution/index.tsx | 2 +- .../shared/charts/annotations/index.tsx | 45 ------- .../shared/charts/timeseries_chart.tsx | 38 +++++- .../transaction_breakdown_chart_contents.tsx | 36 +++++- .../charts/transaction_charts/index.tsx | 119 +++++++++--------- .../transaction_error_rate_chart/index.tsx | 1 + .../public/context/annotations_context.tsx | 49 ++++++++ .../apm/public/hooks/use_annotations.ts | 34 ++--- 9 files changed, 194 insertions(+), 137 deletions(-) delete mode 100644 x-pack/plugins/apm/public/components/shared/charts/annotations/index.tsx create mode 100644 x-pack/plugins/apm/public/context/annotations_context.tsx diff --git a/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx b/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx index 99316e3520a76..159f111bee04c 100644 --- a/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx +++ b/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx @@ -90,7 +90,12 @@ export function ErrorDistribution({ distribution, title }: Props) { showOverlappingTicks tickFormat={xFormatter} /> - + formatYShort(value)} /> ({ - dataValue: annotation['@timestamp'], - header: asAbsoluteDateTime(annotation['@timestamp']), - details: `${i18n.translate('xpack.apm.chart.annotation.version', { - defaultMessage: 'Version', - })} ${annotation.text}`, - }))} - style={{ line: { strokeWidth: 1, stroke: color, opacity: 1 } }} - marker={} - markerPosition={Position.Top} - /> - ); -} diff --git a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx index c4f5abe104aa9..ea6f2a4a233e5 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx @@ -5,28 +5,35 @@ */ import { + AnnotationDomainTypes, AreaSeries, Axis, Chart, CurveType, LegendItemListener, + LineAnnotation, LineSeries, niceTimeFormatter, Placement, Position, ScaleType, Settings, + YDomainRange, } from '@elastic/charts'; +import { EuiIcon } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import moment from 'moment'; import React, { useEffect } from 'react'; import { useHistory } from 'react-router-dom'; import { useChartTheme } from '../../../../../observability/public'; +import { asAbsoluteDateTime } from '../../../../common/utils/formatters'; import { TimeSeries } from '../../../../typings/timeseries'; import { FETCH_STATUS } from '../../../hooks/useFetcher'; +import { useTheme } from '../../../hooks/useTheme'; import { useUrlParams } from '../../../hooks/useUrlParams'; +import { useAnnotations } from '../../../hooks/use_annotations'; import { useChartPointerEvent } from '../../../hooks/use_chart_pointer_event'; import { unit } from '../../../style/variables'; -import { Annotations } from './annotations'; import { ChartContainer } from './chart_container'; import { onBrushEnd } from './helper/helper'; @@ -45,6 +52,7 @@ interface Props { */ yTickFormat?: (y: number) => string; showAnnotations?: boolean; + yDomain?: YDomainRange; } export function TimeseriesChart({ @@ -56,12 +64,16 @@ export function TimeseriesChart({ yLabelFormat, yTickFormat, showAnnotations = true, + yDomain, }: Props) { const history = useHistory(); const chartRef = React.createRef(); + const { annotations } = useAnnotations(); const chartTheme = useChartTheme(); const { pointerEvent, setPointerEvent } = useChartPointerEvent(); const { urlParams } = useUrlParams(); + const theme = useTheme(); + const { start, end } = urlParams; useEffect(() => { @@ -83,6 +95,8 @@ export function TimeseriesChart({ y === null || y === undefined ); + const annotationColor = theme.eui.euiColorSecondary; + return ( @@ -108,17 +122,35 @@ export function TimeseriesChart({ position={Position.Bottom} showOverlappingTicks tickFormat={xFormatter} + gridLine={{ visible: false }} /> - {showAnnotations && } + {showAnnotations && ( + ({ + dataValue: annotation['@timestamp'], + header: asAbsoluteDateTime(annotation['@timestamp']), + details: `${i18n.translate('xpack.apm.chart.annotation.version', { + defaultMessage: 'Version', + })} ${annotation.text}`, + }))} + style={{ + line: { strokeWidth: 1, stroke: annotationColor, opacity: 1 }, + }} + marker={} + markerPosition={Position.Top} + /> + )} {timeseries.map((serie) => { const Series = serie.type === 'area' ? AreaSeries : LineSeries; diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx index 04c07c01442a4..20056a6831adf 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_breakdown_chart/transaction_breakdown_chart_contents.tsx @@ -5,27 +5,35 @@ */ import { + AnnotationDomainTypes, AreaSeries, Axis, Chart, CurveType, + LineAnnotation, niceTimeFormatter, Placement, Position, ScaleType, Settings, } from '@elastic/charts'; +import { EuiIcon } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import moment from 'moment'; import React, { useEffect } from 'react'; import { useHistory } from 'react-router-dom'; import { useChartTheme } from '../../../../../../observability/public'; -import { asPercent } from '../../../../../common/utils/formatters'; +import { + asAbsoluteDateTime, + asPercent, +} from '../../../../../common/utils/formatters'; import { TimeSeries } from '../../../../../typings/timeseries'; import { FETCH_STATUS } from '../../../../hooks/useFetcher'; +import { useTheme } from '../../../../hooks/useTheme'; import { useUrlParams } from '../../../../hooks/useUrlParams'; +import { useAnnotations } from '../../../../hooks/use_annotations'; import { useChartPointerEvent } from '../../../../hooks/use_chart_pointer_event'; import { unit } from '../../../../style/variables'; -import { Annotations } from '../../charts/annotations'; import { ChartContainer } from '../../charts/chart_container'; import { onBrushEnd } from '../../charts/helper/helper'; @@ -44,9 +52,11 @@ export function TransactionBreakdownChartContents({ }: Props) { const history = useHistory(); const chartRef = React.createRef(); + const { annotations } = useAnnotations(); const chartTheme = useChartTheme(); const { pointerEvent, setPointerEvent } = useChartPointerEvent(); const { urlParams } = useUrlParams(); + const theme = useTheme(); const { start, end } = urlParams; useEffect(() => { @@ -64,6 +74,8 @@ export function TransactionBreakdownChartContents({ const xFormatter = niceTimeFormatter([min, max]); + const annotationColor = theme.eui.euiColorSecondary; + return ( @@ -85,6 +97,7 @@ export function TransactionBreakdownChartContents({ position={Position.Bottom} showOverlappingTicks tickFormat={xFormatter} + gridLine={{ visible: false }} /> asPercent(y ?? 0, 1)} /> - {showAnnotations && } + {showAnnotations && ( + ({ + dataValue: annotation['@timestamp'], + header: asAbsoluteDateTime(annotation['@timestamp']), + details: `${i18n.translate('xpack.apm.chart.annotation.version', { + defaultMessage: 'Version', + })} ${annotation.text}`, + }))} + style={{ + line: { strokeWidth: 1, stroke: annotationColor, opacity: 1 }, + }} + marker={} + markerPosition={Position.Top} + /> + )} {timeseries?.length ? ( timeseries.map((serie) => { diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx index 221f17bb9e1d5..3f8071ec39f0f 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_charts/index.tsx @@ -20,13 +20,14 @@ import { TRANSACTION_ROUTE_CHANGE, } from '../../../../../common/transaction_types'; import { asTransactionRate } from '../../../../../common/utils/formatters'; +import { AnnotationsContextProvider } from '../../../../context/annotations_context'; import { ChartPointerEventContextProvider } from '../../../../context/chart_pointer_event_context'; import { LicenseContext } from '../../../../context/LicenseContext'; import { IUrlParams } from '../../../../context/UrlParamsContext/types'; import { FETCH_STATUS } from '../../../../hooks/useFetcher'; import { ITransactionChartData } from '../../../../selectors/chart_selectors'; -import { TransactionBreakdownChart } from '../transaction_breakdown_chart'; import { TimeseriesChart } from '../timeseries_chart'; +import { TransactionBreakdownChart } from '../transaction_breakdown_chart'; import { TransactionErrorRateChart } from '../transaction_error_rate_chart/'; import { getResponseTimeTickFormatter } from './helper'; import { MLHeader } from './ml_header'; @@ -51,65 +52,69 @@ export function TransactionCharts({ return ( <> - - - - - - - - {responseTimeLabel(transactionType)} - - - - {(license) => ( - - )} - - - { - if (serie) { - toggleSerie(serie); - } - }} - /> - - + + + + + + + + + {responseTimeLabel(transactionType)} + + + + {(license) => ( + + )} + + + { + if (serie) { + toggleSerie(serie); + } + }} + /> + + - - - - {tpmLabel(transactionType)} - - - - - + + + + {tpmLabel(transactionType)} + + + + + - + - - - - - - - - - + + + + + + + + + + ); } diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx index b9028ff2e9e8c..00472df95c4b1 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx @@ -91,6 +91,7 @@ export function TransactionErrorRateChart({ ]} yLabelFormat={yLabelFormat} yTickFormat={yTickFormat} + yDomain={{ min: 0, max: 1 }} /> ); diff --git a/x-pack/plugins/apm/public/context/annotations_context.tsx b/x-pack/plugins/apm/public/context/annotations_context.tsx new file mode 100644 index 0000000000000..4e09a3d227b11 --- /dev/null +++ b/x-pack/plugins/apm/public/context/annotations_context.tsx @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { createContext } from 'react'; +import { useParams } from 'react-router-dom'; +import { Annotation } from '../../common/annotations'; +import { useFetcher } from '../hooks/useFetcher'; +import { useUrlParams } from '../hooks/useUrlParams'; +import { callApmApi } from '../services/rest/createCallApmApi'; + +export const AnnotationsContext = createContext({ annotations: [] } as { + annotations: Annotation[]; +}); + +const INITIAL_STATE = { annotations: [] }; + +export function AnnotationsContextProvider({ + children, +}: { + children: React.ReactNode; +}) { + const { serviceName } = useParams<{ serviceName?: string }>(); + const { urlParams, uiFilters } = useUrlParams(); + const { start, end } = urlParams; + const { environment } = uiFilters; + + const { data = INITIAL_STATE } = useFetcher(() => { + if (start && end && serviceName) { + return callApmApi({ + endpoint: 'GET /api/apm/services/{serviceName}/annotation/search', + params: { + path: { + serviceName, + }, + query: { + start, + end, + environment, + }, + }, + }); + } + }, [start, end, environment, serviceName]); + + return ; +} diff --git a/x-pack/plugins/apm/public/hooks/use_annotations.ts b/x-pack/plugins/apm/public/hooks/use_annotations.ts index e8f6785706a91..1cd9a7e65dda2 100644 --- a/x-pack/plugins/apm/public/hooks/use_annotations.ts +++ b/x-pack/plugins/apm/public/hooks/use_annotations.ts @@ -3,36 +3,16 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { useParams } from 'react-router-dom'; -import { callApmApi } from '../services/rest/createCallApmApi'; -import { useFetcher } from './useFetcher'; -import { useUrlParams } from './useUrlParams'; -const INITIAL_STATE = { annotations: [] }; +import { useContext } from 'react'; +import { AnnotationsContext } from '../context/annotations_context'; export function useAnnotations() { - const { serviceName } = useParams<{ serviceName?: string }>(); - const { urlParams, uiFilters } = useUrlParams(); - const { start, end } = urlParams; - const { environment } = uiFilters; + const context = useContext(AnnotationsContext); - const { data = INITIAL_STATE } = useFetcher(() => { - if (start && end && serviceName) { - return callApmApi({ - endpoint: 'GET /api/apm/services/{serviceName}/annotation/search', - params: { - path: { - serviceName, - }, - query: { - start, - end, - environment, - }, - }, - }); - } - }, [start, end, environment, serviceName]); + if (!context) { + throw new Error('Missing Annotations context provider'); + } - return data; + return context; } From 284b1046df6981f0da1e743680bf99247523627e Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 25 Nov 2020 10:13:36 -0800 Subject: [PATCH 05/37] [Fleet] Support input-level vars & templates (#83878) * Fix bug creating new policy on the fly * Adjust UI for input with vars but no streams * Revert "Fix bug creating new policy on the fly" This reverts commit 34f7014d6977cf73b792a64bcd9bf7c54b72cf42. * Add `compiled_input` field and compile input template, if any. Make compilation method names more generic (instead of only for streams). Add testts * Add compiled input to generated agent yaml * Don't return empty streams in agent yaml when there aren't any * Update missed assertion Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../package_policies_to_agent_inputs.test.ts | 42 +++++- .../package_policies_to_agent_inputs.ts | 33 +++-- .../fleet/common/types/models/agent_policy.ts | 2 +- .../plugins/fleet/common/types/models/epm.ts | 1 + .../common/types/models/package_policy.ts | 1 + .../package_policy_input_config.tsx | 24 ++-- .../components/package_policy_input_panel.tsx | 32 +++-- x-pack/plugins/fleet/server/mocks.ts | 2 +- .../routes/package_policy/handlers.test.ts | 2 +- .../fleet/server/saved_objects/index.ts | 1 + .../server/services/epm/agent/agent.test.ts | 14 +- .../fleet/server/services/epm/agent/agent.ts | 35 ++--- .../server/services/package_policy.test.ts | 128 +++++++++++++++++- .../fleet/server/services/package_policy.ts | 71 +++++++--- .../apps/endpoint/policy_details.ts | 1 - 15 files changed, 306 insertions(+), 83 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts index f721afb639141..a370f92e97fe1 100644 --- a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts +++ b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.test.ts @@ -100,7 +100,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { ).toEqual([]); }); - it('returns agent inputs', () => { + it('returns agent inputs with streams', () => { expect( storedPackagePoliciesToAgentInputs([ { @@ -143,6 +143,46 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { ]); }); + it('returns agent inputs without streams', () => { + expect( + storedPackagePoliciesToAgentInputs([ + { + ...mockPackagePolicy, + package: { + name: 'mock-package', + title: 'Mock package', + version: '0.0.0', + }, + inputs: [ + { + ...mockInput, + compiled_input: { + inputVar: 'input-value', + }, + streams: [], + }, + ], + }, + ]) + ).toEqual([ + { + id: 'some-uuid', + name: 'mock-package-policy', + revision: 1, + type: 'test-logs', + data_stream: { namespace: 'default' }, + use_output: 'default', + meta: { + package: { + name: 'mock-package', + version: '0.0.0', + }, + }, + inputVar: 'input-value', + }, + ]); + }); + it('returns agent inputs without disabled streams', () => { expect( storedPackagePoliciesToAgentInputs([ diff --git a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts index e74256ce732a6..d780fb791aa8e 100644 --- a/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts +++ b/x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts @@ -33,20 +33,25 @@ export const storedPackagePoliciesToAgentInputs = ( acc[key] = value; return acc; }, {} as { [k: string]: any }), - streams: input.streams - .filter((stream) => stream.enabled) - .map((stream) => { - const fullStream: FullAgentPolicyInputStream = { - id: stream.id, - data_stream: stream.data_stream, - ...stream.compiled_stream, - ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { - acc[key] = value; - return acc; - }, {} as { [k: string]: any }), - }; - return fullStream; - }), + ...(input.compiled_input || {}), + ...(input.streams.length + ? { + streams: input.streams + .filter((stream) => stream.enabled) + .map((stream) => { + const fullStream: FullAgentPolicyInputStream = { + id: stream.id, + data_stream: stream.data_stream, + ...stream.compiled_stream, + ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { + acc[key] = value; + return acc; + }, {} as { [k: string]: any }), + }; + return fullStream; + }), + } + : {}), }; if (packagePolicy.package) { diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts index f43f65fb317f3..75bb2998f2d92 100644 --- a/x-pack/plugins/fleet/common/types/models/agent_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/agent_policy.ts @@ -49,7 +49,7 @@ export interface FullAgentPolicyInput { package?: Pick; [key: string]: unknown; }; - streams: FullAgentPolicyInputStream[]; + streams?: FullAgentPolicyInputStream[]; [key: string]: any; } diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 7a6f6232b2d4f..53e507f6fb494 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -121,6 +121,7 @@ export interface RegistryInput { title: string; description?: string; vars?: RegistryVarsEntry[]; + template_path?: string; } export interface RegistryStream { diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index ae16899a4b6f9..6da98a51ef1ff 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -42,6 +42,7 @@ export interface NewPackagePolicyInput { export interface PackagePolicyInput extends Omit { streams: PackagePolicyInputStream[]; + compiled_input?: any; } export interface NewPackagePolicy { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx index 75000ad7e1d3b..9015cd09f78a3 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_config.tsx @@ -27,6 +27,7 @@ const FlexItemWithMaxWidth = styled(EuiFlexItem)` `; export const PackagePolicyInputConfig: React.FunctionComponent<{ + hasInputStreams: boolean; packageInputVars?: RegistryVarsEntry[]; packagePolicyInput: NewPackagePolicyInput; updatePackagePolicyInput: (updatedInput: Partial) => void; @@ -34,6 +35,7 @@ export const PackagePolicyInputConfig: React.FunctionComponent<{ forceShowErrors?: boolean; }> = memo( ({ + hasInputStreams, packageInputVars, packagePolicyInput, updatePackagePolicyInput, @@ -82,15 +84,19 @@ export const PackagePolicyInputConfig: React.FunctionComponent<{ /> - - -

- -

- + {hasInputStreams ? ( + <> + + +

+ +

+ + + ) : null} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx index 79ff0cc29850c..8e242980ce807 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/package_policy_input_panel.tsx @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import React, { useState, Fragment, memo } from 'react'; +import React, { useState, Fragment, memo, useMemo } from 'react'; import styled from 'styled-components'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -85,16 +85,23 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ const errorCount = countValidationErrors(inputValidationResults); const hasErrors = forceShowErrors && errorCount; - const inputStreams = packageInputStreams - .map((packageInputStream) => { - return { - packageInputStream, - packagePolicyInputStream: packagePolicyInput.streams.find( - (stream) => stream.data_stream.dataset === packageInputStream.data_stream.dataset - ), - }; - }) - .filter((stream) => Boolean(stream.packagePolicyInputStream)); + const hasInputStreams = useMemo(() => packageInputStreams.length > 0, [ + packageInputStreams.length, + ]); + const inputStreams = useMemo( + () => + packageInputStreams + .map((packageInputStream) => { + return { + packageInputStream, + packagePolicyInputStream: packagePolicyInput.streams.find( + (stream) => stream.data_stream.dataset === packageInputStream.data_stream.dataset + ), + }; + }) + .filter((stream) => Boolean(stream.packagePolicyInputStream)), + [packageInputStreams, packagePolicyInput.streams] + ); return ( <> @@ -179,13 +186,14 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ {isShowingStreams && packageInput.vars && packageInput.vars.length ? ( - + {hasInputStreams ? : } ) : null} diff --git a/x-pack/plugins/fleet/server/mocks.ts b/x-pack/plugins/fleet/server/mocks.ts index 91098c87c312a..bc3e89ef6d3ce 100644 --- a/x-pack/plugins/fleet/server/mocks.ts +++ b/x-pack/plugins/fleet/server/mocks.ts @@ -25,7 +25,7 @@ export const createAppContextStartContractMock = (): FleetAppContext => { export const createPackagePolicyServiceMock = () => { return { - assignPackageStream: jest.fn(), + compilePackagePolicyInputs: jest.fn(), buildPackagePolicyFromPackage: jest.fn(), bulkCreate: jest.fn(), create: jest.fn(), diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index f47b8499a1b69..fee74e39c833a 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -22,7 +22,7 @@ jest.mock('../../services/package_policy', (): { } => { return { packagePolicyService: { - assignPackageStream: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)), + compilePackagePolicyInputs: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)), buildPackagePolicyFromPackage: jest.fn(), bulkCreate: jest.fn(), create: jest.fn((soClient, callCluster, newData) => diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 9d85a151efbbf..201ca1c7a97bc 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -242,6 +242,7 @@ const getSavedObjectTypes = ( enabled: { type: 'boolean' }, vars: { type: 'flattened' }, config: { type: 'flattened' }, + compiled_input: { type: 'flattened' }, streams: { type: 'nested', properties: { diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts index 54b40400bb4e7..dba6f442d76e2 100644 --- a/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.test.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { createStream } from './agent'; +import { compileTemplate } from './agent'; -describe('createStream', () => { +describe('compileTemplate', () => { it('should work', () => { const streamTemplate = ` input: log @@ -27,7 +27,7 @@ hidden_password: {{password}} password: { type: 'password', value: '' }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'log', paths: ['/usr/local/var/log/nginx/access.log'], @@ -67,7 +67,7 @@ foo: bar password: { type: 'password', value: '' }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'redis/metrics', metricsets: ['key'], @@ -114,7 +114,7 @@ hidden_password: {{password}} tags: { value: ['foo', 'bar', 'forwarded'] }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'log', paths: ['/usr/local/var/log/nginx/access.log'], @@ -133,7 +133,7 @@ hidden_password: {{password}} tags: { value: ['foo', 'bar'] }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'log', paths: ['/usr/local/var/log/nginx/access.log'], @@ -157,7 +157,7 @@ input: logs }, }; - const output = createStream(vars, streamTemplate); + const output = compileTemplate(vars, streamTemplate); expect(output).toEqual({ input: 'logs', }); diff --git a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts index eeadac6e168b1..400a688722f99 100644 --- a/x-pack/plugins/fleet/server/services/epm/agent/agent.ts +++ b/x-pack/plugins/fleet/server/services/epm/agent/agent.ts @@ -10,27 +10,30 @@ import { PackagePolicyConfigRecord } from '../../../../common'; const handlebars = Handlebars.create(); -export function createStream(variables: PackagePolicyConfigRecord, streamTemplate: string) { - const { vars, yamlValues } = buildTemplateVariables(variables, streamTemplate); +export function compileTemplate(variables: PackagePolicyConfigRecord, templateStr: string) { + const { vars, yamlValues } = buildTemplateVariables(variables, templateStr); - const template = handlebars.compile(streamTemplate, { noEscape: true }); - let stream = template(vars); - stream = replaceRootLevelYamlVariables(yamlValues, stream); + const template = handlebars.compile(templateStr, { noEscape: true }); + let compiledTemplate = template(vars); + compiledTemplate = replaceRootLevelYamlVariables(yamlValues, compiledTemplate); - const yamlFromStream = safeLoad(stream, {}); + const yamlFromCompiledTemplate = safeLoad(compiledTemplate, {}); // Hack to keep empty string ('') values around in the end yaml because // `safeLoad` replaces empty strings with null - const patchedYamlFromStream = Object.entries(yamlFromStream).reduce((acc, [key, value]) => { - if (value === null && typeof vars[key] === 'string' && vars[key].trim() === '') { - acc[key] = ''; - } else { - acc[key] = value; - } - return acc; - }, {} as { [k: string]: any }); + const patchedYamlFromCompiledTemplate = Object.entries(yamlFromCompiledTemplate).reduce( + (acc, [key, value]) => { + if (value === null && typeof vars[key] === 'string' && vars[key].trim() === '') { + acc[key] = ''; + } else { + acc[key] = value; + } + return acc; + }, + {} as { [k: string]: any } + ); - return replaceVariablesInYaml(yamlValues, patchedYamlFromStream); + return replaceVariablesInYaml(yamlValues, patchedYamlFromCompiledTemplate); } function isValidKey(key: string) { @@ -54,7 +57,7 @@ function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) return yaml; } -function buildTemplateVariables(variables: PackagePolicyConfigRecord, streamTemplate: string) { +function buildTemplateVariables(variables: PackagePolicyConfigRecord, templateStr: string) { const yamlValues: { [k: string]: any } = {}; const vars = Object.entries(variables).reduce((acc, [key, recordEntry]) => { // support variables with . like key.patterns diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index 6ae76c56436d5..30a980ab07f70 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -25,7 +25,16 @@ paths: }, ]; } - return []; + return [ + { + buffer: Buffer.from(` +hosts: +{{#each hosts}} +- {{this}} +{{/each}} +`), + }, + ]; } jest.mock('./epm/packages/assets', () => { @@ -47,9 +56,9 @@ jest.mock('./epm/registry', () => { }); describe('Package policy service', () => { - describe('assignPackageStream', () => { + describe('compilePackagePolicyInputs', () => { it('should work with config variables from the stream', async () => { - const inputs = await packagePolicyService.assignPackageStream( + const inputs = await packagePolicyService.compilePackagePolicyInputs( ({ data_streams: [ { @@ -110,7 +119,7 @@ describe('Package policy service', () => { }); it('should work with config variables at the input level', async () => { - const inputs = await packagePolicyService.assignPackageStream( + const inputs = await packagePolicyService.compilePackagePolicyInputs( ({ data_streams: [ { @@ -169,6 +178,117 @@ describe('Package policy service', () => { }, ]); }); + + it('should work with an input with a template and no streams', async () => { + const inputs = await packagePolicyService.compilePackagePolicyInputs( + ({ + data_streams: [], + policy_templates: [ + { + inputs: [{ type: 'log', template_path: 'some_template_path.yml' }], + }, + ], + } as unknown) as PackageInfo, + [ + { + type: 'log', + enabled: true, + vars: { + hosts: { + value: ['localhost'], + }, + }, + streams: [], + }, + ] + ); + + expect(inputs).toEqual([ + { + type: 'log', + enabled: true, + vars: { + hosts: { + value: ['localhost'], + }, + }, + compiled_input: { + hosts: ['localhost'], + }, + streams: [], + }, + ]); + }); + + it('should work with an input with a template and streams', async () => { + const inputs = await packagePolicyService.compilePackagePolicyInputs( + ({ + data_streams: [ + { + dataset: 'package.dataset1', + type: 'logs', + streams: [{ input: 'log', template_path: 'some_template_path.yml' }], + }, + ], + policy_templates: [ + { + inputs: [{ type: 'log', template_path: 'some_template_path.yml' }], + }, + ], + } as unknown) as PackageInfo, + [ + { + type: 'log', + enabled: true, + vars: { + hosts: { + value: ['localhost'], + }, + paths: { + value: ['/var/log/set.log'], + }, + }, + streams: [ + { + id: 'datastream01', + data_stream: { dataset: 'package.dataset1', type: 'logs' }, + enabled: true, + }, + ], + }, + ] + ); + + expect(inputs).toEqual([ + { + type: 'log', + enabled: true, + vars: { + hosts: { + value: ['localhost'], + }, + paths: { + value: ['/var/log/set.log'], + }, + }, + compiled_input: { + hosts: ['localhost'], + }, + streams: [ + { + id: 'datastream01', + data_stream: { dataset: 'package.dataset1', type: 'logs' }, + enabled: true, + compiled_stream: { + metricset: ['dataset1'], + paths: ['/var/log/set.log'], + type: 'log', + }, + }, + ], + }, + ]); + }); }); describe('update', () => { diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 0f78c97a6f2bd..7b8952bdea2cd 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -31,7 +31,7 @@ import { outputService } from './output'; import * as Registry from './epm/registry'; import { getPackageInfo, getInstallation, ensureInstalledPackage } from './epm/packages'; import { getAssetsData } from './epm/packages/assets'; -import { createStream } from './epm/agent/agent'; +import { compileTemplate } from './epm/agent/agent'; import { normalizeKuery } from './saved_object'; const SAVED_OBJECT_TYPE = PACKAGE_POLICY_SAVED_OBJECT_TYPE; @@ -92,7 +92,7 @@ class PackagePolicyService { } } - inputs = await this.assignPackageStream(pkgInfo, inputs); + inputs = await this.compilePackagePolicyInputs(pkgInfo, inputs); } const isoDate = new Date().toISOString(); @@ -285,7 +285,7 @@ class PackagePolicyService { pkgVersion: packagePolicy.package.version, }); - inputs = await this.assignPackageStream(pkgInfo, inputs); + inputs = await this.compilePackagePolicyInputs(pkgInfo, inputs); } await soClient.update( @@ -374,14 +374,20 @@ class PackagePolicyService { } } - public async assignPackageStream( + public async compilePackagePolicyInputs( pkgInfo: PackageInfo, inputs: PackagePolicyInput[] ): Promise { const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version); - const inputsPromises = inputs.map((input) => - _assignPackageStreamToInput(registryPkgInfo, pkgInfo, input) - ); + const inputsPromises = inputs.map(async (input) => { + const compiledInput = await _compilePackagePolicyInput(registryPkgInfo, pkgInfo, input); + const compiledStreams = await _compilePackageStreams(registryPkgInfo, pkgInfo, input); + return { + ...input, + compiled_input: compiledInput, + streams: compiledStreams, + }; + }); return Promise.all(inputsPromises); } @@ -396,20 +402,53 @@ function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyI }; } -async function _assignPackageStreamToInput( +async function _compilePackagePolicyInput( + registryPkgInfo: RegistryPackage, + pkgInfo: PackageInfo, + input: PackagePolicyInput +) { + if (!input.enabled || !pkgInfo.policy_templates?.[0].inputs) { + return undefined; + } + + const packageInputs = pkgInfo.policy_templates[0].inputs; + const packageInput = packageInputs.find((pkgInput) => pkgInput.type === input.type); + if (!packageInput) { + throw new Error(`Input template not found, unable to find input type ${input.type}`); + } + + if (!packageInput.template_path) { + return undefined; + } + + const [pkgInputTemplate] = await getAssetsData(registryPkgInfo, (path: string) => + path.endsWith(`/agent/input/${packageInput.template_path!}`) + ); + + if (!pkgInputTemplate || !pkgInputTemplate.buffer) { + throw new Error(`Unable to load input template at /agent/input/${packageInput.template_path!}`); + } + + return compileTemplate( + // Populate template variables from input vars + Object.assign({}, input.vars), + pkgInputTemplate.buffer.toString() + ); +} + +async function _compilePackageStreams( registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, input: PackagePolicyInput ) { const streamsPromises = input.streams.map((stream) => - _assignPackageStreamToStream(registryPkgInfo, pkgInfo, input, stream) + _compilePackageStream(registryPkgInfo, pkgInfo, input, stream) ); - const streams = await Promise.all(streamsPromises); - return { ...input, streams }; + return await Promise.all(streamsPromises); } -async function _assignPackageStreamToStream( +async function _compilePackageStream( registryPkgInfo: RegistryPackage, pkgInfo: PackageInfo, input: PackagePolicyInput, @@ -442,22 +481,22 @@ async function _assignPackageStreamToStream( throw new Error(`Stream template path not found for dataset ${datasetPath}`); } - const [pkgStream] = await getAssetsData( + const [pkgStreamTemplate] = await getAssetsData( registryPkgInfo, (path: string) => path.endsWith(streamFromPkg.template_path), datasetPath ); - if (!pkgStream || !pkgStream.buffer) { + if (!pkgStreamTemplate || !pkgStreamTemplate.buffer) { throw new Error( `Unable to load stream template ${streamFromPkg.template_path} for dataset ${datasetPath}` ); } - const yaml = createStream( + const yaml = compileTemplate( // Populate template variables from input vars and stream vars Object.assign({}, input.vars, stream.vars), - pkgStream.buffer.toString() + pkgStreamTemplate.buffer.toString() ); stream.compiled_stream = yaml; diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index b3c130ea1e5dc..46085b0db3063 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -249,7 +249,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }, }, }, - streams: [], type: 'endpoint', use_output: 'default', }, From 497511272308d2aae3f81e26121588aa73dfdd24 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 25 Nov 2020 12:12:28 -0700 Subject: [PATCH 06/37] [cli/dev] log a warning when --no-base-path is used with --dev (#84354) Co-authored-by: spalger --- src/cli/cluster/cluster_manager.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/cli/cluster/cluster_manager.ts b/src/cli/cluster/cluster_manager.ts index 7a14f617b5d5a..b0f7cded938dd 100644 --- a/src/cli/cluster/cluster_manager.ts +++ b/src/cli/cluster/cluster_manager.ts @@ -73,6 +73,19 @@ export class ClusterManager { this.inReplMode = !!opts.repl; this.basePathProxy = basePathProxy; + if (!this.basePathProxy) { + this.log.warn( + '====================================================================================================' + ); + this.log.warn( + 'no-base-path', + 'Running Kibana in dev mode with --no-base-path disables several useful features and is not recommended' + ); + this.log.warn( + '====================================================================================================' + ); + } + // run @kbn/optimizer and write it's state to kbnOptimizerReady$ if (opts.disableOptimizer) { this.kbnOptimizerReady$.next(true); From 5fda30001f69d536bfa7f0f84ecd5f6e7df1dcc8 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Wed, 25 Nov 2020 14:47:11 -0500 Subject: [PATCH 07/37] [Security Solution][Resolver] Add support for predefined schemas for endpoint and winlogbeat (#84103) * Refactoring entity route to return schema * Refactoring frontend middleware to pick off id field from entity route * Refactoring schema and adding name and comments * Adding name to schema mocks * Fixing type issue --- .../common/endpoint/types/index.ts | 39 ++++- .../mocks/no_ancestors_two_children.ts | 13 +- ..._children_in_index_called_awesome_index.ts | 13 +- ...ith_related_events_and_cursor_on_origin.ts | 13 +- ..._children_with_related_events_on_origin.ts | 13 +- .../one_node_with_paginated_related_events.ts | 13 +- .../store/middleware/resolver_tree_fetcher.ts | 2 +- .../server/endpoint/routes/resolver/entity.ts | 147 ++++++++++++------ .../resolver/tree/queries/descendants.ts | 8 +- .../routes/resolver/tree/queries/lifecycle.ts | 8 +- .../routes/resolver/tree/queries/stats.ts | 8 +- .../routes/resolver/tree/utils/fetch.test.ts | 9 +- .../routes/resolver/tree/utils/fetch.ts | 25 +-- .../routes/resolver/tree/utils/index.ts | 27 +--- .../apis/resolver/common.ts | 25 +-- .../apis/resolver/entity.ts | 9 +- .../apis/resolver/tree.ts | 12 +- 17 files changed, 259 insertions(+), 125 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index cd5c60e2698cb..d6be83d7cbbe3 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -870,10 +870,47 @@ export interface SafeLegacyEndpointEvent { }>; } +/** + * The fields to use to identify nodes within a resolver tree. + */ +export interface ResolverSchema { + /** + * the ancestry field should be set to a field that contains an order array representing + * the ancestors of a node. + */ + ancestry?: string; + /** + * id represents the field to use as the unique ID for a node. + */ + id: string; + /** + * field to use for the name of the node + */ + name?: string; + /** + * parent represents the field that is the edge between two nodes. + */ + parent: string; +} + /** * The response body for the resolver '/entity' index API */ -export type ResolverEntityIndex = Array<{ entity_id: string }>; +export type ResolverEntityIndex = Array<{ + /** + * A name for the schema that is being used (e.g. endpoint, winlogbeat, etc) + */ + name: string; + /** + * The schema to pass to the /tree api and other backend requests, based on the contents of the document found using + * the _id + */ + schema: ResolverSchema; + /** + * Unique ID value for the requested document using the `_id` field passed to the /entity route + */ + id: string; +}>; /** * Takes a @kbn/config-schema 'schema' type and returns a type that represents valid inputs. diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts index 09625e5726b1d..472fdc79d1f02 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children.ts @@ -99,7 +99,18 @@ export function noAncestorsTwoChildren(): { dataAccessLayer: DataAccessLayer; me * Get entities matching a document. */ entities(): Promise { - return Promise.resolve([{ entity_id: metadata.entityIDs.origin }]); + return Promise.resolve([ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]); }, }, }; diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts index 3bbe4bcf51060..b085738d3fd2e 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_in_index_called_awesome_index.ts @@ -115,7 +115,18 @@ export function noAncestorsTwoChildenInIndexCalledAwesomeIndex(): { entities({ indices }): Promise { // Only return values if the `indices` array contains exactly `'awesome_index'` if (indices.length === 1 && indices[0] === 'awesome_index') { - return Promise.resolve([{ entity_id: metadata.entityIDs.origin }]); + return Promise.resolve([ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]); } return Promise.resolve([]); }, diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts index 7682165ac5e94..43704db358d7e 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_and_cursor_on_origin.ts @@ -140,7 +140,18 @@ export function noAncestorsTwoChildrenWithRelatedEventsOnOriginWithOneAfterCurso * Get entities matching a document. */ async entities(): Promise { - return [{ entity_id: metadata.entityIDs.origin }]; + return [ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]; }, }, }; diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts index 837d824db8748..c4d538d2eed94 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/no_ancestors_two_children_with_related_events_on_origin.ts @@ -112,7 +112,18 @@ export function noAncestorsTwoChildrenWithRelatedEventsOnOrigin(): { * Get entities matching a document. */ async entities(): Promise { - return [{ entity_id: metadata.entityIDs.origin }]; + return [ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]; }, }, }; diff --git a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts index 01477ff16868e..7849776ed1378 100644 --- a/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts +++ b/x-pack/plugins/security_solution/public/resolver/data_access_layer/mocks/one_node_with_paginated_related_events.ts @@ -103,7 +103,18 @@ export function oneNodeWithPaginatedEvents(): { * Get entities matching a document. */ async entities(): Promise { - return [{ entity_id: metadata.entityIDs.origin }]; + return [ + { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + id: metadata.entityIDs.origin, + }, + ]; }, }, }; diff --git a/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts b/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts index ef4ca2380ebf4..aecdd6b92a463 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts @@ -56,7 +56,7 @@ export function ResolverTreeFetcher( }); return; } - const entityIDToFetch = matchingEntities[0].entity_id; + const entityIDToFetch = matchingEntities[0].id; result = await dataAccessLayer.resolverTree( entityIDToFetch, lastRequestAbortController.signal diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts index 510bb6c545558..c731692e6fb89 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/entity.ts @@ -3,10 +3,70 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { RequestHandler } from 'kibana/server'; +import _ from 'lodash'; +import { RequestHandler, SearchResponse } from 'kibana/server'; import { TypeOf } from '@kbn/config-schema'; +import { ApiResponse } from '@elastic/elasticsearch'; import { validateEntities } from '../../../../common/endpoint/schema/resolver'; -import { ResolverEntityIndex } from '../../../../common/endpoint/types'; +import { ResolverEntityIndex, ResolverSchema } from '../../../../common/endpoint/types'; + +interface SupportedSchema { + /** + * A name for the schema being used + */ + name: string; + + /** + * A constraint to search for in the documented returned by Elasticsearch + */ + constraint: { field: string; value: string }; + + /** + * Schema to return to the frontend so that it can be passed in to call to the /tree API + */ + schema: ResolverSchema; +} + +/** + * This structure defines the preset supported schemas for a resolver graph. We'll probably want convert this + * implementation to something similar to how row renderers is implemented. + */ +const supportedSchemas: SupportedSchema[] = [ + { + name: 'endpoint', + constraint: { + field: 'agent.type', + value: 'endpoint', + }, + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, + }, + { + name: 'winlogbeat', + constraint: { + field: 'agent.type', + value: 'winlogbeat', + }, + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + name: 'process.name', + }, + }, +]; + +function getFieldAsString(doc: unknown, field: string): string | undefined { + const value = _.get(doc, field); + if (value === undefined) { + return undefined; + } + + return String(value); +} /** * This is used to get an 'entity_id' which is an internal-to-Resolver concept, from an `_id`, which @@ -18,61 +78,46 @@ export function handleEntities(): RequestHandler + > = await context.core.elasticsearch.client.asCurrentUser.search({ + ignore_unavailable: true, + index: indices, + body: { + // only return 1 match at most + size: 1, + query: { + bool: { + filter: [ { - _source: { - process?: { - entity_id?: string; - }; - }; - } - ]; - }; - } - - const queryResponse: ExpectedQueryResponse = await context.core.elasticsearch.legacy.client.callAsCurrentUser( - 'search', - { - ignoreUnavailable: true, - index: indices, - body: { - // only return process.entity_id - _source: 'process.entity_id', - // only return 1 match at most - size: 1, - query: { - bool: { - filter: [ - { - // only return documents with the matching _id - ids: { - values: _id, - }, + // only return documents with the matching _id + ids: { + values: _id, }, - ], - }, + }, + ], }, }, - } - ); + }, + }); const responseBody: ResolverEntityIndex = []; - for (const hit of queryResponse.hits.hits) { - // check that the field is defined and that is not an empty string - if (hit._source.process?.entity_id) { - responseBody.push({ - entity_id: hit._source.process.entity_id, - }); + for (const hit of queryResponse.body.hits.hits) { + for (const supportedSchema of supportedSchemas) { + const fieldValue = getFieldAsString(hit._source, supportedSchema.constraint.field); + const id = getFieldAsString(hit._source, supportedSchema.schema.id); + // check that the constraint and id fields are defined and that the id field is not an empty string + if ( + fieldValue?.toLowerCase() === supportedSchema.constraint.value.toLowerCase() && + id !== undefined && + id !== '' + ) { + responseBody.push({ + name: supportedSchema.name, + schema: supportedSchema.schema, + id, + }); + } } } return response.ok({ body: responseBody }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts index 405429cc24191..3baf3a8667529 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts @@ -6,12 +6,12 @@ import { SearchResponse } from 'elasticsearch'; import { ApiResponse } from '@elastic/elasticsearch'; import { IScopedClusterClient } from 'src/core/server'; -import { FieldsObject } from '../../../../../../common/endpoint/types'; +import { FieldsObject, ResolverSchema } from '../../../../../../common/endpoint/types'; import { JsonObject, JsonValue } from '../../../../../../../../../src/plugins/kibana_utils/common'; -import { NodeID, Schema, Timerange, docValueFields } from '../utils/index'; +import { NodeID, Timerange, docValueFields } from '../utils/index'; interface DescendantsParams { - schema: Schema; + schema: ResolverSchema; indexPatterns: string | string[]; timerange: Timerange; } @@ -20,7 +20,7 @@ interface DescendantsParams { * Builds a query for retrieving descendants of a node. */ export class DescendantsQuery { - private readonly schema: Schema; + private readonly schema: ResolverSchema; private readonly indexPatterns: string | string[]; private readonly timerange: Timerange; private readonly docValueFields: JsonValue[]; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts index 606a4538ba88c..5253806be66ba 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts @@ -6,12 +6,12 @@ import { SearchResponse } from 'elasticsearch'; import { ApiResponse } from '@elastic/elasticsearch'; import { IScopedClusterClient } from 'src/core/server'; -import { FieldsObject } from '../../../../../../common/endpoint/types'; +import { FieldsObject, ResolverSchema } from '../../../../../../common/endpoint/types'; import { JsonObject, JsonValue } from '../../../../../../../../../src/plugins/kibana_utils/common'; -import { NodeID, Schema, Timerange, docValueFields } from '../utils/index'; +import { NodeID, Timerange, docValueFields } from '../utils/index'; interface LifecycleParams { - schema: Schema; + schema: ResolverSchema; indexPatterns: string | string[]; timerange: Timerange; } @@ -20,7 +20,7 @@ interface LifecycleParams { * Builds a query for retrieving descendants of a node. */ export class LifecycleQuery { - private readonly schema: Schema; + private readonly schema: ResolverSchema; private readonly indexPatterns: string | string[]; private readonly timerange: Timerange; private readonly docValueFields: JsonValue[]; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts index 33dcdce8987f5..117cc3647dd0e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts @@ -7,8 +7,8 @@ import { SearchResponse } from 'elasticsearch'; import { ApiResponse } from '@elastic/elasticsearch'; import { IScopedClusterClient } from 'src/core/server'; import { JsonObject } from '../../../../../../../../../src/plugins/kibana_utils/common'; -import { EventStats } from '../../../../../../common/endpoint/types'; -import { NodeID, Schema, Timerange } from '../utils/index'; +import { EventStats, ResolverSchema } from '../../../../../../common/endpoint/types'; +import { NodeID, Timerange } from '../utils/index'; interface AggBucket { key: string; @@ -26,7 +26,7 @@ interface CategoriesAgg extends AggBucket { } interface StatsParams { - schema: Schema; + schema: ResolverSchema; indexPatterns: string | string[]; timerange: Timerange; } @@ -35,7 +35,7 @@ interface StatsParams { * Builds a query for retrieving descendants of a node. */ export class StatsQuery { - private readonly schema: Schema; + private readonly schema: ResolverSchema; private readonly indexPatterns: string | string[]; private readonly timerange: Timerange; constructor({ schema, indexPatterns, timerange }: StatsParams) { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts index 8105f1125d01d..d5e0af9dea239 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.test.ts @@ -18,14 +18,17 @@ import { DescendantsQuery } from '../queries/descendants'; import { StatsQuery } from '../queries/stats'; import { IScopedClusterClient } from 'src/core/server'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; -import { FieldsObject, ResolverNode } from '../../../../../../common/endpoint/types'; -import { Schema } from './index'; +import { + FieldsObject, + ResolverNode, + ResolverSchema, +} from '../../../../../../common/endpoint/types'; jest.mock('../queries/descendants'); jest.mock('../queries/lifecycle'); jest.mock('../queries/stats'); -function formatResponse(results: FieldsObject[], schema: Schema): ResolverNode[] { +function formatResponse(results: FieldsObject[], schema: ResolverSchema): ResolverNode[] { return results.map((node) => { return { id: getIDField(node, schema) ?? '', diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts index eaecad6c47970..356357082d6ee 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts @@ -8,9 +8,14 @@ import { firstNonNullValue, values, } from '../../../../../../common/endpoint/models/ecs_safety_helpers'; -import { ECSField, ResolverNode, FieldsObject } from '../../../../../../common/endpoint/types'; +import { + ECSField, + ResolverNode, + FieldsObject, + ResolverSchema, +} from '../../../../../../common/endpoint/types'; import { DescendantsQuery } from '../queries/descendants'; -import { Schema, NodeID } from './index'; +import { NodeID } from './index'; import { LifecycleQuery } from '../queries/lifecycle'; import { StatsQuery } from '../queries/stats'; @@ -26,7 +31,7 @@ export interface TreeOptions { from: string; to: string; }; - schema: Schema; + schema: ResolverSchema; nodes: NodeID[]; indexPatterns: string[]; } @@ -98,7 +103,7 @@ export class Fetcher { private static getNextAncestorsToFind( results: FieldsObject[], - schema: Schema, + schema: ResolverSchema, levelsLeft: number ): NodeID[] { const nodesByID = results.reduce((accMap: Map, result: FieldsObject) => { @@ -216,7 +221,7 @@ export class Fetcher { export function getLeafNodes( results: FieldsObject[], nodes: Array, - schema: Schema + schema: ResolverSchema ): NodeID[] { let largestAncestryArray = 0; const nodesToQueryNext: Map> = new Map(); @@ -269,7 +274,7 @@ export function getLeafNodes( * @param obj the doc value fields retrieved from a document returned by Elasticsearch * @param schema the schema used for identifying connections between documents */ -export function getIDField(obj: FieldsObject, schema: Schema): NodeID | undefined { +export function getIDField(obj: FieldsObject, schema: ResolverSchema): NodeID | undefined { const id: ECSField = obj[schema.id]; return firstNonNullValue(id); } @@ -281,7 +286,7 @@ export function getIDField(obj: FieldsObject, schema: Schema): NodeID | undefine * @param obj the doc value fields retrieved from a document returned by Elasticsearch * @param schema the schema used for identifying connections between documents */ -export function getNameField(obj: FieldsObject, schema: Schema): string | undefined { +export function getNameField(obj: FieldsObject, schema: ResolverSchema): string | undefined { if (!schema.name) { return undefined; } @@ -297,12 +302,12 @@ export function getNameField(obj: FieldsObject, schema: Schema): string | undefi * @param obj the doc value fields retrieved from a document returned by Elasticsearch * @param schema the schema used for identifying connections between documents */ -export function getParentField(obj: FieldsObject, schema: Schema): NodeID | undefined { +export function getParentField(obj: FieldsObject, schema: ResolverSchema): NodeID | undefined { const parent: ECSField = obj[schema.parent]; return firstNonNullValue(parent); } -function getAncestryField(obj: FieldsObject, schema: Schema): NodeID[] | undefined { +function getAncestryField(obj: FieldsObject, schema: ResolverSchema): NodeID[] | undefined { if (!schema.ancestry) { return undefined; } @@ -324,7 +329,7 @@ function getAncestryField(obj: FieldsObject, schema: Schema): NodeID[] | undefin * @param obj the doc value fields retrieved from a document returned by Elasticsearch * @param schema the schema used for identifying connections between documents */ -export function getAncestryAsArray(obj: FieldsObject, schema: Schema): NodeID[] { +export function getAncestryAsArray(obj: FieldsObject, schema: ResolverSchema): NodeID[] { const ancestry = getAncestryField(obj, schema); if (!ancestry || ancestry.length <= 0) { const parentField = getParentField(obj, schema); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts index 21a49e268310b..be08b4390a69c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/index.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ResolverSchema } from '../../../../../../common/endpoint/types'; + /** * Represents a time range filter */ @@ -17,29 +19,6 @@ export interface Timerange { */ export type NodeID = string | number; -/** - * The fields to use to identify nodes within a resolver tree. - */ -export interface Schema { - /** - * the ancestry field should be set to a field that contains an order array representing - * the ancestors of a node. - */ - ancestry?: string; - /** - * id represents the field to use as the unique ID for a node. - */ - id: string; - /** - * field to use for the name of the node - */ - name?: string; - /** - * parent represents the field that is the edge between two nodes. - */ - parent: string; -} - /** * Returns the doc value fields filter to use in queries to limit the number of fields returned in the * query response. @@ -49,7 +28,7 @@ export interface Schema { * @param schema is the node schema information describing how relationships are formed between nodes * in the resolver graph. */ -export function docValueFields(schema: Schema): Array<{ field: string }> { +export function docValueFields(schema: ResolverSchema): Array<{ field: string }> { const filter = [{ field: '@timestamp' }, { field: schema.id }, { field: schema.parent }]; if (schema.ancestry) { filter.push({ field: schema.ancestry }); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts index b4e98d7d4b95e..3cc833c6a2475 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts @@ -6,16 +6,14 @@ import _ from 'lodash'; import expect from '@kbn/expect'; import { firstNonNullValue } from '../../../../plugins/security_solution/common/endpoint/models/ecs_safety_helpers'; -import { - NodeID, - Schema, -} from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils'; +import { NodeID } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils'; import { SafeResolverChildNode, SafeResolverLifecycleNode, SafeResolverEvent, ResolverNodeStats, ResolverNode, + ResolverSchema, } from '../../../../plugins/security_solution/common/endpoint/types'; import { parentEntityIDSafeVersion, @@ -41,7 +39,7 @@ const createLevels = ({ descendantsByParent: Map>; levels: Array>; currentNodes: Map | undefined; - schema: Schema; + schema: ResolverSchema; }): Array> => { if (!currentNodes || currentNodes.size === 0) { return levels; @@ -98,7 +96,7 @@ export interface APIResponse { * @param node a resolver node * @param schema the schema that was used to retrieve this resolver node */ -export const getID = (node: ResolverNode | undefined, schema: Schema): NodeID => { +export const getID = (node: ResolverNode | undefined, schema: ResolverSchema): NodeID => { const id = firstNonNullValue(node?.data[schema.id]); if (!id) { throw new Error(`Unable to find id ${schema.id} in node: ${JSON.stringify(node)}`); @@ -106,7 +104,10 @@ export const getID = (node: ResolverNode | undefined, schema: Schema): NodeID => return id; }; -const getParentInternal = (node: ResolverNode | undefined, schema: Schema): NodeID | undefined => { +const getParentInternal = ( + node: ResolverNode | undefined, + schema: ResolverSchema +): NodeID | undefined => { if (node) { return firstNonNullValue(node?.data[schema.parent]); } @@ -119,7 +120,7 @@ const getParentInternal = (node: ResolverNode | undefined, schema: Schema): Node * @param node a resolver node * @param schema the schema that was used to retrieve this resolver node */ -export const getParent = (node: ResolverNode | undefined, schema: Schema): NodeID => { +export const getParent = (node: ResolverNode | undefined, schema: ResolverSchema): NodeID => { const parent = getParentInternal(node, schema); if (!parent) { throw new Error(`Unable to find parent ${schema.parent} in node: ${JSON.stringify(node)}`); @@ -138,7 +139,7 @@ export const getParent = (node: ResolverNode | undefined, schema: Schema): NodeI const createTreeFromResponse = ( treeExpectations: TreeExpectation[], nodes: ResolverNode[], - schema: Schema + schema: ResolverSchema ) => { const nodesByID = new Map(); const nodesByParent = new Map>(); @@ -206,7 +207,7 @@ const verifyAncestry = ({ genTree, }: { responseTrees: APIResponse; - schema: Schema; + schema: ResolverSchema; genTree: Tree; }) => { const allGenNodes = new Map([ @@ -277,7 +278,7 @@ const verifyChildren = ({ genTree, }: { responseTrees: APIResponse; - schema: Schema; + schema: ResolverSchema; genTree: Tree; }) => { const allGenNodes = new Map([ @@ -358,7 +359,7 @@ export const verifyTree = ({ }: { expectations: TreeExpectation[]; response: ResolverNode[]; - schema: Schema; + schema: ResolverSchema; genTree: Tree; relatedEventsCategories?: RelatedEventInfo[]; }) => { diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts index 7fbba4e04798d..2607b934e7df2 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity.ts @@ -29,8 +29,15 @@ export default function ({ getService }: FtrProviderContext) { ); expect(body).eql([ { + name: 'endpoint', + schema: { + id: 'process.entity_id', + parent: 'process.parent.entity_id', + ancestry: 'process.Ext.ancestry', + name: 'process.name', + }, // this value is from the es archive - entity_id: + id: 'MTIwNWY1NWQtODRkYS00MzkxLWIyNWQtYTNkNGJmNDBmY2E1LTc1NTItMTMyNDM1NDY1MTQuNjI0MjgxMDA=', }, ]); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts index 646a666629ac9..7a1210c6b762f 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts @@ -5,8 +5,10 @@ */ import expect from '@kbn/expect'; import { getNameField } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch'; -import { Schema } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/tree/utils'; -import { ResolverNode } from '../../../../plugins/security_solution/common/endpoint/types'; +import { + ResolverNode, + ResolverSchema, +} from '../../../../plugins/security_solution/common/endpoint/types'; import { parentEntityIDSafeVersion, timestampSafeVersion, @@ -44,18 +46,18 @@ export default function ({ getService }: FtrProviderContext) { ancestryArraySize: 2, }; - const schemaWithAncestry: Schema = { + const schemaWithAncestry: ResolverSchema = { ancestry: 'process.Ext.ancestry', id: 'process.entity_id', parent: 'process.parent.entity_id', }; - const schemaWithoutAncestry: Schema = { + const schemaWithoutAncestry: ResolverSchema = { id: 'process.entity_id', parent: 'process.parent.entity_id', }; - const schemaWithName: Schema = { + const schemaWithName: ResolverSchema = { id: 'process.entity_id', parent: 'process.parent.entity_id', name: 'process.name', From dfa9c75021f1872b31f770c4009cc2e0d556dbfe Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Wed, 25 Nov 2020 15:15:46 -0500 Subject: [PATCH 08/37] Fix issues with show_license_expiration (#84361) --- .../cluster/overview/elasticsearch_panel.js | 71 +++++++++++-------- .../alerts/license_expiration_alert.test.ts | 28 ++++++++ .../server/alerts/license_expiration_alert.ts | 9 ++- 3 files changed, 77 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js b/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js index 7e85d62c4bbd6..ded309ce64e2e 100644 --- a/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js +++ b/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js @@ -230,6 +230,46 @@ export function ElasticsearchPanel(props) { return null; }; + const showLicense = () => { + if (!props.showLicenseExpiration) { + return null; + } + return ( + + + + + + + + + {capitalize(props.license.type)} + + + + + {props.license.expiry_date_in_millis === undefined ? ( + '' + ) : ( + + )} + + + + + + ); + }; + const statusColorMap = { green: 'success', yellow: 'warning', @@ -325,36 +365,7 @@ export function ElasticsearchPanel(props) { {formatNumber(get(nodes, 'jvm.max_uptime_in_millis'), 'time_since')} {showMlJobs()} - - - - - - - - {capitalize(props.license.type)} - - - - - {props.license.expiry_date_in_millis === undefined ? ( - '' - ) : ( - - )} - - - - + {showLicense()} diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts index 74c300d971898..b82b4c235acba 100644 --- a/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts @@ -76,6 +76,7 @@ describe('LicenseExpirationAlert', () => { const monitoringCluster = null; const config = { ui: { + show_license_expiration: true, ccs: { enabled: true }, container: { elasticsearch: { enabled: false } }, metricbeat: { index: 'metricbeat-*' }, @@ -282,5 +283,32 @@ describe('LicenseExpirationAlert', () => { state: 'resolved', }); }); + + it('should not fire actions if we are not showing license expiration', async () => { + const alert = new LicenseExpirationAlert(); + const customConfig = { + ...config, + ui: { + ...config.ui, + show_license_expiration: false, + }, + }; + alert.initializeAlertType( + getUiSettingsService as any, + monitoringCluster as any, + getLogger as any, + customConfig as any, + kibanaUrl, + false + ); + const type = alert.getAlertType(); + await type.executor({ + ...executorOptions, + // @ts-ignore + params: alert.defaultParams, + } as any); + expect(replaceState).not.toHaveBeenCalledWith({}); + expect(scheduleActions).not.toHaveBeenCalled(); + }); }); }); diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.ts index 00846e9cf7599..9692d95bfc6fe 100644 --- a/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/license_expiration_alert.ts @@ -18,7 +18,7 @@ import { LegacyAlert, CommonAlertParams, } from '../../common/types/alerts'; -import { AlertInstance } from '../../../alerts/server'; +import { AlertExecutorOptions, AlertInstance } from '../../../alerts/server'; import { INDEX_ALERTS, ALERT_LICENSE_EXPIRATION, @@ -64,6 +64,13 @@ export class LicenseExpirationAlert extends BaseAlert { AlertingDefaults.ALERT_TYPE.context.actionPlain, ]; + protected async execute(options: AlertExecutorOptions): Promise { + if (!this.config.ui.show_license_expiration) { + return; + } + return await super.execute(options); + } + protected async fetchData( params: CommonAlertParams, callCluster: any, From 71f77862e7a6fa00c248c854a7814a2331d22841 Mon Sep 17 00:00:00 2001 From: Dan Panzarella Date: Wed, 25 Nov 2020 15:29:23 -0500 Subject: [PATCH 09/37] [Security Solution] Add Endpoint policy feature checks (#83972) --- .../common/endpoint/models/policy_config.ts | 5 + .../common/license/license.ts | 33 +++--- .../common/license/policy_config.test.ts | 110 ++++++++++++++++++ .../common/license/policy_config.ts | 66 +++++++++++ .../policy/store/policy_details/middleware.ts | 7 +- 5 files changed, 198 insertions(+), 23 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/license/policy_config.test.ts create mode 100644 x-pack/plugins/security_solution/common/license/policy_config.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts index 890def5b63d4a..22037c021701f 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts @@ -68,3 +68,8 @@ export const factory = (): PolicyConfig => { }, }; }; + +/** + * Reflects what string the Endpoint will use when message field is default/empty + */ +export const DefaultMalwareMessage = 'Elastic Security { action } { filename }'; diff --git a/x-pack/plugins/security_solution/common/license/license.ts b/x-pack/plugins/security_solution/common/license/license.ts index 96c1a14ceb1f4..2d424ab9c960a 100644 --- a/x-pack/plugins/security_solution/common/license/license.ts +++ b/x-pack/plugins/security_solution/common/license/license.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { Observable, Subscription } from 'rxjs'; -import { ILicense } from '../../../licensing/common/types'; +import { ILicense, LicenseType } from '../../../licensing/common/types'; // Generic license service class that works with the license observable // Both server and client plugins instancates a singleton version of this class @@ -36,25 +36,20 @@ export class LicenseService { return this.observable; } - public isGoldPlus() { - return ( - this.licenseInformation?.isAvailable && - this.licenseInformation?.isActive && - this.licenseInformation?.hasAtLeast('gold') - ); + public isAtLeast(level: LicenseType): boolean { + return isAtLeast(this.licenseInformation, level); } - public isPlatinumPlus() { - return ( - this.licenseInformation?.isAvailable && - this.licenseInformation?.isActive && - this.licenseInformation?.hasAtLeast('platinum') - ); + public isGoldPlus(): boolean { + return this.isAtLeast('gold'); } - public isEnterprise() { - return ( - this.licenseInformation?.isAvailable && - this.licenseInformation?.isActive && - this.licenseInformation?.hasAtLeast('enterprise') - ); + public isPlatinumPlus(): boolean { + return this.isAtLeast('platinum'); + } + public isEnterprise(): boolean { + return this.isAtLeast('enterprise'); } } + +export const isAtLeast = (license: ILicense | null, level: LicenseType): boolean => { + return license !== null && license.isAvailable && license.isActive && license.hasAtLeast(level); +}; diff --git a/x-pack/plugins/security_solution/common/license/policy_config.test.ts b/x-pack/plugins/security_solution/common/license/policy_config.test.ts new file mode 100644 index 0000000000000..6923bf00055f6 --- /dev/null +++ b/x-pack/plugins/security_solution/common/license/policy_config.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + isEndpointPolicyValidForLicense, + unsetPolicyFeaturesAboveLicenseLevel, +} from './policy_config'; +import { DefaultMalwareMessage, factory } from '../endpoint/models/policy_config'; +import { licenseMock } from '../../../licensing/common/licensing.mock'; + +describe('policy_config and licenses', () => { + const Platinum = licenseMock.createLicense({ license: { type: 'platinum', mode: 'platinum' } }); + const Gold = licenseMock.createLicense({ license: { type: 'gold', mode: 'gold' } }); + const Basic = licenseMock.createLicense({ license: { type: 'basic', mode: 'basic' } }); + + describe('isEndpointPolicyValidForLicense', () => { + it('allows malware notification to be disabled with a Platinum license', () => { + const policy = factory(); + policy.windows.popup.malware.enabled = false; // make policy change + const valid = isEndpointPolicyValidForLicense(policy, Platinum); + expect(valid).toBeTruthy(); + }); + it('blocks windows malware notification changes below Platinum licenses', () => { + const policy = factory(); + policy.windows.popup.malware.enabled = false; // make policy change + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); + + it('blocks mac malware notification changes below Platinum licenses', () => { + const policy = factory(); + policy.mac.popup.malware.enabled = false; // make policy change + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); + + it('allows malware notification message changes with a Platinum license', () => { + const policy = factory(); + policy.windows.popup.malware.message = 'BOOM'; // make policy change + const valid = isEndpointPolicyValidForLicense(policy, Platinum); + expect(valid).toBeTruthy(); + }); + it('blocks windows malware notification message changes below Platinum licenses', () => { + const policy = factory(); + policy.windows.popup.malware.message = 'BOOM'; // make policy change + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); + it('blocks mac malware notification message changes below Platinum licenses', () => { + const policy = factory(); + policy.mac.popup.malware.message = 'BOOM'; // make policy change + let valid = isEndpointPolicyValidForLicense(policy, Gold); + expect(valid).toBeFalsy(); + + valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeFalsy(); + }); + + it('allows default policyConfig with Basic', () => { + const policy = factory(); + const valid = isEndpointPolicyValidForLicense(policy, Basic); + expect(valid).toBeTruthy(); + }); + }); + + describe('unsetPolicyFeaturesAboveLicenseLevel', () => { + it('does not change any fields with a Platinum license', () => { + const policy = factory(); + const popupMessage = 'WOOP WOOP'; + policy.windows.popup.malware.message = popupMessage; + policy.mac.popup.malware.message = popupMessage; + policy.windows.popup.malware.enabled = false; + + const retPolicy = unsetPolicyFeaturesAboveLicenseLevel(policy, Platinum); + expect(retPolicy.windows.popup.malware.enabled).toBeFalsy(); + expect(retPolicy.windows.popup.malware.message).toEqual(popupMessage); + expect(retPolicy.mac.popup.malware.message).toEqual(popupMessage); + }); + it('resets Platinum-paid fields for lower license tiers', () => { + const defaults = factory(); // reference + const policy = factory(); // what we will modify, and should be reset + const popupMessage = 'WOOP WOOP'; + policy.windows.popup.malware.message = popupMessage; + policy.mac.popup.malware.message = popupMessage; + policy.windows.popup.malware.enabled = false; + + const retPolicy = unsetPolicyFeaturesAboveLicenseLevel(policy, Gold); + expect(retPolicy.windows.popup.malware.enabled).toEqual( + defaults.windows.popup.malware.enabled + ); + expect(retPolicy.windows.popup.malware.message).not.toEqual(popupMessage); + expect(retPolicy.mac.popup.malware.message).not.toEqual(popupMessage); + + // need to invert the test, since it could be either value + expect(['', DefaultMalwareMessage]).toContain(retPolicy.windows.popup.malware.message); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/common/license/policy_config.ts b/x-pack/plugins/security_solution/common/license/policy_config.ts new file mode 100644 index 0000000000000..da2260ad55e8b --- /dev/null +++ b/x-pack/plugins/security_solution/common/license/policy_config.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ILicense } from '../../../licensing/common/types'; +import { isAtLeast } from './license'; +import { PolicyConfig } from '../endpoint/types'; +import { DefaultMalwareMessage, factory } from '../endpoint/models/policy_config'; + +/** + * Given an endpoint package policy, verifies that all enabled features that + * require a certain license level have a valid license for them. + */ +export const isEndpointPolicyValidForLicense = ( + policy: PolicyConfig, + license: ILicense | null +): boolean => { + if (isAtLeast(license, 'platinum')) { + return true; // currently, platinum allows all features + } + + const defaults = factory(); + + // only platinum or higher may disable malware notification + if ( + policy.windows.popup.malware.enabled !== defaults.windows.popup.malware.enabled || + policy.mac.popup.malware.enabled !== defaults.mac.popup.malware.enabled + ) { + return false; + } + + // Only Platinum or higher may change the malware message (which can be blank or what Endpoint defaults) + if ( + [policy.windows, policy.mac].some( + (p) => p.popup.malware.message !== '' && p.popup.malware.message !== DefaultMalwareMessage + ) + ) { + return false; + } + + return true; +}; + +/** + * Resets paid features in a PolicyConfig back to default values + * when unsupported by the given license level. + */ +export const unsetPolicyFeaturesAboveLicenseLevel = ( + policy: PolicyConfig, + license: ILicense | null +): PolicyConfig => { + if (isAtLeast(license, 'platinum')) { + return policy; + } + + const defaults = factory(); + // set any license-gated features back to the defaults + policy.windows.popup.malware.enabled = defaults.windows.popup.malware.enabled; + policy.mac.popup.malware.enabled = defaults.mac.popup.malware.enabled; + policy.windows.popup.malware.message = defaults.windows.popup.malware.message; + policy.mac.popup.malware.message = defaults.mac.popup.malware.message; + + return policy; +}; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts index 36649d22f730c..f039324b3af64 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts @@ -5,6 +5,7 @@ */ import { IHttpFetchError } from 'kibana/public'; +import { DefaultMalwareMessage } from '../../../../../../common/endpoint/models/policy_config'; import { PolicyDetailsState, UpdatePolicyResponse } from '../../types'; import { policyIdFromParams, @@ -38,10 +39,8 @@ export const policyDetailsMiddlewareFactory: ImmutableMiddlewareFactory Date: Wed, 25 Nov 2020 13:34:59 -0700 Subject: [PATCH 10/37] [basePathProxy] include query in redirect (#84356) Co-authored-by: spalger --- src/core/server/http/base_path_proxy_server.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/server/http/base_path_proxy_server.ts b/src/core/server/http/base_path_proxy_server.ts index 42841377e7369..737aab00cff0e 100644 --- a/src/core/server/http/base_path_proxy_server.ts +++ b/src/core/server/http/base_path_proxy_server.ts @@ -199,8 +199,13 @@ export class BasePathProxyServer { const isGet = request.method === 'get'; const isBasepathLike = oldBasePath.length === 3; + const newUrl = Url.format({ + pathname: `${this.httpConfig.basePath}/${kbnPath}`, + query: request.query, + }); + return isGet && isBasepathLike && shouldRedirectFromOldBasePath(kbnPath) - ? responseToolkit.redirect(`${this.httpConfig.basePath}/${kbnPath}`) + ? responseToolkit.redirect(newUrl) : responseToolkit.response('Not Found').code(404); }, method: '*', From 459263fd596b91f35c71f025ad9990dde794a9b5 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 25 Nov 2020 13:29:57 -0800 Subject: [PATCH 11/37] [Fleet] Support URL query state in agent logs UI (#84298) * Initial attempt at URL state * Break into smaller files * Handle invalid date range expressions --- .../components/agent_logs/agent_logs.tsx | 278 ++++++++++++++++++ .../components/agent_logs/constants.tsx | 9 + .../components/agent_logs/index.tsx | 277 ++++------------- 3 files changed, 339 insertions(+), 225 deletions(-) create mode 100644 x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx new file mode 100644 index 0000000000000..00deeff89503f --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx @@ -0,0 +1,278 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React, { memo, useMemo, useState, useCallback, useEffect } from 'react'; +import styled from 'styled-components'; +import url from 'url'; +import { encode } from 'rison-node'; +import { stringify } from 'query-string'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiSuperDatePicker, + EuiFilterGroup, + EuiPanel, + EuiButtonEmpty, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import semverGte from 'semver/functions/gte'; +import semverCoerce from 'semver/functions/coerce'; +import { createStateContainerReactHelpers } from '../../../../../../../../../../../src/plugins/kibana_utils/public'; +import { RedirectAppLinks } from '../../../../../../../../../../../src/plugins/kibana_react/public'; +import { TimeRange, esKuery } from '../../../../../../../../../../../src/plugins/data/public'; +import { LogStream } from '../../../../../../../../../infra/public'; +import { Agent } from '../../../../../types'; +import { useStartServices } from '../../../../../hooks'; +import { DEFAULT_DATE_RANGE } from './constants'; +import { DatasetFilter } from './filter_dataset'; +import { LogLevelFilter } from './filter_log_level'; +import { LogQueryBar } from './query_bar'; +import { buildQuery } from './build_query'; +import { SelectLogLevel } from './select_log_level'; + +const WrapperFlexGroup = styled(EuiFlexGroup)` + height: 100%; +`; + +const DatePickerFlexItem = styled(EuiFlexItem)` + max-width: 312px; +`; + +export interface AgentLogsProps { + agent: Agent; + state: AgentLogsState; +} + +export interface AgentLogsState { + start: string; + end: string; + logLevels: string[]; + datasets: string[]; + query: string; +} + +export const AgentLogsUrlStateHelper = createStateContainerReactHelpers(); + +export const AgentLogsUI: React.FunctionComponent = memo(({ agent, state }) => { + const { data, application, http } = useStartServices(); + const { update: updateState } = AgentLogsUrlStateHelper.useTransitions(); + + // Util to convert date expressions (returned by datepicker) to timestamps (used by LogStream) + const getDateRangeTimestamps = useCallback( + (timeRange: TimeRange) => { + const { min, max } = data.query.timefilter.timefilter.calculateBounds(timeRange); + return min && max + ? { + start: min.valueOf(), + end: max.valueOf(), + } + : undefined; + }, + [data.query.timefilter.timefilter] + ); + + const tryUpdateDateRange = useCallback( + (timeRange: TimeRange) => { + const timestamps = getDateRangeTimestamps(timeRange); + if (timestamps) { + updateState({ + start: timeRange.from, + end: timeRange.to, + }); + } + }, + [getDateRangeTimestamps, updateState] + ); + + const [dateRangeTimestamps, setDateRangeTimestamps] = useState<{ start: number; end: number }>( + getDateRangeTimestamps({ + from: state.start, + to: state.end, + }) || + getDateRangeTimestamps({ + from: DEFAULT_DATE_RANGE.start, + to: DEFAULT_DATE_RANGE.end, + })! + ); + + // Attempts to parse for timestamps when start/end date expressions change + // If invalid date expressions, set expressions back to default + // Otherwise set the new timestamps + useEffect(() => { + const timestampsFromDateRange = getDateRangeTimestamps({ + from: state.start, + to: state.end, + }); + if (!timestampsFromDateRange) { + tryUpdateDateRange({ + from: DEFAULT_DATE_RANGE.start, + to: DEFAULT_DATE_RANGE.end, + }); + } else { + setDateRangeTimestamps(timestampsFromDateRange); + } + }, [state.start, state.end, getDateRangeTimestamps, tryUpdateDateRange]); + + // Query validation helper + const isQueryValid = useCallback((testQuery: string) => { + try { + esKuery.fromKueryExpression(testQuery); + return true; + } catch (err) { + return false; + } + }, []); + + // User query state + const [draftQuery, setDraftQuery] = useState(state.query); + const [isDraftQueryValid, setIsDraftQueryValid] = useState(isQueryValid(state.query)); + const onUpdateDraftQuery = useCallback( + (newDraftQuery: string, runQuery?: boolean) => { + setDraftQuery(newDraftQuery); + if (isQueryValid(newDraftQuery)) { + setIsDraftQueryValid(true); + if (runQuery) { + updateState({ query: newDraftQuery }); + } + } else { + setIsDraftQueryValid(false); + } + }, + [isQueryValid, updateState] + ); + + // Build final log stream query from agent id, datasets, log levels, and user input + const logStreamQuery = useMemo( + () => + buildQuery({ + agentId: agent.id, + datasets: state.datasets, + logLevels: state.logLevels, + userQuery: state.query, + }), + [agent.id, state.datasets, state.logLevels, state.query] + ); + + // Generate URL to pass page state to Logs UI + const viewInLogsUrl = useMemo( + () => + http.basePath.prepend( + url.format({ + pathname: '/app/logs/stream', + search: stringify( + { + logPosition: encode({ + start: state.start, + end: state.end, + streamLive: false, + }), + logFilter: encode({ + expression: logStreamQuery, + kind: 'kuery', + }), + }, + { sort: false, encode: false } + ), + }) + ), + [http.basePath, state.start, state.end, logStreamQuery] + ); + + const agentVersion = agent.local_metadata?.elastic?.agent?.version; + const isLogLevelSelectionAvailable = useMemo(() => { + if (!agentVersion) { + return false; + } + const agentVersionWithPrerelease = semverCoerce(agentVersion)?.version; + if (!agentVersionWithPrerelease) { + return false; + } + return semverGte(agentVersionWithPrerelease, '7.11.0'); + }, [agentVersion]); + + return ( + + + + + + + + + { + const currentDatasets = [...state.datasets]; + const datasetPosition = currentDatasets.indexOf(dataset); + if (datasetPosition >= 0) { + currentDatasets.splice(datasetPosition, 1); + updateState({ datasets: currentDatasets }); + } else { + updateState({ datasets: [...state.datasets, dataset] }); + } + }} + /> + { + const currentLevels = [...state.logLevels]; + const levelPosition = currentLevels.indexOf(level); + if (levelPosition >= 0) { + currentLevels.splice(levelPosition, 1); + updateState({ logLevels: currentLevels }); + } else { + updateState({ logLevels: [...state.logLevels, level] }); + } + }} + /> + + + + { + tryUpdateDateRange({ + from: start, + to: end, + }); + }} + /> + + + + + + + + + + + + + + + + {isLogLevelSelectionAvailable && ( + + + + )} + + ); +}); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx index ea98de3560246..89fe1a916605d 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/constants.tsx @@ -3,6 +3,8 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { AgentLogsState } from './agent_logs'; + export const AGENT_LOG_INDEX_PATTERN = 'logs-elastic_agent-*,logs-elastic_agent.*-*'; export const AGENT_DATASET = 'elastic_agent'; export const AGENT_DATASET_PATTERN = 'elastic_agent.*'; @@ -24,6 +26,13 @@ export const DEFAULT_DATE_RANGE = { start: 'now-1d', end: 'now', }; +export const DEFAULT_LOGS_STATE: AgentLogsState = { + start: DEFAULT_DATE_RANGE.start, + end: DEFAULT_DATE_RANGE.end, + logLevels: [], + datasets: [AGENT_DATASET], + query: '', +}; export const AGENT_LOG_LEVELS = { ERROR: 'error', diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/index.tsx index bed857c073099..0d888a88ec2cb 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/index.tsx @@ -3,236 +3,63 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import React, { memo, useMemo, useState, useCallback } from 'react'; -import styled from 'styled-components'; -import url from 'url'; -import { encode } from 'rison-node'; -import { stringify } from 'query-string'; +import React, { memo, useEffect, useState } from 'react'; import { - EuiFlexGroup, - EuiFlexItem, - EuiSuperDatePicker, - EuiFilterGroup, - EuiPanel, - EuiButtonEmpty, -} from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n/react'; -import semverGte from 'semver/functions/gte'; -import semverCoerce from 'semver/functions/coerce'; -import { RedirectAppLinks } from '../../../../../../../../../../../src/plugins/kibana_react/public'; -import { TimeRange, esKuery } from '../../../../../../../../../../../src/plugins/data/public'; -import { LogStream } from '../../../../../../../../../infra/public'; -import { Agent } from '../../../../../types'; -import { useStartServices } from '../../../../../hooks'; -import { AGENT_DATASET, DEFAULT_DATE_RANGE } from './constants'; -import { DatasetFilter } from './filter_dataset'; -import { LogLevelFilter } from './filter_log_level'; -import { LogQueryBar } from './query_bar'; -import { buildQuery } from './build_query'; -import { SelectLogLevel } from './select_log_level'; + createStateContainer, + syncState, + createKbnUrlStateStorage, + INullableBaseStateContainer, + PureTransition, + getStateFromKbnUrl, +} from '../../../../../../../../../../../src/plugins/kibana_utils/public'; +import { DEFAULT_LOGS_STATE } from './constants'; +import { AgentLogsUI, AgentLogsProps, AgentLogsState, AgentLogsUrlStateHelper } from './agent_logs'; -const WrapperFlexGroup = styled(EuiFlexGroup)` - height: 100%; -`; +const stateStorageKey = '_q'; -const DatePickerFlexItem = styled(EuiFlexItem)` - max-width: 312px; -`; +const stateContainer = createStateContainer< + AgentLogsState, + { + update: PureTransition]>; + } +>( + { + ...DEFAULT_LOGS_STATE, + ...getStateFromKbnUrl(stateStorageKey, window.location.href), + }, + { + update: (state) => (updatedState) => ({ ...state, ...updatedState }), + } +); -export const AgentLogs: React.FunctionComponent<{ agent: Agent }> = memo(({ agent }) => { - const { data, application, http } = useStartServices(); +const AgentLogsConnected = AgentLogsUrlStateHelper.connect((state) => ({ + state: state || DEFAULT_LOGS_STATE, +}))(AgentLogsUI); - // Util to convert date expressions (returned by datepicker) to timestamps (used by LogStream) - const getDateRangeTimestamps = useCallback( - (timeRange: TimeRange) => { - const { min, max } = data.query.timefilter.timefilter.calculateBounds(timeRange); - return min && max - ? { - startTimestamp: min.valueOf(), - endTimestamp: max.valueOf(), - } - : undefined; - }, - [data.query.timefilter.timefilter] - ); +export const AgentLogs: React.FunctionComponent> = memo( + ({ agent }) => { + const [isSyncReady, setIsSyncReady] = useState(false); - // Initial time range filter - const [dateRange, setDateRange] = useState<{ - startExpression: string; - endExpression: string; - startTimestamp: number; - endTimestamp: number; - }>({ - startExpression: DEFAULT_DATE_RANGE.start, - endExpression: DEFAULT_DATE_RANGE.end, - ...getDateRangeTimestamps({ from: DEFAULT_DATE_RANGE.start, to: DEFAULT_DATE_RANGE.end })!, - }); + useEffect(() => { + const stateStorage = createKbnUrlStateStorage(); + const { start, stop } = syncState({ + storageKey: stateStorageKey, + stateContainer: stateContainer as INullableBaseStateContainer, + stateStorage, + }); + start(); + setIsSyncReady(true); - const tryUpdateDateRange = useCallback( - (timeRange: TimeRange) => { - const timestamps = getDateRangeTimestamps(timeRange); - if (timestamps) { - setDateRange({ - startExpression: timeRange.from, - endExpression: timeRange.to, - ...timestamps, - }); - } - }, - [getDateRangeTimestamps] - ); + return () => { + stop(); + stateContainer.set(DEFAULT_LOGS_STATE); + }; + }, []); - // Filters - const [selectedLogLevels, setSelectedLogLevels] = useState([]); - const [selectedDatasets, setSelectedDatasets] = useState([AGENT_DATASET]); - - // User query state - const [query, setQuery] = useState(''); - const [draftQuery, setDraftQuery] = useState(''); - const [isDraftQueryValid, setIsDraftQueryValid] = useState(true); - const onUpdateDraftQuery = useCallback((newDraftQuery: string, runQuery?: boolean) => { - setDraftQuery(newDraftQuery); - try { - esKuery.fromKueryExpression(newDraftQuery); - setIsDraftQueryValid(true); - if (runQuery) { - setQuery(newDraftQuery); - } - } catch (err) { - setIsDraftQueryValid(false); - } - }, []); - - // Build final log stream query from agent id, datasets, log levels, and user input - const logStreamQuery = useMemo( - () => - buildQuery({ - agentId: agent.id, - datasets: selectedDatasets, - logLevels: selectedLogLevels, - userQuery: query, - }), - [agent.id, query, selectedDatasets, selectedLogLevels] - ); - - // Generate URL to pass page state to Logs UI - const viewInLogsUrl = useMemo( - () => - http.basePath.prepend( - url.format({ - pathname: '/app/logs/stream', - search: stringify( - { - logPosition: encode({ - start: dateRange.startExpression, - end: dateRange.endExpression, - streamLive: false, - }), - logFilter: encode({ - expression: logStreamQuery, - kind: 'kuery', - }), - }, - { sort: false, encode: false } - ), - }) - ), - [logStreamQuery, dateRange.endExpression, dateRange.startExpression, http.basePath] - ); - - const agentVersion = agent.local_metadata?.elastic?.agent?.version; - const isLogLevelSelectionAvailable = useMemo(() => { - if (!agentVersion) { - return false; - } - const agentVersionWithPrerelease = semverCoerce(agentVersion)?.version; - if (!agentVersionWithPrerelease) { - return false; - } - return semverGte(agentVersionWithPrerelease, '7.11.0'); - }, [agentVersion]); - - return ( - - - - - - - - - { - const currentLevels = [...selectedDatasets]; - const levelPosition = currentLevels.indexOf(level); - if (levelPosition >= 0) { - currentLevels.splice(levelPosition, 1); - setSelectedDatasets(currentLevels); - } else { - setSelectedDatasets([...selectedDatasets, level]); - } - }} - /> - { - const currentLevels = [...selectedLogLevels]; - const levelPosition = currentLevels.indexOf(level); - if (levelPosition >= 0) { - currentLevels.splice(levelPosition, 1); - setSelectedLogLevels(currentLevels); - } else { - setSelectedLogLevels([...selectedLogLevels, level]); - } - }} - /> - - - - { - tryUpdateDateRange({ - from: start, - to: end, - }); - }} - /> - - - - - - - - - - - - - - - - {isLogLevelSelectionAvailable && ( - - - - )} - - ); -}); + return ( + + {isSyncReady ? : null} + + ); + } +); From fac792778e5dada6c672c9ef082db837629a847e Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Thu, 26 Nov 2020 09:32:04 +0100 Subject: [PATCH 12/37] [fleet] Add config options to accepted docker env vars (#84338) --- .../os_packages/docker_generator/resources/bin/kibana-docker | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker index 4c833f5be6c5b..3e440c89b82d8 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker @@ -166,6 +166,9 @@ kibana_vars=( xpack.code.security.gitProtocolWhitelist xpack.encryptedSavedObjects.encryptionKey xpack.encryptedSavedObjects.keyRotation.decryptionOnlyKeys + xpack.fleet.agents.elasticsearch.host + xpack.fleet.agents.kibana.host + xpack.fleet.agents.tlsCheckDisabled xpack.graph.enabled xpack.graph.canEditDrillDownUrls xpack.graph.savePolicy From a4c8dca02147dc3e3f9fef86c4e8f26320ed2058 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Thu, 26 Nov 2020 10:08:45 +0100 Subject: [PATCH 13/37] [Uptime] Fix headers io-ts type (#84089) --- x-pack/plugins/uptime/common/runtime_types/ping/ping.ts | 6 +++++- .../uptime/public/components/monitor/ping_list/headers.tsx | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts b/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts index 9e5cd7641b65d..f9dde011b25fe 100644 --- a/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts +++ b/x-pack/plugins/uptime/common/runtime_types/ping/ping.ts @@ -86,6 +86,10 @@ export const MonitorType = t.intersection([ export type Monitor = t.TypeOf; +export const PingHeadersType = t.record(t.string, t.union([t.string, t.array(t.string)])); + +export type PingHeaders = t.TypeOf; + export const PingType = t.intersection([ t.type({ timestamp: t.string, @@ -135,7 +139,7 @@ export const PingType = t.intersection([ bytes: t.number, redirects: t.array(t.string), status_code: t.number, - headers: t.record(t.string, t.string), + headers: PingHeadersType, }), version: t.string, }), diff --git a/x-pack/plugins/uptime/public/components/monitor/ping_list/headers.tsx b/x-pack/plugins/uptime/public/components/monitor/ping_list/headers.tsx index 52fe26a7e08ca..a8cd8e2a26f98 100644 --- a/x-pack/plugins/uptime/public/components/monitor/ping_list/headers.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/ping_list/headers.tsx @@ -7,9 +7,10 @@ import React from 'react'; import { EuiAccordion, EuiDescriptionList, EuiSpacer, EuiText } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { PingHeaders as HeadersProp } from '../../../../common/runtime_types'; interface Props { - headers: Record; + headers: HeadersProp; } export const PingHeaders = ({ headers }: Props) => { From d2552c426d6b0385ab075284c06232df8d25ab68 Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Thu, 26 Nov 2020 14:08:47 +0300 Subject: [PATCH 14/37] fix identation in list (#84301) --- .../plugin/migrating-legacy-plugins-examples.asciidoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc b/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc index 469f7a4f3adb1..8a0e487971b20 100644 --- a/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc +++ b/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc @@ -902,8 +902,9 @@ The most significant changes on the Kibana side for the consumers are the follow ===== User client accessor Internal /current user client accessors has been renamed and are now properties instead of functions: -** `callAsInternalUser('ping')` -> `asInternalUser.ping()` -** `callAsCurrentUser('ping')` -> `asCurrentUser.ping()` + +* `callAsInternalUser('ping')` -> `asInternalUser.ping()` +* `callAsCurrentUser('ping')` -> `asCurrentUser.ping()` * the API now reflects the `Client`’s instead of leveraging the string-based endpoint names the `LegacyAPICaller` was using. From 4004dd4012b569a72f53d09a482958f62de64383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Thu, 26 Nov 2020 12:25:05 +0100 Subject: [PATCH 15/37] [Application Usage] Update `schema` with new `fleet` rename (#84327) --- .../server/collectors/application_usage/schema.ts | 2 +- src/plugins/telemetry/schema/oss_plugins.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts index 2e79cdaa7fc6b..76141f098a065 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts @@ -73,7 +73,7 @@ export const applicationUsageSchema = { logs: commonSchema, metrics: commonSchema, infra: commonSchema, // It's a forward app so we'll likely never report it - ingestManager: commonSchema, + fleet: commonSchema, lens: commonSchema, maps: commonSchema, ml: commonSchema, diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index a1eae69ffaed0..3d79d7c6cf0e1 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -616,7 +616,7 @@ } } }, - "ingestManager": { + "fleet": { "properties": { "clicks_total": { "type": "long" From 36ab99e546ead4f0a3e688d2575512172f94470b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 26 Nov 2020 14:01:53 +0100 Subject: [PATCH 16/37] [Logs UI] Limit the height of the "view in context" container (#83178) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../infra/public/pages/logs/stream/page_view_log_in_context.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx b/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx index 4ac3d15a82222..8a4081288b283 100644 --- a/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx +++ b/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx @@ -84,6 +84,7 @@ const LogInContextWrapper = euiStyled.div<{ width: number | string; height: numb padding: 16px; width: ${(props) => (typeof props.width === 'number' ? `${props.width}px` : props.width)}; height: ${(props) => (typeof props.height === 'number' ? `${props.height}px` : props.height)}; + max-height: 75vh; // Same as EuiModal `; const LogEntryContext: React.FC<{ context: LogEntry['context'] }> = ({ context }) => { From b246701d39a582b4ea3c32f9a3ca8ed8fcdb2480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 26 Nov 2020 14:18:42 +0100 Subject: [PATCH 17/37] [Logs UI] Polish the UI for the log entry examples in the anomaly table (#82139) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../log_entry_examples/log_entry_examples.tsx | 1 + .../log_entry_examples_empty_indicator.tsx | 2 +- .../log_entry_examples_failure_indicator.tsx | 2 +- .../sections/anomalies/expanded_row.tsx | 12 +++++++----- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx index 2ec9922d94555..2d15068e51da5 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx @@ -46,4 +46,5 @@ const Wrapper = euiStyled.div` flex-direction: column; flex: 1 0 0%; overflow: hidden; + padding-top: 1px; // Buffer for the "Reload" buttons' hover state `; diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx index 1d6028ed032a2..d3d309948529f 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx @@ -10,7 +10,7 @@ import React from 'react'; export const LogEntryExampleMessagesEmptyIndicator: React.FunctionComponent<{ onReload: () => void; }> = ({ onReload }) => ( - + void; }> = ({ onRetry }) => ( - + - +

{examplesTitle}

 + + Date: Thu, 26 Nov 2020 15:45:21 +0100 Subject: [PATCH 18/37] [Discover] Fix navigating back when changing index pattern (#84061) --- .../public/application/angular/discover.js | 6 +++--- .../discover/_indexpattern_without_timefield.ts | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 272c2f2ca6187..7059593c0c4e7 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -252,7 +252,8 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise if (!_.isEqual(newStatePartial, oldStatePartial)) { $scope.$evalAsync(async () => { if (oldStatePartial.index !== newStatePartial.index) { - //in case of index switch the route has currently to be reloaded, legacy + //in case of index pattern switch the route has currently to be reloaded, legacy + $route.reload(); return; } @@ -289,8 +290,7 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise $scope.state.sort, config.get(MODIFY_COLUMNS_ON_SWITCH) ); - await replaceUrlAppState(nextAppState); - $route.reload(); + await setAppState(nextAppState); } }; diff --git a/test/functional/apps/discover/_indexpattern_without_timefield.ts b/test/functional/apps/discover/_indexpattern_without_timefield.ts index 677b27c31bd86..20d783690277f 100644 --- a/test/functional/apps/discover/_indexpattern_without_timefield.ts +++ b/test/functional/apps/discover/_indexpattern_without_timefield.ts @@ -20,6 +20,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { + const browser = getService('browser'); const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); const security = getService('security'); @@ -50,5 +51,21 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { throw new Error('Expected timepicker to exist'); } }); + it('should switch between with and without timefield using the browser back button', async () => { + await PageObjects.discover.selectIndexPattern('without-timefield'); + if (await PageObjects.timePicker.timePickerExists()) { + throw new Error('Expected timepicker not to exist'); + } + + await PageObjects.discover.selectIndexPattern('with-timefield'); + if (!(await PageObjects.timePicker.timePickerExists())) { + throw new Error('Expected timepicker to exist'); + } + // Navigating back to discover + await browser.goBack(); + if (await PageObjects.timePicker.timePickerExists()) { + throw new Error('Expected timepicker not to exist'); + } + }); }); } From 93b0273ccddb89aed744ef1d64eafd390dcf9090 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Thu, 26 Nov 2020 18:04:27 +0300 Subject: [PATCH 19/37] TSVB offsets (#83051) Closes: #40299 Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../response_processors/series/time_shift.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js index 14de6aa18f872..c00b0894073d2 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js @@ -17,7 +17,7 @@ * under the License. */ -import _ from 'lodash'; +import { startsWith } from 'lodash'; import moment from 'moment'; export function timeShift(resp, panel, series) { @@ -26,13 +26,15 @@ export function timeShift(resp, panel, series) { const matches = series.offset_time.match(/^([+-]?[\d]+)([shmdwMy]|ms)$/); if (matches) { - const offsetValue = Number(matches[1]); + const offsetValue = matches[1]; const offsetUnit = matches[2]; - const offset = moment.duration(offsetValue, offsetUnit).valueOf(); results.forEach((item) => { - if (_.startsWith(item.id, series.id)) { - item.data = item.data.map(([time, value]) => [time + offset, value]); + if (startsWith(item.id, series.id)) { + item.data = item.data.map((row) => [ + moment.utc(row[0]).add(offsetValue, offsetUnit).valueOf(), + row[1], + ]); } }); } From 4d8b7f9084f46bc4694ea8d78aee2b13a049a80c Mon Sep 17 00:00:00 2001 From: Joe Portner <5295965+jportner@users.noreply.github.com> Date: Thu, 26 Nov 2020 10:51:10 -0500 Subject: [PATCH 20/37] Improve short-url redirect validation (#84366) --- .../routes/lib/short_url_assert_valid.test.ts | 45 ++++++++++--------- .../routes/lib/short_url_assert_valid.ts | 12 +++-- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts b/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts index 02a5e123b6481..232429ac4ec33 100644 --- a/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts +++ b/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts @@ -19,36 +19,39 @@ import { shortUrlAssertValid } from './short_url_assert_valid'; +const PROTOCOL_ERROR = /^Short url targets cannot have a protocol/; +const HOSTNAME_ERROR = /^Short url targets cannot have a hostname/; +const PATH_ERROR = /^Short url target path must be in the format/; + describe('shortUrlAssertValid()', () => { const invalid = [ - ['protocol', 'http://localhost:5601/app/kibana'], - ['protocol', 'https://localhost:5601/app/kibana'], - ['protocol', 'mailto:foo@bar.net'], - ['protocol', 'javascript:alert("hi")'], // eslint-disable-line no-script-url - ['hostname', 'localhost/app/kibana'], - ['hostname and port', 'local.host:5601/app/kibana'], - ['hostname and auth', 'user:pass@localhost.net/app/kibana'], - ['path traversal', '/app/../../not-kibana'], - ['deep path', '/app/kibana/foo'], - ['deep path', '/app/kibana/foo/bar'], - ['base path', '/base/app/kibana'], + ['protocol', 'http://localhost:5601/app/kibana', PROTOCOL_ERROR], + ['protocol', 'https://localhost:5601/app/kibana', PROTOCOL_ERROR], + ['protocol', 'mailto:foo@bar.net', PROTOCOL_ERROR], + ['protocol', 'javascript:alert("hi")', PROTOCOL_ERROR], // eslint-disable-line no-script-url + ['hostname', 'localhost/app/kibana', PATH_ERROR], // according to spec, this is not a valid URL -- you cannot specify a hostname without a protocol + ['hostname and port', 'local.host:5601/app/kibana', PROTOCOL_ERROR], // parser detects 'local.host' as the protocol + ['hostname and auth', 'user:pass@localhost.net/app/kibana', PROTOCOL_ERROR], // parser detects 'user' as the protocol + ['path traversal', '/app/../../not-kibana', PATH_ERROR], // fails because there are >2 path parts + ['path traversal', '/../not-kibana', PATH_ERROR], // fails because first path part is not 'app' + ['deep path', '/app/kibana/foo', PATH_ERROR], // fails because there are >2 path parts + ['deeper path', '/app/kibana/foo/bar', PATH_ERROR], // fails because there are >2 path parts + ['base path', '/base/app/kibana', PATH_ERROR], // fails because there are >2 path parts + ['path with an extra leading slash', '//foo/app/kibana', HOSTNAME_ERROR], // parser detects 'foo' as the hostname + ['path with an extra leading slash', '///app/kibana', HOSTNAME_ERROR], // parser detects '' as the hostname + ['path without app', '/foo/kibana', PATH_ERROR], // fails because first path part is not 'app' + ['path without appId', '/app/', PATH_ERROR], // fails because there is only one path part (leading and trailing slashes are trimmed) ]; - invalid.forEach(([desc, url]) => { - it(`fails when url has ${desc}`, () => { - try { - shortUrlAssertValid(url); - throw new Error(`expected assertion to throw`); - } catch (err) { - if (!err || !err.isBoom) { - throw err; - } - } + invalid.forEach(([desc, url, error]) => { + it(`fails when url has ${desc as string}`, () => { + expect(() => shortUrlAssertValid(url as string)).toThrowError(error); }); }); const valid = [ '/app/kibana', + '/app/kibana/', // leading and trailing slashes are trimmed '/app/monitoring#angular/route', '/app/text#document-id', '/app/some?with=query', diff --git a/src/plugins/share/server/routes/lib/short_url_assert_valid.ts b/src/plugins/share/server/routes/lib/short_url_assert_valid.ts index 581410359322f..773e3acdcb902 100644 --- a/src/plugins/share/server/routes/lib/short_url_assert_valid.ts +++ b/src/plugins/share/server/routes/lib/short_url_assert_valid.ts @@ -22,18 +22,22 @@ import { trim } from 'lodash'; import Boom from '@hapi/boom'; export function shortUrlAssertValid(url: string) { - const { protocol, hostname, pathname } = parse(url); + const { protocol, hostname, pathname } = parse( + url, + false /* parseQueryString */, + true /* slashesDenoteHost */ + ); - if (protocol) { + if (protocol !== null) { throw Boom.notAcceptable(`Short url targets cannot have a protocol, found "${protocol}"`); } - if (hostname) { + if (hostname !== null) { throw Boom.notAcceptable(`Short url targets cannot have a hostname, found "${hostname}"`); } const pathnameParts = trim(pathname === null ? undefined : pathname, '/').split('/'); - if (pathnameParts.length !== 2) { + if (pathnameParts.length !== 2 || pathnameParts[0] !== 'app' || !pathnameParts[1]) { throw Boom.notAcceptable( `Short url target path must be in the format "/app/{{appId}}", found "${pathname}"` ); From 51f75a5655224d77c97b944f8b603ff34ec92714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yulia=20=C4=8Cech?= <6585477+yuliacech@users.noreply.github.com> Date: Thu, 26 Nov 2020 17:04:24 +0100 Subject: [PATCH 21/37] Added data streams privileges to better control delete actions in UI (#83573) * Added data streams privileges to better control delete actions in UI * Fix type check issues * Change data streams privileges request * Fixed type check issue * Fixed api integration test * Cleaned up not needed code * Renamed some data streams and added a default value for stats find Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../helpers/test_subjects.ts | 2 + .../home/data_streams_tab.helpers.ts | 12 + .../home/data_streams_tab.test.ts | 73 ++++++ .../common/lib/data_stream_serialization.ts | 2 + .../common/types/data_streams.ts | 16 +- .../data_stream_detail_panel.tsx | 2 +- .../data_stream_list/data_stream_list.tsx | 9 +- .../data_stream_table/data_stream_table.tsx | 6 +- .../server/client/elasticsearch.ts | 22 -- .../plugins/index_management/server/plugin.ts | 3 +- .../component_templates/privileges.test.ts | 2 + .../api/data_streams/register_get_route.ts | 221 ++++++++++++------ .../index_management/server/shared_imports.ts | 2 +- .../plugins/index_management/server/types.ts | 3 +- .../index_management/data_streams.ts | 9 + 15 files changed, 271 insertions(+), 113 deletions(-) diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts b/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts index 04843cae6a57e..e8105ac2937c0 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts @@ -13,6 +13,8 @@ export type TestSubjects = | 'createTemplateButton' | 'dataStreamsEmptyPromptTemplateLink' | 'dataStreamTable' + | 'deleteDataStreamsButton' + | 'deleteDataStreamButton' | 'deleteSystemTemplateCallOut' | 'deleteTemplateButton' | 'deleteTemplatesConfirmation' diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts index 4e0486e55720d..9c92af30097a2 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts @@ -24,6 +24,7 @@ export interface DataStreamsTabTestBed extends TestBed { clickNameAt: (index: number) => void; clickIndicesAt: (index: number) => void; clickDeleteActionAt: (index: number) => void; + selectDataStream: (name: string, selected: boolean) => void; clickConfirmDelete: () => void; clickDeleteDataStreamButton: () => void; clickDetailPanelIndexTemplateLink: () => void; @@ -125,6 +126,13 @@ export const setup = async (overridingDependencies: any = {}): Promise { + const { + form: { selectCheckBox }, + } = testBed; + selectCheckBox(`checkboxSelectRow-${name}`, selected); + }; + const findDeleteConfirmationModal = () => { const { find } = testBed; return find('deleteDataStreamsConfirmation'); @@ -194,6 +202,7 @@ export const setup = async (overridingDependencies: any = {}): Promise): DataSt indexTemplateName: 'indexTemplate', storageSize: '1b', maxTimeStamp: 420, + privileges: { + delete_index: true, + }, ...dataStream, }); diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts index 8ce307c103f4c..91502621d50c5 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts @@ -449,4 +449,77 @@ describe('Data Streams tab', () => { expect(tableCellsValues).toEqual([['', 'non-managed-data-stream', 'green', '1', 'Delete']]); }); }); + + describe('data stream privileges', () => { + describe('delete', () => { + const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers; + + const dataStreamWithDelete = createDataStreamPayload({ + name: 'dataStreamWithDelete', + privileges: { delete_index: true }, + }); + const dataStreamNoDelete = createDataStreamPayload({ + name: 'dataStreamNoDelete', + privileges: { delete_index: false }, + }); + + beforeEach(async () => { + setLoadDataStreamsResponse([dataStreamWithDelete, dataStreamNoDelete]); + + testBed = await setup({ history: createMemoryHistory() }); + await act(async () => { + testBed.actions.goToDataStreamsList(); + }); + testBed.component.update(); + }); + + test('displays/hides delete button depending on data streams privileges', async () => { + const { table } = testBed; + const { tableCellsValues } = table.getMetaData('dataStreamTable'); + + expect(tableCellsValues).toEqual([ + ['', 'dataStreamNoDelete', 'green', '1', ''], + ['', 'dataStreamWithDelete', 'green', '1', 'Delete'], + ]); + }); + + test('displays/hides delete action depending on data streams privileges', async () => { + const { + actions: { selectDataStream }, + find, + } = testBed; + + selectDataStream('dataStreamNoDelete', true); + expect(find('deleteDataStreamsButton').exists()).toBeFalsy(); + + selectDataStream('dataStreamWithDelete', true); + expect(find('deleteDataStreamsButton').exists()).toBeFalsy(); + + selectDataStream('dataStreamNoDelete', false); + expect(find('deleteDataStreamsButton').exists()).toBeTruthy(); + }); + + test('displays delete button in detail panel', async () => { + const { + actions: { clickNameAt }, + find, + } = testBed; + setLoadDataStreamResponse(dataStreamWithDelete); + await clickNameAt(1); + + expect(find('deleteDataStreamButton').exists()).toBeTruthy(); + }); + + test('hides delete button in detail panel', async () => { + const { + actions: { clickNameAt }, + find, + } = testBed; + setLoadDataStreamResponse(dataStreamNoDelete); + await clickNameAt(0); + + expect(find('deleteDataStreamButton').exists()).toBeFalsy(); + }); + }); + }); }); diff --git a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts b/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts index 2d8e038d2a60f..fe7db99c98db1 100644 --- a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts +++ b/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts @@ -18,6 +18,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS store_size: storageSize, maximum_timestamp: maxTimeStamp, _meta, + privileges, } = dataStreamFromEs; return { @@ -37,6 +38,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS storageSize, maxTimeStamp, _meta, + privileges, }; } diff --git a/x-pack/plugins/index_management/common/types/data_streams.ts b/x-pack/plugins/index_management/common/types/data_streams.ts index adb7104043fbb..fdfe6278eb985 100644 --- a/x-pack/plugins/index_management/common/types/data_streams.ts +++ b/x-pack/plugins/index_management/common/types/data_streams.ts @@ -10,13 +10,19 @@ interface TimestampFieldFromEs { type TimestampField = TimestampFieldFromEs; -interface MetaFieldFromEs { +interface MetaFromEs { managed_by: string; package: any; managed: boolean; } -type MetaField = MetaFieldFromEs; +type Meta = MetaFromEs; + +interface PrivilegesFromEs { + delete_index: boolean; +} + +type Privileges = PrivilegesFromEs; export type HealthFromEs = 'GREEN' | 'YELLOW' | 'RED'; @@ -25,12 +31,13 @@ export interface DataStreamFromEs { timestamp_field: TimestampFieldFromEs; indices: DataStreamIndexFromEs[]; generation: number; - _meta?: MetaFieldFromEs; + _meta?: MetaFromEs; status: HealthFromEs; template: string; ilm_policy?: string; store_size?: string; maximum_timestamp?: number; + privileges: PrivilegesFromEs; } export interface DataStreamIndexFromEs { @@ -50,7 +57,8 @@ export interface DataStream { ilmPolicyName?: string; storageSize?: string; maxTimeStamp?: number; - _meta?: MetaField; + _meta?: Meta; + privileges: Privileges; } export interface DataStreamIndex { diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx index 05d7e97745b9e..ec47b2c062aa9 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx @@ -290,7 +290,7 @@ export const DataStreamDetailPanel: React.FunctionComponent = ({ - {!isLoading && !error ? ( + {!isLoading && !error && dataStream?.privileges.delete_index ? ( { const { isDeepLink } = extractQueryParams(search); + const decodedDataStreamName = attemptToURIDecode(dataStreamName); const { core: { getUrlForApp }, @@ -241,8 +242,8 @@ export const DataStreamList: React.FunctionComponent { history.push(`/${Section.DataStreams}`); diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx index c1fd33a39569c..7a3e719d013c8 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx @@ -162,6 +162,7 @@ export const DataStreamTable: React.FunctionComponent = ({ }, isPrimary: true, 'data-test-subj': 'deleteDataStream', + available: ({ privileges: { delete_index: deleteIndex } }: DataStream) => deleteIndex, }, ], }); @@ -188,9 +189,10 @@ export const DataStreamTable: React.FunctionComponent = ({ incremental: true, }, toolsLeft: - selection.length > 0 ? ( + selection.length > 0 && + selection.every((dataStream: DataStream) => dataStream.privileges.delete_index) ? ( setDataStreamsToDelete(selection.map(({ name }: DataStream) => name))} color="danger" > diff --git a/x-pack/plugins/index_management/server/client/elasticsearch.ts b/x-pack/plugins/index_management/server/client/elasticsearch.ts index ed5ede07479ca..8b7749335131e 100644 --- a/x-pack/plugins/index_management/server/client/elasticsearch.ts +++ b/x-pack/plugins/index_management/server/client/elasticsearch.ts @@ -11,28 +11,6 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any) const dataManagement = Client.prototype.dataManagement.prototype; // Data streams - dataManagement.getDataStreams = ca({ - urls: [ - { - fmt: '/_data_stream', - }, - ], - method: 'GET', - }); - - dataManagement.getDataStream = ca({ - urls: [ - { - fmt: '/_data_stream/<%=name%>', - req: { - name: { - type: 'string', - }, - }, - }, - ], - method: 'GET', - }); // We don't allow the user to create a data stream in the UI or API. We're just adding this here // to enable the API integration tests. diff --git a/x-pack/plugins/index_management/server/plugin.ts b/x-pack/plugins/index_management/server/plugin.ts index ae9633f3e22b9..3d70140fa60b7 100644 --- a/x-pack/plugins/index_management/server/plugin.ts +++ b/x-pack/plugins/index_management/server/plugin.ts @@ -24,7 +24,7 @@ import { PLUGIN } from '../common'; import { Dependencies } from './types'; import { ApiRoutes } from './routes'; import { License, IndexDataEnricher } from './services'; -import { isEsError } from './shared_imports'; +import { isEsError, handleEsError } from './shared_imports'; import { elasticsearchJsPlugin } from './client/elasticsearch'; export interface DataManagementContext { @@ -110,6 +110,7 @@ export class IndexMgmtServerPlugin implements Plugin { indexDataEnricher: mockedIndexDataEnricher, lib: { isEsError: jest.fn(), + handleEsError: jest.fn(), }, }); @@ -123,6 +124,7 @@ describe('GET privileges', () => { indexDataEnricher: mockedIndexDataEnricher, lib: { isEsError: jest.fn(), + handleEsError: jest.fn(), }, }); diff --git a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts index fa93d9bd0c563..d19383d892cbd 100644 --- a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts +++ b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts @@ -6,122 +6,189 @@ import { schema, TypeOf } from '@kbn/config-schema'; +import { ElasticsearchClient } from 'kibana/server'; import { deserializeDataStream, deserializeDataStreamList } from '../../../../common/lib'; +import { DataStreamFromEs } from '../../../../common/types'; import { RouteDependencies } from '../../../types'; import { addBasePath } from '../index'; -const querySchema = schema.object({ - includeStats: schema.maybe(schema.oneOf([schema.literal('true'), schema.literal('false')])), -}); +interface PrivilegesFromEs { + username: string; + has_all_requested: boolean; + cluster: Record; + index: Record>; + application: Record; +} + +interface StatsFromEs { + data_stream: string; + store_size: string; + maximum_timestamp: number; +} -export function registerGetAllRoute({ router, license, lib: { isEsError } }: RouteDependencies) { +const enhanceDataStreams = ({ + dataStreams, + dataStreamsStats, + dataStreamsPrivileges, +}: { + dataStreams: DataStreamFromEs[]; + dataStreamsStats?: StatsFromEs[]; + dataStreamsPrivileges?: PrivilegesFromEs; +}): DataStreamFromEs[] => { + return dataStreams.map((dataStream: DataStreamFromEs) => { + let enhancedDataStream = { ...dataStream }; + + if (dataStreamsStats) { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { store_size, maximum_timestamp } = + dataStreamsStats.find( + ({ data_stream: statsName }: { data_stream: string }) => statsName === dataStream.name + ) || {}; + + enhancedDataStream = { + ...enhancedDataStream, + store_size, + maximum_timestamp, + }; + } + + enhancedDataStream = { + ...enhancedDataStream, + privileges: { + delete_index: dataStreamsPrivileges + ? dataStreamsPrivileges.index[dataStream.name].delete_index + : true, + }, + }; + + return enhancedDataStream; + }); +}; + +const getDataStreamsStats = (client: ElasticsearchClient, name = '*') => { + return client.transport.request({ + path: `/_data_stream/${encodeURIComponent(name)}/_stats`, + method: 'GET', + querystring: { + human: true, + }, + }); +}; + +const getDataStreamsPrivileges = (client: ElasticsearchClient, names: string[]) => { + return client.security.hasPrivileges({ + body: { + index: [ + { + names, + privileges: ['delete_index'], + }, + ], + }, + }); +}; + +export function registerGetAllRoute({ + router, + license, + lib: { handleEsError }, + config, +}: RouteDependencies) { + const querySchema = schema.object({ + includeStats: schema.maybe(schema.oneOf([schema.literal('true'), schema.literal('false')])), + }); router.get( { path: addBasePath('/data_streams'), validate: { query: querySchema } }, - license.guardApiRoute(async (ctx, req, res) => { - const { callAsCurrentUser } = ctx.dataManagement!.client; + license.guardApiRoute(async (ctx, req, response) => { + const { asCurrentUser } = ctx.core.elasticsearch.client; const includeStats = (req.query as TypeOf).includeStats === 'true'; try { - const { data_streams: dataStreams } = await callAsCurrentUser( - 'dataManagement.getDataStreams' - ); - - if (includeStats) { - const { - data_streams: dataStreamsStats, - } = await ctx.core.elasticsearch.legacy.client.callAsCurrentUser('transport.request', { - path: '/_data_stream/*/_stats', - method: 'GET', - query: { - human: true, - }, - }); + let { + body: { data_streams: dataStreams }, + } = await asCurrentUser.indices.getDataStream(); - // Merge stats into data streams. - for (let i = 0; i < dataStreams.length; i++) { - const dataStream = dataStreams[i]; + let dataStreamsStats; + let dataStreamsPrivileges; - // eslint-disable-next-line @typescript-eslint/naming-convention - const { store_size, maximum_timestamp } = dataStreamsStats.find( - ({ data_stream: statsName }: { data_stream: string }) => statsName === dataStream.name - ); - - dataStreams[i] = { - ...dataStream, - store_size, - maximum_timestamp, - }; - } + if (includeStats) { + ({ + body: { data_streams: dataStreamsStats }, + } = await getDataStreamsStats(asCurrentUser)); } - return res.ok({ body: deserializeDataStreamList(dataStreams) }); - } catch (error) { - if (isEsError(error)) { - return res.customError({ - statusCode: error.statusCode, - body: error, - }); + if (config.isSecurityEnabled() && dataStreams.length > 0) { + ({ body: dataStreamsPrivileges } = await getDataStreamsPrivileges( + asCurrentUser, + dataStreams.map((dataStream: DataStreamFromEs) => dataStream.name) + )); } - return res.internalError({ body: error }); + dataStreams = enhanceDataStreams({ + dataStreams, + dataStreamsStats, + dataStreamsPrivileges, + }); + + return response.ok({ body: deserializeDataStreamList(dataStreams) }); + } catch (error) { + return handleEsError({ error, response }); } }) ); } -export function registerGetOneRoute({ router, license, lib: { isEsError } }: RouteDependencies) { +export function registerGetOneRoute({ + router, + license, + lib: { handleEsError }, + config, +}: RouteDependencies) { const paramsSchema = schema.object({ name: schema.string(), }); - router.get( { path: addBasePath('/data_streams/{name}'), validate: { params: paramsSchema }, }, - license.guardApiRoute(async (ctx, req, res) => { + license.guardApiRoute(async (ctx, req, response) => { const { name } = req.params as TypeOf; - const { callAsCurrentUser } = ctx.dataManagement!.client; + const { asCurrentUser } = ctx.core.elasticsearch.client; try { const [ - { data_streams: dataStream }, - { data_streams: dataStreamsStats }, + { + body: { data_streams: dataStreams }, + }, + { + body: { data_streams: dataStreamsStats }, + }, ] = await Promise.all([ - callAsCurrentUser('dataManagement.getDataStream', { - name, - }), - ctx.core.elasticsearch.legacy.client.callAsCurrentUser('transport.request', { - path: `/_data_stream/${encodeURIComponent(name)}/_stats`, - method: 'GET', - query: { - human: true, - }, - }), + asCurrentUser.indices.getDataStream({ name }), + getDataStreamsStats(asCurrentUser, name), ]); - if (dataStream[0]) { - // eslint-disable-next-line @typescript-eslint/naming-convention - const { store_size, maximum_timestamp } = dataStreamsStats[0]; - dataStream[0] = { - ...dataStream[0], - store_size, - maximum_timestamp, - }; - const body = deserializeDataStream(dataStream[0]); - return res.ok({ body }); - } + if (dataStreams[0]) { + let dataStreamsPrivileges; + if (config.isSecurityEnabled()) { + ({ body: dataStreamsPrivileges } = await getDataStreamsPrivileges(asCurrentUser, [ + dataStreams[0].name, + ])); + } - return res.notFound(); - } catch (e) { - if (isEsError(e)) { - return res.customError({ - statusCode: e.statusCode, - body: e, + const enhancedDataStreams = enhanceDataStreams({ + dataStreams, + dataStreamsStats, + dataStreamsPrivileges, }); + const body = deserializeDataStream(enhancedDataStreams[0]); + return response.ok({ body }); } - // Case: default - return res.internalError({ body: e }); + + return response.notFound(); + } catch (error) { + return handleEsError({ error, response }); } }) ); diff --git a/x-pack/plugins/index_management/server/shared_imports.ts b/x-pack/plugins/index_management/server/shared_imports.ts index 454beda5394c7..0606f474897b5 100644 --- a/x-pack/plugins/index_management/server/shared_imports.ts +++ b/x-pack/plugins/index_management/server/shared_imports.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { isEsError } from '../../../../src/plugins/es_ui_shared/server'; +export { isEsError, handleEsError } from '../../../../src/plugins/es_ui_shared/server'; diff --git a/x-pack/plugins/index_management/server/types.ts b/x-pack/plugins/index_management/server/types.ts index 7aa91629f0a47..177dedeb87bb4 100644 --- a/x-pack/plugins/index_management/server/types.ts +++ b/x-pack/plugins/index_management/server/types.ts @@ -8,7 +8,7 @@ import { PluginSetupContract as FeaturesPluginSetup } from '../../features/serve import { LicensingPluginSetup } from '../../licensing/server'; import { SecurityPluginSetup } from '../../security/server'; import { License, IndexDataEnricher } from './services'; -import { isEsError } from './shared_imports'; +import { isEsError, handleEsError } from './shared_imports'; export interface Dependencies { security: SecurityPluginSetup; @@ -25,6 +25,7 @@ export interface RouteDependencies { indexDataEnricher: IndexDataEnricher; lib: { isEsError: typeof isEsError; + handleEsError: typeof handleEsError; }; } diff --git a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts index f4b947336e044..6cf1a40a4d5a1 100644 --- a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts +++ b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts @@ -77,6 +77,9 @@ export default function ({ getService }: FtrProviderContext) { expect(dataStreams).to.eql([ { name: testDataStreamName, + privileges: { + delete_index: true, + }, timeStampField: { name: '@timestamp' }, indices: [ { @@ -105,6 +108,9 @@ export default function ({ getService }: FtrProviderContext) { expect(dataStreams.length).to.be(1); expect(dataStreamWithoutStorageSize).to.eql({ name: testDataStreamName, + privileges: { + delete_index: true, + }, timeStampField: { name: '@timestamp' }, indices: [ { @@ -132,6 +138,9 @@ export default function ({ getService }: FtrProviderContext) { expect(dataStreamWithoutStorageSize).to.eql({ name: testDataStreamName, + privileges: { + delete_index: true, + }, timeStampField: { name: '@timestamp' }, indices: [ { From ee5c9bceebf1b0ddd3f4da371e8009d455e7b3f6 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 26 Nov 2020 20:34:06 +0100 Subject: [PATCH 22/37] Upgrade fp-ts to 2.8.6 (#83866) * Upgrade fp-ts to 2.8.6 * reduce import size from io-ts * removed unused imports * remove usage of fpts from alerts Co-authored-by: Gidi Meir Morris Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../plugins/alerts/public/alert_api.test.ts | 66 +------------------ x-pack/plugins/alerts/public/alert_api.ts | 41 ++---------- .../public/application/lib/alert_api.ts | 6 +- yarn.lock | 6 +- 4 files changed, 14 insertions(+), 105 deletions(-) diff --git a/x-pack/plugins/alerts/public/alert_api.test.ts b/x-pack/plugins/alerts/public/alert_api.test.ts index 3ee67b79b7bda..0fa2e7f25323b 100644 --- a/x-pack/plugins/alerts/public/alert_api.test.ts +++ b/x-pack/plugins/alerts/public/alert_api.test.ts @@ -6,7 +6,7 @@ import { AlertType } from '../common'; import { httpServiceMock } from '../../../../src/core/public/mocks'; -import { loadAlert, loadAlertState, loadAlertType, loadAlertTypes } from './alert_api'; +import { loadAlert, loadAlertType, loadAlertTypes } from './alert_api'; import uuid from 'uuid'; const http = httpServiceMock.createStartContract(); @@ -114,67 +114,3 @@ describe('loadAlert', () => { expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}`); }); }); - -describe('loadAlertState', () => { - test('should call get API with base parameters', async () => { - const alertId = uuid.v4(); - const resolvedValue = { - alertTypeState: { - some: 'value', - }, - alertInstances: { - first_instance: {}, - second_instance: {}, - }, - }; - http.get.mockResolvedValueOnce(resolvedValue); - - expect(await loadAlertState({ http, alertId })).toEqual(resolvedValue); - expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}/state`); - }); - - test('should parse AlertInstances', async () => { - const alertId = uuid.v4(); - const resolvedValue = { - alertTypeState: { - some: 'value', - }, - alertInstances: { - first_instance: { - state: {}, - meta: { - lastScheduledActions: { - group: 'first_group', - date: '2020-02-09T23:15:41.941Z', - }, - }, - }, - }, - }; - http.get.mockResolvedValueOnce(resolvedValue); - - expect(await loadAlertState({ http, alertId })).toEqual({ - ...resolvedValue, - alertInstances: { - first_instance: { - state: {}, - meta: { - lastScheduledActions: { - group: 'first_group', - date: new Date('2020-02-09T23:15:41.941Z'), - }, - }, - }, - }, - }); - expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}/state`); - }); - - test('should handle empty response from api', async () => { - const alertId = uuid.v4(); - http.get.mockResolvedValueOnce(''); - - expect(await loadAlertState({ http, alertId })).toEqual({}); - expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}/state`); - }); -}); diff --git a/x-pack/plugins/alerts/public/alert_api.ts b/x-pack/plugins/alerts/public/alert_api.ts index 5b7cd2128f386..753158ed1ed42 100644 --- a/x-pack/plugins/alerts/public/alert_api.ts +++ b/x-pack/plugins/alerts/public/alert_api.ts @@ -5,15 +5,9 @@ */ import { HttpSetup } from 'kibana/public'; -import * as t from 'io-ts'; -import { pipe } from 'fp-ts/lib/pipeable'; -import { fold } from 'fp-ts/lib/Either'; -import { findFirst } from 'fp-ts/lib/Array'; -import { isNone } from 'fp-ts/lib/Option'; - import { i18n } from '@kbn/i18n'; -import { BASE_ALERT_API_PATH, alertStateSchema } from '../common'; -import { Alert, AlertType, AlertTaskState } from '../common'; +import { BASE_ALERT_API_PATH } from '../common'; +import type { Alert, AlertType } from '../common'; export async function loadAlertTypes({ http }: { http: HttpSetup }): Promise { return await http.get(`${BASE_ALERT_API_PATH}/list_alert_types`); @@ -26,10 +20,10 @@ export async function loadAlertType({ http: HttpSetup; id: AlertType['id']; }): Promise { - const maybeAlertType = findFirst((type) => type.id === id)( - await http.get(`${BASE_ALERT_API_PATH}/list_alert_types`) - ); - if (isNone(maybeAlertType)) { + const maybeAlertType = ((await http.get( + `${BASE_ALERT_API_PATH}/list_alert_types` + )) as AlertType[]).find((type) => type.id === id); + if (!maybeAlertType) { throw new Error( i18n.translate('xpack.alerts.loadAlertType.missingAlertTypeError', { defaultMessage: 'Alert type "{id}" is not registered.', @@ -39,7 +33,7 @@ export async function loadAlertType({ }) ); } - return maybeAlertType.value; + return maybeAlertType; } export async function loadAlert({ @@ -51,24 +45,3 @@ export async function loadAlert({ }): Promise { return await http.get(`${BASE_ALERT_API_PATH}/alert/${alertId}`); } - -type EmptyHttpResponse = ''; -export async function loadAlertState({ - http, - alertId, -}: { - http: HttpSetup; - alertId: string; -}): Promise { - return await http - .get(`${BASE_ALERT_API_PATH}/alert/${alertId}/state`) - .then((state: AlertTaskState | EmptyHttpResponse) => (state ? state : {})) - .then((state: AlertTaskState) => { - return pipe( - alertStateSchema.decode(state), - fold((e: t.Errors) => { - throw new Error(`Alert "${alertId}" has invalid state`); - }, t.identity) - ); - }); -} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts b/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts index 7c2f50211d4af..d34481850ca4a 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts @@ -5,7 +5,7 @@ */ import { HttpSetup } from 'kibana/public'; -import * as t from 'io-ts'; +import { Errors, identity } from 'io-ts'; import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; import { pick } from 'lodash'; @@ -48,9 +48,9 @@ export async function loadAlertState({ .then((state: AlertTaskState) => { return pipe( alertStateSchema.decode(state), - fold((e: t.Errors) => { + fold((e: Errors) => { throw new Error(`Alert "${alertId}" has invalid state`); - }, t.identity) + }, identity) ); }); } diff --git a/yarn.lock b/yarn.lock index 1cde1266ca38f..1f5eba54fb835 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14045,9 +14045,9 @@ fp-ts@^1.0.0: integrity sha512-fWwnAgVlTsV26Ruo9nx+fxNHIm6l1puE1VJ/C0XJ3nRQJJJIgRHYw6sigB3MuNFZL1o4fpGlhwFhcbxHK0RsOA== fp-ts@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.3.1.tgz#8068bfcca118227932941101e062134d7ecd9119" - integrity sha512-KevPBnYt0aaJiuUzmU9YIxjrhC9AgJ8CLtLlXmwArovlNTeYM5NtEoKd86B0wHd7FIbzeE8sNXzCoYIOr7e6Iw== + version "2.8.6" + resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.8.6.tgz#1a0e6c3f29f5b0fbfa3120f034ea266aa73c811b" + integrity sha512-fGGpKf/Jy3UT4s16oM+hr/8F5QXFcZ+20NAvaZXH5Y5jsiLPMDCaNqffXq0z1Kr6ZUJj0346cH9tq+cI2SoJ4w== fragment-cache@^0.2.1: version "0.2.1" From f99766e7ab838c1424f7633a3c5182705abf0eaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Fri, 27 Nov 2020 10:19:11 +0100 Subject: [PATCH 23/37] [APM] Fix missing `service.node.name` (#84269) --- .../app/service_node_metrics/index.tsx | 25 ++++------------- .../public/hooks/useServiceMetricCharts.ts | 6 ++-- .../server/lib/metrics/by_agent/default.ts | 4 +-- .../by_agent/java/gc/get_gc_rate_chart.ts | 16 +++++++---- .../by_agent/java/gc/get_gc_time_chart.ts | 16 +++++++---- .../by_agent/java/heap_memory/index.ts | 14 ++++++---- .../server/lib/metrics/by_agent/java/index.ts | 28 +++++++++++-------- .../by_agent/java/non_heap_memory/index.ts | 14 ++++++---- .../by_agent/java/thread_count/index.ts | 14 ++++++---- .../lib/metrics/by_agent/shared/cpu/index.ts | 14 ++++++---- .../metrics/by_agent/shared/memory/index.ts | 14 ++++++---- .../get_metrics_chart_data_by_agent.ts | 2 +- .../apm/server/lib/metrics/queries.test.ts | 10 +++---- 13 files changed, 98 insertions(+), 79 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx index a74ff574bc0c8..a886c3f29d57c 100644 --- a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx @@ -59,7 +59,11 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) { const { urlParams, uiFilters } = useUrlParams(); const { serviceName, serviceNodeName } = match.params; const { agentName } = useAgentName(); - const { data } = useServiceMetricCharts(urlParams, agentName); + const { data } = useServiceMetricCharts( + urlParams, + agentName, + serviceNodeName + ); const { start, end } = urlParams; const { data: { host, containerId } = INITIAL_DATA, status } = useFetcher( @@ -177,25 +181,6 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) { )} - {agentName && ( - - - {data.charts.map((chart) => ( - - - - - - ))} - - - - )} {agentName && ( diff --git a/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts b/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts index d264ad6069db3..7a54c6ffc6dbe 100644 --- a/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts +++ b/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts @@ -17,7 +17,8 @@ const INITIAL_DATA: MetricsChartsByAgentAPIResponse = { export function useServiceMetricCharts( urlParams: IUrlParams, - agentName?: string + agentName?: string, + serviceNodeName?: string ) { const { serviceName } = useParams<{ serviceName?: string }>(); const { start, end } = urlParams; @@ -30,6 +31,7 @@ export function useServiceMetricCharts( params: { path: { serviceName }, query: { + serviceNodeName, start, end, agentName, @@ -39,7 +41,7 @@ export function useServiceMetricCharts( }); } }, - [serviceName, start, end, agentName, uiFilters] + [serviceName, start, end, agentName, serviceNodeName, uiFilters] ); return { diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts index fbcbc9f12791f..71bc1018b619b 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts @@ -13,8 +13,8 @@ export async function getDefaultMetricsCharts( serviceName: string ) { const charts = await Promise.all([ - getCPUChartData(setup, serviceName), - getMemoryChartData(setup, serviceName), + getCPUChartData({ setup, serviceName }), + getMemoryChartData({ setup, serviceName }), ]); return { charts }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts index 7cedeb828e3b7..8e68c229c555c 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts @@ -30,11 +30,15 @@ const chartBase: ChartBase = { series, }; -const getGcRateChart = ( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) => { +function getGcRateChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformGcMetrics({ setup, serviceName, @@ -42,6 +46,6 @@ const getGcRateChart = ( chartBase, fieldName: METRIC_JAVA_GC_COUNT, }); -}; +} export { getGcRateChart }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts index f21d3d8e7c056..e7a83e71a22e9 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts @@ -30,11 +30,15 @@ const chartBase: ChartBase = { series, }; -const getGcTimeChart = ( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) => { +function getGcTimeChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformGcMetrics({ setup, serviceName, @@ -42,6 +46,6 @@ const getGcTimeChart = ( chartBase, fieldName: METRIC_JAVA_GC_TIME, }); -}; +} export { getGcTimeChart }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts index eb79897f9f055..4f087594acc21 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts @@ -50,11 +50,15 @@ const chartBase: ChartBase = { series, }; -export async function getHeapMemoryChart( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getHeapMemoryChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts index d4084701f0f49..4a1f3f572b3a5 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts @@ -13,19 +13,23 @@ import { getMemoryChartData } from '../shared/memory'; import { getGcRateChart } from './gc/get_gc_rate_chart'; import { getGcTimeChart } from './gc/get_gc_time_chart'; -export async function getJavaMetricsCharts( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getJavaMetricsCharts({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { const charts = await Promise.all([ - getCPUChartData(setup, serviceName, serviceNodeName), - getMemoryChartData(setup, serviceName, serviceNodeName), - getHeapMemoryChart(setup, serviceName, serviceNodeName), - getNonHeapMemoryChart(setup, serviceName, serviceNodeName), - getThreadCountChart(setup, serviceName, serviceNodeName), - getGcRateChart(setup, serviceName, serviceNodeName), - getGcTimeChart(setup, serviceName, serviceNodeName), + getCPUChartData({ setup, serviceName, serviceNodeName }), + getMemoryChartData({ setup, serviceName, serviceNodeName }), + getHeapMemoryChart({ setup, serviceName, serviceNodeName }), + getNonHeapMemoryChart({ setup, serviceName, serviceNodeName }), + getThreadCountChart({ setup, serviceName, serviceNodeName }), + getGcRateChart({ setup, serviceName, serviceNodeName }), + getGcTimeChart({ setup, serviceName, serviceNodeName }), ]); return { charts }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts index 50cc449da3c15..dc2684f707d17 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts @@ -47,11 +47,15 @@ const chartBase: ChartBase = { series, }; -export async function getNonHeapMemoryChart( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getNonHeapMemoryChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts index 0062f0a423970..b42ab61f5c766 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts @@ -39,11 +39,15 @@ const chartBase: ChartBase = { series, }; -export async function getThreadCountChart( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getThreadCountChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts index ca642aa12fff1..83311caf99516 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts @@ -51,11 +51,15 @@ const chartBase: ChartBase = { series, }; -export async function getCPUChartData( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getCPUChartData({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { const metricsChart = await fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts index e6ee47cc815ef..8fd234903d58e 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts @@ -68,11 +68,15 @@ export const percentCgroupMemoryUsedScript = { `, }; -export async function getMemoryChartData( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getMemoryChartData({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { const cgroupResponse = await fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts b/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts index 72cd65deebff6..1fc510b77c856 100644 --- a/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts +++ b/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts @@ -25,7 +25,7 @@ export async function getMetricsChartDataByAgent({ }): Promise { switch (agentName) { case 'java': { - return getJavaMetricsCharts(setup, serviceName, serviceNodeName); + return getJavaMetricsCharts({ setup, serviceName, serviceNodeName }); } default: { diff --git a/x-pack/plugins/apm/server/lib/metrics/queries.test.ts b/x-pack/plugins/apm/server/lib/metrics/queries.test.ts index fc24377ebf390..0bf3d3daaad66 100644 --- a/x-pack/plugins/apm/server/lib/metrics/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/metrics/queries.test.ts @@ -21,7 +21,7 @@ describe('metrics queries', () => { const createTests = (serviceNodeName?: string) => { it('fetches cpu chart data', async () => { mock = await inspectSearchParams((setup) => - getCPUChartData(setup, 'foo', serviceNodeName) + getCPUChartData({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); @@ -29,7 +29,7 @@ describe('metrics queries', () => { it('fetches memory chart data', async () => { mock = await inspectSearchParams((setup) => - getMemoryChartData(setup, 'foo', serviceNodeName) + getMemoryChartData({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); @@ -37,7 +37,7 @@ describe('metrics queries', () => { it('fetches heap memory chart data', async () => { mock = await inspectSearchParams((setup) => - getHeapMemoryChart(setup, 'foo', serviceNodeName) + getHeapMemoryChart({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); @@ -45,7 +45,7 @@ describe('metrics queries', () => { it('fetches non heap memory chart data', async () => { mock = await inspectSearchParams((setup) => - getNonHeapMemoryChart(setup, 'foo', serviceNodeName) + getNonHeapMemoryChart({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); @@ -53,7 +53,7 @@ describe('metrics queries', () => { it('fetches thread count chart data', async () => { mock = await inspectSearchParams((setup) => - getThreadCountChart(setup, 'foo', serviceNodeName) + getThreadCountChart({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); From 31c1ca0026ffd17db16ff46214cb367e4f2fa513 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 27 Nov 2020 15:14:42 +0000 Subject: [PATCH 24/37] skip flaky suite (#84445) --- test/api_integration/apis/saved_objects/migrations.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/api_integration/apis/saved_objects/migrations.ts b/test/api_integration/apis/saved_objects/migrations.ts index 99a58620b17f5..b5fa16558514a 100644 --- a/test/api_integration/apis/saved_objects/migrations.ts +++ b/test/api_integration/apis/saved_objects/migrations.ts @@ -54,7 +54,8 @@ function getLogMock() { export default ({ getService }: FtrProviderContext) => { const esClient = getService('es'); - describe('Kibana index migration', () => { + // FLAKY: https://github.com/elastic/kibana/issues/84445 + describe.skip('Kibana index migration', () => { before(() => esClient.indices.delete({ index: '.migrate-*' })); it('Migrates an existing index that has never been migrated before', async () => { From d21e4f5af41369289023d2ead593b7a6d0bec436 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 28 Nov 2020 03:49:04 +0000 Subject: [PATCH 25/37] skip flaky suite (#84440) --- x-pack/test/functional/apps/grok_debugger/grok_debugger.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/grok_debugger/grok_debugger.js b/x-pack/test/functional/apps/grok_debugger/grok_debugger.js index 771ca2088c055..8513429639e58 100644 --- a/x-pack/test/functional/apps/grok_debugger/grok_debugger.js +++ b/x-pack/test/functional/apps/grok_debugger/grok_debugger.js @@ -11,7 +11,8 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['grokDebugger']); - describe('grok debugger app', function () { + // FLAKY: https://github.com/elastic/kibana/issues/84440 + describe.skip('grok debugger app', function () { this.tags('includeFirefox'); before(async () => { await esArchiver.load('empty_kibana'); From bdf7b88b45329b29d8affcb5a0dc5452ed97210e Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sun, 29 Nov 2020 22:10:23 -0500 Subject: [PATCH 26/37] [Security Solution][Detections] Handle dupes when processing threshold rules (#83062) * Fix threshold rule synthetic signal generation * Use top_hits aggregation * Find signals and aggregate over search terms * Exclude dupes * Fixes to algorithm * Sync timestamps with events/signals * Add timestampOverride * Revert changes in signal creation * Simplify query, return 10k buckets * Account for when threshold.field is not supplied * Ensure we're getting the last event when threshold.field is not provided * Add missing import * Handle case where threshold field not supplied * Fix type errors * Handle non-ECS fields * Regorganize * Address comments * Fix type error * Add unit test for buildBulkBody on threshold results * Add threshold_count back to mapping (and deprecate) * Timestamp fixes Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../routes/index/get_signals_template.ts | 2 +- .../routes/index/signals_mapping.json | 10 ++ .../signals/build_bulk_body.test.ts | 109 ++++++++++++++++++ .../signals/build_bulk_body.ts | 1 + .../detection_engine/signals/build_signal.ts | 4 +- .../signals/bulk_create_threshold_signals.ts | 10 +- .../find_previous_threshold_signals.ts | 87 ++++++++++++++ .../signals/find_threshold_signals.ts | 1 + .../signals/signal_rule_alert_type.ts | 45 +++++++- .../lib/detection_engine/signals/types.ts | 14 ++- .../endpoint/resolver/signals/mappings.json | 10 ++ .../es_archives/export_rule/mappings.json | 10 ++ 12 files changed, 296 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index d1a9b701b2c9d..8ea1faa84cfba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -7,7 +7,7 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; -export const SIGNALS_TEMPLATE_VERSION = 2; +export const SIGNALS_TEMPLATE_VERSION = 3; export const MIN_EQL_RULE_INDEX_VERSION = 2; export const getSignalsTemplate = (index: string) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 4e9477f3f2f73..96868e62ea978 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -337,6 +337,16 @@ "threshold_count": { "type": "float" }, + "threshold_result": { + "properties": { + "count": { + "type": "long" + }, + "value": { + "type": "keyword" + } + } + }, "depth": { "type": "integer" } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index ad060a9304e84..cd61fbcfd0fc7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -123,6 +123,115 @@ describe('buildBulkBody', () => { expect(fakeSignalSourceHit).toEqual(expected); }); + test('bulk body builds well-defined body with threshold results', () => { + const sampleParams = sampleRuleAlertParams(); + const baseDoc = sampleDocNoSortId(); + const doc: SignalSourceHit = { + ...baseDoc, + _source: { + ...baseDoc._source, + threshold_result: { + count: 5, + value: 'abcd', + }, + }, + }; + delete doc._source.source; + const fakeSignalSourceHit = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error + delete fakeSignalSourceHit['@timestamp']; + const expected: Omit & { someKey: 'someValue' } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2020-04-20T21:27:45+0000', + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + throttle: 'no_actions', + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + version: 1, + created_at: fakeSignalSourceHit.signal.rule?.created_at, + updated_at: fakeSignalSourceHit.signal.rule?.updated_at, + exceptions_list: getListArrayMock(), + }, + threshold_result: { + count: 5, + value: 'abcd', + }, + depth: 1, + }, + }; + expect(fakeSignalSourceHit).toEqual(expected); + }); + test('bulk body builds original_event if it exists on the event to begin with', () => { const sampleParams = sampleRuleAlertParams(); const doc = sampleDocNoSortId(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index e50956e9ef752..e6e6d2e0f5fa9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -71,6 +71,7 @@ export const buildBulkBody = ({ ...buildSignal([doc], rule), ...additionalSignalFields(doc), }; + delete doc._source.threshold_result; const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index b36a1cbb4a6b3..9cf2462526cfc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -96,9 +96,9 @@ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => export const additionalSignalFields = (doc: BaseSignalHit) => { return { parent: buildParent(removeClashes(doc)), - original_time: doc._source['@timestamp'], + original_time: doc._source['@timestamp'], // This field has already been replaced with timestampOverride, if provided. original_event: doc._source.event ?? undefined, - threshold_count: doc._source.threshold_count ?? undefined, + threshold_result: doc._source.threshold_result, original_signal: doc._source.signal != null && !isEventTypeSignal(doc) ? doc._source.signal : undefined, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index edaaa345d8a69..a98aae4ec8107 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -149,7 +149,10 @@ const getTransformedHits = ( const source = { '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), - threshold_count: totalResults, + threshold_result: { + count: totalResults, + value: ruleId, + }, ...getThresholdSignalQueryFields(hit, filter), }; @@ -176,7 +179,10 @@ const getTransformedHits = ( const source = { '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), - threshold_count: docCount, + threshold_result: { + count: docCount, + value: get(threshold.field, hit._source), + }, ...getThresholdSignalQueryFields(hit, filter), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts new file mode 100644 index 0000000000000..960693bc703d6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; +import { singleSearchAfter } from './single_search_after'; + +import { AlertServices } from '../../../../../alerts/server'; +import { Logger } from '../../../../../../../src/core/server'; +import { SignalSearchResponse } from './types'; +import { BuildRuleMessage } from './rule_messages'; + +interface FindPreviousThresholdSignalsParams { + from: string; + to: string; + indexPattern: string[]; + services: AlertServices; + logger: Logger; + ruleId: string; + bucketByField: string; + timestampOverride: TimestampOverrideOrUndefined; + buildRuleMessage: BuildRuleMessage; +} + +export const findPreviousThresholdSignals = async ({ + from, + to, + indexPattern, + services, + logger, + ruleId, + bucketByField, + timestampOverride, + buildRuleMessage, +}: FindPreviousThresholdSignalsParams): Promise<{ + searchResult: SignalSearchResponse; + searchDuration: string; + searchErrors: string[]; +}> => { + const aggregations = { + threshold: { + terms: { + field: 'signal.threshold_result.value', + }, + aggs: { + lastSignalTimestamp: { + max: { + field: 'signal.original_time', // timestamp of last event captured by bucket + }, + }, + }, + }, + }; + + const filter = { + bool: { + must: [ + { + term: { + 'signal.rule.rule_id': ruleId, + }, + }, + { + term: { + 'signal.rule.threshold.field': bucketByField, + }, + }, + ], + }, + }; + + return singleSearchAfter({ + aggregations, + searchAfterSortId: undefined, + timestampOverride, + index: indexPattern, + from, + to, + services, + logger, + filter, + pageSize: 0, + buildRuleMessage, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 01e4812b9c8bf..7141b61a23e6e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -51,6 +51,7 @@ export const findThresholdSignals = async ({ terms: { field: threshold.field, min_doc_count: threshold.value, + size: 10000, // max 10k buckets }, aggs: { // Get the most recent hit per bucket diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 003626e319007..f0b1825c7cc99 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,6 +8,7 @@ import { Logger, KibanaRequest } from 'src/core/server'; +import { Filter } from 'src/plugins/data/common'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, @@ -29,6 +30,7 @@ import { RuleAlertAttributes, EqlSignalSearchResponse, BaseSignalHit, + ThresholdQueryBucket, } from './types'; import { getGapBetweenRuns, @@ -46,6 +48,7 @@ import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; import { findMlSignals } from './find_ml_signals'; import { findThresholdSignals } from './find_threshold_signals'; +import { findPreviousThresholdSignals } from './find_previous_threshold_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals'; import { @@ -300,6 +303,46 @@ export const signalRulesAlertType = ({ lists: exceptionItems ?? [], }); + const { + searchResult: previousSignals, + searchErrors: previousSearchErrors, + } = await findPreviousThresholdSignals({ + indexPattern: [outputIndex], + from, + to, + services, + logger, + ruleId, + bucketByField: threshold.field, + timestampOverride, + buildRuleMessage, + }); + + previousSignals.aggregations.threshold.buckets.forEach((bucket: ThresholdQueryBucket) => { + esFilter.bool.filter.push(({ + bool: { + must_not: { + bool: { + must: [ + { + term: { + [threshold.field ?? 'signal.rule.rule_id']: bucket.key, + }, + }, + { + range: { + [timestampOverride ?? '@timestamp']: { + lte: bucket.lastSignalTimestamp.value_as_string, + }, + }, + }, + ], + }, + }, + }, + } as unknown) as Filter); + }); + const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ inputIndexPattern: inputIndex, from, @@ -349,7 +392,7 @@ export const signalRulesAlertType = ({ }), createSearchAfterReturnType({ success, - errors: [...errors, ...searchErrors], + errors: [...errors, ...previousSearchErrors, ...searchErrors], createdSignalsCount: createdItemsCount, bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], }), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index cda3c97c08531..9e81797b14731 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -46,6 +46,11 @@ export interface SignalsStatusParams { status: Status; } +export interface ThresholdResult { + count: number; + value: string; +} + export interface SignalSource { [key: string]: SearchTypes; // TODO: SignalSource is being used as the type for documents matching detection engine queries, but they may not @@ -67,6 +72,7 @@ export interface SignalSource { // signal.depth doesn't exist on pre-7.10 signals depth?: number; }; + threshold_result?: ThresholdResult; } export interface BulkItem { @@ -156,7 +162,7 @@ export interface Signal { original_time?: string; original_event?: SearchTypes; status: Status; - threshold_count?: SearchTypes; + threshold_result?: ThresholdResult; original_signal?: SearchTypes; depth: number; } @@ -239,3 +245,9 @@ export interface SearchAfterAndBulkCreateReturnType { export interface ThresholdAggregationBucket extends TermAggregationBucket { top_threshold_hits: BaseSearchResponse; } + +export interface ThresholdQueryBucket extends TermAggregationBucket { + lastSignalTimestamp: { + value_as_string: string; + }; +} diff --git a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json index ad77961a41445..3c6042e8efd24 100644 --- a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json @@ -2582,6 +2582,16 @@ }, "threshold_count": { "type": "float" + }, + "threshold_result": { + "properties": { + "count": { + "type": "long" + }, + "value": { + "type": "keyword" + } + } } } }, diff --git a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json index 757121df53d44..f701f811b244b 100644 --- a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json +++ b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json @@ -5131,6 +5131,16 @@ }, "threshold_count": { "type": "float" + }, + "threshold_result": { + "properties": { + "count": { + "type": "long" + }, + "value": { + "type": "keyword" + } + } } } }, From 767286e2da8c94e46007e3327471b5a05d813bc8 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 30 Nov 2020 09:34:33 +0000 Subject: [PATCH 27/37] [ML] Fix spaces job ID check (#84404) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../ml/server/lib/ml_client/ml_client.ts | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts b/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts index a75ba46a9b22a..8cfb066c9d092 100644 --- a/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts +++ b/x-pack/plugins/ml/server/lib/ml_client/ml_client.ts @@ -39,15 +39,19 @@ export function getMlClient( const jobIds = jobType === 'anomaly-detector' ? getADJobIdsFromRequest(p) : getDFAJobIdsFromRequest(p); if (jobIds.length) { - const filteredJobIds = await jobSavedObjectService.filterJobIdsForSpace(jobType, jobIds); - let missingIds = jobIds.filter((j) => filteredJobIds.indexOf(j) === -1); - if (allowWildcards === true && missingIds.join().match('\\*') !== null) { - // filter out wildcard ids from the error - missingIds = missingIds.filter((id) => id.match('\\*') === null); - } - if (missingIds.length) { - throw new MLJobNotFound(`No known job with id '${missingIds.join(',')}'`); - } + await checkIds(jobType, jobIds, allowWildcards); + } + } + + async function checkIds(jobType: JobType, jobIds: string[], allowWildcards: boolean = false) { + const filteredJobIds = await jobSavedObjectService.filterJobIdsForSpace(jobType, jobIds); + let missingIds = jobIds.filter((j) => filteredJobIds.indexOf(j) === -1); + if (allowWildcards === true && missingIds.join().match('\\*') !== null) { + // filter out wildcard ids from the error + missingIds = missingIds.filter((id) => id.match('\\*') === null); + } + if (missingIds.length) { + throw new MLJobNotFound(`No known job with id '${missingIds.join(',')}'`); } } @@ -59,8 +63,17 @@ export function getMlClient( if (ids.length) { // find all groups from unfiltered jobs const responseGroupIds = [...new Set(allJobs.map((j) => j.groups ?? []).flat())]; - // work out which ids requested are actually groups - const requestedGroupIds = ids.filter((id) => responseGroupIds.includes(id)); + + // work out which ids requested are actually groups and which are jobs + const requestedGroupIds: string[] = []; + const requestedJobIds: string[] = []; + ids.forEach((id) => { + if (responseGroupIds.includes(id)) { + requestedGroupIds.push(id); + } else { + requestedJobIds.push(id); + } + }); // find all groups from filtered jobs const groupIdsFromFilteredJobs = [ @@ -77,10 +90,15 @@ export function getMlClient( ); if (groupsIdsThatDidNotMatch.length) { - // is there are group ids which were requested but didn't + // if there are group ids which were requested but didn't // exist in filtered jobs, list them in an error throw new MLJobNotFound(`No known job with id '${groupsIdsThatDidNotMatch.join(',')}'`); } + + // check the remaining jobs ids + if (requestedJobIds.length) { + await checkIds('anomaly-detector', requestedJobIds, true); + } } } From 454635228e032250f0be62e22403003a06f4de37 Mon Sep 17 00:00:00 2001 From: MadameSheema Date: Mon, 30 Nov 2020 10:37:42 +0100 Subject: [PATCH 28/37] [Security Solution] Exceptions Cypress tests (#81759) * improves 'Creates and activates a new custom rule' test * fixes constant problem * improves 'Creates and activates a new custom rule with override option' test * improves 'Creates and activates a new threshold rule' test * refactor * fixes type check issue * improves assertions * removes unused code * changes variables for constants * improves 'waitForTheRuleToBeExecuted' test * improves readability * fixes jenkins error * refactor * blah * more things * finishes 'Creates an exception from rule details and deletes the excpetion' implementation * implements 'Creates an exception from an alert and deletes the exception' * updates VALUES_INPUT locator * updates archiver * refactor * improves the code * fixes CI error * renames exceptions archive * refactor * fixes merge issue * fixes CI issue * debug * refactor * improves test data * removes signals index after the execution * removes unused line * removes unused variable * refactors 'numberOfauditbeatExceptionsAlerts' constant to camel case * simplifies the archive * waits for the rule to be executed after navigating to opened alerts tab * cleaning data * fixes tests flakiness * cleans test data * refactors code * removes unsused archives * cleans data * simplifies data * fixes CI issue Co-authored-by: Elastic Machine Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../alerts_detection_exceptions.spec.ts | 178 + .../cypress/objects/exception.ts | 17 + .../cypress/screens/alerts.ts | 2 + .../cypress/screens/exceptions.ts | 24 + .../cypress/screens/rule_details.ts | 14 +- .../security_solution/cypress/tasks/alerts.ts | 34 +- .../cypress/tasks/api_calls.ts | 45 + .../cypress/tasks/create_new_rule.ts | 31 +- .../cypress/tasks/rule_details.ts | 92 + .../timeline_actions/alert_context_menu.tsx | 4 +- .../detection_engine/rules/details/index.tsx | 4 + .../auditbeat_for_exceptions/data.json.gz | Bin 0 -> 1112 bytes .../auditbeat_for_exceptions/mappings.json | 3577 +++++++++++++++++ .../auditbeat_for_exceptions2/data.json.gz | Bin 0 -> 1113 bytes .../auditbeat_for_exceptions2/mappings.json | 3577 +++++++++++++++++ .../auditbeat_for_exceptions3/data.json.gz | Bin 0 -> 1506 bytes 16 files changed, 7572 insertions(+), 27 deletions(-) create mode 100644 x-pack/plugins/security_solution/cypress/integration/alerts_detection_exceptions.spec.ts create mode 100644 x-pack/plugins/security_solution/cypress/objects/exception.ts create mode 100644 x-pack/plugins/security_solution/cypress/screens/exceptions.ts create mode 100644 x-pack/plugins/security_solution/cypress/tasks/api_calls.ts create mode 100644 x-pack/plugins/security_solution/cypress/tasks/rule_details.ts create mode 100644 x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions/data.json.gz create mode 100644 x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions/mappings.json create mode 100644 x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions2/data.json.gz create mode 100644 x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions2/mappings.json create mode 100644 x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions3/data.json.gz diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_exceptions.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_exceptions.spec.ts new file mode 100644 index 0000000000000..b1d7163ac70e0 --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_exceptions.spec.ts @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { exception } from '../objects/exception'; +import { newRule } from '../objects/rule'; + +import { RULE_STATUS } from '../screens/create_new_rule'; +import { SERVER_SIDE_EVENT_COUNT } from '../screens/timeline'; + +import { + addExceptionFromFirstAlert, + goToClosedAlerts, + goToManageAlertsDetectionRules, + goToOpenedAlerts, + waitForAlertsIndexToBeCreated, +} from '../tasks/alerts'; +import { createCustomRule, deleteCustomRule, removeSignalsIndex } from '../tasks/api_calls'; +import { goToRuleDetails } from '../tasks/alerts_detection_rules'; +import { waitForAlertsToPopulate } from '../tasks/create_new_rule'; +import { esArchiverLoad, esArchiverUnload } from '../tasks/es_archiver'; +import { loginAndWaitForPageWithoutDateRange } from '../tasks/login'; +import { + activatesRule, + addsException, + addsExceptionFromRuleSettings, + goToAlertsTab, + goToExceptionsTab, + removeException, + waitForTheRuleToBeExecuted, +} from '../tasks/rule_details'; +import { refreshPage } from '../tasks/security_header'; + +import { DETECTIONS_URL } from '../urls/navigation'; + +const NUMBER_OF_AUDITBEAT_EXCEPTIONS_ALERTS = 1; + +describe('Exceptions', () => { + beforeEach(() => { + loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); + waitForAlertsIndexToBeCreated(); + createCustomRule(newRule); + goToManageAlertsDetectionRules(); + goToRuleDetails(); + + cy.get(RULE_STATUS).should('have.text', '—'); + + esArchiverLoad('auditbeat_for_exceptions'); + activatesRule(); + waitForTheRuleToBeExecuted(); + waitForAlertsToPopulate(); + refreshPage(); + + cy.get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((numberOfInitialAlertsText) => { + cy.wrap(parseInt(numberOfInitialAlertsText, 10)).should( + 'eql', + NUMBER_OF_AUDITBEAT_EXCEPTIONS_ALERTS + ); + }); + }); + + afterEach(() => { + esArchiverUnload('auditbeat_for_exceptions'); + esArchiverUnload('auditbeat_for_exceptions2'); + removeSignalsIndex(); + deleteCustomRule(); + }); + context('From rule', () => { + it('Creates an exception and deletes it', () => { + goToExceptionsTab(); + addsExceptionFromRuleSettings(exception); + esArchiverLoad('auditbeat_for_exceptions2'); + waitForTheRuleToBeExecuted(); + goToAlertsTab(); + refreshPage(); + + cy.get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((numberOfAlertsAfterCreatingExceptionText) => { + cy.wrap(parseInt(numberOfAlertsAfterCreatingExceptionText, 10)).should('eql', 0); + }); + + goToClosedAlerts(); + refreshPage(); + + cy.get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((numberOfClosedAlertsAfterCreatingExceptionText) => { + cy.wrap(parseInt(numberOfClosedAlertsAfterCreatingExceptionText, 10)).should( + 'eql', + NUMBER_OF_AUDITBEAT_EXCEPTIONS_ALERTS + ); + }); + + goToOpenedAlerts(); + waitForTheRuleToBeExecuted(); + refreshPage(); + + cy.get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((numberOfOpenedAlertsAfterCreatingExceptionText) => { + cy.wrap(parseInt(numberOfOpenedAlertsAfterCreatingExceptionText, 10)).should('eql', 0); + }); + + goToExceptionsTab(); + removeException(); + refreshPage(); + goToAlertsTab(); + waitForTheRuleToBeExecuted(); + waitForAlertsToPopulate(); + refreshPage(); + + cy.get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((numberOfAlertsAfterRemovingExceptionsText) => { + cy.wrap(parseInt(numberOfAlertsAfterRemovingExceptionsText, 10)).should( + 'eql', + NUMBER_OF_AUDITBEAT_EXCEPTIONS_ALERTS + ); + }); + }); + }); + + context('From alert', () => { + it('Creates an exception and deletes it', () => { + addExceptionFromFirstAlert(); + addsException(exception); + esArchiverLoad('auditbeat_for_exceptions2'); + + cy.get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((numberOfAlertsAfterCreatingExceptionText) => { + cy.wrap(parseInt(numberOfAlertsAfterCreatingExceptionText, 10)).should('eql', 0); + }); + + goToClosedAlerts(); + refreshPage(); + + cy.get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((numberOfClosedAlertsAfterCreatingExceptionText) => { + cy.wrap(parseInt(numberOfClosedAlertsAfterCreatingExceptionText, 10)).should( + 'eql', + NUMBER_OF_AUDITBEAT_EXCEPTIONS_ALERTS + ); + }); + + goToOpenedAlerts(); + waitForTheRuleToBeExecuted(); + refreshPage(); + + cy.get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((numberOfOpenedAlertsAfterCreatingExceptionText) => { + cy.wrap(parseInt(numberOfOpenedAlertsAfterCreatingExceptionText, 10)).should('eql', 0); + }); + + goToExceptionsTab(); + removeException(); + goToAlertsTab(); + waitForTheRuleToBeExecuted(); + waitForAlertsToPopulate(); + refreshPage(); + + cy.get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((numberOfAlertsAfterRemovingExceptionsText) => { + cy.wrap(parseInt(numberOfAlertsAfterRemovingExceptionsText, 10)).should( + 'eql', + NUMBER_OF_AUDITBEAT_EXCEPTIONS_ALERTS + ); + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/cypress/objects/exception.ts b/x-pack/plugins/security_solution/cypress/objects/exception.ts new file mode 100644 index 0000000000000..775a83dee2543 --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/objects/exception.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface Exception { + field: string; + operator: string; + values: string[]; +} + +export const exception: Exception = { + field: 'host.name', + operator: 'is', + values: ['suricata-iowa'], +}; diff --git a/x-pack/plugins/security_solution/cypress/screens/alerts.ts b/x-pack/plugins/security_solution/cypress/screens/alerts.ts index ed05874bd4c4d..2c80d02cad83d 100644 --- a/x-pack/plugins/security_solution/cypress/screens/alerts.ts +++ b/x-pack/plugins/security_solution/cypress/screens/alerts.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +export const ADD_EXCEPTION_BTN = '[data-test-subj="addExceptionButton"]'; + export const ALERTS = '[data-test-subj="event"]'; export const ALERT_CHECKBOX = '[data-test-subj="select-event-container"] .euiCheckbox__input'; diff --git a/x-pack/plugins/security_solution/cypress/screens/exceptions.ts b/x-pack/plugins/security_solution/cypress/screens/exceptions.ts new file mode 100644 index 0000000000000..4e263903a34a6 --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/screens/exceptions.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const ADD_EXCEPTIONS_BTN = '[data-test-subj="exceptionsHeaderAddExceptionBtn"]'; + +export const CLOSE_ALERTS_CHECKBOX = + '[data-test-subj="bulk-close-alert-on-add-add-exception-checkbox"]'; + +export const CONFIRM_BTN = '[data-test-subj="add-exception-confirm-button"]'; + +export const FIELD_INPUT = + '[data-test-subj="fieldAutocompleteComboBox"] [data-test-subj="comboBoxInput"]'; + +export const FIELD_INPUT_RESULT = '.euiFilterSelectItem'; + +export const LOADING_SPINNER = '[data-test-subj="loading-spinner"]'; + +export const OPERATOR_INPUT = '[data-test-subj="operatorAutocompleteComboBox"]'; + +export const VALUES_INPUT = + '[data-test-subj="valuesAutocompleteMatch"] [data-test-subj="comboBoxInput"]'; diff --git a/x-pack/plugins/security_solution/cypress/screens/rule_details.ts b/x-pack/plugins/security_solution/cypress/screens/rule_details.ts index 8cf0dfb5f6661..8e93d5dcd6315 100644 --- a/x-pack/plugins/security_solution/cypress/screens/rule_details.ts +++ b/x-pack/plugins/security_solution/cypress/screens/rule_details.ts @@ -15,6 +15,8 @@ export const ABOUT_DETAILS = export const ADDITIONAL_LOOK_BACK_DETAILS = 'Additional look-back time'; +export const ALERTS_TAB = '[data-test-subj="alertsTab"]'; + export const ANOMALY_SCORE_DETAILS = 'Anomaly score'; export const CUSTOM_QUERY_DETAILS = 'Custom query'; @@ -22,11 +24,13 @@ export const CUSTOM_QUERY_DETAILS = 'Custom query'; export const DEFINITION_DETAILS = '[data-test-subj=definitionRule] [data-test-subj="listItemColumnStepRuleDescription"]'; +export const DELETE_RULE = '[data-test-subj=rules-details-delete-rule]'; + export const DETAILS_DESCRIPTION = '.euiDescriptionList__description'; export const DETAILS_TITLE = '.euiDescriptionList__title'; -export const DELETE_RULE = '[data-test-subj=rules-details-delete-rule]'; +export const EXCEPTIONS_TAB = '[data-test-subj="exceptionsTab"]'; export const FALSE_POSITIVES_DETAILS = 'False positive examples'; @@ -42,6 +46,8 @@ export const MACHINE_LEARNING_JOB_STATUS = '[data-test-subj="machineLearningJobS export const MITRE_ATTACK_DETAILS = 'MITRE ATT&CK'; +export const REFRESH_BUTTON = '[data-test-subj="refreshButton"]'; + export const RULE_ABOUT_DETAILS_HEADER_TOGGLE = '[data-test-subj="stepAboutDetailsToggle"]'; export const RULE_NAME_HEADER = '[data-test-subj="header-page-title"]'; @@ -54,6 +60,12 @@ export const RISK_SCORE_OVERRIDE_DETAILS = 'Risk score override'; export const REFERENCE_URLS_DETAILS = 'Reference URLs'; +export const REMOVE_EXCEPTION_BTN = '[data-test-subj="exceptionsViewerDeleteBtn"]'; + +export const RULE_SWITCH = '[data-test-subj="ruleSwitch"]'; + +export const RULE_SWITCH_LOADER = '[data-test-subj="rule-switch-loader"]'; + export const RULE_TYPE_DETAILS = 'Rule type'; export const RUNS_EVERY_DETAILS = 'Runs every'; diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts index c846ced2febfd..39e57f39a145d 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts @@ -5,28 +5,34 @@ */ import { - CLOSED_ALERTS_FILTER_BTN, - EXPAND_ALERT_BTN, - LOADING_ALERTS_PANEL, - MANAGE_ALERT_DETECTION_RULES_BTN, - OPENED_ALERTS_FILTER_BTN, - SEND_ALERT_TO_TIMELINE_BTN, + ADD_EXCEPTION_BTN, + ALERT_RISK_SCORE_HEADER, ALERTS, ALERT_CHECKBOX, - TIMELINE_CONTEXT_MENU_BTN, CLOSE_ALERT_BTN, - TAKE_ACTION_POPOVER_BTN, CLOSE_SELECTED_ALERTS_BTN, + CLOSED_ALERTS_FILTER_BTN, + EXPAND_ALERT_BTN, IN_PROGRESS_ALERTS_FILTER_BTN, - OPEN_ALERT_BTN, - OPEN_SELECTED_ALERTS_BTN, + LOADING_ALERTS_PANEL, + MANAGE_ALERT_DETECTION_RULES_BTN, MARK_ALERT_IN_PROGRESS_BTN, MARK_SELECTED_ALERTS_IN_PROGRESS_BTN, - ALERT_RISK_SCORE_HEADER, + OPEN_ALERT_BTN, + OPEN_SELECTED_ALERTS_BTN, + OPENED_ALERTS_FILTER_BTN, + SEND_ALERT_TO_TIMELINE_BTN, + TAKE_ACTION_POPOVER_BTN, + TIMELINE_CONTEXT_MENU_BTN, } from '../screens/alerts'; import { REFRESH_BUTTON } from '../screens/security_header'; import { TIMELINE_COLUMN_SPINNER } from '../screens/timeline'; +export const addExceptionFromFirstAlert = () => { + cy.get(TIMELINE_CONTEXT_MENU_BTN).first().click(); + cy.get(ADD_EXCEPTION_BTN).click(); +}; + export const closeFirstAlert = () => { cy.get(TIMELINE_CONTEXT_MENU_BTN).first().click({ force: true }); cy.get(CLOSE_ALERT_BTN).click(); @@ -43,6 +49,9 @@ export const expandFirstAlert = () => { export const goToClosedAlerts = () => { cy.get(CLOSED_ALERTS_FILTER_BTN).click(); + cy.get(REFRESH_BUTTON).should('not.have.text', 'Updating'); + cy.get(REFRESH_BUTTON).should('have.text', 'Refresh'); + cy.get(TIMELINE_COLUMN_SPINNER).should('not.exist'); }; export const goToManageAlertsDetectionRules = () => { @@ -51,6 +60,9 @@ export const goToManageAlertsDetectionRules = () => { export const goToOpenedAlerts = () => { cy.get(OPENED_ALERTS_FILTER_BTN).click({ force: true }); + cy.get(REFRESH_BUTTON).should('not.have.text', 'Updating'); + cy.get(REFRESH_BUTTON).should('have.text', 'Refresh'); + cy.get(TIMELINE_COLUMN_SPINNER).should('not.exist'); }; export const openFirstAlert = () => { diff --git a/x-pack/plugins/security_solution/cypress/tasks/api_calls.ts b/x-pack/plugins/security_solution/cypress/tasks/api_calls.ts new file mode 100644 index 0000000000000..4c42eb26cf62f --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/tasks/api_calls.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CustomRule } from '../objects/rule'; + +export const createCustomRule = (rule: CustomRule) => { + cy.request({ + method: 'POST', + url: 'api/detection_engine/rules', + body: { + rule_id: 'rule_testing', + risk_score: parseInt(rule.riskScore, 10), + description: rule.description, + interval: '10s', + name: rule.name, + severity: rule.severity.toLocaleLowerCase(), + type: 'query', + from: 'now-17520h', + index: ['exceptions-*'], + query: rule.customQuery, + language: 'kuery', + enabled: false, + }, + headers: { 'kbn-xsrf': 'cypress-creds' }, + }); +}; + +export const deleteCustomRule = () => { + cy.request({ + method: 'DELETE', + url: 'api/detection_engine/rules?rule_id=rule_testing', + headers: { 'kbn-xsrf': 'cypress-creds' }, + }); +}; + +export const removeSignalsIndex = () => { + cy.request({ + method: 'DELETE', + url: `api/detection_engine/index`, + headers: { 'kbn-xsrf': 'delete-signals' }, + }); +}; diff --git a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts index 251a7ccc4b9c9..9b809dbe524ae 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts @@ -11,7 +11,6 @@ import { OverrideRule, ThresholdRule, } from '../objects/rule'; -import { NUMBER_OF_ALERTS } from '../screens/alerts'; import { ABOUT_CONTINUE_BTN, ABOUT_EDIT_TAB, @@ -65,6 +64,7 @@ import { EQL_QUERY_PREVIEW_HISTOGRAM, EQL_QUERY_VALIDATION_SPINNER, } from '../screens/create_new_rule'; +import { SERVER_SIDE_EVENT_COUNT } from '../screens/timeline'; import { NOTIFICATION_TOASTS, TOAST_ERROR_CLASS } from '../screens/shared'; import { TIMELINE } from '../screens/timelines'; import { refreshPage } from './security_header'; @@ -273,6 +273,22 @@ export const selectThresholdRuleType = () => { cy.get(THRESHOLD_TYPE).click({ force: true }); }; +export const waitForAlertsToPopulate = async () => { + cy.waitUntil( + () => { + refreshPage(); + return cy + .get(SERVER_SIDE_EVENT_COUNT) + .invoke('text') + .then((countText) => { + const alertCount = parseInt(countText, 10) || 0; + return alertCount > 0; + }); + }, + { interval: 500, timeout: 12000 } + ); +}; + export const waitForTheRuleToBeExecuted = () => { cy.waitUntil(() => { cy.get(REFRESH_BUTTON).click(); @@ -283,19 +299,6 @@ export const waitForTheRuleToBeExecuted = () => { }); }; -export const waitForAlertsToPopulate = async () => { - cy.waitUntil(() => { - refreshPage(); - return cy - .get(NUMBER_OF_ALERTS) - .invoke('text') - .then((countText) => { - const alertCount = parseInt(countText, 10) || 0; - return alertCount > 0; - }); - }); -}; - export const selectEqlRuleType = () => { cy.get(EQL_TYPE).click({ force: true }); }; diff --git a/x-pack/plugins/security_solution/cypress/tasks/rule_details.ts b/x-pack/plugins/security_solution/cypress/tasks/rule_details.ts new file mode 100644 index 0000000000000..9dd23a1dfa71f --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/tasks/rule_details.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Exception } from '../objects/exception'; +import { RULE_STATUS } from '../screens/create_new_rule'; +import { + ADD_EXCEPTIONS_BTN, + CLOSE_ALERTS_CHECKBOX, + CONFIRM_BTN, + FIELD_INPUT, + LOADING_SPINNER, + OPERATOR_INPUT, + VALUES_INPUT, +} from '../screens/exceptions'; +import { + ALERTS_TAB, + EXCEPTIONS_TAB, + REFRESH_BUTTON, + REMOVE_EXCEPTION_BTN, + RULE_SWITCH, +} from '../screens/rule_details'; + +export const activatesRule = () => { + cy.server(); + cy.route('PATCH', '**/api/detection_engine/rules/_bulk_update').as('bulk_update'); + cy.get(RULE_SWITCH).should('be.visible'); + cy.get(RULE_SWITCH).click(); + cy.wait('@bulk_update').then((response) => { + cy.wrap(response.status).should('eql', 200); + }); +}; + +export const deactivatesRule = () => { + cy.get(RULE_SWITCH).should('be.visible'); + cy.get(RULE_SWITCH).click(); +}; + +export const addsException = (exception: Exception) => { + cy.get(LOADING_SPINNER).should('exist'); + cy.get(LOADING_SPINNER).should('not.exist'); + cy.get(FIELD_INPUT).should('exist'); + cy.get(FIELD_INPUT).type(`${exception.field}{enter}`); + cy.get(OPERATOR_INPUT).type(`${exception.operator}{enter}`); + exception.values.forEach((value) => { + cy.get(VALUES_INPUT).type(`${value}{enter}`); + }); + cy.get(CLOSE_ALERTS_CHECKBOX).click({ force: true }); + cy.get(CONFIRM_BTN).click(); + cy.get(CONFIRM_BTN).should('have.attr', 'disabled'); + cy.get(CONFIRM_BTN).should('not.have.attr', 'disabled'); +}; + +export const addsExceptionFromRuleSettings = (exception: Exception) => { + cy.get(ADD_EXCEPTIONS_BTN).click(); + cy.get(LOADING_SPINNER).should('exist'); + cy.get(LOADING_SPINNER).should('not.exist'); + cy.get(LOADING_SPINNER).should('exist'); + cy.get(LOADING_SPINNER).should('not.exist'); + cy.get(FIELD_INPUT).should('be.visible'); + cy.get(FIELD_INPUT).type(`${exception.field}{enter}`); + cy.get(OPERATOR_INPUT).type(`${exception.operator}{enter}`); + exception.values.forEach((value) => { + cy.get(VALUES_INPUT).type(`${value}{enter}`); + }); + cy.get(CLOSE_ALERTS_CHECKBOX).click({ force: true }); + cy.get(CONFIRM_BTN).click(); + cy.get(CONFIRM_BTN).should('have.attr', 'disabled'); + cy.get(CONFIRM_BTN).should('not.have.attr', 'disabled'); +}; + +export const goToAlertsTab = () => { + cy.get(ALERTS_TAB).click(); +}; + +export const goToExceptionsTab = () => { + cy.get(EXCEPTIONS_TAB).click(); +}; + +export const removeException = () => { + cy.get(REMOVE_EXCEPTION_BTN).click(); +}; + +export const waitForTheRuleToBeExecuted = async () => { + let status = ''; + while (status !== 'succeeded') { + cy.get(REFRESH_BUTTON).click({ force: true }); + status = await cy.get(RULE_STATUS).invoke('text').promisify(); + } +}; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx index fcef88b3f189a..cf8204478a955 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx @@ -334,7 +334,9 @@ const AlertContextMenuComponent: React.FC = ({ onClick={handleAddExceptionClick} disabled={!canUserCRUD || !hasIndexWrite || !areExceptionsAllowed} > - {i18n.ACTION_ADD_EXCEPTION} + + {i18n.ACTION_ADD_EXCEPTION} + ); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx index d7cc389507463..ba676835d60f1 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx @@ -110,16 +110,19 @@ const getRuleDetailsTabs = (rule: Rule | null) => { id: RuleDetailTabs.alerts, name: detectionI18n.ALERT, disabled: false, + dataTestSubj: 'alertsTab', }, { id: RuleDetailTabs.exceptions, name: i18n.EXCEPTIONS_TAB, disabled: !canUseExceptions, + dataTestSubj: 'exceptionsTab', }, { id: RuleDetailTabs.failures, name: i18n.FAILURE_HISTORY_TAB, disabled: false, + dataTestSubj: 'failureHistoryTab', }, ]; }; @@ -263,6 +266,7 @@ export const RuleDetailsPageComponent: FC = ({ isSelected={tab.id === ruleDetailTab} disabled={tab.disabled} key={tab.id} + data-test-subj={tab.dataTestSubj} > {tab.name} diff --git a/x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions/data.json.gz b/x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions/data.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..4139fd9d28f46afe02883236ac50d20fdd697ce5 GIT binary patch literal 1112 zcmV-e1gHBSiwFqq?!aFF17u-zVJ>QOZ*Bm+R!wi)I1s((R|G!o0+m9_`ZTb>B1jL# zqUfPr6v$#B(B#PBS|SCCvf~!{?4&IgBY-aP92&F$?z-{M7zcVsLnuU?E`45 zv+3n}R9f@dS+^$*omq;JTit*iMj?)Dsw>=su_bg5+QwjaRVtf#oyT#$u(9l!zb`ou zUXv}axjvQOq!`ZCcL6mO8su@{!zD~dh1X~Y#ym5d;uPT|PNM{;MVvUn#%-w?SNn^z zPebSv7qe|HxCew`181PD_l~yn9x}siE{GmG zz0k;Lp^<43MoAo=e}B4}Iht(`wDY%hScgNr#5y?4NJr?o{*?fIlxkmAwLH3YQKDfd z)yefSyr?QpT{lKF=|B&?mG&f_u~!$Iiahsl0l`Lfz=f|4Nth(bqnl;VN8j-nE>|S$ z>ycKcPMd~0Hp7wu!MAsq!89P?p+sfV)}ZYc6{$OL@>hlewOQAyhgqhbl6~u^8tI4J z5-&5Mq}20jtb|Q^rp#z)L+xG-anE^LrXH856q=anQT!XQg`z!K>=idXCIOen1!07= z$_qrQHBL#C6cM6Df{j#GG)}h_RI4e+|8a+HS-320kfY5eMhI^Vl{Uj8PF^SRUtbEQ zdE;nfA`#h`F!Xlh|DI9$Gc-WQ;1b4oN9Ck$((rni1qNGAgm2x05@6Y|N`}i4BfJi* zcr0q*-`ig3-nX?i(Unxqpur1vcgW2>9N+%-V2iwc;9@6#ca&_#qx0-8OI3}@bo0B?5tkAI16_L+Z;zGMgdQN4eD)_uT#Pi`WB2A&|rt`7uri8icJ9W3^;;udECn?rGs#ohy%3Temxh3ho z?DlXh8G*)}XOUi1nTsDB{mO%=_<~a-Et_O*1Ri<%1%gfGOXrmhD4cEKBQRq|jDlVn1r6i20k@qX~T$xW^#c7N7GG|r e|6W1qyD4>k_R6}se81eh-24D#-rx9%3;+P03Laws literal 0 HcmV?d00001 diff --git a/x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions/mappings.json b/x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions/mappings.json new file mode 100644 index 0000000000000..4e5c6e9955310 --- /dev/null +++ b/x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions/mappings.json @@ -0,0 +1,3577 @@ +{ + "type": "index", + "value": { + "aliases": { + "exceptions-8.0.0": { + "is_write_index": false + }, + "beats": { + }, + "siem-read-alias": { + } + }, + "index": "exceptions-8.0.0-2019.08.30-000021", + "mappings": { + "_meta": { + "beat": "auditbeat", + "version": "8.0.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "labels.*" + } + }, + { + "container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "container.labels.*" + } + }, + { + "fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + }, + { + "docker.container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.container.labels.*" + } + }, + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "auditd": { + "properties": { + "data": { + "properties": { + "a0": { + "ignore_above": 1024, + "type": "keyword" + }, + "a1": { + "ignore_above": 1024, + "type": "keyword" + }, + "a2": { + "ignore_above": 1024, + "type": "keyword" + }, + "a3": { + "ignore_above": 1024, + "type": "keyword" + }, + "a[0-3]": { + "ignore_above": 1024, + "type": "keyword" + }, + "acct": { + "ignore_above": 1024, + "type": "keyword" + }, + "acl": { + "ignore_above": 1024, + "type": "keyword" + }, + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "added": { + "ignore_above": 1024, + "type": "keyword" + }, + "addr": { + "ignore_above": 1024, + "type": "keyword" + }, + "apparmor": { + "ignore_above": 1024, + "type": "keyword" + }, + "arch": { + "ignore_above": 1024, + "type": "keyword" + }, + "argc": { + "ignore_above": 1024, + "type": "keyword" + }, + "audit_backlog_limit": { + "ignore_above": 1024, + "type": "keyword" + }, + "audit_backlog_wait_time": { + "ignore_above": 1024, + "type": "keyword" + }, + "audit_enabled": { + "ignore_above": 1024, + "type": "keyword" + }, + "audit_failure": { + "ignore_above": 1024, + "type": "keyword" + }, + "banners": { + "ignore_above": 1024, + "type": "keyword" + }, + "bool": { + "ignore_above": 1024, + "type": "keyword" + }, + "bus": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_fe": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_fi": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_fp": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_fver": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_pe": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_pi": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_pp": { + "ignore_above": 1024, + "type": "keyword" + }, + "capability": { + "ignore_above": 1024, + "type": "keyword" + }, + "cgroup": { + "ignore_above": 1024, + "type": "keyword" + }, + "changed": { + "ignore_above": 1024, + "type": "keyword" + }, + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "cmd": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "compat": { + "ignore_above": 1024, + "type": "keyword" + }, + "daddr": { + "ignore_above": 1024, + "type": "keyword" + }, + "data": { + "ignore_above": 1024, + "type": "keyword" + }, + "default-context": { + "ignore_above": 1024, + "type": "keyword" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "dir": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "dmac": { + "ignore_above": 1024, + "type": "keyword" + }, + "dport": { + "ignore_above": 1024, + "type": "keyword" + }, + "enforcing": { + "ignore_above": 1024, + "type": "keyword" + }, + "entries": { + "ignore_above": 1024, + "type": "keyword" + }, + "exit": { + "ignore_above": 1024, + "type": "keyword" + }, + "fam": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "fd": { + "ignore_above": 1024, + "type": "keyword" + }, + "fe": { + "ignore_above": 1024, + "type": "keyword" + }, + "feature": { + "ignore_above": 1024, + "type": "keyword" + }, + "fi": { + "ignore_above": 1024, + "type": "keyword" + }, + "file": { + "ignore_above": 1024, + "type": "keyword" + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "format": { + "ignore_above": 1024, + "type": "keyword" + }, + "fp": { + "ignore_above": 1024, + "type": "keyword" + }, + "fver": { + "ignore_above": 1024, + "type": "keyword" + }, + "grantors": { + "ignore_above": 1024, + "type": "keyword" + }, + "grp": { + "ignore_above": 1024, + "type": "keyword" + }, + "hook": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "icmp_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "igid": { + "ignore_above": 1024, + "type": "keyword" + }, + "img-ctx": { + "ignore_above": 1024, + "type": "keyword" + }, + "info": { + "ignore_above": 1024, + "type": "keyword" + }, + "inif": { + "ignore_above": 1024, + "type": "keyword" + }, + "ino": { + "ignore_above": 1024, + "type": "keyword" + }, + "inode_gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "inode_uid": { + "ignore_above": 1024, + "type": "keyword" + }, + "invalid_context": { + "ignore_above": 1024, + "type": "keyword" + }, + "ioctlcmd": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "ipid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ipx-net": { + "ignore_above": 1024, + "type": "keyword" + }, + "items": { + "ignore_above": 1024, + "type": "keyword" + }, + "iuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "ksize": { + "ignore_above": 1024, + "type": "keyword" + }, + "laddr": { + "ignore_above": 1024, + "type": "keyword" + }, + "len": { + "ignore_above": 1024, + "type": "keyword" + }, + "list": { + "ignore_above": 1024, + "type": "keyword" + }, + "lport": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "macproto": { + "ignore_above": 1024, + "type": "keyword" + }, + "maj": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "ignore_above": 1024, + "type": "keyword" + }, + "minor": { + "ignore_above": 1024, + "type": "keyword" + }, + "model": { + "ignore_above": 1024, + "type": "keyword" + }, + "msg": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "nargs": { + "ignore_above": 1024, + "type": "keyword" + }, + "net": { + "ignore_above": 1024, + "type": "keyword" + }, + "new": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-chardev": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-disk": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-enabled": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-fs": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-level": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-log_passwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-mem": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-net": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-range": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-rng": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-role": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-seuser": { + "ignore_above": 1024, + "type": "keyword" + }, + "new-vcpu": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_lock": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_pe": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_pi": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_pp": { + "ignore_above": 1024, + "type": "keyword" + }, + "nlnk-fam": { + "ignore_above": 1024, + "type": "keyword" + }, + "nlnk-grp": { + "ignore_above": 1024, + "type": "keyword" + }, + "nlnk-pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "oauid": { + "ignore_above": 1024, + "type": "keyword" + }, + "obj": { + "ignore_above": 1024, + "type": "keyword" + }, + "obj_gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "obj_uid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ocomm": { + "ignore_above": 1024, + "type": "keyword" + }, + "oflag": { + "ignore_above": 1024, + "type": "keyword" + }, + "old": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-chardev": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-disk": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-enabled": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-fs": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-level": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-log_passwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-mem": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-net": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-range": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-rng": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-role": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-seuser": { + "ignore_above": 1024, + "type": "keyword" + }, + "old-vcpu": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_enforcing": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_lock": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_pa": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_pe": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_pi": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_pp": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_prom": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_val": { + "ignore_above": 1024, + "type": "keyword" + }, + "op": { + "ignore_above": 1024, + "type": "keyword" + }, + "operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "opid": { + "ignore_above": 1024, + "type": "keyword" + }, + "oses": { + "ignore_above": 1024, + "type": "keyword" + }, + "outif": { + "ignore_above": 1024, + "type": "keyword" + }, + "pa": { + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "ignore_above": 1024, + "type": "keyword" + }, + "per": { + "ignore_above": 1024, + "type": "keyword" + }, + "perm": { + "ignore_above": 1024, + "type": "keyword" + }, + "perm_mask": { + "ignore_above": 1024, + "type": "keyword" + }, + "permissive": { + "ignore_above": 1024, + "type": "keyword" + }, + "pfs": { + "ignore_above": 1024, + "type": "keyword" + }, + "pi": { + "ignore_above": 1024, + "type": "keyword" + }, + "pp": { + "ignore_above": 1024, + "type": "keyword" + }, + "printer": { + "ignore_above": 1024, + "type": "keyword" + }, + "profile": { + "ignore_above": 1024, + "type": "keyword" + }, + "prom": { + "ignore_above": 1024, + "type": "keyword" + }, + "proto": { + "ignore_above": 1024, + "type": "keyword" + }, + "qbytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "range": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "removed": { + "ignore_above": 1024, + "type": "keyword" + }, + "res": { + "ignore_above": 1024, + "type": "keyword" + }, + "resrc": { + "ignore_above": 1024, + "type": "keyword" + }, + "rport": { + "ignore_above": 1024, + "type": "keyword" + }, + "sauid": { + "ignore_above": 1024, + "type": "keyword" + }, + "scontext": { + "ignore_above": 1024, + "type": "keyword" + }, + "selected-context": { + "ignore_above": 1024, + "type": "keyword" + }, + "seperm": { + "ignore_above": 1024, + "type": "keyword" + }, + "seperms": { + "ignore_above": 1024, + "type": "keyword" + }, + "seqno": { + "ignore_above": 1024, + "type": "keyword" + }, + "seresult": { + "ignore_above": 1024, + "type": "keyword" + }, + "ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "seuser": { + "ignore_above": 1024, + "type": "keyword" + }, + "sig": { + "ignore_above": 1024, + "type": "keyword" + }, + "sigev_signo": { + "ignore_above": 1024, + "type": "keyword" + }, + "smac": { + "ignore_above": 1024, + "type": "keyword" + }, + "socket": { + "properties": { + "addr": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "ignore_above": 1024, + "type": "keyword" + }, + "saddr": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "spid": { + "ignore_above": 1024, + "type": "keyword" + }, + "sport": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "subj": { + "ignore_above": 1024, + "type": "keyword" + }, + "success": { + "ignore_above": 1024, + "type": "keyword" + }, + "syscall": { + "ignore_above": 1024, + "type": "keyword" + }, + "table": { + "ignore_above": 1024, + "type": "keyword" + }, + "tclass": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcontext": { + "ignore_above": 1024, + "type": "keyword" + }, + "terminal": { + "ignore_above": 1024, + "type": "keyword" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + }, + "unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "uri": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "val": { + "ignore_above": 1024, + "type": "keyword" + }, + "ver": { + "ignore_above": 1024, + "type": "keyword" + }, + "virt": { + "ignore_above": 1024, + "type": "keyword" + }, + "vm": { + "ignore_above": 1024, + "type": "keyword" + }, + "vm-ctx": { + "ignore_above": 1024, + "type": "keyword" + }, + "vm-pid": { + "ignore_above": 1024, + "type": "keyword" + }, + "watch": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "message_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "paths": { + "properties": { + "cap_fe": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_fi": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_fp": { + "ignore_above": 1024, + "type": "keyword" + }, + "cap_fver": { + "ignore_above": 1024, + "type": "keyword" + }, + "dev": { + "ignore_above": 1024, + "type": "keyword" + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "item": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "nametype": { + "ignore_above": 1024, + "type": "keyword" + }, + "obj_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "obj_level": { + "ignore_above": 1024, + "type": "keyword" + }, + "obj_role": { + "ignore_above": 1024, + "type": "keyword" + }, + "obj_user": { + "ignore_above": 1024, + "type": "keyword" + }, + "objtype": { + "ignore_above": 1024, + "type": "keyword" + }, + "ogid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ouid": { + "ignore_above": 1024, + "type": "keyword" + }, + "rdev": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "result": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequence": { + "type": "long" + }, + "session": { + "ignore_above": 1024, + "type": "keyword" + }, + "summary": { + "properties": { + "actor": { + "properties": { + "primary": { + "ignore_above": 1024, + "type": "keyword" + }, + "secondary": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "how": { + "ignore_above": 1024, + "type": "keyword" + }, + "object": { + "properties": { + "primary": { + "ignore_above": 1024, + "type": "keyword" + }, + "secondary": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "user": { + "properties": { + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "user": { + "properties": { + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "docker": { + "properties": { + "container": { + "properties": { + "labels": { + "type": "object" + } + } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "type": "object" + }, + "file": { + "properties": { + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "origin": { + "fields": { + "raw": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "selinux": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "role": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "setgid": { + "type": "boolean" + }, + "setuid": { + "type": "boolean" + }, + "size": { + "type": "long" + }, + "target_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "flow": { + "properties": { + "complete": { + "type": "boolean" + }, + "final": { + "type": "boolean" + } + } + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "geoip": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "properties": { + "blake2b_256": { + "ignore_above": 1024, + "type": "keyword" + }, + "blake2b_384": { + "ignore_above": 1024, + "type": "keyword" + }, + "blake2b_512": { + "ignore_above": 1024, + "type": "keyword" + }, + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha224": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha384": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha3_224": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha3_256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha3_384": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha3_512": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512_224": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512_256": { + "ignore_above": 1024, + "type": "keyword" + }, + "xxh64": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "containerized": { + "type": "boolean" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "jolokia": { + "properties": { + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "secured": { + "type": "boolean" + }, + "server": { + "properties": { + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kubernetes": { + "properties": { + "annotations": { + "type": "object" + }, + "container": { + "properties": { + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "deployment": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pod": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "replicaset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "statefulset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "observer": { + "properties": { + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "sha1": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + } + } + }, + "title": { + "ignore_above": 1024, + "type": "keyword" + }, + "working_directory": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "related": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "user": { + "properties": { + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "socket": { + "properties": { + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "user": { + "properties": { + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "system": { + "properties": { + "audit": { + "properties": { + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "boottime": { + "type": "date" + }, + "containerized": { + "type": "boolean" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "timezone": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "offset": { + "properties": { + "sec": { + "type": "long" + } + } + } + } + }, + "uptime": { + "type": "long" + } + } + }, + "newsocket": { + "properties": { + "egid": { + "type": "long" + }, + "euid": { + "type": "long" + }, + "gid": { + "type": "long" + }, + "internal_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel_sock_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "type": "long" + } + } + }, + "package": { + "properties": { + "arch": { + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "installtime": { + "type": "date" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "release": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "summary": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "socket": { + "properties": { + "egid": { + "type": "long" + }, + "euid": { + "type": "long" + }, + "gid": { + "type": "long" + }, + "internal_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel_sock_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "type": "long" + } + } + }, + "user": { + "properties": { + "dir": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "properties": { + "last_changed": { + "type": "date" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "shell": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_information": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "audit": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "effective": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "filesystem": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "name_map": { + "type": "object" + }, + "saved": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "selinux": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "role": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "terminal": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "settings": { + "index": { + "lifecycle": { + "indexing_complete": "true", + "name": "auditbeat-8.0.0", + "rollover_alias": "auditbeat-8.0.0" + }, + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "number_of_replicas": "0", + "number_of_shards": "1", + "query": { + "default_field": [ + "message", + "tags", + "agent.ephemeral_id", + "agent.id", + "agent.name", + "agent.type", + "agent.version", + "client.address", + "client.domain", + "client.geo.city_name", + "client.geo.continent_name", + "client.geo.country_iso_code", + "client.geo.country_name", + "client.geo.name", + "client.geo.region_iso_code", + "client.geo.region_name", + "client.mac", + "client.user.email", + "client.user.full_name", + "client.user.group.id", + "client.user.group.name", + "client.user.hash", + "client.user.id", + "client.user.name", + "cloud.account.id", + "cloud.availability_zone", + "cloud.instance.id", + "cloud.instance.name", + "cloud.machine.type", + "cloud.provider", + "cloud.region", + "container.id", + "container.image.name", + "container.image.tag", + "container.name", + "container.runtime", + "destination.address", + "destination.domain", + "destination.geo.city_name", + "destination.geo.continent_name", + "destination.geo.country_iso_code", + "destination.geo.country_name", + "destination.geo.name", + "destination.geo.region_iso_code", + "destination.geo.region_name", + "destination.mac", + "destination.user.email", + "destination.user.full_name", + "destination.user.group.id", + "destination.user.group.name", + "destination.user.hash", + "destination.user.id", + "destination.user.name", + "ecs.version", + "error.code", + "error.id", + "error.message", + "event.action", + "event.category", + "event.dataset", + "event.hash", + "event.id", + "event.kind", + "event.module", + "event.original", + "event.outcome", + "event.timezone", + "event.type", + "file.device", + "file.extension", + "file.gid", + "file.group", + "file.inode", + "file.mode", + "file.owner", + "file.path", + "file.target_path", + "file.type", + "file.uid", + "geo.city_name", + "geo.continent_name", + "geo.country_iso_code", + "geo.country_name", + "geo.name", + "geo.region_iso_code", + "geo.region_name", + "group.id", + "group.name", + "host.architecture", + "host.geo.city_name", + "host.geo.continent_name", + "host.geo.country_iso_code", + "host.geo.country_name", + "host.geo.name", + "host.geo.region_iso_code", + "host.geo.region_name", + "host.hostname", + "host.id", + "host.mac", + "host.name", + "host.os.family", + "host.os.full", + "host.os.kernel", + "host.os.name", + "host.os.platform", + "host.os.version", + "host.type", + "host.user.email", + "host.user.full_name", + "host.user.group.id", + "host.user.group.name", + "host.user.hash", + "host.user.id", + "host.user.name", + "http.request.body.content", + "http.request.method", + "http.request.referrer", + "http.response.body.content", + "http.version", + "log.level", + "log.original", + "network.application", + "network.community_id", + "network.direction", + "network.iana_number", + "network.name", + "network.protocol", + "network.transport", + "network.type", + "observer.geo.city_name", + "observer.geo.continent_name", + "observer.geo.country_iso_code", + "observer.geo.country_name", + "observer.geo.name", + "observer.geo.region_iso_code", + "observer.geo.region_name", + "observer.hostname", + "observer.mac", + "observer.os.family", + "observer.os.full", + "observer.os.kernel", + "observer.os.name", + "observer.os.platform", + "observer.os.version", + "observer.serial_number", + "observer.type", + "observer.vendor", + "observer.version", + "organization.id", + "organization.name", + "os.family", + "os.full", + "os.kernel", + "os.name", + "os.platform", + "os.version", + "process.args", + "process.executable", + "process.name", + "process.title", + "process.working_directory", + "server.address", + "server.domain", + "server.geo.city_name", + "server.geo.continent_name", + "server.geo.country_iso_code", + "server.geo.country_name", + "server.geo.name", + "server.geo.region_iso_code", + "server.geo.region_name", + "server.mac", + "server.user.email", + "server.user.full_name", + "server.user.group.id", + "server.user.group.name", + "server.user.hash", + "server.user.id", + "server.user.name", + "service.ephemeral_id", + "service.id", + "service.name", + "service.state", + "service.type", + "service.version", + "source.address", + "source.domain", + "source.geo.city_name", + "source.geo.continent_name", + "source.geo.country_iso_code", + "source.geo.country_name", + "source.geo.name", + "source.geo.region_iso_code", + "source.geo.region_name", + "source.mac", + "source.user.email", + "source.user.full_name", + "source.user.group.id", + "source.user.group.name", + "source.user.hash", + "source.user.id", + "source.user.name", + "url.domain", + "url.fragment", + "url.full", + "url.original", + "url.password", + "url.path", + "url.query", + "url.scheme", + "url.username", + "user.email", + "user.full_name", + "user.group.id", + "user.group.name", + "user.hash", + "user.id", + "user.name", + "user_agent.device.name", + "user_agent.name", + "user_agent.original", + "user_agent.os.family", + "user_agent.os.full", + "user_agent.os.kernel", + "user_agent.os.name", + "user_agent.os.platform", + "user_agent.os.version", + "user_agent.version", + "agent.hostname", + "error.type", + "cloud.project.id", + "host.os.build", + "kubernetes.pod.name", + "kubernetes.pod.uid", + "kubernetes.namespace", + "kubernetes.node.name", + "kubernetes.replicaset.name", + "kubernetes.deployment.name", + "kubernetes.statefulset.name", + "kubernetes.container.name", + "kubernetes.container.image", + "jolokia.agent.version", + "jolokia.agent.id", + "jolokia.server.product", + "jolokia.server.version", + "jolokia.server.vendor", + "jolokia.url", + "raw", + "file.origin", + "file.selinux.user", + "file.selinux.role", + "file.selinux.domain", + "file.selinux.level", + "user.audit.id", + "user.audit.name", + "user.effective.id", + "user.effective.name", + "user.effective.group.id", + "user.effective.group.name", + "user.filesystem.id", + "user.filesystem.name", + "user.filesystem.group.id", + "user.filesystem.group.name", + "user.saved.id", + "user.saved.name", + "user.saved.group.id", + "user.saved.group.name", + "user.selinux.user", + "user.selinux.role", + "user.selinux.domain", + "user.selinux.level", + "user.selinux.category", + "source.path", + "destination.path", + "auditd.message_type", + "auditd.session", + "auditd.result", + "auditd.summary.actor.primary", + "auditd.summary.actor.secondary", + "auditd.summary.object.type", + "auditd.summary.object.primary", + "auditd.summary.object.secondary", + "auditd.summary.how", + "auditd.paths.inode", + "auditd.paths.dev", + "auditd.paths.obj_user", + "auditd.paths.obj_role", + "auditd.paths.obj_domain", + "auditd.paths.obj_level", + "auditd.paths.objtype", + "auditd.paths.ouid", + "auditd.paths.rdev", + "auditd.paths.nametype", + "auditd.paths.ogid", + "auditd.paths.item", + "auditd.paths.mode", + "auditd.paths.name", + "auditd.data.action", + "auditd.data.minor", + "auditd.data.acct", + "auditd.data.addr", + "auditd.data.cipher", + "auditd.data.id", + "auditd.data.entries", + "auditd.data.kind", + "auditd.data.ksize", + "auditd.data.spid", + "auditd.data.arch", + "auditd.data.argc", + "auditd.data.major", + "auditd.data.unit", + "auditd.data.table", + "auditd.data.terminal", + "auditd.data.grantors", + "auditd.data.direction", + "auditd.data.op", + "auditd.data.tty", + "auditd.data.syscall", + "auditd.data.data", + "auditd.data.family", + "auditd.data.mac", + "auditd.data.pfs", + "auditd.data.items", + "auditd.data.a0", + "auditd.data.a1", + "auditd.data.a2", + "auditd.data.a3", + "auditd.data.hostname", + "auditd.data.lport", + "auditd.data.rport", + "auditd.data.exit", + "auditd.data.fp", + "auditd.data.laddr", + "auditd.data.sport", + "auditd.data.capability", + "auditd.data.nargs", + "auditd.data.new-enabled", + "auditd.data.audit_backlog_limit", + "auditd.data.dir", + "auditd.data.cap_pe", + "auditd.data.model", + "auditd.data.new_pp", + "auditd.data.old-enabled", + "auditd.data.oauid", + "auditd.data.old", + "auditd.data.banners", + "auditd.data.feature", + "auditd.data.vm-ctx", + "auditd.data.opid", + "auditd.data.seperms", + "auditd.data.seresult", + "auditd.data.new-rng", + "auditd.data.old-net", + "auditd.data.sigev_signo", + "auditd.data.ino", + "auditd.data.old_enforcing", + "auditd.data.old-vcpu", + "auditd.data.range", + "auditd.data.res", + "auditd.data.added", + "auditd.data.fam", + "auditd.data.nlnk-pid", + "auditd.data.subj", + "auditd.data.a[0-3]", + "auditd.data.cgroup", + "auditd.data.kernel", + "auditd.data.ocomm", + "auditd.data.new-net", + "auditd.data.permissive", + "auditd.data.class", + "auditd.data.compat", + "auditd.data.fi", + "auditd.data.changed", + "auditd.data.msg", + "auditd.data.dport", + "auditd.data.new-seuser", + "auditd.data.invalid_context", + "auditd.data.dmac", + "auditd.data.ipx-net", + "auditd.data.iuid", + "auditd.data.macproto", + "auditd.data.obj", + "auditd.data.ipid", + "auditd.data.new-fs", + "auditd.data.vm-pid", + "auditd.data.cap_pi", + "auditd.data.old-auid", + "auditd.data.oses", + "auditd.data.fd", + "auditd.data.igid", + "auditd.data.new-disk", + "auditd.data.parent", + "auditd.data.len", + "auditd.data.oflag", + "auditd.data.uuid", + "auditd.data.code", + "auditd.data.nlnk-grp", + "auditd.data.cap_fp", + "auditd.data.new-mem", + "auditd.data.seperm", + "auditd.data.enforcing", + "auditd.data.new-chardev", + "auditd.data.old-rng", + "auditd.data.outif", + "auditd.data.cmd", + "auditd.data.hook", + "auditd.data.new-level", + "auditd.data.sauid", + "auditd.data.sig", + "auditd.data.audit_backlog_wait_time", + "auditd.data.printer", + "auditd.data.old-mem", + "auditd.data.perm", + "auditd.data.old_pi", + "auditd.data.state", + "auditd.data.format", + "auditd.data.new_gid", + "auditd.data.tcontext", + "auditd.data.maj", + "auditd.data.watch", + "auditd.data.device", + "auditd.data.grp", + "auditd.data.bool", + "auditd.data.icmp_type", + "auditd.data.new_lock", + "auditd.data.old_prom", + "auditd.data.acl", + "auditd.data.ip", + "auditd.data.new_pi", + "auditd.data.default-context", + "auditd.data.inode_gid", + "auditd.data.new-log_passwd", + "auditd.data.new_pe", + "auditd.data.selected-context", + "auditd.data.cap_fver", + "auditd.data.file", + "auditd.data.net", + "auditd.data.virt", + "auditd.data.cap_pp", + "auditd.data.old-range", + "auditd.data.resrc", + "auditd.data.new-range", + "auditd.data.obj_gid", + "auditd.data.proto", + "auditd.data.old-disk", + "auditd.data.audit_failure", + "auditd.data.inif", + "auditd.data.vm", + "auditd.data.flags", + "auditd.data.nlnk-fam", + "auditd.data.old-fs", + "auditd.data.old-ses", + "auditd.data.seqno", + "auditd.data.fver", + "auditd.data.qbytes", + "auditd.data.seuser", + "auditd.data.cap_fe", + "auditd.data.new-vcpu", + "auditd.data.old-level", + "auditd.data.old_pp", + "auditd.data.daddr", + "auditd.data.old-role", + "auditd.data.ioctlcmd", + "auditd.data.smac", + "auditd.data.apparmor", + "auditd.data.fe", + "auditd.data.perm_mask", + "auditd.data.ses", + "auditd.data.cap_fi", + "auditd.data.obj_uid", + "auditd.data.reason", + "auditd.data.list", + "auditd.data.old_lock", + "auditd.data.bus", + "auditd.data.old_pe", + "auditd.data.new-role", + "auditd.data.prom", + "auditd.data.uri", + "auditd.data.audit_enabled", + "auditd.data.old-log_passwd", + "auditd.data.old-seuser", + "auditd.data.per", + "auditd.data.scontext", + "auditd.data.tclass", + "auditd.data.ver", + "auditd.data.new", + "auditd.data.val", + "auditd.data.img-ctx", + "auditd.data.old-chardev", + "auditd.data.old_val", + "auditd.data.success", + "auditd.data.inode_uid", + "auditd.data.removed", + "auditd.data.socket.port", + "auditd.data.socket.saddr", + "auditd.data.socket.addr", + "auditd.data.socket.family", + "auditd.data.socket.path", + "geoip.continent_name", + "geoip.city_name", + "geoip.region_name", + "geoip.country_iso_code", + "hash.blake2b_256", + "hash.blake2b_384", + "hash.blake2b_512", + "hash.md5", + "hash.sha1", + "hash.sha224", + "hash.sha256", + "hash.sha384", + "hash.sha3_224", + "hash.sha3_256", + "hash.sha3_384", + "hash.sha3_512", + "hash.sha512", + "hash.sha512_224", + "hash.sha512_256", + "hash.xxh64", + "event.origin", + "user.entity_id", + "user.terminal", + "process.entity_id", + "socket.entity_id", + "system.audit.host.timezone.name", + "system.audit.host.hostname", + "system.audit.host.id", + "system.audit.host.architecture", + "system.audit.host.mac", + "system.audit.host.os.platform", + "system.audit.host.os.name", + "system.audit.host.os.family", + "system.audit.host.os.version", + "system.audit.host.os.kernel", + "system.audit.package.entity_id", + "system.audit.package.name", + "system.audit.package.version", + "system.audit.package.release", + "system.audit.package.arch", + "system.audit.package.license", + "system.audit.package.summary", + "system.audit.package.url", + "system.audit.user.name", + "system.audit.user.uid", + "system.audit.user.gid", + "system.audit.user.dir", + "system.audit.user.shell", + "system.audit.user.user_information", + "system.audit.user.password.type", + "fields.*" + ] + }, + "refresh_interval": "5s" + } + } + } +} diff --git a/x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions2/data.json.gz b/x-pack/test/security_solution_cypress/es_archives/auditbeat_for_exceptions2/data.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..0fdcb7d783ea67d71a6286e74d7392c593cd6f41 GIT binary patch literal 1113 zcmV-f1g85RiwFpX?!aFF17u-zVJ>QOZ*Bm+R$Fh|HWYr(uL%6K0hL0^x*8ZT1nq!g zDEiP01u_fxkId-sE-7c7< zi+65ePnzDXe76twkuzsq!{dj%J=}lpzVi16_+swYe+=%nUM8&X9GKOnS9@e1WJ!o}k`-uja* z2;(KNOVB+*3yL)BQ_VRpX;S47Wm#QBX+=;}mZ*qOQ8H_w2(jXt(8fPWZ3JltG&F}c z+VKq$#CJI*W_dvG3gmM8z9Cv?m=k_nX?a| zEzYKw>rrXLCuiLp4QRWRBEPx;JB(tSxKvlT2V+O*4vdSz{;E_i^Ri6Sa%N-Mvv6Os zC%hpW-f(kVf0JT3SFa;#E40YtD1=Mcj*6hs9*kvSH^n)^S(+yq&Z{)@g00(JGp_a* zZ=aUXCoU%2TyPgSa&<7GSd`3X=#=~ysF^Pm8lnxSk*nj)@I7asocES?(;hO*Z7zs8 zb$X_e$xI{jDo(OAKL7r3GfOnt9;gnt^;m~vJjXiNPe@1Tx&DO!bC7Dct{Zvq>!QTK zR;r`#V|h{6oceBz7^1-py_N1Hfw5N?oT{=6Z~?(i>VT_I9kMvfvPVD5j-Px_$MCr# zS>K#!b=1Z-)U#QZ3<$ox!vv-QfdC~c+pYoQwx~&?!OLG;3N&`zs2(Pnwo3M0nCe77 zbOR^udW)H7vA!&qwfYKVI-^K~9@iArIJogT%%0h=j0ki}kc+hY=Od6*GK z$m_B~q+a5jBw3XpT4mTuWlht3Q$xKNbNp|A*ye@H!3HH-tx|;W%2H`HJmTzi68{OM zV4OEiRyGoml?_90NB-{_WjI3vbPO(GjB6@Kb(5Ai!z?h^a3VtM4wL}LhDA19mKfn> z$SSuaBQOZ*BnHS8Z?GHW2=vUlI6Ihm{H`krZWMz!0Qs zi>)ozm$m3H7!*m_oUNBil%2H5e;+BoM3NORNt<+-KO{EEJ09@d0mRDOYF+eMyIMWJT06iANp1#0ua50aE#FyQV*i= z<3`?|=TnuLmIAw^ikV7P$rqEg%@i@=M^phqk`UmCL%WPcUd!&ie985K$9kd7#pH9IHSek`AgqPf_Xu>|mD!kPolv*j$U=K5q7V^GeS&C& zNlVxs8xF_r#AWNIl#7|p_Ac6bK10A+B6%r+KdhlB^S_la!=|bQuZ$L;(ki`He5s`> z-RY?)_M<2*zIDdUI@SZB^anyoBhM$;+x)+AI2qi#UWuxEZ3}nELqoVz%RSg}v9W)Y zDYMGU`6OB7tF~P>PHLK$7wt0^8A+l=`9}i0__|o){p7GDEmT$Y^J7}|0drDUlwlj6e6 z@+|0jb+wYWE}O#j;6=&Ztb6UjE`%F$-WTe3j}O%jZ4Byy z$F+2QQrJ|q?y8IEw21+H+*_^(mxmAQ%vjd2U2dm-L$3?b9l1=LEfX$fxyJnno+tLh z5==1-sG<<|1vl2T7B7$b->#feSAFQ*v}7G^|6U$l9vwLaH=RZbzdMc0CQl!D8nG?^ z?WPed6(%9%Ku}^w24$__vE)F-k_f?A43q1p5!-ZvxsG`@P;g-&L=1%j2_*txKa2pU zj$BBBB#C4D)cB!hksuh%KI~gv;c)T=1IZUs_*Jurvb{P2omU?oLXTH6mW|%1bxLA;^J1_*BnCG-6($`l((KkELy(X9FnQk!BK=njG}h27@kGo zIG~H=Whz*ADmGqlgiXA*1YTn|Z@~31OjIST0U&)v~o>v%G`$w`%_X0W4E~ IK#CXu0N{}9eE Date: Mon, 30 Nov 2020 14:08:50 +0300 Subject: [PATCH 29/37] TSVB field list performance issue on using annotations (#84407) * TSVB field list performance issue on using annotations * Add AbortController to fetchFields and change translation id in annotations_editor * Rename fetchFields to debouncedFetchFields Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../vis_type_timeseries/common/constants.ts | 1 + .../components/aggs/field_select.js | 2 +- .../components/annotations_editor.js | 17 +++++++-- .../application/components/vis_editor.js | 15 +++++++- .../public/application/lib/fetch_fields.js | 38 +++++++++++-------- .../server/routes/fields.ts | 3 +- 6 files changed, 54 insertions(+), 22 deletions(-) diff --git a/src/plugins/vis_type_timeseries/common/constants.ts b/src/plugins/vis_type_timeseries/common/constants.ts index bfcb5e8e15b9d..019afcba60000 100644 --- a/src/plugins/vis_type_timeseries/common/constants.ts +++ b/src/plugins/vis_type_timeseries/common/constants.ts @@ -22,4 +22,5 @@ export const INDEXES_SEPARATOR = ','; export const AUTO_INTERVAL = 'auto'; export const ROUTES = { VIS_DATA: '/api/metrics/vis/data', + FIELDS: '/api/metrics/fields', }; diff --git a/src/plugins/vis_type_timeseries/public/application/components/aggs/field_select.js b/src/plugins/vis_type_timeseries/public/application/components/aggs/field_select.js index a4168fd1398f1..b1ff749494b10 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/aggs/field_select.js +++ b/src/plugins/vis_type_timeseries/public/application/components/aggs/field_select.js @@ -91,7 +91,7 @@ function FieldSelectUi({ } FieldSelectUi.defaultProps = { - indexPattern: '*', + indexPattern: '', disabled: false, restrict: [], placeholder: i18n.translate('visTypeTimeseries.fieldSelect.selectFieldPlaceholder', { diff --git a/src/plugins/vis_type_timeseries/public/application/components/annotations_editor.js b/src/plugins/vis_type_timeseries/public/application/components/annotations_editor.js index 34132e5f72716..3d38aa72fc271 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/annotations_editor.js +++ b/src/plugins/vis_type_timeseries/public/application/components/annotations_editor.js @@ -43,13 +43,14 @@ import { EuiCode, EuiText, } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; function newAnnotation() { return { id: uuid.v1(), color: '#F00', - index_pattern: '*', + index_pattern: '', time_field: '@timestamp', icon: 'fa-tag', ignore_global_filters: 1, @@ -84,7 +85,7 @@ export class AnnotationsEditor extends Component { const defaults = { fields: '', template: '', - index_pattern: '*', + index_pattern: '', query_string: { query: '', language: getDefaultQueryLanguage() }, }; const model = { ...defaults, ...row }; @@ -100,6 +101,8 @@ export class AnnotationsEditor extends Component { const htmlId = htmlIdGenerator(model.id); const handleAdd = collectionActions.handleAdd.bind(null, this.props, newAnnotation); const handleDelete = collectionActions.handleDelete.bind(null, this.props, model); + const defaultIndexPattern = this.props.model.default_index_pattern; + return (
@@ -120,14 +123,22 @@ export class AnnotationsEditor extends Component { label={ } + helpText={ + defaultIndexPattern && + !model.index_pattern && + i18n.translate('visTypeTimeseries.annotationsEditor.searchByDefaultIndex', { + defaultMessage: 'Default index pattern is used. To query all indexes use *', + }) + } fullWidth > diff --git a/src/plugins/vis_type_timeseries/public/application/components/vis_editor.js b/src/plugins/vis_type_timeseries/public/application/components/vis_editor.js index 083ccaf8e5073..2cc50bd7efaeb 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/vis_editor.js +++ b/src/plugins/vis_type_timeseries/public/application/components/vis_editor.js @@ -76,6 +76,19 @@ export class VisEditor extends Component { }); }, VIS_STATE_DEBOUNCE_DELAY); + debouncedFetchFields = debounce( + (extractedIndexPatterns) => { + if (this.abortControllerFetchFields) { + this.abortControllerFetchFields.abort(); + } + this.abortControllerFetchFields = new AbortController(); + + return fetchFields(extractedIndexPatterns, this.abortControllerFetchFields.signal); + }, + VIS_STATE_DEBOUNCE_DELAY, + { leading: true } + ); + handleChange = (partialModel) => { if (isEmpty(partialModel)) { return; @@ -94,7 +107,7 @@ export class VisEditor extends Component { const extractedIndexPatterns = extractIndexPatterns(nextModel); if (!isEqual(this.state.extractedIndexPatterns, extractedIndexPatterns)) { - fetchFields(extractedIndexPatterns).then((visFields) => + this.debouncedFetchFields(extractedIndexPatterns).then((visFields) => this.setState({ visFields, extractedIndexPatterns, diff --git a/src/plugins/vis_type_timeseries/public/application/lib/fetch_fields.js b/src/plugins/vis_type_timeseries/public/application/lib/fetch_fields.js index 6dd56a58418c5..a32ab71f36357 100644 --- a/src/plugins/vis_type_timeseries/public/application/lib/fetch_fields.js +++ b/src/plugins/vis_type_timeseries/public/application/lib/fetch_fields.js @@ -19,34 +19,40 @@ import { i18n } from '@kbn/i18n'; import { extractIndexPatterns } from '../../../common/extract_index_patterns'; import { getCoreStart } from '../../services'; +import { ROUTES } from '../../../common/constants'; -export async function fetchFields(indexPatterns = ['*']) { +export async function fetchFields(indexPatterns = [], signal) { const patterns = Array.isArray(indexPatterns) ? indexPatterns : [indexPatterns]; try { const indexFields = await Promise.all( - patterns.map((pattern) => { - return getCoreStart().http.get('/api/metrics/fields', { + patterns.map((pattern) => + getCoreStart().http.get(ROUTES.FIELDS, { query: { index: pattern, }, - }); - }) + signal, + }) + ) ); - const fields = patterns.reduce((cumulatedFields, currentPattern, index) => { - return { + + return patterns.reduce( + (cumulatedFields, currentPattern, index) => ({ ...cumulatedFields, [currentPattern]: indexFields[index], - }; - }, {}); - return fields; - } catch (error) { - getCoreStart().notifications.toasts.addDanger({ - title: i18n.translate('visTypeTimeseries.fetchFields.loadIndexPatternFieldsErrorMessage', { - defaultMessage: 'Unable to load index_pattern fields', }), - text: error.message, - }); + {} + ); + } catch (error) { + if (error.name !== 'AbortError') { + getCoreStart().notifications.toasts.addDanger({ + title: i18n.translate('visTypeTimeseries.fetchFields.loadIndexPatternFieldsErrorMessage', { + defaultMessage: 'Unable to load index_pattern fields', + }), + text: error.message, + }); + } } + return []; } export async function fetchIndexPatternFields({ params, fields = {} }) { diff --git a/src/plugins/vis_type_timeseries/server/routes/fields.ts b/src/plugins/vis_type_timeseries/server/routes/fields.ts index f9a600fa4b1f3..a9a890845d154 100644 --- a/src/plugins/vis_type_timeseries/server/routes/fields.ts +++ b/src/plugins/vis_type_timeseries/server/routes/fields.ts @@ -21,11 +21,12 @@ import { isBoom } from '@hapi/boom'; import { schema } from '@kbn/config-schema'; import { getFields } from '../lib/get_fields'; import { Framework } from '../plugin'; +import { ROUTES } from '../../common/constants'; export const fieldsRoutes = (framework: Framework) => { framework.router.get( { - path: '/api/metrics/fields', + path: ROUTES.FIELDS, validate: { query: schema.object({ index: schema.string() }), }, From 4c59629fdcb6177d5d3009badb73a80ecf2fcbac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Mon, 30 Nov 2020 12:29:47 +0100 Subject: [PATCH 30/37] [Elasticsearch Migration] Update docs re UsageCollection (#84322) --- ...migrating-legacy-plugins-examples.asciidoc | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc b/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc index 8a0e487971b20..a033bbd26a1a7 100644 --- a/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc +++ b/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc @@ -1143,6 +1143,77 @@ router.get( ); ---- +==== Accessing the client from a collector's `fetch` method + +At the moment, the `fetch` method's context receives preconfigured +<> for Elasticsearch and SavedObjects. +To help in the transition, both, the legacy (`callCluster`) and new clients are provided, +but we strongly discourage using the deprecated legacy ones for any new implementation. + +[source,typescript] +---- +usageCollection.makeUsageCollector({ + type: 'my-collector', + isReady: async () => true, // Logic to confirm the `fetch` method is ready to be called + schema: {...}, + async fetch(context) { + const { callCluster, esClient, soClient } = context; + + // Before: + const result = callCluster('search', options) + + // After: + const { body: result } = esClient.search(options); + + return result; + } +}); +---- + +Regarding the `soClient`, it is encouraged to use it instead of the plugin's owned SavedObject's repository +as we used to do in the past. + +Before: + +[source,typescript] +---- +function getUsageCollector( + usageCollection: UsageCollectionSetup, + getSavedObjectsRepository: () => ISavedObjectsRepository | undefined +) { + usageCollection.makeUsageCollector({ + type: 'my-collector', + isReady: () => typeof getSavedObjectsRepository() !== 'undefined', + schema: {...}, + async fetch() { + const savedObjectsRepository = getSavedObjectsRepository(); + + const { attributes: result } = await savedObjectsRepository.get('my-so-type', 'my-so-id'); + + return result; + } + }); +} +---- + +After: + +[source,typescript] +---- +function getUsageCollector(usageCollection: UsageCollectionSetup) { + usageCollection.makeUsageCollector({ + type: 'my-collector', + isReady: () => true, + schema: {...}, + async fetch({ soClient }) { + const { attributes: result } = await soClient.get('my-so-type', 'my-so-id'); + + return result; + } + }); +} +---- + ==== Creating a custom client Note that the `plugins` option is no longer available on the new From d11ca6c1dac64a778d9e6009c992a9c6bb113b75 Mon Sep 17 00:00:00 2001 From: Fabien Baligand Date: Mon, 30 Nov 2020 12:30:30 +0100 Subject: [PATCH 31/37] [Visualizations] Adds visConfig.title and uiState to build pipeline function (#84456) * add visConfig.title to pipeline visualization function - To be consistent with visualizations built with "buildPipelineVisFunction". - To provide "title" information for visualizations. * add uiState to pipeline visualization function Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- src/plugins/visualizations/public/legacy/build_pipeline.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plugins/visualizations/public/legacy/build_pipeline.ts b/src/plugins/visualizations/public/legacy/build_pipeline.ts index d412ec918a71a..0c244876ca6a3 100644 --- a/src/plugins/visualizations/public/legacy/build_pipeline.ts +++ b/src/plugins/visualizations/public/legacy/build_pipeline.ts @@ -301,8 +301,10 @@ export const buildPipeline = async (vis: Vis, params: BuildPipelineParams) => { } else { const visConfig = { ...vis.params }; visConfig.dimensions = schemas; + visConfig.title = title; pipeline += `visualization type='${vis.type.name}' ${prepareJson('visConfig', visConfig)} + ${prepareJson('uiState', uiState)} metricsAtAllLevels=${vis.isHierarchical()} partialRows=${vis.params.showPartialRows || false} `; if (indexPattern) { From cb55898c98445e08ddac594fd943d1de3cdf19f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Mon, 30 Nov 2020 12:30:47 +0100 Subject: [PATCH 32/37] [UsageCollection] Remove `formatBulkUpload` and other unused APIs (#84313) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...emetry_application_usage_collector.test.ts | 2 +- .../collectors/core/core_usage_collector.ts | 2 +- .../server/collectors/core/index.test.ts | 2 +- .../server/collectors/kibana/index.test.ts | 21 +----- .../kibana/kibana_usage_collector.ts | 17 +---- .../collectors/management/index.test.ts | 2 +- .../server/collectors/ops_stats/index.test.ts | 2 +- .../server/collectors/ui_metric/index.test.ts | 2 +- .../server/collector/collector.test.ts | 44 ------------- .../server/collector/collector.ts | 59 +++-------------- .../server/collector/collector_set.test.ts | 64 +++---------------- .../server/collector/collector_set.ts | 57 +++++++---------- .../server/collector/index.ts | 2 +- .../server/collector/usage_collector.ts | 33 ++-------- src/plugins/usage_collection/server/plugin.ts | 4 +- .../server/usage_collection.mock.ts | 7 -- .../__tests__/bulk_uploader.js | 11 ---- .../collectors/get_settings_collector.ts | 1 - .../collectors/get_usage_collector.ts | 2 +- .../telemetry_collection/get_kibana_stats.ts | 1 - ...egister_monitoring_telemetry_collection.ts | 1 - .../usage/reporting_usage_collector.test.ts | 1 - .../server/usage/reporting_usage_collector.ts | 29 +-------- .../spaces_usage_collector.ts | 24 +------ 24 files changed, 64 insertions(+), 326 deletions(-) diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts index 6cb104416ef58..47dc26e0ab3d8 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts @@ -40,7 +40,7 @@ describe('telemetry_application_usage', () => { const logger = loggingSystemMock.createLogger(); - let collector: Collector; + let collector: Collector; const usageCollectionMock = createUsageCollectionSetupMock(); usageCollectionMock.makeUsageCollector.mockImplementation((config) => { diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts index 297baf016e9e6..3fd011b0bded2 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts @@ -24,7 +24,7 @@ export function getCoreUsageCollector( usageCollection: UsageCollectionSetup, getCoreUsageDataService: () => CoreUsageDataStart ) { - return usageCollection.makeUsageCollector({ + return usageCollection.makeUsageCollector({ type: 'core', isReady: () => typeof getCoreUsageDataService() !== 'undefined', schema: { diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts index e31437a744e29..335d73fb60847 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts @@ -29,7 +29,7 @@ import { CoreUsageData } from 'src/core/server/'; const logger = loggingSystemMock.createLogger(); describe('telemetry_core', () => { - let collector: Collector; + let collector: Collector; const usageCollectionMock = createUsageCollectionSetupMock(); usageCollectionMock.makeUsageCollector.mockImplementation((config) => { diff --git a/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts index 16a60eca60f47..83cac1d456a3a 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts @@ -31,7 +31,7 @@ import { registerKibanaUsageCollector } from './'; const logger = loggingSystemMock.createLogger(); describe('telemetry_kibana', () => { - let collector: Collector; + let collector: Collector; const usageCollectionMock = createUsageCollectionSetupMock(); usageCollectionMock.makeUsageCollector.mockImplementation((config) => { @@ -66,23 +66,4 @@ describe('telemetry_kibana', () => { timelion_sheet: { total: 0 }, }); }); - - test('formatForBulkUpload', async () => { - const resultFromFetch = { - index: '.kibana-tests', - dashboard: { total: 0 }, - visualization: { total: 0 }, - search: { total: 0 }, - index_pattern: { total: 0 }, - graph_workspace: { total: 0 }, - timelion_sheet: { total: 0 }, - }; - - expect(collector.formatForBulkUpload!(resultFromFetch)).toStrictEqual({ - type: 'kibana_stats', - payload: { - usage: resultFromFetch, - }, - }); - }); }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/kibana/kibana_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/kibana/kibana_usage_collector.ts index d292b2d5ace0e..6c2e0a2c926ad 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/kibana/kibana_usage_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/kibana/kibana_usage_collector.ts @@ -21,7 +21,6 @@ import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; import { SharedGlobalConfig } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { KIBANA_STATS_TYPE } from '../../../common/constants'; import { getSavedObjectsCounts, KibanaSavedObjectCounts } from './get_saved_object_counts'; interface KibanaUsage extends KibanaSavedObjectCounts { @@ -32,7 +31,7 @@ export function getKibanaUsageCollector( usageCollection: UsageCollectionSetup, legacyConfig$: Observable ) { - return usageCollection.makeUsageCollector({ + return usageCollection.makeUsageCollector({ type: 'kibana', isReady: () => true, schema: { @@ -53,20 +52,6 @@ export function getKibanaUsageCollector( ...(await getSavedObjectsCounts(callCluster, index)), }; }, - - /* - * Format the response data into a model for internal upload - * 1. Make this data part of the "kibana_stats" type - * 2. Organize the payload in the usage namespace of the data payload (usage.index, etc) - */ - formatForBulkUpload: (result) => { - return { - type: KIBANA_STATS_TYPE, - payload: { - usage: result, - }, - }; - }, }); } diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/management/index.test.ts index 0aafee01cf49d..bd63c5b40735a 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/index.test.ts @@ -29,7 +29,7 @@ import { registerManagementUsageCollector } from './'; const logger = loggingSystemMock.createLogger(); describe('telemetry_application_usage_collector', () => { - let collector: Collector; + let collector: Collector; const usageCollectionMock = createUsageCollectionSetupMock(); usageCollectionMock.makeUsageCollector.mockImplementation((config) => { diff --git a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts index 8db7010e64026..1aa60e2c8e961 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts @@ -31,7 +31,7 @@ import { loggingSystemMock } from '../../../../../core/server/mocks'; const logger = loggingSystemMock.createLogger(); describe('telemetry_ops_stats', () => { - let collector: Collector; + let collector: Collector; const usageCollectionMock = createUsageCollectionSetupMock(); usageCollectionMock.makeStatsCollector.mockImplementation((config) => { diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts index 90e3b7110e1dc..d852e505027fa 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts @@ -29,7 +29,7 @@ import { registerUiMetricUsageCollector } from './'; const logger = loggingSystemMock.createLogger(); describe('telemetry_ui_metric', () => { - let collector: Collector; + let collector: Collector; const usageCollectionMock = createUsageCollectionSetupMock(); usageCollectionMock.makeUsageCollector.mockImplementation((config) => { diff --git a/src/plugins/usage_collection/server/collector/collector.test.ts b/src/plugins/usage_collection/server/collector/collector.test.ts index 875414fbeec48..fe52b9b2ca03e 100644 --- a/src/plugins/usage_collection/server/collector/collector.test.ts +++ b/src/plugins/usage_collection/server/collector/collector.test.ts @@ -19,7 +19,6 @@ import { loggingSystemMock } from '../../../../core/server/mocks'; import { Collector } from './collector'; -import { UsageCollector } from './usage_collector'; const logger = loggingSystemMock.createLogger(); @@ -88,49 +87,6 @@ describe('collector', () => { }); }); - describe('formatForBulkUpload', () => { - it('should use the default formatter', () => { - const fetchOutput = { testPass: 100 }; - const collector = new Collector(logger, { - type: 'my_test_collector', - isReady: () => false, - fetch: () => fetchOutput, - }); - expect(collector.formatForBulkUpload(fetchOutput)).toStrictEqual({ - type: 'my_test_collector', - payload: fetchOutput, - }); - }); - - it('should use a custom formatter', () => { - const fetchOutput = { testPass: 100 }; - const collector = new Collector(logger, { - type: 'my_test_collector', - isReady: () => false, - fetch: () => fetchOutput, - formatForBulkUpload: (a) => ({ type: 'other_value', payload: { nested: a } }), - }); - expect(collector.formatForBulkUpload(fetchOutput)).toStrictEqual({ - type: 'other_value', - payload: { nested: fetchOutput }, - }); - }); - - it("should use UsageCollector's default formatter", () => { - const fetchOutput = { testPass: 100 }; - const collector = new UsageCollector(logger, { - type: 'my_test_collector', - isReady: () => false, - fetch: () => fetchOutput, - schema: { testPass: { type: 'long' } }, - }); - expect(collector.formatForBulkUpload(fetchOutput)).toStrictEqual({ - type: 'kibana_stats', - payload: { usage: { my_test_collector: fetchOutput } }, - }); - }); - }); - describe('schema TS validations', () => { // These tests below are used to ensure types inference is working as expected. // We don't intend to test any logic as such, just the relation between the types in `fetch` and `schema`. diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index 797fdaa06a620..8e86bc3d1cd26 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -26,8 +26,6 @@ import { KibanaRequest, } from 'src/core/server'; -export type CollectorFormatForBulkUpload = (result: T) => { type: string; payload: U }; - export type AllowedSchemaNumberTypes = 'long' | 'integer' | 'short' | 'byte' | 'double' | 'float'; export type AllowedSchemaTypes = AllowedSchemaNumberTypes | 'keyword' | 'text' | 'boolean' | 'date'; @@ -87,7 +85,7 @@ export type CollectorFetchMethod< TReturn, ExtraOptions extends object = {} > = ( - this: Collector & ExtraOptions, // Specify the context of `this` for this.log and others to become available + this: Collector & ExtraOptions, // Specify the context of `this` for this.log and others to become available context: CollectorFetchContext ) => Promise | TReturn; @@ -108,7 +106,6 @@ export type CollectorOptionsFetchExtendedContext< export type CollectorOptions< TFetchReturn = unknown, - UFormatBulkUploadPayload = TFetchReturn, // TODO: Once we remove bulk_uploader's dependency on usageCollection, we'll be able to remove this type WithKibanaRequest extends boolean = boolean, ExtraOptions extends object = {} > = { @@ -130,13 +127,6 @@ export type CollectorOptions< * @param collectorFetchContext {@link CollectorFetchContext} */ fetch: CollectorFetchMethod; - /** - * A hook for allowing the fetched data payload to be organized into a typed - * data model for internal bulk upload. See defaultFormatterForBulkUpload for - * a generic example. - * @deprecated Used only by the Legacy Monitoring collection (to be removed in 8.0) - */ - formatForBulkUpload?: CollectorFormatForBulkUpload; } & ExtraOptions & (WithKibanaRequest extends true // If enforced to true via Types, the config must be enforced ? { @@ -146,28 +136,16 @@ export type CollectorOptions< extendFetchContext?: CollectorOptionsFetchExtendedContext; }); -export class Collector< - TFetchReturn, - UFormatBulkUploadPayload = TFetchReturn, - ExtraOptions extends object = {} -> { +export class Collector { public readonly extendFetchContext: CollectorOptionsFetchExtendedContext; - public readonly type: CollectorOptions['type']; - public readonly init?: CollectorOptions['init']; + public readonly type: CollectorOptions['type']; + public readonly init?: CollectorOptions['init']; public readonly fetch: CollectorFetchMethod; - public readonly isReady: CollectorOptions['isReady']; - private readonly _formatForBulkUpload?: CollectorFormatForBulkUpload< - TFetchReturn, - UFormatBulkUploadPayload - >; - /* - * @param {Object} logger - logger object - * @param {String} options.type - property name as the key for the data - * @param {Function} options.init (optional) - initialization function - * @param {Function} options.fetch - function to query data - * @param {Function} options.formatForBulkUpload - optional - * @param {Function} options.isReady - method that returns a boolean or Promise of a boolean to indicate the collector is ready to report data - * @param {Function} options.rest - optional other properties + public readonly isReady: CollectorOptions['isReady']; + /** + * @private Constructor of a Collector. It should be called via the CollectorSet factory methods: `makeStatsCollector` and `makeUsageCollector` + * @param log {@link Logger} + * @param collectorDefinition {@link CollectorOptions} */ constructor( public readonly log: Logger, @@ -175,11 +153,10 @@ export class Collector< type, init, fetch, - formatForBulkUpload, isReady, extendFetchContext = {}, ...options - }: CollectorOptions + }: CollectorOptions ) { if (type === undefined) { throw new Error('Collector must be instantiated with a options.type string property'); @@ -200,21 +177,5 @@ export class Collector< this.fetch = fetch; this.isReady = typeof isReady === 'function' ? isReady : () => true; this.extendFetchContext = extendFetchContext; - this._formatForBulkUpload = formatForBulkUpload; - } - - public formatForBulkUpload(result: TFetchReturn) { - if (this._formatForBulkUpload) { - return this._formatForBulkUpload(result); - } else { - return this.defaultFormatterForBulkUpload(result); - } - } - - protected defaultFormatterForBulkUpload(result: TFetchReturn) { - return { - type: this.type, - payload: (result as unknown) as UFormatBulkUploadPayload, - }; } } diff --git a/src/plugins/usage_collection/server/collector/collector_set.test.ts b/src/plugins/usage_collection/server/collector/collector_set.test.ts index fc17ce131430c..90a69043e0635 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.test.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.test.ts @@ -20,7 +20,7 @@ import { noop } from 'lodash'; import { Collector } from './collector'; import { CollectorSet } from './collector_set'; -import { UsageCollector, UsageCollectorOptions } from './usage_collector'; +import { UsageCollector } from './usage_collector'; import { elasticsearchServiceMock, loggingSystemMock, @@ -163,29 +163,6 @@ describe('CollectorSet', () => { }, ]); }); - - it('should infer the types from the implementations of fetch and formatForBulkUpload', async () => { - const collectors = new CollectorSet({ logger }); - collectors.registerCollector( - new Collector(logger, { - type: 'MY_TEST_COLLECTOR', - fetch: () => ({ test: 1 }), - formatForBulkUpload: (result) => ({ - type: 'MY_TEST_COLLECTOR', - payload: { test: result.test * 2 }, - }), - isReady: () => true, - }) - ); - - const result = await collectors.bulkFetch(mockCallCluster, mockEsClient, mockSoClient, req); - expect(result).toStrictEqual([ - { - type: 'MY_TEST_COLLECTOR', - result: { test: 1 }, // It matches the return of `fetch`. `formatForBulkUpload` is used later on - }, - ]); - }); }); describe('toApiFieldNames', () => { @@ -253,29 +230,6 @@ describe('CollectorSet', () => { }); }); - describe('isUsageCollector', () => { - const collectorOptions: UsageCollectorOptions = { - type: 'MY_TEST_COLLECTOR', - fetch: () => ({ test: 1 }), - isReady: () => true, - schema: { test: { type: 'long' } }, - }; - - it('returns true only for UsageCollector instances', () => { - const collectors = new CollectorSet({ logger }); - const usageCollector = new UsageCollector(logger, collectorOptions); - const collector = new Collector(logger, collectorOptions); - const randomClass = new (class Random {})(); - expect(collectors.isUsageCollector(usageCollector)).toEqual(true); - expect(collectors.isUsageCollector(collector)).toEqual(false); - expect(collectors.isUsageCollector(randomClass)).toEqual(false); - expect(collectors.isUsageCollector({})).toEqual(false); - expect(collectors.isUsageCollector(null)).toEqual(false); - expect(collectors.isUsageCollector('')).toEqual(false); - expect(collectors.isUsageCollector(void 0)).toEqual(false); - }); - }); - describe('makeStatsCollector', () => { const collectorSet = new CollectorSet({ logger }); test('TS should hide kibanaRequest when not opted-in', () => { @@ -402,7 +356,7 @@ describe('CollectorSet', () => { return { test: kibanaRequest ? 1 : 0 }; }, }); - collectorSet.makeUsageCollector<{ test: number }, unknown, false>({ + collectorSet.makeUsageCollector<{ test: number }, false>({ type: 'MY_TEST_COLLECTOR', isReady: () => true, schema: { test: { type: 'long' } }, @@ -415,7 +369,7 @@ describe('CollectorSet', () => { kibanaRequest: false, }, }); - collectorSet.makeUsageCollector<{ test: number }, unknown, false>({ + collectorSet.makeUsageCollector<{ test: number }, false>({ type: 'MY_TEST_COLLECTOR', isReady: () => true, schema: { test: { type: 'long' } }, @@ -442,7 +396,7 @@ describe('CollectorSet', () => { kibanaRequest: true, }, }); - collectorSet.makeUsageCollector<{ test: number }, unknown, false>({ + collectorSet.makeUsageCollector<{ test: number }, false>({ type: 'MY_TEST_COLLECTOR', isReady: () => true, schema: { test: { type: 'long' } }, @@ -460,7 +414,7 @@ describe('CollectorSet', () => { test('TS should allow `true` when types explicitly declare `true` and do not allow `false` or undefined', () => { // false is the default when at least 1 type is specified - collectorSet.makeUsageCollector<{ test: number }, unknown, true>({ + collectorSet.makeUsageCollector<{ test: number }, true>({ type: 'MY_TEST_COLLECTOR', isReady: () => true, schema: { test: { type: 'long' } }, @@ -472,7 +426,7 @@ describe('CollectorSet', () => { kibanaRequest: true, }, }); - collectorSet.makeUsageCollector<{ test: number }, unknown, true>({ + collectorSet.makeUsageCollector<{ test: number }, true>({ type: 'MY_TEST_COLLECTOR', isReady: () => true, schema: { test: { type: 'long' } }, @@ -485,7 +439,7 @@ describe('CollectorSet', () => { kibanaRequest: false, }, }); - collectorSet.makeUsageCollector<{ test: number }, unknown, true>({ + collectorSet.makeUsageCollector<{ test: number }, true>({ type: 'MY_TEST_COLLECTOR', isReady: () => true, schema: { test: { type: 'long' } }, @@ -498,7 +452,7 @@ describe('CollectorSet', () => { kibanaRequest: undefined, }, }); - collectorSet.makeUsageCollector<{ test: number }, unknown, true>({ + collectorSet.makeUsageCollector<{ test: number }, true>({ type: 'MY_TEST_COLLECTOR', isReady: () => true, schema: { test: { type: 'long' } }, @@ -509,7 +463,7 @@ describe('CollectorSet', () => { // @ts-expect-error extendFetchContext: {}, }); - collectorSet.makeUsageCollector<{ test: number }, unknown, true>( + collectorSet.makeUsageCollector<{ test: number }, true>( // @ts-expect-error { type: 'MY_TEST_COLLECTOR', diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index cda4ce36d4e23..3555b05518fdb 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -29,8 +29,7 @@ import { import { Collector, CollectorOptions } from './collector'; import { UsageCollector, UsageCollectorOptions } from './usage_collector'; -type AnyCollector = Collector; -type AnyUsageCollector = UsageCollector; +type AnyCollector = Collector; interface CollectorSetConfig { logger: Logger; @@ -38,6 +37,22 @@ interface CollectorSetConfig { collectors?: AnyCollector[]; } +/** + * Public interface of the CollectorSet (makes it easier to mock only the public methods) + */ +export type CollectorSetPublic = Pick< + CollectorSet, + | 'makeStatsCollector' + | 'makeUsageCollector' + | 'registerCollector' + | 'getCollectorByType' + | 'areAllCollectorsReady' + | 'bulkFetch' + | 'bulkFetchUsage' + | 'toObject' + | 'toApiFieldNames' +>; + export class CollectorSet { private _waitingForAllCollectorsTimestamp?: number; private readonly logger: Logger; @@ -55,13 +70,12 @@ export class CollectorSet { */ public makeStatsCollector = < TFetchReturn, - TFormatForBulkUpload, WithKibanaRequest extends boolean, ExtraOptions extends object = {} >( - options: CollectorOptions + options: CollectorOptions ) => { - return new Collector(this.logger, options); + return new Collector(this.logger, options); }; /** @@ -70,32 +84,23 @@ export class CollectorSet { */ public makeUsageCollector = < TFetchReturn, - TFormatForBulkUpload = { usage: { [key: string]: TFetchReturn } }, // TODO: Right now, users will need to explicitly claim `true` for TS to allow `kibanaRequest` usage. // If we improve `telemetry-check-tools` so plugins do not need to specify TFetchReturn, // we'll be able to remove the type defaults and TS will successfully infer the config value as provided in JS. WithKibanaRequest extends boolean = false, ExtraOptions extends object = {} >( - options: UsageCollectorOptions< - TFetchReturn, - TFormatForBulkUpload, - WithKibanaRequest, - ExtraOptions - > + options: UsageCollectorOptions ) => { - return new UsageCollector( - this.logger, - options - ); + return new UsageCollector(this.logger, options); }; /** * Registers a collector to be used when collecting all the usage and stats data * @param collector Collector to be added to the set (previously created via `makeUsageCollector` or `makeStatsCollector`) */ - public registerCollector = ( - collector: Collector + public registerCollector = ( + collector: Collector ) => { // check instanceof if (!(collector instanceof Collector)) { @@ -118,10 +123,6 @@ export class CollectorSet { return [...this.collectors.values()].find((c) => c.type === type); }; - public isUsageCollector = (x: AnyUsageCollector | any): x is AnyUsageCollector => { - return x instanceof UsageCollector; - }; - public areAllCollectorsReady = async (collectorSet: CollectorSet = this) => { if (!(collectorSet instanceof CollectorSet)) { throw new Error( @@ -205,7 +206,7 @@ export class CollectorSet { /* * @return {new CollectorSet} */ - public getFilteredCollectorSet = (filter: (col: AnyCollector) => boolean) => { + private getFilteredCollectorSet = (filter: (col: AnyCollector) => boolean) => { const filtered = [...this.collectors.values()].filter(filter); return this.makeCollectorSetFromArray(filtered); }; @@ -267,16 +268,6 @@ export class CollectorSet { }, {}); }; - // TODO: remove - public map = (mapFn: any) => { - return [...this.collectors.values()].map(mapFn); - }; - - // TODO: remove - public some = (someFn: any) => { - return [...this.collectors.values()].some(someFn); - }; - private makeCollectorSetFromArray = (collectors: AnyCollector[]) => { return new CollectorSet({ logger: this.logger, diff --git a/src/plugins/usage_collection/server/collector/index.ts b/src/plugins/usage_collection/server/collector/index.ts index 2f8be884a8a7b..1bbb995154690 100644 --- a/src/plugins/usage_collection/server/collector/index.ts +++ b/src/plugins/usage_collection/server/collector/index.ts @@ -17,7 +17,7 @@ * under the License. */ -export { CollectorSet } from './collector_set'; +export { CollectorSet, CollectorSetPublic } from './collector_set'; export { Collector, AllowedSchemaTypes, diff --git a/src/plugins/usage_collection/server/collector/usage_collector.ts b/src/plugins/usage_collection/server/collector/usage_collector.ts index a042ea113d5cc..57f6d109d6d20 100644 --- a/src/plugins/usage_collection/server/collector/usage_collector.ts +++ b/src/plugins/usage_collection/server/collector/usage_collector.ts @@ -18,43 +18,24 @@ */ import { Logger } from 'src/core/server'; -import { KIBANA_STATS_TYPE } from '../../common/constants'; import { Collector, CollectorOptions } from './collector'; // Enforce the `schema` property for UsageCollectors export type UsageCollectorOptions< TFetchReturn = unknown, - UFormatBulkUploadPayload = { usage: { [key: string]: TFetchReturn } }, WithKibanaRequest extends boolean = false, ExtraOptions extends object = {} -> = CollectorOptions & - Required, 'schema'>>; +> = CollectorOptions & + Required, 'schema'>>; -export class UsageCollector< +export class UsageCollector extends Collector< TFetchReturn, - UFormatBulkUploadPayload = { usage: { [key: string]: TFetchReturn } }, - ExtraOptions extends object = {} -> extends Collector { + ExtraOptions +> { constructor( - public readonly log: Logger, - collectorOptions: UsageCollectorOptions< - TFetchReturn, - UFormatBulkUploadPayload, - any, - ExtraOptions - > + log: Logger, + collectorOptions: UsageCollectorOptions ) { super(log, collectorOptions); } - - protected defaultFormatterForBulkUpload(result: TFetchReturn) { - return { - type: KIBANA_STATS_TYPE, - payload: ({ - usage: { - [this.type]: result, - }, - } as unknown) as UFormatBulkUploadPayload, - }; - } } diff --git a/src/plugins/usage_collection/server/plugin.ts b/src/plugins/usage_collection/server/plugin.ts index 9a8876446d01e..31e4f0ea253ea 100644 --- a/src/plugins/usage_collection/server/plugin.ts +++ b/src/plugins/usage_collection/server/plugin.ts @@ -27,10 +27,10 @@ import { Plugin, } from 'src/core/server'; import { ConfigType } from './config'; -import { CollectorSet } from './collector'; +import { CollectorSet, CollectorSetPublic } from './collector'; import { setupRoutes } from './routes'; -export type UsageCollectionSetup = CollectorSet; +export type UsageCollectionSetup = CollectorSetPublic; export class UsageCollectionPlugin implements Plugin { private readonly logger: Logger; private savedObjects?: ISavedObjectsRepository; diff --git a/src/plugins/usage_collection/server/usage_collection.mock.ts b/src/plugins/usage_collection/server/usage_collection.mock.ts index 05dae8fa85164..fb0a2e56ff3c9 100644 --- a/src/plugins/usage_collection/server/usage_collection.mock.ts +++ b/src/plugins/usage_collection/server/usage_collection.mock.ts @@ -37,13 +37,6 @@ export const createUsageCollectionSetupMock = () => { bulkFetch: jest.fn(), bulkFetchUsage: jest.fn(), getCollectorByType: jest.fn(), - getFilteredCollectorSet: jest.fn(), - // @ts-ignore jest.fn doesn't play nice with type guards - isUsageCollector: jest.fn(), - makeCollectorSetFromArray: jest.fn(), - map: jest.fn(), - maximumWaitTimeForAllCollectorsInS: 0, - some: jest.fn(), toApiFieldNames: jest.fn(), toObject: jest.fn(), makeStatsCollector: jest.fn().mockImplementation((cfg) => new Collector(logger, cfg)), diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js index da12bde966091..1aa9e49bc5f35 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js @@ -71,7 +71,6 @@ describe('BulkUploader', () => { type: 'type_collector_test', fetch: noop, // empty payloads, isReady: () => true, - formatForBulkUpload: (result) => result, }, ]); @@ -111,13 +110,11 @@ describe('BulkUploader', () => { type: 'type_collector_test', fetch: noop, // empty payloads, isReady: () => false, - formatForBulkUpload: (result) => result, }, { type: 'type_collector_test2', fetch: noop, // empty payloads, isReady: () => true, - formatForBulkUpload: (result) => result, }, ]); @@ -153,7 +150,6 @@ describe('BulkUploader', () => { { fetch: () => ({ type: 'type_collector_test', result: { testData: 12345 } }), isReady: () => true, - formatForBulkUpload: (result) => result, }, ]); const uploader = new BulkUploader({ ...server, interval: FETCH_INTERVAL }); @@ -191,13 +187,11 @@ describe('BulkUploader', () => { { fetch: usageCollectorFetch, isReady: () => true, - formatForBulkUpload: (result) => result, isUsageCollector: true, }, { fetch: collectorFetch, isReady: () => true, - formatForBulkUpload: (result) => result, isUsageCollector: false, }, ]); @@ -223,7 +217,6 @@ describe('BulkUploader', () => { { fetch: usageCollectorFetch, isReady: () => true, - formatForBulkUpload: (result) => result, isUsageCollector: true, }, ]); @@ -253,13 +246,11 @@ describe('BulkUploader', () => { { fetch: statsCollectorFetch, isReady: () => true, - formatForBulkUpload: (result) => result, isUsageCollector: false, }, { fetch: usageCollectorFetch, isReady: () => true, - formatForBulkUpload: (result) => result, isUsageCollector: true, }, ]); @@ -290,13 +281,11 @@ describe('BulkUploader', () => { { fetch: usageCollectorFetch, isReady: () => true, - formatForBulkUpload: (result) => result, isUsageCollector: true, }, { fetch: collectorFetch, isReady: () => true, - formatForBulkUpload: (result) => result, isUsageCollector: false, }, ]); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts index 858c50790fc2e..44ae0147f6c52 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts @@ -89,7 +89,6 @@ export function getSettingsCollector( ) { return usageCollection.makeStatsCollector< EmailSettingData | undefined, - unknown, false, KibanaSettingsCollectorExtraOptions >({ diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts index 038042f109817..e5171c4901dd4 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_usage_collector.ts @@ -20,7 +20,7 @@ export function getMonitoringUsageCollector( config: MonitoringConfig, legacyEsClient: ILegacyClusterClient ) { - return usageCollection.makeUsageCollector({ + return usageCollection.makeUsageCollector({ type: 'monitoring', isReady: () => true, schema: { diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts index 3bf1d087b973a..57e75d9c9c12e 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts @@ -114,7 +114,6 @@ export function getUsageStats(rawStats: SearchResponse) { } = currUsage; // Stats filtered by telemetry collectors need to be flattened since they're pulled in a generic way. - // A plugin might not provide flat stats if it implements formatForBulkUpload in its collector. // e.g: we want `xpack.reporting` to just be `reporting` const plugins = { ...pluginsTop, ...xpack }; diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_telemetry_collection.ts b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_telemetry_collection.ts index 91d6c2374acba..8351c30a4f712 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_telemetry_collection.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_telemetry_collection.ts @@ -23,7 +23,6 @@ export function registerMonitoringTelemetryCollection( ) { const monitoringStatsCollector = usageCollection.makeStatsCollector< UsageStatsPayload[], - unknown, true, MonitoringCollectorOptions >({ diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts index fff18353c58b0..7cae5e9b6f956 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts @@ -463,7 +463,6 @@ describe('Ready for collection observable', () => { expect(args).toMatchInlineSnapshot(` Object { "fetch": [Function], - "formatForBulkUpload": [Function], "isReady": [Function], "schema": Object { "PNG": Object { diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts index 2ef7a7995b839..f4209730b68ce 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts @@ -14,16 +14,6 @@ import { getReportingUsage } from './get_reporting_usage'; import { ReportingUsageType } from './types'; import { reportingSchema } from './schema'; -// places the reporting data as kibana stats -const METATYPE = 'kibana_stats'; - -interface XpackBulkUpload { - usage: { - xpack: { - reporting: ReportingUsageType; - }; - }; -} /* * @return {Object} kibana usage stats type collection object */ @@ -34,7 +24,7 @@ export function getReportingUsageCollector( exportTypesRegistry: ExportTypesRegistry, isReady: () => Promise ) { - return usageCollection.makeUsageCollector({ + return usageCollection.makeUsageCollector({ type: 'reporting', fetch: ({ callCluster }: CollectorFetchContext) => { const config = reporting.getConfig(); @@ -42,23 +32,6 @@ export function getReportingUsageCollector( }, isReady, schema: reportingSchema, - /* - * Format the response data into a model for internal upload - * 1. Make this data part of the "kibana_stats" type - * 2. Organize the payload in the usage.xpack.reporting namespace of the data payload - */ - formatForBulkUpload: (result: ReportingUsageType) => { - return { - type: METATYPE, - payload: { - usage: { - xpack: { - reporting: result, - }, - }, - }, - }; - }, }); } diff --git a/x-pack/plugins/spaces/server/usage_collection/spaces_usage_collector.ts b/x-pack/plugins/spaces/server/usage_collection/spaces_usage_collector.ts index 0e31c930a926b..d563a4a9b100d 100644 --- a/x-pack/plugins/spaces/server/usage_collection/spaces_usage_collector.ts +++ b/x-pack/plugins/spaces/server/usage_collection/spaces_usage_collector.ts @@ -8,7 +8,6 @@ import { LegacyCallAPIOptions } from 'src/core/server'; import { take } from 'rxjs/operators'; import { CollectorFetchContext, UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { Observable } from 'rxjs'; -import { KIBANA_STATS_TYPE_MONITORING } from '../../../monitoring/common/constants'; import { PluginsSetup } from '../plugin'; type CallCluster = ( @@ -146,11 +145,6 @@ interface CollectorDeps { licensing: PluginsSetup['licensing']; } -interface BulkUpload { - usage: { - spaces: UsageStats; - }; -} /* * @param {Object} server * @return {Object} kibana usage stats type collection object @@ -159,7 +153,7 @@ export function getSpacesUsageCollector( usageCollection: UsageCollectionSetup, deps: CollectorDeps ) { - return usageCollection.makeUsageCollector({ + return usageCollection.makeUsageCollector({ type: 'spaces', isReady: () => true, schema: { @@ -202,22 +196,6 @@ export function getSpacesUsageCollector( ...usageStats, } as UsageStats; }, - - /* - * Format the response data into a model for internal upload - * 1. Make this data part of the "kibana_stats" type - * 2. Organize the payload in the usage.xpack.spaces namespace of the data payload - */ - formatForBulkUpload: (result: UsageStats) => { - return { - type: KIBANA_STATS_TYPE_MONITORING, - payload: { - usage: { - spaces: result, - }, - }, - }; - }, }); } From 090359ca5a1c99911cbe3ea74efb9402c8413296 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 30 Nov 2020 12:24:03 +0000 Subject: [PATCH 33/37] [ML] Renaming saved object repair to sync (#84311) * [ML] Remaiming saved object repair to sync * removing 0 from sync lists Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../plugins/ml/common/types/saved_objects.ts | 2 +- .../index.ts | 2 +- .../job_spaces_sync_flyout.tsx} | 72 +++++++++---------- .../sync_list.tsx} | 70 +++++++++--------- .../jobs_list_page/jobs_list_page.tsx | 16 ++--- .../services/ml_api_service/saved_objects.ts | 8 +-- x-pack/plugins/ml/server/routes/apidoc.json | 2 +- .../plugins/ml/server/routes/saved_objects.ts | 20 +++--- .../ml/server/routes/schemas/saved_objects.ts | 2 +- .../plugins/ml/server/saved_objects/index.ts | 2 +- .../initialization/initialization.ts | 4 +- .../space_overrides/space_overrides.ts | 2 +- .../saved_objects/{repair.ts => sync.ts} | 10 +-- 13 files changed, 101 insertions(+), 111 deletions(-) rename x-pack/plugins/ml/public/application/components/{job_spaces_repair => job_spaces_sync}/index.ts (78%) rename x-pack/plugins/ml/public/application/components/{job_spaces_repair/job_spaces_repair_flyout.tsx => job_spaces_sync/job_spaces_sync_flyout.tsx} (59%) rename x-pack/plugins/ml/public/application/components/{job_spaces_repair/repair_list.tsx => job_spaces_sync/sync_list.tsx} (60%) rename x-pack/plugins/ml/server/saved_objects/{repair.ts => sync.ts} (97%) diff --git a/x-pack/plugins/ml/common/types/saved_objects.ts b/x-pack/plugins/ml/common/types/saved_objects.ts index d6c9ad758e8c6..aa3220747e849 100644 --- a/x-pack/plugins/ml/common/types/saved_objects.ts +++ b/x-pack/plugins/ml/common/types/saved_objects.ts @@ -11,7 +11,7 @@ export interface SavedObjectResult { [jobId: string]: { success: boolean; error?: any }; } -export interface RepairSavedObjectResponse { +export interface SyncSavedObjectResponse { savedObjectsCreated: SavedObjectResult; savedObjectsDeleted: SavedObjectResult; datafeedsAdded: SavedObjectResult; diff --git a/x-pack/plugins/ml/public/application/components/job_spaces_repair/index.ts b/x-pack/plugins/ml/public/application/components/job_spaces_sync/index.ts similarity index 78% rename from x-pack/plugins/ml/public/application/components/job_spaces_repair/index.ts rename to x-pack/plugins/ml/public/application/components/job_spaces_sync/index.ts index 3a9c22c1f3688..314071d5be1d8 100644 --- a/x-pack/plugins/ml/public/application/components/job_spaces_repair/index.ts +++ b/x-pack/plugins/ml/public/application/components/job_spaces_sync/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { JobSpacesRepairFlyout } from './job_spaces_repair_flyout'; +export { JobSpacesSyncFlyout } from './job_spaces_sync_flyout'; diff --git a/x-pack/plugins/ml/public/application/components/job_spaces_repair/job_spaces_repair_flyout.tsx b/x-pack/plugins/ml/public/application/components/job_spaces_sync/job_spaces_sync_flyout.tsx similarity index 59% rename from x-pack/plugins/ml/public/application/components/job_spaces_repair/job_spaces_repair_flyout.tsx rename to x-pack/plugins/ml/public/application/components/job_spaces_sync/job_spaces_sync_flyout.tsx index 47d3fe065dd66..db80d0aa330f8 100644 --- a/x-pack/plugins/ml/public/application/components/job_spaces_repair/job_spaces_repair_flyout.tsx +++ b/x-pack/plugins/ml/public/application/components/job_spaces_sync/job_spaces_sync_flyout.tsx @@ -23,34 +23,31 @@ import { } from '@elastic/eui'; import { ml } from '../../services/ml_api_service'; -import { - RepairSavedObjectResponse, - SavedObjectResult, -} from '../../../../common/types/saved_objects'; -import { RepairList } from './repair_list'; +import { SyncSavedObjectResponse, SavedObjectResult } from '../../../../common/types/saved_objects'; +import { SyncList } from './sync_list'; import { useToastNotificationService } from '../../services/toast_notification_service'; interface Props { onClose: () => void; } -export const JobSpacesRepairFlyout: FC = ({ onClose }) => { +export const JobSpacesSyncFlyout: FC = ({ onClose }) => { const { displayErrorToast, displaySuccessToast } = useToastNotificationService(); const [loading, setLoading] = useState(false); - const [repairable, setRepairable] = useState(false); - const [repairResp, setRepairResp] = useState(null); + const [canSync, setCanSync] = useState(false); + const [syncResp, setSyncResp] = useState(null); - async function loadRepairList(simulate: boolean = true) { + async function loadSyncList(simulate: boolean = true) { setLoading(true); try { - const resp = await ml.savedObjects.repairSavedObjects(simulate); - setRepairResp(resp); + const resp = await ml.savedObjects.syncSavedObjects(simulate); + setSyncResp(resp); const count = Object.values(resp).reduce((acc, cur) => acc + Object.keys(cur).length, 0); - setRepairable(count > 0); + setCanSync(count > 0); setLoading(false); return resp; } catch (error) { - // this shouldn't be hit as errors are returned per-repair task + // this shouldn't be hit as errors are returned per-sync task // as part of the response displayErrorToast(error); setLoading(false); @@ -59,32 +56,33 @@ export const JobSpacesRepairFlyout: FC = ({ onClose }) => { } useEffect(() => { - loadRepairList(); + loadSyncList(); }, []); - async function repair() { - if (repairable) { - // perform the repair - const resp = await loadRepairList(false); - // check simulate the repair again to check that all - // items have been repaired. - await loadRepairList(true); + async function sync() { + if (canSync) { + // perform the sync + const resp = await loadSyncList(false); + // check simulate the sync again to check that all + // items have been synchronized. + await loadSyncList(true); if (resp === null) { return; } const { successCount, errorCount } = getResponseCounts(resp); if (errorCount > 0) { - const title = i18n.translate('xpack.ml.management.repairSavedObjectsFlyout.repair.error', { - defaultMessage: 'Some jobs cannot be repaired.', + const title = i18n.translate('xpack.ml.management.syncSavedObjectsFlyout.sync.error', { + defaultMessage: 'Some jobs cannot be synchronized.', }); displayErrorToast(resp as any, title); return; } displaySuccessToast( - i18n.translate('xpack.ml.management.repairSavedObjectsFlyout.repair.success', { - defaultMessage: '{successCount} {successCount, plural, one {job} other {jobs}} repaired', + i18n.translate('xpack.ml.management.syncSavedObjectsFlyout.sync.success', { + defaultMessage: + '{successCount} {successCount, plural, one {job} other {jobs}} synchronized', values: { successCount }, }) ); @@ -98,8 +96,8 @@ export const JobSpacesRepairFlyout: FC = ({ onClose }) => {

@@ -108,33 +106,29 @@ export const JobSpacesRepairFlyout: FC = ({ onClose }) => { - + - + @@ -145,7 +139,7 @@ export const JobSpacesRepairFlyout: FC = ({ onClose }) => { ); }; -function getResponseCounts(resp: RepairSavedObjectResponse) { +function getResponseCounts(resp: SyncSavedObjectResponse) { let successCount = 0; let errorCount = 0; Object.values(resp).forEach((result: SavedObjectResult) => { diff --git a/x-pack/plugins/ml/public/application/components/job_spaces_repair/repair_list.tsx b/x-pack/plugins/ml/public/application/components/job_spaces_sync/sync_list.tsx similarity index 60% rename from x-pack/plugins/ml/public/application/components/job_spaces_repair/repair_list.tsx rename to x-pack/plugins/ml/public/application/components/job_spaces_sync/sync_list.tsx index 3eab255ba34e6..573842c2f35e7 100644 --- a/x-pack/plugins/ml/public/application/components/job_spaces_repair/repair_list.tsx +++ b/x-pack/plugins/ml/public/application/components/job_spaces_sync/sync_list.tsx @@ -9,38 +9,36 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { EuiText, EuiTitle, EuiAccordion, EuiTextColor, EuiHorizontalRule } from '@elastic/eui'; -import { RepairSavedObjectResponse } from '../../../../common/types/saved_objects'; +import { SyncSavedObjectResponse } from '../../../../common/types/saved_objects'; -export const RepairList: FC<{ repairItems: RepairSavedObjectResponse | null }> = ({ - repairItems, -}) => { - if (repairItems === null) { +export const SyncList: FC<{ syncItems: SyncSavedObjectResponse | null }> = ({ syncItems }) => { + if (syncItems === null) { return null; } return ( <> - + - + - + - + ); }; -const SavedObjectsCreated: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repairItems }) => { - const items = Object.keys(repairItems.savedObjectsCreated); +const SavedObjectsCreated: FC<{ syncItems: SyncSavedObjectResponse }> = ({ syncItems }) => { + const items = Object.keys(syncItems.savedObjectsCreated); const title = ( <> @@ -48,7 +46,7 @@ const SavedObjectsCreated: FC<{ repairItems: RepairSavedObjectResponse }> = ({ r

@@ -59,7 +57,7 @@ const SavedObjectsCreated: FC<{ repairItems: RepairSavedObjectResponse }> = ({ r

@@ -67,11 +65,11 @@ const SavedObjectsCreated: FC<{ repairItems: RepairSavedObjectResponse }> = ({ r ); - return ; + return ; }; -const SavedObjectsDeleted: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repairItems }) => { - const items = Object.keys(repairItems.savedObjectsDeleted); +const SavedObjectsDeleted: FC<{ syncItems: SyncSavedObjectResponse }> = ({ syncItems }) => { + const items = Object.keys(syncItems.savedObjectsDeleted); const title = ( <> @@ -79,7 +77,7 @@ const SavedObjectsDeleted: FC<{ repairItems: RepairSavedObjectResponse }> = ({ r

@@ -90,7 +88,7 @@ const SavedObjectsDeleted: FC<{ repairItems: RepairSavedObjectResponse }> = ({ r

@@ -98,11 +96,11 @@ const SavedObjectsDeleted: FC<{ repairItems: RepairSavedObjectResponse }> = ({ r ); - return ; + return ; }; -const DatafeedsAdded: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repairItems }) => { - const items = Object.keys(repairItems.datafeedsAdded); +const DatafeedsAdded: FC<{ syncItems: SyncSavedObjectResponse }> = ({ syncItems }) => { + const items = Object.keys(syncItems.datafeedsAdded); const title = ( <> @@ -110,7 +108,7 @@ const DatafeedsAdded: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repair

@@ -121,7 +119,7 @@ const DatafeedsAdded: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repair

@@ -129,11 +127,11 @@ const DatafeedsAdded: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repair ); - return ; + return ; }; -const DatafeedsRemoved: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repairItems }) => { - const items = Object.keys(repairItems.datafeedsRemoved); +const DatafeedsRemoved: FC<{ syncItems: SyncSavedObjectResponse }> = ({ syncItems }) => { + const items = Object.keys(syncItems.datafeedsRemoved); const title = ( <> @@ -141,7 +139,7 @@ const DatafeedsRemoved: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repa

@@ -152,7 +150,7 @@ const DatafeedsRemoved: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repa

@@ -160,23 +158,21 @@ const DatafeedsRemoved: FC<{ repairItems: RepairSavedObjectResponse }> = ({ repa ); - return ; + return ; }; -const RepairItem: FC<{ id: string; title: JSX.Element; items: string[] }> = ({ +const SyncItem: FC<{ id: string; title: JSX.Element; items: string[] }> = ({ id, title, items, }) => ( - {items.length && ( -

    - {items.map((item) => ( -
  • {item}
    • - ))} -
- )} +
    + {items.map((item) => ( +
  • {item}
    • + ))} +
); diff --git a/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/jobs_list_page.tsx b/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/jobs_list_page.tsx index 8ad18e2b821b6..18d45c02a3c95 100644 --- a/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/jobs_list_page.tsx +++ b/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/jobs_list_page.tsx @@ -39,7 +39,7 @@ import { DataFrameAnalyticsList } from '../../../../data_frame_analytics/pages/a import { AccessDeniedPage } from '../access_denied_page'; import { SharePluginStart } from '../../../../../../../../../src/plugins/share/public'; import { SpacesPluginStart } from '../../../../../../../spaces/public'; -import { JobSpacesRepairFlyout } from '../../../../components/job_spaces_repair'; +import { JobSpacesSyncFlyout } from '../../../../components/job_spaces_sync'; import { getDefaultAnomalyDetectionJobsListState } from '../../../../jobs/jobs_list/jobs'; import { getMlGlobalServices } from '../../../../app'; import { ListingPageUrlState } from '../../../../../../common/types/common'; @@ -125,7 +125,7 @@ export const JobsListPage: FC<{ const spacesEnabled = spaces !== undefined; const [initialized, setInitialized] = useState(false); const [accessDenied, setAccessDenied] = useState(false); - const [showRepairFlyout, setShowRepairFlyout] = useState(false); + const [showSyncFlyout, setShowSyncFlyout] = useState(false); const [isMlEnabledInSpace, setIsMlEnabledInSpace] = useState(false); const tabs = useTabs(isMlEnabledInSpace, spacesEnabled); const [currentTabId, setCurrentTabId] = useState(tabs[0].id); @@ -184,8 +184,8 @@ export const JobsListPage: FC<{ ); } - function onCloseRepairFlyout() { - setShowRepairFlyout(false); + function onCloseSyncFlyout() { + setShowSyncFlyout(false); } if (accessDenied) { @@ -244,12 +244,12 @@ export const JobsListPage: FC<{ {spacesEnabled && ( <> - setShowRepairFlyout(true)}> - {i18n.translate('xpack.ml.management.jobsList.repairFlyoutButton', { - defaultMessage: 'Repair saved objects', + setShowSyncFlyout(true)}> + {i18n.translate('xpack.ml.management.jobsList.syncFlyoutButton', { + defaultMessage: 'Synchronize saved objects', })} - {showRepairFlyout && } + {showSyncFlyout && } )} diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/saved_objects.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/saved_objects.ts index b47cf3f62871c..e821fa3da4d66 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/saved_objects.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/saved_objects.ts @@ -11,7 +11,7 @@ import { HttpService } from '../http_service'; import { basePath } from './index'; import { JobType, - RepairSavedObjectResponse, + SyncSavedObjectResponse, SavedObjectResult, JobsSpacesResponse, } from '../../../../common/types/saved_objects'; @@ -40,9 +40,9 @@ export const savedObjectsApiProvider = (httpService: HttpService) => ({ }); }, - repairSavedObjects(simulate: boolean = false) { - return httpService.http({ - path: `${basePath()}/saved_objects/repair`, + syncSavedObjects(simulate: boolean = false) { + return httpService.http({ + path: `${basePath()}/saved_objects/sync`, method: 'GET', query: { simulate }, }); diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index 5672824f3d040..85df7228fe929 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -145,7 +145,7 @@ "JobSavedObjects", "SavedObjectsStatus", - "RepairJobSavedObjects", + "SyncJobSavedObjects", "InitializeJobSavedObjects", "AssignJobsToSpaces", "RemoveJobsFromSpaces", diff --git a/x-pack/plugins/ml/server/routes/saved_objects.ts b/x-pack/plugins/ml/server/routes/saved_objects.ts index 3ba69b0d6b505..57c6084d9971b 100644 --- a/x-pack/plugins/ml/server/routes/saved_objects.ts +++ b/x-pack/plugins/ml/server/routes/saved_objects.ts @@ -6,8 +6,8 @@ import { wrapError } from '../client/error_wrapper'; import { RouteInitialization, SavedObjectsRouteDeps } from '../types'; -import { checksFactory, repairFactory } from '../saved_objects'; -import { jobsAndSpaces, repairJobObjects, jobTypeSchema } from './schemas/saved_objects'; +import { checksFactory, syncSavedObjectsFactory } from '../saved_objects'; +import { jobsAndSpaces, syncJobObjects, jobTypeSchema } from './schemas/saved_objects'; import { jobIdsSchema } from './schemas/job_service_schema'; /** @@ -50,8 +50,8 @@ export function savedObjectsRoutes( /** * @apiGroup JobSavedObjects * - * @api {get} /api/ml/saved_objects/repair Repair job saved objects - * @apiName RepairJobSavedObjects + * @api {get} /api/ml/saved_objects/sync Sync job saved objects + * @apiName SyncJobSavedObjects * @apiDescription Create saved objects for jobs which are missing them. * Delete saved objects for jobs which no longer exist. * Update missing datafeed ids in saved objects for datafeeds which exist. @@ -60,9 +60,9 @@ export function savedObjectsRoutes( */ router.get( { - path: '/api/ml/saved_objects/repair', + path: '/api/ml/saved_objects/sync', validate: { - query: repairJobObjects, + query: syncJobObjects, }, options: { tags: ['access:ml:canCreateJob', 'access:ml:canCreateDataFrameAnalytics'], @@ -71,8 +71,8 @@ export function savedObjectsRoutes( routeGuard.fullLicenseAPIGuard(async ({ client, request, response, jobSavedObjectService }) => { try { const { simulate } = request.query; - const { repairJobs } = repairFactory(client, jobSavedObjectService); - const savedObjects = await repairJobs(simulate); + const { syncSavedObjects } = syncSavedObjectsFactory(client, jobSavedObjectService); + const savedObjects = await syncSavedObjects(simulate); return response.ok({ body: savedObjects, @@ -95,7 +95,7 @@ export function savedObjectsRoutes( { path: '/api/ml/saved_objects/initialize', validate: { - query: repairJobObjects, + query: syncJobObjects, }, options: { tags: ['access:ml:canCreateJob', 'access:ml:canCreateDataFrameAnalytics'], @@ -104,7 +104,7 @@ export function savedObjectsRoutes( routeGuard.fullLicenseAPIGuard(async ({ client, request, response, jobSavedObjectService }) => { try { const { simulate } = request.query; - const { initSavedObjects } = repairFactory(client, jobSavedObjectService); + const { initSavedObjects } = syncSavedObjectsFactory(client, jobSavedObjectService); const savedObjects = await initSavedObjects(simulate); return response.ok({ diff --git a/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts b/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts index 6b8c64714a82c..c2d091bd16052 100644 --- a/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts +++ b/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts @@ -12,7 +12,7 @@ export const jobsAndSpaces = schema.object({ spaces: schema.arrayOf(schema.string()), }); -export const repairJobObjects = schema.object({ simulate: schema.maybe(schema.boolean()) }); +export const syncJobObjects = schema.object({ simulate: schema.maybe(schema.boolean()) }); export const jobTypeSchema = schema.object({ jobType: schema.string(), diff --git a/x-pack/plugins/ml/server/saved_objects/index.ts b/x-pack/plugins/ml/server/saved_objects/index.ts index fdd675f40d083..a082f3c3ec54a 100644 --- a/x-pack/plugins/ml/server/saved_objects/index.ts +++ b/x-pack/plugins/ml/server/saved_objects/index.ts @@ -7,6 +7,6 @@ export { setupSavedObjects } from './saved_objects'; export { JobObject, JobSavedObjectService, jobSavedObjectServiceFactory } from './service'; export { checksFactory } from './checks'; -export { repairFactory } from './repair'; +export { syncSavedObjectsFactory } from './sync'; export { jobSavedObjectsInitializationFactory } from './initialization'; export { savedObjectClientsFactory } from './util'; diff --git a/x-pack/plugins/ml/server/saved_objects/initialization/initialization.ts b/x-pack/plugins/ml/server/saved_objects/initialization/initialization.ts index 5edf35c033177..a81fc925b824b 100644 --- a/x-pack/plugins/ml/server/saved_objects/initialization/initialization.ts +++ b/x-pack/plugins/ml/server/saved_objects/initialization/initialization.ts @@ -6,7 +6,7 @@ import { IScopedClusterClient, CoreStart, SavedObjectsClientContract } from 'kibana/server'; import { savedObjectClientsFactory } from '../util'; -import { repairFactory } from '../repair'; +import { syncSavedObjectsFactory } from '../sync'; import { jobSavedObjectServiceFactory, JobObject } from '../service'; import { mlLog } from '../../lib/log'; import { ML_SAVED_OBJECT_TYPE } from '../../../common/types/saved_objects'; @@ -58,7 +58,7 @@ export function jobSavedObjectsInitializationFactory( // create space overrides for specific jobs const jobSpaceOverrides = await createJobSpaceOverrides(client); // initialize jobs - const { initSavedObjects } = repairFactory(client, jobSavedObjectService); + const { initSavedObjects } = syncSavedObjectsFactory(client, jobSavedObjectService); const { jobs } = await initSavedObjects(false, jobSpaceOverrides); mlLog.info(`${jobs.length} job saved objects initialized`); } catch (error) { diff --git a/x-pack/plugins/ml/server/saved_objects/initialization/space_overrides/space_overrides.ts b/x-pack/plugins/ml/server/saved_objects/initialization/space_overrides/space_overrides.ts index d8c713888051f..969024ee58b51 100644 --- a/x-pack/plugins/ml/server/saved_objects/initialization/space_overrides/space_overrides.ts +++ b/x-pack/plugins/ml/server/saved_objects/initialization/space_overrides/space_overrides.ts @@ -5,7 +5,7 @@ */ import { IScopedClusterClient } from 'kibana/server'; -import type { JobSpaceOverrides } from '../../repair'; +import type { JobSpaceOverrides } from '../../sync'; import { logJobsSpaces } from './logs'; import { metricsJobsSpaces } from './metrics'; diff --git a/x-pack/plugins/ml/server/saved_objects/repair.ts b/x-pack/plugins/ml/server/saved_objects/sync.ts similarity index 97% rename from x-pack/plugins/ml/server/saved_objects/repair.ts rename to x-pack/plugins/ml/server/saved_objects/sync.ts index 692217e5fac36..16e0520567056 100644 --- a/x-pack/plugins/ml/server/saved_objects/repair.ts +++ b/x-pack/plugins/ml/server/saved_objects/sync.ts @@ -9,7 +9,7 @@ import { IScopedClusterClient } from 'kibana/server'; import type { JobObject, JobSavedObjectService } from './service'; import { JobType, - RepairSavedObjectResponse, + SyncSavedObjectResponse, InitializeSavedObjectResponse, } from '../../common/types/saved_objects'; import { checksFactory } from './checks'; @@ -23,14 +23,14 @@ export interface JobSpaceOverrides { }; } -export function repairFactory( +export function syncSavedObjectsFactory( client: IScopedClusterClient, jobSavedObjectService: JobSavedObjectService ) { const { checkStatus } = checksFactory(client, jobSavedObjectService); - async function repairJobs(simulate: boolean = false) { - const results: RepairSavedObjectResponse = { + async function syncSavedObjects(simulate: boolean = false) { + const results: SyncSavedObjectResponse = { savedObjectsCreated: {}, savedObjectsDeleted: {}, datafeedsAdded: {}, @@ -232,5 +232,5 @@ export function repairFactory( return results; } - return { checkStatus, repairJobs, initSavedObjects }; + return { checkStatus, syncSavedObjects, initSavedObjects }; } From 668ec44e3c10c2bf229b688253e8b1da0e10a945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Mon, 30 Nov 2020 13:25:28 +0100 Subject: [PATCH 34/37] [APM] Adjust time formats based on the difference between start and end (#84470) --- .../common/utils/formatters/datetime.test.ts | 26 ++++++++++++++++--- .../apm/common/utils/formatters/datetime.ts | 8 +++--- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/apm/common/utils/formatters/datetime.test.ts b/x-pack/plugins/apm/common/utils/formatters/datetime.test.ts index 733fb7bb5eea1..76f01c170ba99 100644 --- a/x-pack/plugins/apm/common/utils/formatters/datetime.test.ts +++ b/x-pack/plugins/apm/common/utils/formatters/datetime.test.ts @@ -73,19 +73,37 @@ describe('date time formatters', () => { const dateRange = asRelativeDateTimeRange(start, end); expect(dateRange).toEqual('Oct 29, 2019, 10:01 - 15:01 (UTC+1)'); }); - }); - describe('MMM D, YYYY, HH:mm:ss - HH:mm:ss (UTC)', () => { it('range: 14 minutes', () => { const start = formatDateToTimezone('2019-10-29 10:01:01'); const end = formatDateToTimezone('2019-10-29 10:15:01'); const dateRange = asRelativeDateTimeRange(start, end); - expect(dateRange).toEqual('Oct 29, 2019, 10:01:01 - 10:15:01 (UTC+1)'); + expect(dateRange).toEqual('Oct 29, 2019, 10:01 - 10:15 (UTC+1)'); }); it('range: 5 minutes', () => { const start = formatDateToTimezone('2019-10-29 10:01:01'); const end = formatDateToTimezone('2019-10-29 10:06:01'); const dateRange = asRelativeDateTimeRange(start, end); - expect(dateRange).toEqual('Oct 29, 2019, 10:01:01 - 10:06:01 (UTC+1)'); + expect(dateRange).toEqual('Oct 29, 2019, 10:01 - 10:06 (UTC+1)'); + }); + it('range: 1 minute', () => { + const start = formatDateToTimezone('2019-10-29 10:01:01'); + const end = formatDateToTimezone('2019-10-29 10:02:01'); + const dateRange = asRelativeDateTimeRange(start, end); + expect(dateRange).toEqual('Oct 29, 2019, 10:01 - 10:02 (UTC+1)'); + }); + }); + describe('MMM D, YYYY, HH:mm:ss - HH:mm:ss (UTC)', () => { + it('range: 50 seconds', () => { + const start = formatDateToTimezone('2019-10-29 10:01:01'); + const end = formatDateToTimezone('2019-10-29 10:01:50'); + const dateRange = asRelativeDateTimeRange(start, end); + expect(dateRange).toEqual('Oct 29, 2019, 10:01:01 - 10:01:50 (UTC+1)'); + }); + it('range: 10 seconds', () => { + const start = formatDateToTimezone('2019-10-29 10:01:01'); + const end = formatDateToTimezone('2019-10-29 10:01:11'); + const dateRange = asRelativeDateTimeRange(start, end); + expect(dateRange).toEqual('Oct 29, 2019, 10:01:01 - 10:01:11 (UTC+1)'); }); }); describe('MMM D, YYYY, HH:mm:ss.SSS - HH:mm:ss.SSS (UTC)', () => { diff --git a/x-pack/plugins/apm/common/utils/formatters/datetime.ts b/x-pack/plugins/apm/common/utils/formatters/datetime.ts index da08b3f49edce..6da71512befc5 100644 --- a/x-pack/plugins/apm/common/utils/formatters/datetime.ts +++ b/x-pack/plugins/apm/common/utils/formatters/datetime.ts @@ -80,14 +80,14 @@ function getFormatsAccordingToDateDifference( return { dateFormat: dateFormatWithDays }; } - if (getDateDifference(start, end, 'hours') >= 5) { + if (getDateDifference(start, end, 'minutes') >= 1) { return { dateFormat: dateFormatWithDays, timeFormat: getTimeFormat('minutes'), }; } - if (getDateDifference(start, end, 'minutes') >= 5) { + if (getDateDifference(start, end, 'seconds') >= 10) { return { dateFormat: dateFormatWithDays, timeFormat: getTimeFormat('seconds'), @@ -121,8 +121,8 @@ export function asAbsoluteDateTime( * | >= 5 years | YYYY - YYYY | * | >= 5 months | MMM YYYY - MMM YYYY | * | > 1 day | MMM D, YYYY - MMM D, YYYY | - * | >= 5 hours | MMM D, YYYY, HH:mm - HH:mm (UTC) | - * | >= 5 minutes | MMM D, YYYY, HH:mm:ss - HH:mm:ss (UTC) | + * | >= 1 minute | MMM D, YYYY, HH:mm - HH:mm (UTC) | + * | >= 10 seconds | MMM D, YYYY, HH:mm:ss - HH:mm:ss (UTC) | * | default | MMM D, YYYY, HH:mm:ss.SSS - HH:mm:ss.SSS (UTC) | * * @param start timestamp From 2e6dca62f1999d9bc69eaf8f76889c62b3365169 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 30 Nov 2020 13:51:11 +0100 Subject: [PATCH 35/37] Uptime overview overhaul (#83406) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../translations/translations/ja-JP.json | 3 - .../translations/translations/zh-CN.json | 3 - .../common/runtime_types/monitor/state.ts | 11 +- .../plugins/uptime/public/apps/uptime_app.tsx | 40 ++- .../status_bar/ssl_certificate.tsx | 2 +- .../overview/filter_group/filter_popover.tsx | 3 +- .../__snapshots__/monitor_list.test.tsx.snap | 309 ++++++++++-------- .../monitor_list_status_column.test.tsx.snap | 248 -------------- .../__tests__/monitor_list.test.tsx | 28 +- .../monitor_status_column.test.tsx.snap | 300 +++++++++++++++++ .../__tests__/monitor_status_column.test.tsx} | 35 +- .../{ => columns}/cert_status_column.tsx | 8 +- .../monitor_list/columns/monitor_name_col.tsx | 78 +++++ .../columns/monitor_status_column.tsx | 195 +++++++++++ .../overview/monitor_list/monitor_list.tsx | 43 +-- .../monitor_list/monitor_list_container.tsx | 1 - .../monitor_list_drawer.test.tsx.snap | 8 +- .../monitor_status_list.test.tsx.snap | 64 ++-- .../monitor_status_row.test.tsx.snap | 48 +-- .../most_recent_error.test.tsx.snap | 44 ++- .../__tests__/integration_group.test.tsx | 4 +- .../__tests__/monitor_list_drawer.test.tsx | 2 +- .../actions_popover/actions_popover.tsx | 7 +- .../monitor_list_drawer/enabled_alerts.tsx | 58 ++-- .../monitor_list_drawer.tsx | 24 +- .../monitor_status_list.tsx | 12 +- .../monitor_status_row.tsx | 39 +-- .../monitor_list_drawer/monitor_url.tsx | 34 ++ .../monitor_list_drawer/most_recent_error.tsx | 36 +- .../monitor_list_drawer/most_recent_run.tsx | 34 ++ .../monitor_list_status_column.tsx | 146 --------- .../public/contexts/uptime_theme_context.tsx | 7 +- .../__tests__/get_apm_href.test.ts | 2 +- .../__tests__/get_infra_href.test.ts | 2 +- .../__tests__/get_logging_href.test.ts | 2 +- .../uptime/public/lib/helper/test_helpers.ts | 22 ++ .../plugins/uptime/public/pages/overview.tsx | 5 +- .../monitor_summary_iterator.test.ts | 2 +- .../search/refine_potential_matches.ts | 1 + .../monitor_states_real_data.snap | 2 + 40 files changed, 1126 insertions(+), 786 deletions(-) delete mode 100644 x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/__snapshots__/monitor_list_status_column.test.tsx.snap create mode 100644 x-pack/plugins/uptime/public/components/overview/monitor_list/columns/__tests__/__snapshots__/monitor_status_column.test.tsx.snap rename x-pack/plugins/uptime/public/components/overview/monitor_list/{__tests__/monitor_list_status_column.test.tsx => columns/__tests__/monitor_status_column.test.tsx} (87%) rename x-pack/plugins/uptime/public/components/overview/monitor_list/{ => columns}/cert_status_column.tsx (86%) create mode 100644 x-pack/plugins/uptime/public/components/overview/monitor_list/columns/monitor_name_col.tsx create mode 100644 x-pack/plugins/uptime/public/components/overview/monitor_list/columns/monitor_status_column.tsx create mode 100644 x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_url.tsx create mode 100644 x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/most_recent_run.tsx delete mode 100644 x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_status_column.tsx create mode 100644 x-pack/plugins/uptime/public/lib/helper/test_helpers.ts diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index ed514eda000aa..261458a8e7793 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -20634,11 +20634,8 @@ "xpack.uptime.monitorList.defineConnector.description": "アラートを有効にするには、デフォルトのアラートアクションコネクターを定義してください。", "xpack.uptime.monitorList.disableDownAlert": "ステータスアラートを無効にする", "xpack.uptime.monitorList.downLineSeries.downLabel": "ダウン", - "xpack.uptime.monitorList.drawer.locations.statusDown": "{locations}でダウン", - "xpack.uptime.monitorList.drawer.locations.statusUp": "{locations}でアップ", "xpack.uptime.monitorList.drawer.missingLocation": "一部のHeartbeatインスタンスには位置情報が定義されていません。Heartbeat構成への{link}。", "xpack.uptime.monitorList.enabledAlerts.noAlert": "このモニターではアラートが有効ではありません。", - "xpack.uptime.monitorList.enabledAlerts.title": "有効なアラート：", "xpack.uptime.monitorList.enableDownAlert": "ステータスアラートを有効にする", "xpack.uptime.monitorList.expandDrawerButton.ariaLabel": "ID {id}のモニターの行を展開", "xpack.uptime.monitorList.geoName.helpLinkAnnotation": "場所を追加", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index a500b63fbf863..f18c899f67652 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -20654,11 +20654,8 @@ "xpack.uptime.monitorList.defineConnector.description": "要开始启用告警，请在以下位置定义默认告警操作连接器", "xpack.uptime.monitorList.disableDownAlert": "禁用状态告警", "xpack.uptime.monitorList.downLineSeries.downLabel": "关闭", - "xpack.uptime.monitorList.drawer.locations.statusDown": "在 {locations} 已关闭", - "xpack.uptime.monitorList.drawer.locations.statusUp": "在 {locations} 正运行", "xpack.uptime.monitorList.drawer.missingLocation": "某些 Heartbeat 实例未定义位置。{link}到您的 Heartbeat 配置。", "xpack.uptime.monitorList.enabledAlerts.noAlert": "没有为此监测启用告警。", - "xpack.uptime.monitorList.enabledAlerts.title": "已启用的告警：", "xpack.uptime.monitorList.enableDownAlert": "启用状态告警", "xpack.uptime.monitorList.expandDrawerButton.ariaLabel": "展开 ID {id} 的监测行", "xpack.uptime.monitorList.geoName.helpLinkAnnotation": "添加位置", diff --git a/x-pack/plugins/uptime/common/runtime_types/monitor/state.ts b/x-pack/plugins/uptime/common/runtime_types/monitor/state.ts index d32c47bb5d3f9..46290b9e53b8e 100644 --- a/x-pack/plugins/uptime/common/runtime_types/monitor/state.ts +++ b/x-pack/plugins/uptime/common/runtime_types/monitor/state.ts @@ -23,9 +23,14 @@ export const StateType = t.intersection([ up: t.number, down: t.number, }), - monitor: t.partial({ - name: t.string, - }), + monitor: t.intersection([ + t.partial({ + name: t.string, + }), + t.type({ + type: t.string, + }), + ]), }), t.partial({ tls: t.partial({ diff --git a/x-pack/plugins/uptime/public/apps/uptime_app.tsx b/x-pack/plugins/uptime/public/apps/uptime_app.tsx index c928ac0dc458f..9535cfdb8c8b0 100644 --- a/x-pack/plugins/uptime/public/apps/uptime_app.tsx +++ b/x-pack/plugins/uptime/public/apps/uptime_app.tsx @@ -32,9 +32,11 @@ import { import { store } from '../state'; import { kibanaService } from '../state/kibana_service'; import { ScopedHistory } from '../../../../../src/core/public'; +import { EuiThemeProvider } from '../../../observability/public'; export interface UptimeAppColors { danger: string; + dangerBehindText: string; success: string; gray: string; range: string; @@ -103,24 +105,26 @@ const Application = (props: UptimeAppProps) => { services={{ ...core, ...plugins, triggersActionsUi: startPlugins.triggersActionsUi }} > - - - - - - - -
- - -
- - - - - - - + + + + + + + + +
+ + +
+ + + + + + + + diff --git a/x-pack/plugins/uptime/public/components/monitor/status_details/status_bar/ssl_certificate.tsx b/x-pack/plugins/uptime/public/components/monitor/status_details/status_bar/ssl_certificate.tsx index ffe4f5d759e03..4c0824b5dfa4f 100644 --- a/x-pack/plugins/uptime/public/components/monitor/status_details/status_bar/ssl_certificate.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/status_details/status_bar/ssl_certificate.tsx @@ -11,7 +11,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { Tls, X509Expiry } from '../../../../../common/runtime_types'; import { CERTIFICATES_ROUTE } from '../../../../../common/constants'; import { MonListDescription, MonListTitle } from './status_bar'; -import { CertStatusColumn } from '../../../overview/monitor_list/cert_status_column'; +import { CertStatusColumn } from '../../../overview/monitor_list/columns/cert_status_column'; interface Props { /** diff --git a/x-pack/plugins/uptime/public/components/overview/filter_group/filter_popover.tsx b/x-pack/plugins/uptime/public/components/overview/filter_group/filter_popover.tsx index da66e4179e1ac..e79c036d54e0e 100644 --- a/x-pack/plugins/uptime/public/components/overview/filter_group/filter_popover.tsx +++ b/x-pack/plugins/uptime/public/components/overview/filter_group/filter_popover.tsx @@ -54,6 +54,7 @@ export const FilterPopover = ({ const mItems = selectedItems.concat(allItems ?? []); const newItems = mItems.filter((item, index) => mItems.indexOf(item) === index); setItems(newItems); + setTempSelectedItems(selectedItems); }, [allItems, selectedItems]); useEffect(() => { @@ -73,7 +74,7 @@ export const FilterPopover = ({ isDisabled={disabled && selectedItems.length === 0} isSelected={tempSelectedItems.length > 0} numFilters={items.length} - numActiveFilters={tempSelectedItems.length} + numActiveFilters={isOpen ? tempSelectedItems.length : selectedItems.length} onClick={() => { setIsOpen(!isOpen); onFilterFieldChange(fieldName, tempSelectedItems); diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/__snapshots__/monitor_list.test.tsx.snap b/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/__snapshots__/monitor_list.test.tsx.snap index 1f5e86dc67386..edd901253f509 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/__snapshots__/monitor_list.test.tsx.snap +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/__tests__/__snapshots__/monitor_list.test.tsx.snap @@ -199,7 +199,9 @@ exports[`MonitorList component MonitorListPagination component renders the pagin Object { "monitor_id": "foo", "state": Object { - "monitor": Object {}, + "monitor": Object { + "type": "http", + }, "summary": Object { "down": 2, "up": 1, @@ -255,7 +257,9 @@ exports[`MonitorList component MonitorListPagination component renders the pagin Object { "monitor_id": "bar", "state": Object { - "monitor": Object {}, + "monitor": Object { + "type": "http", + }, "summary": Object { "down": 0, "up": 2, @@ -507,7 +511,9 @@ exports[`MonitorList component renders error list 1`] = ` Object { "monitor_id": "foo", "state": Object { - "monitor": Object {}, + "monitor": Object { + "type": "http", + }, "summary": Object { "down": 2, "up": 1, @@ -563,7 +569,9 @@ exports[`MonitorList component renders error list 1`] = ` Object { "monitor_id": "bar", "state": Object { - "monitor": Object {}, + "monitor": Object { + "type": "http", + }, "summary": Object { "down": 0, "up": 2, @@ -710,7 +718,9 @@ exports[`MonitorList component renders loading state 1`] = ` Object { "monitor_id": "foo", "state": Object { - "monitor": Object {}, + "monitor": Object { + "type": "http", + }, "summary": Object { "down": 2, "up": 1, @@ -766,7 +776,9 @@ exports[`MonitorList component renders loading state 1`] = ` Object { "monitor_id": "bar", "state": Object { - "monitor": Object {}, + "monitor": Object { + "type": "http", + }, "summary": Object { "down": 0, "up": 2, @@ -820,10 +832,10 @@ exports[`MonitorList component renders loading state 1`] = ` exports[`MonitorList component renders the monitor list 1`] = ` .c3 { - padding-left: 17px; + padding-right: 4px; } -.c5 { +.c4 { padding-top: 12px; } @@ -837,12 +849,6 @@ exports[`MonitorList component renders the monitor list 1`] = ` position: relative; } -.c4 { - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -} - @media (max-width:574px) { .c2 { min-width: 230px; @@ -966,7 +972,7 @@ exports[`MonitorList component renders the monitor list 1`] = `

@@ -1012,6 +1018,7 @@ exports[`MonitorList component renders the monitor list 1`] = ` data-test-subj="tableHeaderCell_state.url.full_2" role="columnheader" scope="col" + style="width:40%" >
-
+
+ + + +
+
+
+
+
- + in 0/1 location,
-
-
+ -
-
- 5m ago -
+ Checked Sept 4, 2020 9:31:38 AM
- +
-
-
- in 0/1 Location -
-
@@ -1172,21 +1176,46 @@ exports[`MonitorList component renders the monitor list 1`] = `
- + + Unnamed - foo + + +
+ +
+
@@ -1395,21 +1415,46 @@ exports[`MonitorList component renders the monitor list 1`] = `
- + + Unnamed - bar + + +
+ +
+
+ />
-
+
+ + + +
+
+
+
+
- + in 0/1 location,
-
-
+ -
-
- 5m ago -
+ Checked Sept 4, 2020 9:31:38 AM
- +
-
-
- in 0/1 Location -
-
+ />
, -] + + Get https://expired.badssl.com: x509: certificate has expired or is not yet valid + + + + `; exports[`MostRecentError component validates props with shallow render 1`] = ` diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/integration_group.test.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/integration_group.test.tsx index 3e7860e4e6ab4..62e34354dd356 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/integration_group.test.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/integration_group.test.tsx @@ -17,7 +17,7 @@ describe('IntegrationGroup', () => { monitor_id: '12345', state: { summary: {}, - monitor: {}, + monitor: { type: 'http' }, summaryPings: [], timestamp: '123', url: {}, @@ -48,7 +48,7 @@ describe('IntegrationGroup', () => { state: { timestamp: 'foo', summaryPings: [], - monitor: {}, + monitor: { type: 'http' }, summary: { up: 0, down: 0, diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_list_drawer.test.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_list_drawer.test.tsx index 4e8ffc64cfe92..6da19b2ccc4d2 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_list_drawer.test.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/__tests__/monitor_list_drawer.test.tsx @@ -17,7 +17,7 @@ describe('MonitorListDrawer component', () => { summary = { monitor_id: 'foo', state: { - monitor: {}, + monitor: { type: 'http' }, summaryPings: [ makePing({ docId: 'foo', diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/actions_popover/actions_popover.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/actions_popover/actions_popover.tsx index 9e96f0ca76535..3e1276b26b727 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/actions_popover/actions_popover.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/actions_popover/actions_popover.tsx @@ -46,7 +46,12 @@ export const ActionsPopoverComponent = ({ iconType="arrowDown" iconSide="right" > - Integrations + {i18n.translate( + 'xpack.uptime.monitorList.observabilityInvestigateColumn.popoverIconButton.label', + { + defaultMessage: 'Investigate', + } + )} } closePopover={() => togglePopoverIsVisible({ id: popoverId, open: false })} diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/enabled_alerts.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/enabled_alerts.tsx index d869c6d78ec11..b39162b70eda4 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/enabled_alerts.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/enabled_alerts.tsx @@ -5,9 +5,17 @@ */ import React, { useContext } from 'react'; -import { EuiCallOut, EuiListGroup, EuiLoadingSpinner, EuiSpacer, EuiText } from '@elastic/eui'; +import { + EuiCallOut, + EuiDescriptionList, + EuiDescriptionListDescription, + EuiDescriptionListTitle, + EuiListGroup, + EuiLoadingSpinner, +} from '@elastic/eui'; import { EuiListGroupItemProps } from '@elastic/eui/src/components/list_group/list_group_item'; import { i18n } from '@kbn/i18n'; +import styled from 'styled-components'; import { UptimeSettingsContext } from '../../../../contexts'; import { Alert } from '../../../../../../triggers_actions_ui/public'; @@ -16,6 +24,15 @@ interface Props { loading: boolean; } +const LinkGroupList = styled(EuiListGroup)` + &&& { + a { + padding-left: 0; + padding-top: 0; + } + } +`; + export const EnabledAlerts = ({ monitorAlerts, loading }: Props) => { const { basePath } = useContext(UptimeSettingsContext); @@ -31,27 +48,24 @@ export const EnabledAlerts = ({ monitorAlerts, loading }: Props) => { }); return ( - <> - - - -

- {i18n.translate('xpack.uptime.monitorList.enabledAlerts.title', { - defaultMessage: 'Enabled alerts:', - description: 'Alerts enabled for this monitor', + + + {i18n.translate('xpack.uptime.monitorList.enabledAlerts.title', { + defaultMessage: 'Enabled alerts', + description: 'Alerts enabled for this monitor', + })} + + + {listItems.length === 0 && !loading && ( + - - - {listItems.length === 0 && !loading && ( - - )} - {loading ? : } - + /> + )} + {loading ? : } + + ); }; diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_list_drawer.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_list_drawer.tsx index 4b359099bc58c..ca7a608986e85 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_list_drawer.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_list_drawer.tsx @@ -6,13 +6,15 @@ import React from 'react'; import styled from 'styled-components'; -import { EuiLink, EuiFlexGroup, EuiFlexItem, EuiIcon, EuiText } from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import { Alert } from '../../../../../../triggers_actions_ui/public'; import { MostRecentError } from './most_recent_error'; import { MonitorStatusList } from './monitor_status_list'; import { MonitorDetails, MonitorSummary } from '../../../../../common/runtime_types'; import { ActionsPopover } from './actions_popover/actions_popover_container'; import { EnabledAlerts } from './enabled_alerts'; -import { Alert } from '../../../../../../triggers_actions_ui/public'; +import { MonitorUrl } from './monitor_url'; +import { MostRecentRun } from './most_recent_run'; const ContainerDiv = styled.div` padding: 10px; @@ -47,19 +49,25 @@ export function MonitorListDrawerComponent({ - - - {monitorUrl} - - - + + + + + + + {/* TODO: add link to details page */} + + + + + {monitorDetails && monitorDetails.error && ( { return ( <> - - + + + + + + + + {(downChecks.has(UNNAMED_LOCATION) || upChecks.has(UNNAMED_LOCATION)) && ( <> diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_status_row.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_status_row.tsx index d0cc71ece989a..5bcb4ef97c3a4 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_status_row.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_status_row.tsx @@ -4,11 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useContext } from 'react'; -import { EuiHealth, EuiSpacer } from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { UptimeThemeContext } from '../../../../contexts'; +import React from 'react'; +import { EuiBadge, EuiSpacer } from '@elastic/eui'; import { UNNAMED_LOCATION, STATUS } from '../../../../../common/constants'; +import { getHealthMessage } from '../columns/monitor_status_column'; interface MonitorStatusRowProps { /** @@ -22,11 +21,7 @@ interface MonitorStatusRowProps { } export const MonitorStatusRow = ({ locationNames, status }: MonitorStatusRowProps) => { - const { - colors: { success, danger }, - } = useContext(UptimeThemeContext); - - const color = status === STATUS.UP ? success : danger; + const color = status === STATUS.UP ? 'secondary' : 'danger'; let checkListArray = [...locationNames]; // If un-named location exists, move it to end @@ -35,29 +30,13 @@ export const MonitorStatusRow = ({ locationNames, status }: MonitorStatusRowProp checkListArray.push(UNNAMED_LOCATION); } - if (locationNames.size === 0) { - return null; - } - const locations = checkListArray.join(', '); return ( - <> - - {status === STATUS.UP ? ( - - ) : ( - - )} - + + {getHealthMessage(status)} + + {locations || '--'} - + ); }; diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_url.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_url.tsx new file mode 100644 index 0000000000000..d988f5a582495 --- /dev/null +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/monitor_url.tsx @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiLink, + EuiDescriptionList, + EuiDescriptionListDescription, + EuiDescriptionListTitle, +} from '@elastic/eui'; + +interface Props { + monitorUrl: string; +} +export const MonitorUrl = ({ monitorUrl }: Props) => { + return ( + + + {i18n.translate('xpack.uptime.monitorList.drawer.url', { + defaultMessage: 'Url', + })} + + + + {monitorUrl} + + + + ); +}; diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/most_recent_error.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/most_recent_error.tsx index e7d9885680340..d611278d91033 100644 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/most_recent_error.tsx +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/most_recent_error.tsx @@ -4,7 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ import React from 'react'; -import { EuiText, EuiSpacer } from '@elastic/eui'; +import { + EuiDescriptionList, + EuiDescriptionListTitle, + EuiDescriptionListDescription, +} from '@elastic/eui'; import moment from 'moment'; import { i18n } from '@kbn/i18n'; import { MonitorPageLink } from '../../../common/monitor_page_link'; @@ -37,21 +41,19 @@ export const MostRecentError = ({ error, monitorId, timestamp }: MostRecentError const timestampStr = timestamp ? moment(new Date(timestamp).valueOf()).fromNow() : ''; return ( - <> - - -

- {i18n.translate('xpack.uptime.monitorList.mostRecentError.title', { - defaultMessage: 'Most recent error ({timestamp})', - values: { timestamp: timestampStr }, - description: 'Most Recent Error title in Monitor List Expanded row', - })} -

- - - - {error?.message} - - + + + {i18n.translate('xpack.uptime.monitorList.mostRecentError.title', { + defaultMessage: 'Most recent error ({timestamp})', + values: { timestamp: timestampStr }, + description: 'Most Recent Error title in Monitor List Expanded row', + })} + + + + {error?.message} + + + ); }; diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/most_recent_run.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/most_recent_run.tsx new file mode 100644 index 0000000000000..cd9108a36f332 --- /dev/null +++ b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_drawer/most_recent_run.tsx @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React from 'react'; +import { + EuiDescriptionList, + EuiDescriptionListTitle, + EuiDescriptionListDescription, + EuiText, +} from '@elastic/eui'; +import moment from 'moment'; +import { i18n } from '@kbn/i18n'; +import { MonitorSummary } from '../../../../../common/runtime_types'; + +interface Props { + summary: MonitorSummary; +} + +export const MostRecentRun = ({ summary }: Props) => { + return ( + + + {i18n.translate('xpack.uptime.monitorList.drawer.mostRecentRun', { + defaultMessage: 'Most recent test run', + })} + + + {moment(summary.state.timestamp).format('LLL').toString()} + + + ); +}; diff --git a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_status_column.tsx b/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_status_column.tsx deleted file mode 100644 index 7140211d18807..0000000000000 --- a/x-pack/plugins/uptime/public/components/overview/monitor_list/monitor_list_status_column.tsx +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React from 'react'; -import moment from 'moment'; -import { i18n } from '@kbn/i18n'; -import styled from 'styled-components'; -import { EuiHealth, EuiFlexGroup, EuiFlexItem, EuiText, EuiToolTip } from '@elastic/eui'; -import { parseTimestamp } from './parse_timestamp'; -import { Ping } from '../../../../common/runtime_types'; -import { - STATUS, - SHORT_TIMESPAN_LOCALE, - UNNAMED_LOCATION, - SHORT_TS_LOCALE, -} from '../../../../common/constants'; - -import * as labels from './translations'; - -interface MonitorListStatusColumnProps { - status: string; - timestamp: string; - summaryPings: Ping[]; -} - -const PaddedSpan = styled.span` - padding-left: 17px; -`; - -const StatusColumnFlexG = styled(EuiFlexGroup)` - @media (max-width: 574px) { - min-width: 230px; - } -`; - -const getHealthColor = (status: string): string => { - switch (status) { - case STATUS.UP: - return 'success'; - case STATUS.DOWN: - return 'danger'; - default: - return ''; - } -}; - -const getHealthMessage = (status: string): string | null => { - switch (status) { - case STATUS.UP: - return labels.UP; - case STATUS.DOWN: - return labels.DOWN; - default: - return null; - } -}; - -const getRelativeShortTimeStamp = (timeStamp: any) => { - const prevLocale: string = moment.locale() ?? 'en'; - - const shortLocale = moment.locale(SHORT_TS_LOCALE) === SHORT_TS_LOCALE; - - if (!shortLocale) { - moment.defineLocale(SHORT_TS_LOCALE, SHORT_TIMESPAN_LOCALE); - } - - const shortTimestamp = parseTimestamp(timeStamp).fromNow(); - - // Reset it so, it does't impact other part of the app - moment.locale(prevLocale); - return shortTimestamp; -}; - -export const getLocationStatus = (summaryPings: Ping[], status: string) => { - const upPings: Set = new Set(); - const downPings: Set = new Set(); - - summaryPings.forEach((summaryPing: Ping) => { - const location = summaryPing?.observer?.geo?.name ?? UNNAMED_LOCATION; - - if (summaryPing.monitor.status === STATUS.UP) { - upPings.add(location); - } else if (summaryPing.monitor.status === STATUS.DOWN) { - downPings.add(location); - } - }); - - // if monitor is down in one dns, it will be considered down so removing it from up list - const absUpChecks: Set = new Set([...upPings].filter((item) => !downPings.has(item))); - - const totalLocations = absUpChecks.size + downPings.size; - let statusMessage = ''; - if (status === STATUS.DOWN) { - statusMessage = `${downPings.size}/${totalLocations}`; - } else { - statusMessage = `${absUpChecks.size}/${totalLocations}`; - } - - if (totalLocations > 1) { - return i18n.translate('xpack.uptime.monitorList.statusColumn.locStatusMessage.multiple', { - defaultMessage: 'in {noLoc} Locations', - values: { noLoc: statusMessage }, - }); - } - - return i18n.translate('xpack.uptime.monitorList.statusColumn.locStatusMessage', { - defaultMessage: 'in {noLoc} Location', - values: { noLoc: statusMessage }, - }); -}; - -export const MonitorListStatusColumn = ({ - status, - summaryPings = [], - timestamp: tsString, -}: MonitorListStatusColumnProps) => { - const timestamp = parseTimestamp(tsString); - return ( - - - - {getHealthMessage(status)} - - - - {timestamp.toLocaleString()} - - } - > - - {getRelativeShortTimeStamp(tsString)} - - - - - - {getLocationStatus(summaryPings, status)} - - - ); -}; diff --git a/x-pack/plugins/uptime/public/contexts/uptime_theme_context.tsx b/x-pack/plugins/uptime/public/contexts/uptime_theme_context.tsx index f0a702b9c0b75..a2f50c44f8ca4 100644 --- a/x-pack/plugins/uptime/public/contexts/uptime_theme_context.tsx +++ b/x-pack/plugins/uptime/public/contexts/uptime_theme_context.tsx @@ -26,6 +26,7 @@ export interface UptimeThemeContextValues { const defaultContext: UptimeThemeContextValues = { colors: { danger: euiLightVars.euiColorDanger, + dangerBehindText: euiDarkVars.euiColorVis9_behindText, mean: euiLightVars.euiColorPrimary, range: euiLightVars.euiFocusBackgroundColor, success: euiLightVars.euiColorSuccess, @@ -49,7 +50,8 @@ export const UptimeThemeContextProvider: React.FC = ({ darkMo let colors: UptimeAppColors; if (darkMode) { colors = { - danger: euiDarkVars.euiColorDanger, + danger: euiDarkVars.euiColorVis9, + dangerBehindText: euiDarkVars.euiColorVis9_behindText, mean: euiDarkVars.euiColorPrimary, gray: euiDarkVars.euiColorLightShade, range: euiDarkVars.euiFocusBackgroundColor, @@ -59,7 +61,8 @@ export const UptimeThemeContextProvider: React.FC = ({ darkMo }; } else { colors = { - danger: euiLightVars.euiColorDanger, + danger: euiLightVars.euiColorVis9, + dangerBehindText: euiLightVars.euiColorVis9_behindText, mean: euiLightVars.euiColorPrimary, gray: euiLightVars.euiColorLightShade, range: euiLightVars.euiFocusBackgroundColor, diff --git a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_apm_href.test.ts b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_apm_href.test.ts index 2444cfbee63d5..bbd389195d9dc 100644 --- a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_apm_href.test.ts +++ b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_apm_href.test.ts @@ -14,7 +14,7 @@ describe('getApmHref', () => { monitor_id: 'foo', state: { summary: {}, - monitor: {}, + monitor: { type: 'http' }, summaryPings: [ makePing({ docId: 'foo', diff --git a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_infra_href.test.ts b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_infra_href.test.ts index b1247ad5b8935..44e29c81f43f8 100644 --- a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_infra_href.test.ts +++ b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_infra_href.test.ts @@ -30,7 +30,7 @@ describe('getInfraHref', () => { summaryPings: [ping], summary: {}, url: {}, - monitor: {}, + monitor: { type: 'http' }, timestamp: '123', }, }; diff --git a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_logging_href.test.ts b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_logging_href.test.ts index d2a7a96a8b6f9..ae94e6cadd541 100644 --- a/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_logging_href.test.ts +++ b/x-pack/plugins/uptime/public/lib/helper/observability_integration/__tests__/get_logging_href.test.ts @@ -38,7 +38,7 @@ describe('getLoggingHref', () => { summary: {}, summaryPings: [ping], timestamp: '123', - monitor: {}, + monitor: { type: 'http' }, url: {}, }, }; diff --git a/x-pack/plugins/uptime/public/lib/helper/test_helpers.ts b/x-pack/plugins/uptime/public/lib/helper/test_helpers.ts new file mode 100644 index 0000000000000..d18f2aa2a4e78 --- /dev/null +++ b/x-pack/plugins/uptime/public/lib/helper/test_helpers.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* global jest */ + +import moment from 'moment'; +import { Moment } from 'moment-timezone'; + +export function mockMoment() { + // avoid timezone issues + jest.spyOn(moment.prototype, 'format').mockImplementation(function (this: Moment) { + return `Sept 4, 2020 9:31:38 AM`; + }); + + // convert relative time to absolute time to avoid timing issues + jest.spyOn(moment.prototype, 'fromNow').mockImplementation(function (this: Moment) { + return `15 minutes ago`; + }); +} diff --git a/x-pack/plugins/uptime/public/pages/overview.tsx b/x-pack/plugins/uptime/public/pages/overview.tsx index 9594819e385c0..e2b8e911b5a88 100644 --- a/x-pack/plugins/uptime/public/pages/overview.tsx +++ b/x-pack/plugins/uptime/public/pages/overview.tsx @@ -10,7 +10,6 @@ import styled from 'styled-components'; import { i18n } from '@kbn/i18n'; import { useDispatch } from 'react-redux'; import { useGetUrlParams } from '../hooks'; -import { stringifyUrlParams } from '../lib/helper/stringify_url_params'; import { PageHeader } from './page_header'; import { IIndexPattern } from '../../../../../src/plugins/data/public'; import { useUpdateKueryString } from '../hooks'; @@ -63,8 +62,6 @@ export const OverviewPageComponent = React.memo( dispatch(getMonitorAlertsAction.get()); }, [dispatch]); - const linkParameters = stringifyUrlParams(params, true); - const heading = i18n.translate('xpack.uptime.overviewPage.headerText', { defaultMessage: 'Overview', description: `The text that will be displayed in the app's heading when the Overview page loads.`, @@ -93,7 +90,7 @@ export const OverviewPageComponent = React.memo( - + ); diff --git a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_summary_iterator.test.ts b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_summary_iterator.test.ts index 8ba5be943304c..6705bcac3c0ff 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_summary_iterator.test.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/monitor_summary_iterator.test.ts @@ -98,7 +98,7 @@ const makeMonitorSummaries = (count: number): MonitorSummary[] => { summaries.push({ monitor_id: id, state: { - monitor: {}, + monitor: { type: 'http' }, timestamp: (123 + i).toString(), url: {}, summaryPings: [], diff --git a/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts b/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts index dc3af2805d13f..bf0c7fce2be75 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts @@ -86,6 +86,7 @@ export const summaryPingsToSummary = (summaryPings: Ping[]): MonitorSummary => { timestamp: latest.timestamp, monitor: { name: latest.monitor?.name, + type: latest.monitor?.type, }, url: latest.url ?? {}, summary: { diff --git a/x-pack/test/api_integration/apis/uptime/rest/__snapshots__/monitor_states_real_data.snap b/x-pack/test/api_integration/apis/uptime/rest/__snapshots__/monitor_states_real_data.snap index 50625683b605d..93abfaf67a009 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/__snapshots__/monitor_states_real_data.snap +++ b/x-pack/test/api_integration/apis/uptime/rest/__snapshots__/monitor_states_real_data.snap @@ -63,6 +63,7 @@ Object { "state": Object { "monitor": Object { "name": "", + "type": "http", }, "observer": Object { "geo": Object { @@ -242,6 +243,7 @@ Object { "state": Object { "monitor": Object { "name": "", + "type": "http", }, "observer": Object { "geo": Object { From 923a5259237c3f123903127341e1c3c587919372 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Mon, 30 Nov 2020 14:13:53 +0100 Subject: [PATCH 36/37] [Discover] Unskip main functional tests (#84300) --- test/functional/apps/discover/_discover.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/functional/apps/discover/_discover.js b/test/functional/apps/discover/_discover.js index fe5c04c001731..2270f3c815aaa 100644 --- a/test/functional/apps/discover/_discover.js +++ b/test/functional/apps/discover/_discover.js @@ -32,8 +32,7 @@ export default function ({ getService, getPageObjects }) { defaultIndex: 'logstash-*', }; - // Failing: See https://github.com/elastic/kibana/issues/82915 - describe.skip('discover test', function describeIndexTests() { + describe('discover test', function describeIndexTests() { before(async function () { log.debug('load kibana index with default index pattern'); await esArchiver.load('discover'); @@ -315,10 +314,13 @@ export default function ({ getService, getPageObjects }) { const getRequestTimestamp = async () => { const requestStats = await inspector.getTableData(); - const requestTimestamp = requestStats.filter((r) => - r[0].includes('Request timestamp') - )[0][1]; - return requestTimestamp; + const requestStatsRow = requestStats.filter( + (r) => r && r[0] && r[0].includes('Request timestamp') + ); + if (!requestStatsRow || !requestStatsRow[0] || !requestStatsRow[0][1]) { + return ''; + } + return requestStatsRow[0][1]; }; const requestTimestampBefore = await getRequestTimestamp(); @@ -327,7 +329,7 @@ export default function ({ getService, getPageObjects }) { log.debug( `Timestamp before: ${requestTimestampBefore}, Timestamp after: ${requestTimestampAfter}` ); - return requestTimestampBefore !== requestTimestampAfter; + return requestTimestampAfter && requestTimestampBefore !== requestTimestampAfter; }); }); From a2b71f8bf901b9c19db1b6a1cc66fad70c90e2ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Mon, 30 Nov 2020 14:20:41 +0100 Subject: [PATCH 37/37] [APM] Service overview: Add throughput chart (#84439) --- x-pack/plugins/apm/common/agent_name.test.ts | 36 +-- x-pack/plugins/apm/common/agent_name.ts | 24 -- .../TransactionDurationAlertTrigger/index.tsx | 4 +- .../index.tsx | 4 +- .../index.tsx | 4 +- .../app/Main/route_config/index.tsx | 43 +-- .../service_details/service_detail_tabs.tsx | 4 +- .../app/service_node_metrics/index.tsx | 4 +- .../components/app/service_overview/index.tsx | 14 +- .../service_overview.test.tsx | 1 + .../service_overview_throughput_chart.tsx | 80 ++++++ .../app/transaction_overview/index.tsx | 10 +- .../transaction_overview.test.tsx | 24 +- .../context/apm_service_context.test.tsx | 70 +++++ .../public/context/apm_service_context.tsx | 72 +++++ ...{useComponentId.tsx => use_apm_service.ts} | 11 +- ...AgentName.ts => use_service_agent_name.ts} | 14 +- ....tsx => use_service_transaction_types.tsx} | 5 +- .../public/hooks/use_transaction_breakdown.ts | 4 +- .../apm/public/hooks/use_transaction_type.ts | 28 -- .../apm/server/lib/services/get_throughput.ts | 84 ++++++ .../apm/server/routes/create_apm_api.ts | 2 + x-pack/plugins/apm/server/routes/services.ts | 31 +++ .../apm_api_integration/basic/tests/index.ts | 3 +- .../services/__snapshots__/throughput.snap | 250 ++++++++++++++++++ .../basic/tests/services/throughput.ts | 85 ++++++ 26 files changed, 750 insertions(+), 161 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/app/service_overview/service_overview_throughput_chart.tsx create mode 100644 x-pack/plugins/apm/public/context/apm_service_context.test.tsx create mode 100644 x-pack/plugins/apm/public/context/apm_service_context.tsx rename x-pack/plugins/apm/public/hooks/{useComponentId.tsx => use_apm_service.ts} (56%) rename x-pack/plugins/apm/public/hooks/{useAgentName.ts => use_service_agent_name.ts} (81%) rename x-pack/plugins/apm/public/hooks/{useServiceTransactionTypes.tsx => use_service_transaction_types.tsx} (86%) delete mode 100644 x-pack/plugins/apm/public/hooks/use_transaction_type.ts create mode 100644 x-pack/plugins/apm/server/lib/services/get_throughput.ts create mode 100644 x-pack/test/apm_api_integration/basic/tests/services/__snapshots__/throughput.snap create mode 100644 x-pack/test/apm_api_integration/basic/tests/services/throughput.ts diff --git a/x-pack/plugins/apm/common/agent_name.test.ts b/x-pack/plugins/apm/common/agent_name.test.ts index f4ac2aa220e89..10afefc264ae9 100644 --- a/x-pack/plugins/apm/common/agent_name.test.ts +++ b/x-pack/plugins/apm/common/agent_name.test.ts @@ -4,43 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - getFirstTransactionType, - isJavaAgentName, - isRumAgentName, -} from './agent_name'; +import { isJavaAgentName, isRumAgentName } from './agent_name'; describe('agent name helpers', () => { - describe('getFirstTransactionType', () => { - describe('with no transaction types', () => { - expect(getFirstTransactionType([])).toBeUndefined(); - }); - - describe('with a non-rum agent', () => { - it('returns "request"', () => { - expect(getFirstTransactionType(['worker', 'request'], 'java')).toEqual( - 'request' - ); - }); - - describe('with no request types', () => { - it('returns the first type', () => { - expect( - getFirstTransactionType(['worker', 'shirker'], 'java') - ).toEqual('worker'); - }); - }); - }); - - describe('with a rum agent', () => { - it('returns "page-load"', () => { - expect( - getFirstTransactionType(['http-request', 'page-load'], 'js-base') - ).toEqual('page-load'); - }); - }); - }); - describe('isJavaAgentName', () => { describe('when the agent name is java', () => { it('returns true', () => { diff --git a/x-pack/plugins/apm/common/agent_name.ts b/x-pack/plugins/apm/common/agent_name.ts index 916fe65684a6b..7fb79aa59595b 100644 --- a/x-pack/plugins/apm/common/agent_name.ts +++ b/x-pack/plugins/apm/common/agent_name.ts @@ -5,10 +5,6 @@ */ import { AgentName } from '../typings/es_schemas/ui/fields/agent'; -import { - TRANSACTION_PAGE_LOAD, - TRANSACTION_REQUEST, -} from './transaction_types'; /* * Agent names can be any string. This list only defines the official agents @@ -50,26 +46,6 @@ export const RUM_AGENT_NAMES: AgentName[] = [ 'opentelemetry/webjs', ]; -function getDefaultTransactionTypeForAgentName(agentName?: string) { - return isRumAgentName(agentName) - ? TRANSACTION_PAGE_LOAD - : TRANSACTION_REQUEST; -} - -export function getFirstTransactionType( - transactionTypes: string[], - agentName?: string -) { - const defaultTransactionType = getDefaultTransactionTypeForAgentName( - agentName - ); - - return ( - transactionTypes.find((type) => type === defaultTransactionType) ?? - transactionTypes[0] - ); -} - export function isJavaAgentName( agentName: string | undefined ): agentName is 'java' { diff --git a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx index ce98354c94c7e..b7220de8079c9 100644 --- a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx +++ b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx @@ -11,7 +11,6 @@ import React from 'react'; import { ForLastExpression } from '../../../../../triggers_actions_ui/public'; import { ALERT_TYPES_CONFIG } from '../../../../common/alert_types'; import { useEnvironments } from '../../../hooks/useEnvironments'; -import { useServiceTransactionTypes } from '../../../hooks/useServiceTransactionTypes'; import { useUrlParams } from '../../../hooks/useUrlParams'; import { ServiceAlertTrigger } from '../ServiceAlertTrigger'; import { PopoverExpression } from '../ServiceAlertTrigger/PopoverExpression'; @@ -22,6 +21,7 @@ import { TransactionTypeField, IsAboveField, } from '../fields'; +import { useApmService } from '../../../hooks/use_apm_service'; interface AlertParams { windowSize: number; @@ -63,7 +63,7 @@ interface Props { export function TransactionDurationAlertTrigger(props: Props) { const { setAlertParams, alertParams, setAlertProperty } = props; const { urlParams } = useUrlParams(); - const transactionTypes = useServiceTransactionTypes(urlParams); + const { transactionTypes } = useApmService(); const { serviceName } = useParams<{ serviceName?: string }>(); const { start, end, transactionType } = urlParams; const { environmentOptions } = useEnvironments({ serviceName, start, end }); diff --git a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAnomalyAlertTrigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAnomalyAlertTrigger/index.tsx index 4f87e13104371..e13ed6c1bcd6f 100644 --- a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAnomalyAlertTrigger/index.tsx +++ b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAnomalyAlertTrigger/index.tsx @@ -10,7 +10,6 @@ import React from 'react'; import { ANOMALY_SEVERITY } from '../../../../../ml/common'; import { ALERT_TYPES_CONFIG } from '../../../../common/alert_types'; import { useEnvironments } from '../../../hooks/useEnvironments'; -import { useServiceTransactionTypes } from '../../../hooks/useServiceTransactionTypes'; import { useUrlParams } from '../../../hooks/useUrlParams'; import { ServiceAlertTrigger } from '../ServiceAlertTrigger'; import { PopoverExpression } from '../ServiceAlertTrigger/PopoverExpression'; @@ -24,6 +23,7 @@ import { ServiceField, TransactionTypeField, } from '../fields'; +import { useApmService } from '../../../hooks/use_apm_service'; interface Params { windowSize: number; @@ -47,7 +47,7 @@ interface Props { export function TransactionDurationAnomalyAlertTrigger(props: Props) { const { setAlertParams, alertParams, setAlertProperty } = props; const { urlParams } = useUrlParams(); - const transactionTypes = useServiceTransactionTypes(urlParams); + const { transactionTypes } = useApmService(); const { serviceName } = useParams<{ serviceName?: string }>(); const { start, end, transactionType } = urlParams; const { environmentOptions } = useEnvironments({ serviceName, start, end }); diff --git a/x-pack/plugins/apm/public/components/alerting/TransactionErrorRateAlertTrigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/TransactionErrorRateAlertTrigger/index.tsx index a9ad212393ac4..464409ed332e8 100644 --- a/x-pack/plugins/apm/public/components/alerting/TransactionErrorRateAlertTrigger/index.tsx +++ b/x-pack/plugins/apm/public/components/alerting/TransactionErrorRateAlertTrigger/index.tsx @@ -8,7 +8,6 @@ import React from 'react'; import { ForLastExpression } from '../../../../../triggers_actions_ui/public'; import { ALERT_TYPES_CONFIG, AlertType } from '../../../../common/alert_types'; import { useEnvironments } from '../../../hooks/useEnvironments'; -import { useServiceTransactionTypes } from '../../../hooks/useServiceTransactionTypes'; import { useUrlParams } from '../../../hooks/useUrlParams'; import { ServiceAlertTrigger } from '../ServiceAlertTrigger'; @@ -19,6 +18,7 @@ import { EnvironmentField, IsAboveField, } from '../fields'; +import { useApmService } from '../../../hooks/use_apm_service'; interface AlertParams { windowSize: number; @@ -38,7 +38,7 @@ interface Props { export function TransactionErrorRateAlertTrigger(props: Props) { const { setAlertParams, alertParams, setAlertProperty } = props; const { urlParams } = useUrlParams(); - const transactionTypes = useServiceTransactionTypes(urlParams); + const { transactionTypes } = useApmService(); const { serviceName } = useParams<{ serviceName?: string }>(); const { start, end, transactionType } = urlParams; const { environmentOptions } = useEnvironments({ serviceName, start, end }); diff --git a/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx b/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx index 63fb69d6d7cbf..ce8f2b0ba611a 100644 --- a/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx +++ b/x-pack/plugins/apm/public/components/app/Main/route_config/index.tsx @@ -7,6 +7,7 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { Redirect, RouteComponentProps } from 'react-router-dom'; +import { ApmServiceContextProvider } from '../../../../context/apm_service_context'; import { UNIDENTIFIED_SERVICE_NODES_LABEL } from '../../../../../common/i18n'; import { SERVICE_NODE_NAME_MISSING } from '../../../../../common/service_nodes'; import { APMRouteDefinition } from '../../../../application/routes'; @@ -227,19 +228,19 @@ export const routes: APMRouteDefinition[] = [ breadcrumb: i18n.translate('xpack.apm.breadcrumb.overviewTitle', { defaultMessage: 'Overview', }), - component: ServiceDetailsOverview, + component: withApmServiceContext(ServiceDetailsOverview), } as APMRouteDefinition<{ serviceName: string }>, // errors { exact: true, path: '/services/:serviceName/errors/:groupId', - component: ErrorGroupDetails, + component: withApmServiceContext(ErrorGroupDetails), breadcrumb: ({ match }) => match.params.groupId, } as APMRouteDefinition<{ groupId: string; serviceName: string }>, { exact: true, path: '/services/:serviceName/errors', - component: ServiceDetailsErrors, + component: withApmServiceContext(ServiceDetailsErrors), breadcrumb: i18n.translate('xpack.apm.breadcrumb.errorsTitle', { defaultMessage: 'Errors', }), @@ -248,7 +249,7 @@ export const routes: APMRouteDefinition[] = [ { exact: true, path: '/services/:serviceName/transactions', - component: ServiceDetailsTransactions, + component: withApmServiceContext(ServiceDetailsTransactions), breadcrumb: i18n.translate('xpack.apm.breadcrumb.transactionsTitle', { defaultMessage: 'Transactions', }), @@ -257,7 +258,7 @@ export const routes: APMRouteDefinition[] = [ { exact: true, path: '/services/:serviceName/metrics', - component: ServiceDetailsMetrics, + component: withApmServiceContext(ServiceDetailsMetrics), breadcrumb: i18n.translate('xpack.apm.breadcrumb.metricsTitle', { defaultMessage: 'Metrics', }), @@ -266,7 +267,7 @@ export const routes: APMRouteDefinition[] = [ { exact: true, path: '/services/:serviceName/nodes', - component: ServiceDetailsNodes, + component: withApmServiceContext(ServiceDetailsNodes), breadcrumb: i18n.translate('xpack.apm.breadcrumb.nodesTitle', { defaultMessage: 'JVMs', }), @@ -275,7 +276,7 @@ export const routes: APMRouteDefinition[] = [ { exact: true, path: '/services/:serviceName/nodes/:serviceNodeName/metrics', - component: ServiceNodeMetrics, + component: withApmServiceContext(ServiceNodeMetrics), breadcrumb: ({ match }) => { const { serviceNodeName } = match.params; @@ -289,12 +290,20 @@ export const routes: APMRouteDefinition[] = [ { exact: true, path: '/services/:serviceName/transactions/view', - component: TransactionDetails, + component: withApmServiceContext(TransactionDetails), breadcrumb: ({ location }) => { const query = toQuery(location.search); return query.transactionName as string; }, }, + { + exact: true, + path: '/services/:serviceName/service-map', + component: withApmServiceContext(ServiceDetailsServiceMap), + breadcrumb: i18n.translate('xpack.apm.breadcrumb.serviceMapTitle', { + defaultMessage: 'Service Map', + }), + }, { exact: true, path: '/link-to/trace/:traceId', @@ -309,14 +318,6 @@ export const routes: APMRouteDefinition[] = [ defaultMessage: 'Service Map', }), }, - { - exact: true, - path: '/services/:serviceName/service-map', - component: ServiceDetailsServiceMap, - breadcrumb: i18n.translate('xpack.apm.breadcrumb.serviceMapTitle', { - defaultMessage: 'Service Map', - }), - }, { exact: true, path: '/settings/customize-ui', @@ -337,3 +338,13 @@ export const routes: APMRouteDefinition[] = [ ), }, ]; + +function withApmServiceContext(WrappedComponent: React.ComponentType) { + return (props: any) => { + return ( + + + + ); + }; +} diff --git a/x-pack/plugins/apm/public/components/app/service_details/service_detail_tabs.tsx b/x-pack/plugins/apm/public/components/app/service_details/service_detail_tabs.tsx index 92eb3753e7989..003bd6ba4c122 100644 --- a/x-pack/plugins/apm/public/components/app/service_details/service_detail_tabs.tsx +++ b/x-pack/plugins/apm/public/components/app/service_details/service_detail_tabs.tsx @@ -9,7 +9,6 @@ import { i18n } from '@kbn/i18n'; import React, { ReactNode } from 'react'; import { isJavaAgentName, isRumAgentName } from '../../../../common/agent_name'; import { enableServiceOverview } from '../../../../common/ui_settings_keys'; -import { useAgentName } from '../../../hooks/useAgentName'; import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; import { useErrorOverviewHref } from '../../shared/Links/apm/ErrorOverviewLink'; import { useMetricOverviewHref } from '../../shared/Links/apm/MetricOverviewLink'; @@ -24,6 +23,7 @@ import { ServiceMetrics } from '../service_metrics'; import { ServiceNodeOverview } from '../ServiceNodeOverview'; import { ServiceOverview } from '../service_overview'; import { TransactionOverview } from '../transaction_overview'; +import { useApmService } from '../../../hooks/use_apm_service'; interface Tab { key: string; @@ -44,7 +44,7 @@ interface Props { } export function ServiceDetailTabs({ serviceName, tab }: Props) { - const { agentName } = useAgentName(); + const { agentName } = useApmService(); const { uiSettings } = useApmPluginContext().core; const overviewTab = { diff --git a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx index a886c3f29d57c..11de40b47ff86 100644 --- a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx @@ -23,10 +23,10 @@ import { RouteComponentProps } from 'react-router-dom'; import styled from 'styled-components'; import { SERVICE_NODE_NAME_MISSING } from '../../../../common/service_nodes'; import { ChartPointerEventContextProvider } from '../../../context/chart_pointer_event_context'; -import { useAgentName } from '../../../hooks/useAgentName'; import { FETCH_STATUS, useFetcher } from '../../../hooks/useFetcher'; import { useServiceMetricCharts } from '../../../hooks/useServiceMetricCharts'; import { useUrlParams } from '../../../hooks/useUrlParams'; +import { useApmService } from '../../../hooks/use_apm_service'; import { px, truncate, unit } from '../../../style/variables'; import { ApmHeader } from '../../shared/ApmHeader'; import { MetricsChart } from '../../shared/charts/metrics_chart'; @@ -58,7 +58,7 @@ type ServiceNodeMetricsProps = RouteComponentProps<{ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) { const { urlParams, uiFilters } = useUrlParams(); const { serviceName, serviceNodeName } = match.params; - const { agentName } = useAgentName(); + const { agentName } = useApmService(); const { data } = useServiceMetricCharts( urlParams, agentName, diff --git a/x-pack/plugins/apm/public/components/app/service_overview/index.tsx b/x-pack/plugins/apm/public/components/app/service_overview/index.tsx index ddf3107a8ab1e..15125128d9781 100644 --- a/x-pack/plugins/apm/public/components/app/service_overview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_overview/index.tsx @@ -21,6 +21,7 @@ import { TransactionErrorRateChart } from '../../shared/charts/transaction_error import { ServiceMapLink } from '../../shared/Links/apm/ServiceMapLink'; import { SearchBar } from '../../shared/search_bar'; import { ServiceOverviewErrorsTable } from './service_overview_errors_table'; +import { ServiceOverviewThroughputChart } from './service_overview_throughput_chart'; import { ServiceOverviewTransactionsTable } from './service_overview_transactions_table'; import { TableLinkFlexItem } from './table_link_flex_item'; @@ -64,18 +65,7 @@ export function ServiceOverview({ - - -

- {i18n.translate( - 'xpack.apm.serviceOverview.trafficChartTitle', - { - defaultMessage: 'Traffic', - } - )} -

- - + diff --git a/x-pack/plugins/apm/public/components/app/service_overview/service_overview.test.tsx b/x-pack/plugins/apm/public/components/app/service_overview/service_overview.test.tsx index e4ef7428ba8d4..b364f027538a6 100644 --- a/x-pack/plugins/apm/public/components/app/service_overview/service_overview.test.tsx +++ b/x-pack/plugins/apm/public/components/app/service_overview/service_overview.test.tsx @@ -72,6 +72,7 @@ describe('ServiceOverview', () => { sort: { direction: 'desc', field: 'test field' }, }, totalItemCount: 0, + throughput: [], }, refetch: () => {}, status: FETCH_STATUS.SUCCESS, diff --git a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_throughput_chart.tsx b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_throughput_chart.tsx new file mode 100644 index 0000000000000..94d92bfbe89dd --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_throughput_chart.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { EuiPanel, EuiTitle } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { useParams } from 'react-router-dom'; +import { asTransactionRate } from '../../../../common/utils/formatters'; +import { useFetcher } from '../../../hooks/useFetcher'; +import { useTheme } from '../../../hooks/useTheme'; +import { useUrlParams } from '../../../hooks/useUrlParams'; +import { useApmService } from '../../../hooks/use_apm_service'; +import { callApmApi } from '../../../services/rest/createCallApmApi'; +import { TimeseriesChart } from '../../shared/charts/timeseries_chart'; + +export function ServiceOverviewThroughputChart({ + height, +}: { + height?: number; +}) { + const theme = useTheme(); + const { serviceName } = useParams<{ serviceName?: string }>(); + const { urlParams, uiFilters } = useUrlParams(); + const { transactionType } = useApmService(); + const { start, end } = urlParams; + + const { data, status } = useFetcher(() => { + if (serviceName && transactionType && start && end) { + return callApmApi({ + endpoint: 'GET /api/apm/services/{serviceName}/throughput', + params: { + path: { + serviceName, + }, + query: { + start, + end, + transactionType, + uiFilters: JSON.stringify(uiFilters), + }, + }, + }); + } + }, [serviceName, start, end, uiFilters, transactionType]); + + return ( + + +

+ {i18n.translate('xpack.apm.serviceOverview.throughtputChartTitle', { + defaultMessage: 'Traffic', + })} +

+ + + + ); +} diff --git a/x-pack/plugins/apm/public/components/app/transaction_overview/index.tsx b/x-pack/plugins/apm/public/components/app/transaction_overview/index.tsx index 45a6114c88afd..28a27c034265a 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_overview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_overview/index.tsx @@ -24,11 +24,9 @@ import { useTrackPageview } from '../../../../../observability/public'; import { Projection } from '../../../../common/projections'; import { TRANSACTION_PAGE_LOAD } from '../../../../common/transaction_types'; import { IUrlParams } from '../../../context/UrlParamsContext/types'; -import { useServiceTransactionTypes } from '../../../hooks/useServiceTransactionTypes'; import { useTransactionCharts } from '../../../hooks/useTransactionCharts'; import { useTransactionList } from '../../../hooks/useTransactionList'; import { useUrlParams } from '../../../hooks/useUrlParams'; -import { useTransactionType } from '../../../hooks/use_transaction_type'; import { TransactionCharts } from '../../shared/charts/transaction_charts'; import { ElasticDocsLink } from '../../shared/Links/ElasticDocsLink'; import { fromQuery, toQuery } from '../../shared/Links/url_helpers'; @@ -39,6 +37,7 @@ import { Correlations } from '../Correlations'; import { TransactionList } from './TransactionList'; import { useRedirect } from './useRedirect'; import { UserExperienceCallout } from './user_experience_callout'; +import { useApmService } from '../../../hooks/use_apm_service'; function getRedirectLocation({ location, @@ -69,8 +68,7 @@ interface TransactionOverviewProps { export function TransactionOverview({ serviceName }: TransactionOverviewProps) { const location = useLocation(); const { urlParams } = useUrlParams(); - const transactionType = useTransactionType(); - const serviceTransactionTypes = useServiceTransactionTypes(urlParams); + const { transactionType, transactionTypes } = useApmService(); // redirect to first transaction type useRedirect(getRedirectLocation({ location, transactionType, urlParams })); @@ -122,9 +120,7 @@ export function TransactionOverview({ serviceName }: TransactionOverviewProps) { - + diff --git a/x-pack/plugins/apm/public/components/app/transaction_overview/transaction_overview.test.tsx b/x-pack/plugins/apm/public/components/app/transaction_overview/transaction_overview.test.tsx index 2d7992feb3760..d4a8b3a46991c 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_overview/transaction_overview.test.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_overview/transaction_overview.test.tsx @@ -11,10 +11,12 @@ import React from 'react'; import { Router } from 'react-router-dom'; import { createKibanaReactContext } from 'src/plugins/kibana_react/public'; import { MockApmPluginContextWrapper } from '../../../context/ApmPluginContext/MockApmPluginContext'; +import { ApmServiceContextProvider } from '../../../context/apm_service_context'; import { UrlParamsProvider } from '../../../context/UrlParamsContext'; import { IUrlParams } from '../../../context/UrlParamsContext/types'; import * as useFetcherHook from '../../../hooks/useFetcher'; -import * as useServiceTransactionTypesHook from '../../../hooks/useServiceTransactionTypes'; +import * as useServiceTransactionTypesHook from '../../../hooks/use_service_transaction_types'; +import * as useServiceAgentNameHook from '../../../hooks/use_service_agent_name'; import { disableConsoleWarning, renderWithTheme, @@ -37,19 +39,23 @@ function setup({ urlParams: IUrlParams; serviceTransactionTypes: string[]; }) { - const defaultLocation = { + history.replace({ pathname: '/services/foo/transactions', search: fromQuery(urlParams), - } as any; - - history.replace({ - ...defaultLocation, }); + // mock transaction types jest .spyOn(useServiceTransactionTypesHook, 'useServiceTransactionTypes') .mockReturnValue(serviceTransactionTypes); + // mock agent + jest.spyOn(useServiceAgentNameHook, 'useServiceAgentName').mockReturnValue({ + agentName: 'nodejs', + error: undefined, + status: useFetcherHook.FETCH_STATUS.SUCCESS, + }); + jest.spyOn(useFetcherHook, 'useFetcher').mockReturnValue({} as any); return renderWithTheme( @@ -57,7 +63,9 @@ function setup({ - + + + @@ -80,7 +88,7 @@ describe('TransactionOverview', () => { jest.clearAllMocks(); }); - describe('when no transaction type is given', () => { + describe('when no transaction type is given in urlParams', () => { it('should redirect to first type', () => { setup({ serviceTransactionTypes: ['firstType', 'secondType'], diff --git a/x-pack/plugins/apm/public/context/apm_service_context.test.tsx b/x-pack/plugins/apm/public/context/apm_service_context.test.tsx new file mode 100644 index 0000000000000..eb08cc22a0549 --- /dev/null +++ b/x-pack/plugins/apm/public/context/apm_service_context.test.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getTransactionType } from './apm_service_context'; + +describe('getTransactionType', () => { + describe('with transaction type in url', () => { + it('returns the transaction type in the url ', () => { + expect( + getTransactionType({ + transactionTypes: ['worker', 'request'], + urlParams: { transactionType: 'custom' }, + agentName: 'nodejs', + }) + ).toBe('custom'); + }); + }); + + describe('with no transaction types', () => { + it('returns undefined', () => { + expect( + getTransactionType({ + transactionTypes: [], + urlParams: {}, + }) + ).toBeUndefined(); + }); + }); + + describe('with a non-rum agent', () => { + describe('with default transaction type', () => { + it('returns "request"', () => { + expect( + getTransactionType({ + transactionTypes: ['worker', 'request'], + urlParams: {}, + agentName: 'nodejs', + }) + ).toEqual('request'); + }); + }); + + describe('with no default transaction type', () => { + it('returns the first type', () => { + expect( + getTransactionType({ + transactionTypes: ['worker', 'custom'], + urlParams: {}, + agentName: 'nodejs', + }) + ).toEqual('worker'); + }); + }); + }); + + describe('with a rum agent', () => { + it('returns "page-load"', () => { + expect( + getTransactionType({ + transactionTypes: ['http-request', 'page-load'], + urlParams: {}, + agentName: 'js-base', + }) + ).toEqual('page-load'); + }); + }); +}); diff --git a/x-pack/plugins/apm/public/context/apm_service_context.tsx b/x-pack/plugins/apm/public/context/apm_service_context.tsx new file mode 100644 index 0000000000000..2f1b33dea5aa6 --- /dev/null +++ b/x-pack/plugins/apm/public/context/apm_service_context.tsx @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { createContext, ReactNode } from 'react'; +import { isRumAgentName } from '../../common/agent_name'; +import { + TRANSACTION_PAGE_LOAD, + TRANSACTION_REQUEST, +} from '../../common/transaction_types'; +import { useServiceTransactionTypes } from '../hooks/use_service_transaction_types'; +import { useUrlParams } from '../hooks/useUrlParams'; +import { useServiceAgentName } from '../hooks/use_service_agent_name'; +import { IUrlParams } from './UrlParamsContext/types'; + +export const APMServiceContext = createContext<{ + agentName?: string; + transactionType?: string; + transactionTypes: string[]; +}>({ transactionTypes: [] }); + +export function ApmServiceContextProvider({ + children, +}: { + children: ReactNode; +}) { + const { urlParams } = useUrlParams(); + const { agentName } = useServiceAgentName(); + const transactionTypes = useServiceTransactionTypes(); + const transactionType = getTransactionType({ + urlParams, + transactionTypes, + agentName, + }); + + return ( + + ); +} + +export function getTransactionType({ + urlParams, + transactionTypes, + agentName, +}: { + urlParams: IUrlParams; + transactionTypes: string[]; + agentName?: string; +}) { + if (urlParams.transactionType) { + return urlParams.transactionType; + } + + if (!agentName || transactionTypes.length === 0) { + return; + } + + // The default transaction type is "page-load" for RUM agents and "request" for all others + const defaultTransactionType = isRumAgentName(agentName) + ? TRANSACTION_PAGE_LOAD + : TRANSACTION_REQUEST; + + // If the default transaction type is not in transactionTypes the first in the list is returned + return transactionTypes.includes(defaultTransactionType) + ? defaultTransactionType + : transactionTypes[0]; +} diff --git a/x-pack/plugins/apm/public/hooks/useComponentId.tsx b/x-pack/plugins/apm/public/hooks/use_apm_service.ts similarity index 56% rename from x-pack/plugins/apm/public/hooks/useComponentId.tsx rename to x-pack/plugins/apm/public/hooks/use_apm_service.ts index c1de5c8ba3971..bc80c3771c39d 100644 --- a/x-pack/plugins/apm/public/hooks/useComponentId.tsx +++ b/x-pack/plugins/apm/public/hooks/use_apm_service.ts @@ -4,12 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useRef } from 'react'; +import { useContext } from 'react'; +import { APMServiceContext } from '../context/apm_service_context'; -let uniqueId = 0; -const getUniqueId = () => uniqueId++; - -export function useComponentId() { - const idRef = useRef(getUniqueId()); - return idRef.current; +export function useApmService() { + return useContext(APMServiceContext); } diff --git a/x-pack/plugins/apm/public/hooks/useAgentName.ts b/x-pack/plugins/apm/public/hooks/use_service_agent_name.ts similarity index 81% rename from x-pack/plugins/apm/public/hooks/useAgentName.ts rename to x-pack/plugins/apm/public/hooks/use_service_agent_name.ts index b226971762fab..199f14532f7b4 100644 --- a/x-pack/plugins/apm/public/hooks/useAgentName.ts +++ b/x-pack/plugins/apm/public/hooks/use_service_agent_name.ts @@ -3,16 +3,16 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + import { useParams } from 'react-router-dom'; import { useFetcher } from './useFetcher'; import { useUrlParams } from './useUrlParams'; -export function useAgentName() { +export function useServiceAgentName() { const { serviceName } = useParams<{ serviceName?: string }>(); const { urlParams } = useUrlParams(); const { start, end } = urlParams; - - const { data: agentName, error, status } = useFetcher( + const { data, error, status } = useFetcher( (callApmApi) => { if (serviceName && start && end) { return callApmApi({ @@ -21,15 +21,11 @@ export function useAgentName() { path: { serviceName }, query: { start, end }, }, - }).then((res) => res.agentName); + }); } }, [serviceName, start, end] ); - return { - agentName, - status, - error, - }; + return { agentName: data?.agentName, status, error }; } diff --git a/x-pack/plugins/apm/public/hooks/useServiceTransactionTypes.tsx b/x-pack/plugins/apm/public/hooks/use_service_transaction_types.tsx similarity index 86% rename from x-pack/plugins/apm/public/hooks/useServiceTransactionTypes.tsx rename to x-pack/plugins/apm/public/hooks/use_service_transaction_types.tsx index 5f778e3d8834b..9d8892ac79b7d 100644 --- a/x-pack/plugins/apm/public/hooks/useServiceTransactionTypes.tsx +++ b/x-pack/plugins/apm/public/hooks/use_service_transaction_types.tsx @@ -5,13 +5,14 @@ */ import { useParams } from 'react-router-dom'; -import { IUrlParams } from '../context/UrlParamsContext/types'; import { useFetcher } from './useFetcher'; +import { useUrlParams } from './useUrlParams'; const INITIAL_DATA = { transactionTypes: [] }; -export function useServiceTransactionTypes(urlParams: IUrlParams) { +export function useServiceTransactionTypes() { const { serviceName } = useParams<{ serviceName?: string }>(); + const { urlParams } = useUrlParams(); const { start, end } = urlParams; const { data = INITIAL_DATA } = useFetcher( (callApmApi) => { diff --git a/x-pack/plugins/apm/public/hooks/use_transaction_breakdown.ts b/x-pack/plugins/apm/public/hooks/use_transaction_breakdown.ts index 686501c1eef4c..f1671ed7aa6d9 100644 --- a/x-pack/plugins/apm/public/hooks/use_transaction_breakdown.ts +++ b/x-pack/plugins/apm/public/hooks/use_transaction_breakdown.ts @@ -7,13 +7,13 @@ import { useParams } from 'react-router-dom'; import { useFetcher } from './useFetcher'; import { useUrlParams } from './useUrlParams'; -import { useTransactionType } from './use_transaction_type'; +import { useApmService } from './use_apm_service'; export function useTransactionBreakdown() { const { serviceName } = useParams<{ serviceName?: string }>(); const { urlParams, uiFilters } = useUrlParams(); const { start, end, transactionName } = urlParams; - const transactionType = useTransactionType(); + const { transactionType } = useApmService(); const { data = { timeseries: undefined }, error, status } = useFetcher( (callApmApi) => { diff --git a/x-pack/plugins/apm/public/hooks/use_transaction_type.ts b/x-pack/plugins/apm/public/hooks/use_transaction_type.ts deleted file mode 100644 index fd4e6516f9ca3..0000000000000 --- a/x-pack/plugins/apm/public/hooks/use_transaction_type.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { getFirstTransactionType } from '../../common/agent_name'; -import { useAgentName } from './useAgentName'; -import { useServiceTransactionTypes } from './useServiceTransactionTypes'; -import { useUrlParams } from './useUrlParams'; - -/** - * Get either the transaction type from the URL parameters, "request" - * (for non-RUM agents), "page-load" (for RUM agents) if this service uses them, - * or the first available transaction type. - */ -export function useTransactionType() { - const { agentName } = useAgentName(); - const { urlParams } = useUrlParams(); - const transactionTypeFromUrlParams = urlParams.transactionType; - const transactionTypes = useServiceTransactionTypes(urlParams); - const firstTransactionType = getFirstTransactionType( - transactionTypes, - agentName - ); - - return transactionTypeFromUrlParams ?? firstTransactionType; -} diff --git a/x-pack/plugins/apm/server/lib/services/get_throughput.ts b/x-pack/plugins/apm/server/lib/services/get_throughput.ts new file mode 100644 index 0000000000000..0ac0ad17ef8fa --- /dev/null +++ b/x-pack/plugins/apm/server/lib/services/get_throughput.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ESFilter } from '../../../../../typings/elasticsearch'; +import { PromiseReturnType } from '../../../../observability/typings/common'; +import { + SERVICE_NAME, + TRANSACTION_TYPE, +} from '../../../common/elasticsearch_fieldnames'; +import { rangeFilter } from '../../../common/utils/range_filter'; +import { + getDocumentTypeFilterForAggregatedTransactions, + getProcessorEventForAggregatedTransactions, +} from '../helpers/aggregated_transactions'; +import { getBucketSize } from '../helpers/get_bucket_size'; +import { Setup, SetupTimeRange } from '../helpers/setup_request'; + +interface Options { + searchAggregatedTransactions: boolean; + serviceName: string; + setup: Setup & SetupTimeRange; + transactionType: string; +} + +type ESResponse = PromiseReturnType; + +function transform(response: ESResponse) { + const buckets = response.aggregations?.throughput?.buckets ?? []; + return buckets.map(({ key: x, doc_count: y }) => ({ x, y })); +} + +async function fetcher({ + searchAggregatedTransactions, + serviceName, + setup, + transactionType, +}: Options) { + const { start, end, apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end }); + const filter: ESFilter[] = [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + { range: rangeFilter(start, end) }, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ...setup.esFilter, + ]; + + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { bool: { filter } }, + aggs: { + throughput: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { min: start, max: end }, + }, + }, + }, + }, + }; + + return apmEventClient.search(params); +} + +export async function getThroughput(options: Options) { + return { + throughput: transform(await fetcher(options)), + }; +} diff --git a/x-pack/plugins/apm/server/routes/create_apm_api.ts b/x-pack/plugins/apm/server/routes/create_apm_api.ts index 019482dd44485..9334ce60a3f9e 100644 --- a/x-pack/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/plugins/apm/server/routes/create_apm_api.ts @@ -22,6 +22,7 @@ import { serviceAnnotationsRoute, serviceAnnotationsCreateRoute, serviceErrorGroupsRoute, + serviceThroughputRoute, serviceTransactionGroupsRoute, } from './services'; import { @@ -117,6 +118,7 @@ const createApmApi = () => { .add(serviceAnnotationsRoute) .add(serviceAnnotationsCreateRoute) .add(serviceErrorGroupsRoute) + .add(serviceThroughputRoute) .add(serviceTransactionGroupsRoute) // Agent configuration diff --git a/x-pack/plugins/apm/server/routes/services.ts b/x-pack/plugins/apm/server/routes/services.ts index 5e02fad2155ad..4c5738ecef581 100644 --- a/x-pack/plugins/apm/server/routes/services.ts +++ b/x-pack/plugins/apm/server/routes/services.ts @@ -20,6 +20,7 @@ import { getSearchAggregatedTransactions } from '../lib/helpers/aggregated_trans import { getServiceErrorGroups } from '../lib/services/get_service_error_groups'; import { toNumberRt } from '../../common/runtime_types/to_number_rt'; import { getServiceTransactionGroups } from '../lib/services/get_service_transaction_groups'; +import { getThroughput } from '../lib/services/get_throughput'; export const servicesRoute = createRoute({ endpoint: 'GET /api/apm/services', @@ -246,6 +247,36 @@ export const serviceErrorGroupsRoute = createRoute({ }, }); +export const serviceThroughputRoute = createRoute({ + endpoint: 'GET /api/apm/services/{serviceName}/throughput', + params: t.type({ + path: t.type({ + serviceName: t.string, + }), + query: t.intersection([ + t.type({ transactionType: t.string }), + uiFiltersRt, + rangeRt, + ]), + }), + options: { tags: ['access:apm'] }, + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + const { serviceName } = context.params.path; + const { transactionType } = context.params.query; + const searchAggregatedTransactions = await getSearchAggregatedTransactions( + setup + ); + + return getThroughput({ + searchAggregatedTransactions, + serviceName, + setup, + transactionType, + }); + }, +}); + export const serviceTransactionGroupsRoute = createRoute({ endpoint: 'GET /api/apm/services/{serviceName}/overview_transaction_groups', params: t.type({ diff --git a/x-pack/test/apm_api_integration/basic/tests/index.ts b/x-pack/test/apm_api_integration/basic/tests/index.ts index e9bc59df96108..27e9528a658a9 100644 --- a/x-pack/test/apm_api_integration/basic/tests/index.ts +++ b/x-pack/test/apm_api_integration/basic/tests/index.ts @@ -16,9 +16,10 @@ export default function apmApiIntegrationTests({ loadTestFile }: FtrProviderCont }); describe('Services', function () { + loadTestFile(require.resolve('./services/agent_name')); loadTestFile(require.resolve('./services/annotations')); + loadTestFile(require.resolve('./services/throughput')); loadTestFile(require.resolve('./services/top_services')); - loadTestFile(require.resolve('./services/agent_name')); loadTestFile(require.resolve('./services/transaction_types')); }); diff --git a/x-pack/test/apm_api_integration/basic/tests/services/__snapshots__/throughput.snap b/x-pack/test/apm_api_integration/basic/tests/services/__snapshots__/throughput.snap new file mode 100644 index 0000000000000..434660cdc2c62 --- /dev/null +++ b/x-pack/test/apm_api_integration/basic/tests/services/__snapshots__/throughput.snap @@ -0,0 +1,250 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Throughput when data is loaded returns the service throughput has the correct throughput 1`] = ` +Array [ + Object { + "x": 1601389800000, + "y": 6, + }, + Object { + "x": 1601389830000, + "y": 0, + }, + Object { + "x": 1601389860000, + "y": 0, + }, + Object { + "x": 1601389890000, + "y": 0, + }, + Object { + "x": 1601389920000, + "y": 3, + }, + Object { + "x": 1601389950000, + "y": 1, + }, + Object { + "x": 1601389980000, + "y": 0, + }, + Object { + "x": 1601390010000, + "y": 0, + }, + Object { + "x": 1601390040000, + "y": 3, + }, + Object { + "x": 1601390070000, + "y": 2, + }, + Object { + "x": 1601390100000, + "y": 0, + }, + Object { + "x": 1601390130000, + "y": 0, + }, + Object { + "x": 1601390160000, + "y": 7, + }, + Object { + "x": 1601390190000, + "y": 3, + }, + Object { + "x": 1601390220000, + "y": 2, + }, + Object { + "x": 1601390250000, + "y": 0, + }, + Object { + "x": 1601390280000, + "y": 0, + }, + Object { + "x": 1601390310000, + "y": 8, + }, + Object { + "x": 1601390340000, + "y": 0, + }, + Object { + "x": 1601390370000, + "y": 0, + }, + Object { + "x": 1601390400000, + "y": 3, + }, + Object { + "x": 1601390430000, + "y": 0, + }, + Object { + "x": 1601390460000, + "y": 0, + }, + Object { + "x": 1601390490000, + "y": 0, + }, + Object { + "x": 1601390520000, + "y": 4, + }, + Object { + "x": 1601390550000, + "y": 3, + }, + Object { + "x": 1601390580000, + "y": 2, + }, + Object { + "x": 1601390610000, + "y": 0, + }, + Object { + "x": 1601390640000, + "y": 1, + }, + Object { + "x": 1601390670000, + "y": 2, + }, + Object { + "x": 1601390700000, + "y": 0, + }, + Object { + "x": 1601390730000, + "y": 0, + }, + Object { + "x": 1601390760000, + "y": 4, + }, + Object { + "x": 1601390790000, + "y": 1, + }, + Object { + "x": 1601390820000, + "y": 1, + }, + Object { + "x": 1601390850000, + "y": 0, + }, + Object { + "x": 1601390880000, + "y": 6, + }, + Object { + "x": 1601390910000, + "y": 0, + }, + Object { + "x": 1601390940000, + "y": 3, + }, + Object { + "x": 1601390970000, + "y": 0, + }, + Object { + "x": 1601391000000, + "y": 4, + }, + Object { + "x": 1601391030000, + "y": 0, + }, + Object { + "x": 1601391060000, + "y": 1, + }, + Object { + "x": 1601391090000, + "y": 0, + }, + Object { + "x": 1601391120000, + "y": 2, + }, + Object { + "x": 1601391150000, + "y": 1, + }, + Object { + "x": 1601391180000, + "y": 2, + }, + Object { + "x": 1601391210000, + "y": 0, + }, + Object { + "x": 1601391240000, + "y": 1, + }, + Object { + "x": 1601391270000, + "y": 0, + }, + Object { + "x": 1601391300000, + "y": 1, + }, + Object { + "x": 1601391330000, + "y": 0, + }, + Object { + "x": 1601391360000, + "y": 1, + }, + Object { + "x": 1601391390000, + "y": 0, + }, + Object { + "x": 1601391420000, + "y": 0, + }, + Object { + "x": 1601391450000, + "y": 0, + }, + Object { + "x": 1601391480000, + "y": 10, + }, + Object { + "x": 1601391510000, + "y": 3, + }, + Object { + "x": 1601391540000, + "y": 1, + }, + Object { + "x": 1601391570000, + "y": 0, + }, + Object { + "x": 1601391600000, + "y": 0, + }, +] +`; diff --git a/x-pack/test/apm_api_integration/basic/tests/services/throughput.ts b/x-pack/test/apm_api_integration/basic/tests/services/throughput.ts new file mode 100644 index 0000000000000..aea4213f7e657 --- /dev/null +++ b/x-pack/test/apm_api_integration/basic/tests/services/throughput.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; +import qs from 'querystring'; +import { first, last } from 'lodash'; +import archives_metadata from '../../../common/archives_metadata'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + const archiveName = 'apm_8.0.0'; + const metadata = archives_metadata[archiveName]; + + describe('Throughput', () => { + describe('when data is not loaded', () => { + it('handles the empty state', async () => { + const response = await supertest.get( + `/api/apm/services/opbeans-java/throughput?${qs.stringify({ + start: metadata.start, + end: metadata.end, + uiFilters: encodeURIComponent('{}'), + transactionType: 'request', + })}` + ); + expect(response.status).to.be(200); + expect(response.body.throughput.length).to.be(0); + }); + }); + + describe('when data is loaded', () => { + before(() => esArchiver.load(archiveName)); + after(() => esArchiver.unload(archiveName)); + + describe('returns the service throughput', () => { + let throughputResponse: { + throughput: Array<{ x: number; y: number | null }>; + }; + before(async () => { + const response = await supertest.get( + `/api/apm/services/opbeans-java/throughput?${qs.stringify({ + start: metadata.start, + end: metadata.end, + uiFilters: encodeURIComponent('{}'), + transactionType: 'request', + })}` + ); + throughputResponse = response.body; + }); + + it('returns some data', () => { + expect(throughputResponse.throughput.length).to.be.greaterThan(0); + + const nonNullDataPoints = throughputResponse.throughput.filter(({ y }) => y !== null); + + expect(nonNullDataPoints.length).to.be.greaterThan(0); + }); + + it('has the correct start date', () => { + expectSnapshot( + new Date(first(throughputResponse.throughput)?.x ?? NaN).toISOString() + ).toMatchInline(`"2020-09-29T14:30:00.000Z"`); + }); + + it('has the correct end date', () => { + expectSnapshot( + new Date(last(throughputResponse.throughput)?.x ?? NaN).toISOString() + ).toMatchInline(`"2020-09-29T15:00:00.000Z"`); + }); + + it('has the correct number of buckets', () => { + expectSnapshot(throughputResponse.throughput.length).toMatchInline(`61`); + }); + + it('has the correct throughput', () => { + expectSnapshot(throughputResponse.throughput).toMatch(); + }); + }); + }); + }); +}