diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f218cffe032b8..dee16cf2fa94c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -184,6 +184,8 @@ /x-pack/test/functional_with_es_ssl/apps/ml/ @elastic/ml-ui /x-pack/test/alerting_api_integration/spaces_only/tests/alerting/ml_rule_types/ @elastic/ml-ui /x-pack/test/alerting_api_integration/spaces_only/tests/alerting/transform_rule_types/ @elastic/ml-ui +/x-pack/test/screenshot_creation/apps/ml_docs @elastic/ml-ui +/x-pack/test/screenshot_creation/services/ml_screenshots.ts @elastic/ml-ui # ML team owns and maintains the transform plugin despite it living in the Data management section. /x-pack/plugins/transform/ @elastic/ml-ui diff --git a/package.json b/package.json index 69bf4f62918aa..f17b4017058d8 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,7 @@ "@elastic/apm-rum": "^5.10.1", "@elastic/apm-rum-react": "^1.3.3", "@elastic/apm-synthtrace": "link:bazel-bin/packages/elastic-apm-synthtrace", - "@elastic/charts": "40.2.0", + "@elastic/charts": "43.1.1", "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath", "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.1.0-canary.2", "@elastic/ems-client": "8.0.0", diff --git a/src/plugins/chart_expressions/expression_gauge/public/components/__snapshots__/gauge_component.test.tsx.snap b/src/plugins/chart_expressions/expression_gauge/public/components/__snapshots__/gauge_component.test.tsx.snap index b588c1d341a75..bd39344807643 100644 --- a/src/plugins/chart_expressions/expression_gauge/public/components/__snapshots__/gauge_component.test.tsx.snap +++ b/src/plugins/chart_expressions/expression_gauge/public/components/__snapshots__/gauge_component.test.tsx.snap @@ -4,11 +4,11 @@ exports[`GaugeComponent renders the chart 1`] = ` - - = memo( } }; - const config: HeatmapSpec['config'] = { - grid: { - stroke: { - width: - args.gridConfig.strokeWidth ?? chartTheme.axes?.gridLine?.horizontal?.strokeWidth ?? 1, - color: - args.gridConfig.strokeColor ?? - chartTheme.axes?.gridLine?.horizontal?.stroke ?? - '#D3DAE6', - }, - cellHeight: { - max: 'fill', - min: 1, + const themeOverrides: PartialTheme = { + legend: { + labelOptions: { + maxLines: args.legend.shouldTruncate ? args.legend?.maxLines ?? 1 : 0, }, }, - cell: { - maxWidth: 'fill', - maxHeight: 'fill', - label: { - visible: args.gridConfig.isCellLabelVisible ?? false, - minFontSize: 8, - maxFontSize: 18, - useGlobalMinFontSize: true, // override the min if there's a different directive upstream + heatmap: { + grid: { + stroke: { + width: + args.gridConfig.strokeWidth ?? + chartTheme.axes?.gridLine?.horizontal?.strokeWidth ?? + 1, + color: + args.gridConfig.strokeColor ?? + chartTheme.axes?.gridLine?.horizontal?.stroke ?? + '#D3DAE6', + }, + cellHeight: { + max: 'fill', + min: 1, + }, }, - border: { - strokeWidth: 0, + cell: { + maxWidth: 'fill', + maxHeight: 'fill', + label: { + visible: args.gridConfig.isCellLabelVisible ?? false, + minFontSize: 8, + maxFontSize: 18, + useGlobalMinFontSize: true, // override the min if there's a different directive upstream + }, + border: { + strokeWidth: 0, + }, + }, + yAxisLabel: { + visible: !!yAxisColumn && args.gridConfig.isYAxisLabelVisible, + // eui color subdued + textColor: chartTheme.axes?.tickLabel?.fill ?? '#6a717d', + padding: yAxisColumn?.name ? 8 : 0, + }, + xAxisLabel: { + visible: Boolean(args.gridConfig.isXAxisLabelVisible && xAxisColumn), + // eui color subdued + textColor: chartTheme.axes?.tickLabel?.fill ?? `#6a717d`, + padding: xAxisColumn?.name ? 8 : 0, + }, + brushMask: { + fill: isDarkTheme ? 'rgb(30,31,35,80%)' : 'rgb(247,247,247,50%)', + }, + brushArea: { + stroke: isDarkTheme ? 'rgb(255, 255, 255)' : 'rgb(105, 112, 125)', }, }, - yAxisLabel: { - visible: !!yAxisColumn && args.gridConfig.isYAxisLabelVisible, - // eui color subdued - textColor: chartTheme.axes?.tickLabel?.fill ?? '#6a717d', - padding: yAxisColumn?.name ? 8 : 0, - name: yAxisColumn?.name ?? '', - ...(yAxisColumn - ? { - formatter: (v: number | string) => - `${formatFactory(yAxisColumn.meta.params).convert(v) ?? ''}`, - } - : {}), - }, - xAxisLabel: { - visible: Boolean(args.gridConfig.isXAxisLabelVisible && xAxisColumn), - // eui color subdued - textColor: chartTheme.axes?.tickLabel?.fill ?? `#6a717d`, - padding: xAxisColumn?.name ? 8 : 0, - formatter: (v: number | string) => `${xValuesFormatter.convert(v) ?? ''}`, - name: xAxisColumn?.name ?? '', - }, - brushMask: { - fill: isDarkTheme ? 'rgb(30,31,35,80%)' : 'rgb(247,247,247,50%)', - }, - brushArea: { - stroke: isDarkTheme ? 'rgb(255, 255, 255)' : 'rgb(105, 112, 125)', - }, - timeZone, }; return ( @@ -456,14 +456,7 @@ export const HeatmapComponent: FC = memo( legendColorPicker={uiState ? legendColorPicker : undefined} debugState={window._echDebugStateFlag ?? false} tooltip={tooltip} - theme={{ - ...chartTheme, - legend: { - labelOptions: { - maxLines: args.legend.shouldTruncate ? args.legend?.maxLines ?? 1 : 0, - }, - }, - }} + theme={[themeOverrides, chartTheme]} xDomain={{ min: dateHistogramMeta && dateHistogramMeta.timeRange @@ -483,6 +476,7 @@ export const HeatmapComponent: FC = memo( type: 'bands', bands, }} + timeZone={timeZone} data={chartData} xAccessor={xAccessor} yAccessor={yAccessor || 'unifiedY'} @@ -490,8 +484,15 @@ export const HeatmapComponent: FC = memo( valueFormatter={valueFormatter} xScale={xScale} ySortPredicate={yAxisColumn ? getSortPredicate(yAxisColumn) : 'dataIndex'} - config={config} xSortPredicate={xAxisColumn ? getSortPredicate(xAxisColumn) : 'dataIndex'} + xAxisLabelName={xAxisColumn?.name} + yAxisLabelName={yAxisColumn?.name} + xAxisLabelFormatter={(v) => `${xValuesFormatter.convert(v) ?? ''}`} + yAxisLabelFormatter={ + yAxisColumn + ? (v) => `${formatFactory(yAxisColumn.meta.params).convert(v) ?? ''}` + : undefined + } /> diff --git a/src/plugins/chart_expressions/expression_pie/public/components/pie_vis_component.tsx b/src/plugins/chart_expressions/expression_pie/public/components/pie_vis_component.tsx index 02e1617b3b294..dff56c34e6c1a 100644 --- a/src/plugins/chart_expressions/expression_pie/public/components/pie_vis_component.tsx +++ b/src/plugins/chart_expressions/expression_pie/public/components/pie_vis_component.tsx @@ -18,6 +18,7 @@ import { TooltipProps, TooltipType, SeriesIdentifier, + PartitionLayout, } from '@elastic/charts'; import { useEuiTheme } from '@elastic/eui'; import { @@ -47,7 +48,7 @@ import { canFilter, getFilterClickData, getFilterEventData, - getConfig, + getPartitionTheme, getColumns, getSplitDimensionAccessor, getColumnByAccessor, @@ -251,8 +252,8 @@ const PieComponent = (props: PieComponentProps) => { return 1; }, [visData.rows, metricColumn]); - const config = useMemo( - () => getConfig(visParams, chartTheme, dimensions, rescaleFactor), + const themeOverrides = useMemo( + () => getPartitionTheme(visParams, chartTheme, dimensions, rescaleFactor), [chartTheme, visParams, dimensions, rescaleFactor] ); const tooltip: TooltipProps = { @@ -369,7 +370,9 @@ const PieComponent = (props: PieComponentProps) => { )} theme={[ // Chart background should be transparent for the usage at Canvas. - { ...chartTheme, background: { color: 'transparent' } }, + { background: { color: 'transparent' } }, + themeOverrides, + chartTheme, { legend: { labelOptions: { @@ -385,6 +388,8 @@ const PieComponent = (props: PieComponentProps) => { id="pie" smallMultiples={SMALL_MULTIPLES_ID} data={visData.rows} + layout={PartitionLayout.sunburst} + specialFirstInnermostSector={false} valueAccessor={(d: Datum) => getSliceValue(d, metricColumn)} percentFormatter={(d: number) => percentFormatter.convert(d / 100)} valueGetter={ @@ -400,7 +405,6 @@ const PieComponent = (props: PieComponentProps) => { : metricFieldFormatter.convert(d) } layers={layers} - config={config} topGroove={!visParams.labels.show ? 0 : undefined} /> diff --git a/src/plugins/chart_expressions/expression_pie/public/utils/get_config.ts b/src/plugins/chart_expressions/expression_pie/public/utils/get_config.ts deleted file mode 100644 index 0da439884ae68..0000000000000 --- a/src/plugins/chart_expressions/expression_pie/public/utils/get_config.ts +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { PartitionConfig, PartitionLayout, RecursivePartial, Theme } from '@elastic/charts'; -import { LabelPositions, PieVisParams, PieContainerDimensions } from '../../common/types'; - -const MAX_SIZE = 1000; - -export const getConfig = ( - visParams: PieVisParams, - chartTheme: RecursivePartial, - dimensions?: PieContainerDimensions, - rescaleFactor: number = 1 -): RecursivePartial => { - // On small multiples we want the labels to only appear inside - const isSplitChart = Boolean(visParams.dimensions.splitColumn || visParams.dimensions.splitRow); - const usingMargin = - dimensions && !isSplitChart - ? { - margin: { - top: (1 - Math.min(1, MAX_SIZE / dimensions?.height)) / 2, - bottom: (1 - Math.min(1, MAX_SIZE / dimensions?.height)) / 2, - left: (1 - Math.min(1, MAX_SIZE / dimensions?.width)) / 2, - right: (1 - Math.min(1, MAX_SIZE / dimensions?.width)) / 2, - }, - } - : null; - - const usingOuterSizeRatio = - dimensions && !isSplitChart - ? { - outerSizeRatio: - // Cap the ratio to 1 and then rescale - rescaleFactor * Math.min(MAX_SIZE / Math.min(dimensions?.width, dimensions?.height), 1), - } - : null; - const config: RecursivePartial = { - partitionLayout: PartitionLayout.sunburst, - fontFamily: chartTheme.barSeriesStyle?.displayValue?.fontFamily, - ...usingOuterSizeRatio, - specialFirstInnermostSector: false, - minFontSize: 10, - maxFontSize: 16, - linkLabel: { - maxCount: 5, - fontSize: 11, - textColor: chartTheme.axes?.axisTitle?.fill, - maxTextLength: visParams.labels.truncate ?? undefined, - }, - sectorLineStroke: chartTheme.lineSeriesStyle?.point?.fill, - sectorLineWidth: 1.5, - circlePadding: 4, - emptySizeRatio: visParams.isDonut ? visParams.emptySizeRatio : 0, - ...usingMargin, - }; - if (!visParams.labels.show) { - // Force all labels to be linked, then prevent links from showing - config.linkLabel = { maxCount: 0, maximumSection: Number.POSITIVE_INFINITY }; - } - - if (visParams.labels.last_level && visParams.labels.show) { - config.linkLabel = { - maxCount: Number.POSITIVE_INFINITY, - maximumSection: Number.POSITIVE_INFINITY, - maxTextLength: visParams.labels.truncate ?? undefined, - }; - } - - if ( - (visParams.labels.position === LabelPositions.INSIDE || isSplitChart) && - visParams.labels.show - ) { - config.linkLabel = { maxCount: 0 }; - } - return config; -}; diff --git a/src/plugins/chart_expressions/expression_pie/public/utils/get_layers.ts b/src/plugins/chart_expressions/expression_pie/public/utils/get_layers.ts index 05512eab72fe0..9268f5631e735 100644 --- a/src/plugins/chart_expressions/expression_pie/public/utils/get_layers.ts +++ b/src/plugins/chart_expressions/expression_pie/public/utils/get_layers.ts @@ -6,13 +6,7 @@ * Side Public License, v 1. */ -import { - Datum, - PartitionFillLabel, - PartitionLayer, - ShapeTreeNode, - ArrayEntry, -} from '@elastic/charts'; +import { Datum, PartitionLayer, ShapeTreeNode, ArrayEntry } from '@elastic/charts'; import { isEqual } from 'lodash'; import type { FieldFormatsStart } from 'src/plugins/field_formats/public'; import { SeriesLayer, PaletteRegistry, lightenColor } from '../../../../charts/public'; @@ -137,7 +131,7 @@ export const getLayers = ( formatter: FieldFormatsStart, syncColors: boolean ): PartitionLayer[] => { - const fillLabel: Partial = { + const fillLabel: PartitionLayer['fillLabel'] = { valueFont: { fontWeight: 700, }, diff --git a/src/plugins/chart_expressions/expression_pie/public/utils/get_config.test.ts b/src/plugins/chart_expressions/expression_pie/public/utils/get_partition_theme.test.ts similarity index 80% rename from src/plugins/chart_expressions/expression_pie/public/utils/get_config.test.ts rename to src/plugins/chart_expressions/expression_pie/public/utils/get_partition_theme.test.ts index 5eaa1bb9b2848..1cccdf8a5e47b 100644 --- a/src/plugins/chart_expressions/expression_pie/public/utils/get_config.test.ts +++ b/src/plugins/chart_expressions/expression_pie/public/utils/get_partition_theme.test.ts @@ -6,19 +6,21 @@ * Side Public License, v 1. */ -import { getConfig } from './get_config'; +import { getPartitionTheme } from './get_partition_theme'; import { createMockPieParams } from '../mocks'; const visParams = createMockPieParams(); describe('getConfig', () => { it('should cap the outerSizeRatio to 1', () => { - expect(getConfig(visParams, {}, { width: 400, height: 400 }).outerSizeRatio).toBe(1); + expect( + getPartitionTheme(visParams, {}, { width: 400, height: 400 }).partition?.outerSizeRatio + ).toBe(1); }); it('should not have outerSizeRatio for split chart', () => { expect( - getConfig( + getPartitionTheme( { ...visParams, dimensions: { @@ -37,11 +39,11 @@ describe('getConfig', () => { }, {}, { width: 400, height: 400 } - ).outerSizeRatio + ).partition?.outerSizeRatio ).toBeUndefined(); expect( - getConfig( + getPartitionTheme( { ...visParams, dimensions: { @@ -60,11 +62,11 @@ describe('getConfig', () => { }, {}, { width: 400, height: 400 } - ).outerSizeRatio + ).partition?.outerSizeRatio ).toBeUndefined(); }); it('should not set outerSizeRatio if dimensions are not defined', () => { - expect(getConfig(visParams, {}).outerSizeRatio).toBeUndefined(); + expect(getPartitionTheme(visParams, {}).partition?.outerSizeRatio).toBeUndefined(); }); }); diff --git a/src/plugins/chart_expressions/expression_pie/public/utils/get_partition_theme.ts b/src/plugins/chart_expressions/expression_pie/public/utils/get_partition_theme.ts new file mode 100644 index 0000000000000..4daaf835fa782 --- /dev/null +++ b/src/plugins/chart_expressions/expression_pie/public/utils/get_partition_theme.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PartialTheme } from '@elastic/charts'; +import { Required } from '@kbn/utility-types'; +import { LabelPositions, PieVisParams, PieContainerDimensions } from '../../common/types'; + +const MAX_SIZE = 1000; + +export const getPartitionTheme = ( + visParams: PieVisParams, + chartTheme: PartialTheme, + dimensions?: PieContainerDimensions, + rescaleFactor: number = 1 +): PartialTheme => { + // On small multiples we want the labels to only appear inside + const isSplitChart = Boolean(visParams.dimensions.splitColumn || visParams.dimensions.splitRow); + const paddingProps: PartialTheme | null = + dimensions && !isSplitChart + ? { + chartPaddings: { + // TODO: simplify ratio logic to be static px units + top: ((1 - Math.min(1, MAX_SIZE / dimensions?.height)) / 2) * dimensions?.height, + bottom: ((1 - Math.min(1, MAX_SIZE / dimensions?.height)) / 2) * dimensions?.height, + left: ((1 - Math.min(1, MAX_SIZE / dimensions?.width)) / 2) * dimensions?.height, + right: ((1 - Math.min(1, MAX_SIZE / dimensions?.width)) / 2) * dimensions?.height, + }, + } + : null; + + const outerSizeRatio: PartialTheme['partition'] | null = + dimensions && !isSplitChart + ? { + outerSizeRatio: + // Cap the ratio to 1 and then rescale + rescaleFactor * Math.min(MAX_SIZE / Math.min(dimensions?.width, dimensions?.height), 1), + } + : null; + const theme: Required = { + chartMargins: { top: 0, bottom: 0, left: 0, right: 0 }, + ...paddingProps, + partition: { + fontFamily: chartTheme.barSeriesStyle?.displayValue?.fontFamily, + ...outerSizeRatio, + minFontSize: 10, + maxFontSize: 16, + linkLabel: { + maxCount: 5, + fontSize: 11, + textColor: chartTheme.axes?.axisTitle?.fill, + maxTextLength: visParams.labels.truncate ?? undefined, + }, + sectorLineStroke: chartTheme.lineSeriesStyle?.point?.fill, + sectorLineWidth: 1.5, + circlePadding: 4, + emptySizeRatio: visParams.isDonut ? visParams.emptySizeRatio : 0, + }, + }; + if (!visParams.labels.show) { + // Force all labels to be linked, then prevent links from showing + theme.partition.linkLabel = { maxCount: 0, maximumSection: Number.POSITIVE_INFINITY }; + } + + if (visParams.labels.last_level && visParams.labels.show) { + theme.partition.linkLabel = { + maxCount: Number.POSITIVE_INFINITY, + maximumSection: Number.POSITIVE_INFINITY, + maxTextLength: visParams.labels.truncate ?? undefined, + }; + } + + if ( + (visParams.labels.position === LabelPositions.INSIDE || isSplitChart) && + visParams.labels.show + ) { + theme.partition.linkLabel = { maxCount: 0 }; + } + + return theme; +}; diff --git a/src/plugins/chart_expressions/expression_pie/public/utils/index.ts b/src/plugins/chart_expressions/expression_pie/public/utils/index.ts index 3ee51003ca1e9..e1b779c511bfc 100644 --- a/src/plugins/chart_expressions/expression_pie/public/utils/index.ts +++ b/src/plugins/chart_expressions/expression_pie/public/utils/index.ts @@ -10,7 +10,7 @@ export { getLayers } from './get_layers'; export { getColorPicker } from './get_color_picker'; export { getLegendActions } from './get_legend_actions'; export { canFilter, getFilterClickData, getFilterEventData } from './filter_helpers'; -export { getConfig } from './get_config'; +export { getPartitionTheme } from './get_partition_theme'; export { getColumns } from './get_columns'; export { getSplitDimensionAccessor } from './get_split_dimension_accessor'; export { getDistinctSeries } from './get_distinct_series'; diff --git a/src/plugins/charts/public/services/theme/theme.test.tsx b/src/plugins/charts/public/services/theme/theme.test.tsx index 079acbb5fefbc..5154c1ce5ad63 100644 --- a/src/plugins/charts/public/services/theme/theme.test.tsx +++ b/src/plugins/charts/public/services/theme/theme.test.tsx @@ -12,11 +12,11 @@ import { take } from 'rxjs/operators'; import { renderHook, act } from '@testing-library/react-hooks'; import { render, act as renderAct } from '@testing-library/react'; +import { LIGHT_THEME, DARK_THEME } from '@elastic/charts'; import { EUI_CHARTS_THEME_DARK, EUI_CHARTS_THEME_LIGHT } from '@elastic/eui/dist/eui_charts_theme'; import { ThemeService } from './theme'; import { coreMock } from '../../../../../core/public/mocks'; -import { LIGHT_THEME, DARK_THEME } from '@elastic/charts'; const { uiSettings: setupMockUiSettings } = coreMock.createSetup(); diff --git a/src/plugins/charts/public/services/theme/theme.ts b/src/plugins/charts/public/services/theme/theme.ts index 1aad4f0ab6328..4397084d890ae 100644 --- a/src/plugins/charts/public/services/theme/theme.ts +++ b/src/plugins/charts/public/services/theme/theme.ts @@ -89,9 +89,8 @@ export class ThemeService { public init(uiSettings: CoreSetup['uiSettings']) { this._uiSettingsDarkMode$ = uiSettings.get$('theme:darkMode'); this._uiSettingsDarkMode$.subscribe((darkMode) => { - this._chartsTheme$.next( - darkMode ? EUI_CHARTS_THEME_DARK.theme : EUI_CHARTS_THEME_LIGHT.theme - ); + const theme = darkMode ? EUI_CHARTS_THEME_DARK.theme : EUI_CHARTS_THEME_LIGHT.theme; + this._chartsTheme$.next(theme); this._chartsBaseTheme$.next(darkMode ? DARK_THEME : LIGHT_THEME); }); } diff --git a/src/plugins/charts/public/static/utils/transform_click_event.ts b/src/plugins/charts/public/static/utils/transform_click_event.ts index d175046b20ebb..ae255455b39b1 100644 --- a/src/plugins/charts/public/static/utils/transform_click_event.ts +++ b/src/plugins/charts/public/static/utils/transform_click_event.ts @@ -28,19 +28,21 @@ export interface BrushTriggerEvent { data: RangeSelectContext['data']; } -type AllSeriesAccessors = Array<[accessor: Accessor | AccessorFn, value: string | number]>; +type AllSeriesAccessors = Array< + [accessor: Accessor | AccessorFn, value: string | number] +>; /** * returns accessor value from string or function accessor * @param datum * @param accessor */ -function getAccessorValue(datum: Datum, accessor: Accessor | AccessorFn) { +function getAccessorValue(datum: D, accessor: Accessor | AccessorFn) { if (typeof accessor === 'function') { return accessor(datum); } - return datum[accessor]; + return (datum as Datum)[accessor]; } /** @@ -259,9 +261,11 @@ export const getFilterFromSeriesFn = /** * Helper function to transform `@elastic/charts` brush event into brush action event */ -export const getBrushFromChartBrushEventFn = - (table: Datatable, xAccessor: Accessor | AccessorFn) => - ({ x: selectedRange }: XYBrushEvent): BrushTriggerEvent => { +export function getBrushFromChartBrushEventFn( + table: Datatable, + xAccessor: Accessor | AccessorFn +) { + return ({ x: selectedRange }: XYBrushEvent): BrushTriggerEvent => { const [start, end] = selectedRange ?? [0, 0]; const range: [number, number] = [start, end]; const column = table.columns.findIndex(({ id }) => validateAccessorId(id, xAccessor)); @@ -275,3 +279,4 @@ export const getBrushFromChartBrushEventFn = name: 'brush', }; }; +} diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 2e746e4ecec93..ec380a0845985 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -206,6 +206,7 @@ export { isEsError, SearchSessionState, SortDirection, + handleResponse, } from './search'; export type { diff --git a/src/plugins/data/public/search/index.ts b/src/plugins/data/public/search/index.ts index 810436dc30b98..6923ec7e8705b 100644 --- a/src/plugins/data/public/search/index.ts +++ b/src/plugins/data/public/search/index.ts @@ -54,6 +54,7 @@ export { waitUntilNextSessionCompletes$, } from './session'; export { getEsPreference } from './es_search'; +export { handleResponse } from './fetch'; export type { SearchInterceptorDeps } from './search_interceptor'; export { SearchInterceptor } from './search_interceptor'; diff --git a/src/plugins/discover/public/components/discover_grid/get_render_cell_value.test.tsx b/src/plugins/discover/public/components/discover_grid/get_render_cell_value.test.tsx index 4479e051b1f26..07ed170258fb1 100644 --- a/src/plugins/discover/public/components/discover_grid/get_render_cell_value.test.tsx +++ b/src/plugins/discover/public/components/discover_grid/get_render_cell_value.test.tsx @@ -96,6 +96,50 @@ describe('Discover grid cell rendering', function () { expect(component.html()).toMatchInlineSnapshot(`"100"`); }); + it('renders bytes column correctly using _source when details is true', () => { + const DiscoverGridCellValue = getRenderCellValueFn( + indexPatternMock, + rowsSource, + rowsSource.map(flatten), + false, + [], + 100 + ); + const component = shallow( + + ); + expect(component.html()).toMatchInlineSnapshot(`"100"`); + }); + + it('renders bytes column correctly using fields when details is true', () => { + const DiscoverGridCellValue = getRenderCellValueFn( + indexPatternMock, + rowsFields, + rowsFields.map(flatten), + false, + [], + 100 + ); + const component = shallow( + + ); + expect(component.html()).toMatchInlineSnapshot(`"100"`); + }); + it('renders _source column correctly', () => { const DiscoverGridCellValue = getRenderCellValueFn( indexPatternMock, @@ -514,13 +558,16 @@ describe('Discover grid cell rendering', function () { /> ); expect(component).toMatchInlineSnapshot(` - - { - "object.value": [ - 100 - ] - } - + `); }); @@ -634,9 +681,15 @@ describe('Discover grid cell rendering', function () { /> ); expect(component).toMatchInlineSnapshot(` - - .gz - + `); const componentWithDetails = shallow( @@ -650,13 +703,14 @@ describe('Discover grid cell rendering', function () { /> ); expect(componentWithDetails).toMatchInlineSnapshot(` - `); }); diff --git a/src/plugins/discover/public/components/discover_grid/get_render_cell_value.tsx b/src/plugins/discover/public/components/discover_grid/get_render_cell_value.tsx index 436281b119bff..5e1a1a7e39db8 100644 --- a/src/plugins/discover/public/components/discover_grid/get_render_cell_value.tsx +++ b/src/plugins/discover/public/components/discover_grid/get_render_cell_value.tsx @@ -8,8 +8,7 @@ import React, { Fragment, useContext, useEffect } from 'react'; import { euiLightVars as themeLight, euiDarkVars as themeDark } from '@kbn/ui-theme'; -import type { DataView } from 'src/plugins/data/common'; - +import type { DataView, DataViewField } from 'src/plugins/data/common'; import { EuiDataGridCellValueElementProps, EuiDescriptionList, @@ -64,89 +63,35 @@ export const getRenderCellValueFn = return -; } - if ( + /** + * when using the fields api this code is used to show top level objects + * this is used for legacy stuff like displaying products of our ecommerce dataset + */ + const useTopLevelObjectColumns = Boolean( useNewFieldsApi && - !field && - row && - row.fields && - !(row.fields as Record)[columnId] - ) { - const innerColumns = Object.fromEntries( - Object.entries(row.fields as Record).filter(([key]) => { - return key.indexOf(`${columnId}.`) === 0; - }) - ); - if (isDetails) { - // nicely formatted JSON for the expanded view - return {JSON.stringify(innerColumns, null, 2)}; - } - - // Put the most important fields first - const highlights: Record = (row.highlight as Record) ?? {}; - const highlightPairs: Array<[string, string]> = []; - const sourcePairs: Array<[string, string]> = []; - Object.entries(innerColumns).forEach(([key, values]) => { - const subField = indexPattern.getFieldByName(key); - const displayKey = indexPattern.fields.getByName - ? indexPattern.fields.getByName(key)?.displayName - : undefined; - const formatter = subField - ? indexPattern.getFormatterForField(subField) - : { convert: (v: unknown, ...rest: unknown[]) => String(v) }; - const formatted = (values as unknown[]) - .map((val: unknown) => - formatter.convert(val, 'html', { - field: subField, - hit: row, - indexPattern, - }) - ) - .join(', '); - const pairs = highlights[key] ? highlightPairs : sourcePairs; - if (displayKey) { - if (fieldsToShow.includes(displayKey)) { - pairs.push([displayKey, formatted]); - } - } else { - pairs.push([key, formatted]); - } - }); - - return ( - // If you change the styling of this list (specifically something that will change the line-height) - // make sure to adjust the img overwrites attached to dscDiscoverGrid__descriptionListDescription - // in discover_grid.scss - - {[...highlightPairs, ...sourcePairs] - .slice(0, maxDocFieldsDisplayed) - .map(([key, value]) => ( - - {key} - - - ))} - - ); - } + !field && + row?.fields && + !(row.fields as Record)[columnId] + ); - if (typeof rowFlattened[columnId] === 'object' && isDetails) { - return ( - } - width={defaultMonacoEditorWidth} - /> + if (isDetails) { + return renderPopoverContent( + row, + rowFlattened, + field, + columnId, + indexPattern, + useTopLevelObjectColumns ); } - if (field && field.type === '_source') { - if (isDetails) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return ; - } - const pairs = formatHit(row, indexPattern, fieldsToShow); + if (field?.type === '_source' || useTopLevelObjectColumns) { + const pairs = useTopLevelObjectColumns + ? getTopLevelObjectPairs(row, columnId, indexPattern, fieldsToShow).slice( + 0, + maxDocFieldsDisplayed + ) + : formatHit(row, indexPattern, fieldsToShow); return ( @@ -163,20 +108,6 @@ export const getRenderCellValueFn = ); } - if (!field?.type && rowFlattened && typeof rowFlattened[columnId] === 'object') { - if (isDetails) { - // nicely formatted JSON for the expanded view - return ( - } - width={defaultMonacoEditorWidth} - /> - ); - } - - return <>{formatFieldValue(rowFlattened[columnId], row, indexPattern, field)}; - } - return ( ); }; + +/** + * Helper function to show top level objects + * this is used for legacy stuff like displaying products of our ecommerce dataset + */ +function getInnerColumns(fields: Record, columnId: string) { + return Object.fromEntries( + Object.entries(fields).filter(([key]) => { + return key.indexOf(`${columnId}.`) === 0; + }) + ); +} + +/** + * Helper function for the cell popover + */ +function renderPopoverContent( + rowRaw: ElasticSearchHit, + rowFlattened: Record, + field: DataViewField | undefined, + columnId: string, + dataView: DataView, + useTopLevelObjectColumns: boolean +) { + if (useTopLevelObjectColumns || field?.type === '_source') { + const json = useTopLevelObjectColumns + ? getInnerColumns(rowRaw.fields as Record, columnId) + : rowRaw; + return ( + } width={defaultMonacoEditorWidth} /> + ); + } + + return ( + + ); +} +/** + * Helper function to show top level objects + * this is used for legacy stuff like displaying products of our ecommerce dataset + */ +function getTopLevelObjectPairs( + row: ElasticSearchHit, + columnId: string, + dataView: DataView, + fieldsToShow: string[] +) { + const innerColumns = getInnerColumns(row.fields as Record, columnId); + // Put the most important fields first + const highlights: Record = (row.highlight as Record) ?? {}; + const highlightPairs: Array<[string, string]> = []; + const sourcePairs: Array<[string, string]> = []; + Object.entries(innerColumns).forEach(([key, values]) => { + const subField = dataView.getFieldByName(key); + const displayKey = dataView.fields.getByName + ? dataView.fields.getByName(key)?.displayName + : undefined; + const formatter = subField + ? dataView.getFormatterForField(subField) + : { convert: (v: unknown, ...rest: unknown[]) => String(v) }; + const formatted = (values as unknown[]) + .map((val: unknown) => + formatter.convert(val, 'html', { + field: subField, + hit: row, + indexPattern: dataView, + }) + ) + .join(', '); + const pairs = highlights[key] ? highlightPairs : sourcePairs; + if (displayKey) { + if (fieldsToShow.includes(displayKey)) { + pairs.push([displayKey, formatted]); + } + } else { + pairs.push([key, formatted]); + } + }); + return [...highlightPairs, ...sourcePairs]; +} diff --git a/src/plugins/inspector/common/adapters/request/request_responder.ts b/src/plugins/inspector/common/adapters/request/request_responder.ts index 1b8da2e57e7f2..1d3a999e4834d 100644 --- a/src/plugins/inspector/common/adapters/request/request_responder.ts +++ b/src/plugins/inspector/common/adapters/request/request_responder.ts @@ -51,7 +51,7 @@ export class RequestResponder { } public finish(status: RequestStatus, response: Response): void { - this.request.time = Date.now() - this.request.startTime; + this.request.time = response.time ?? Date.now() - this.request.startTime; this.request.status = status; this.request.response = response; this.onChange(); diff --git a/src/plugins/inspector/common/adapters/request/types.ts b/src/plugins/inspector/common/adapters/request/types.ts index a204a7aa00a4a..4e6a8d324559f 100644 --- a/src/plugins/inspector/common/adapters/request/types.ts +++ b/src/plugins/inspector/common/adapters/request/types.ts @@ -53,4 +53,5 @@ export interface RequestStatistic { export interface Response { json?: object; + time?: number; } diff --git a/src/plugins/vis_types/timelion/public/components/series/area.tsx b/src/plugins/vis_types/timelion/public/components/series/area.tsx index d149d675d63d7..50c52f69de5bb 100644 --- a/src/plugins/vis_types/timelion/public/components/series/area.tsx +++ b/src/plugins/vis_types/timelion/public/components/series/area.tsx @@ -38,30 +38,32 @@ const getPointFillColor = (points: VisSeries['points'], color: string | undefine ); }; -const getAreaSeriesStyle = ({ color, lines, points }: AreaSeriesComponentProps['visData']) => - ({ - line: { - opacity: isShowLines(lines, points) ? 1 : 0, - stroke: color, - strokeWidth: lines?.lineWidth !== undefined ? Number(lines.lineWidth) : 3, - visible: isShowLines(lines, points), - }, - area: { - fill: color, - opacity: lines?.fill ?? 0, - visible: lines?.show ?? points?.show ?? true, - }, - point: { - fill: getPointFillColor(points, color), - opacity: 1, - radius: points?.radius ?? 3, - stroke: color, - strokeWidth: points?.lineWidth ?? 2, - visible: points?.show ?? false, - shape: points?.symbol === 'cross' ? PointShape.X : points?.symbol, - }, - curve: lines?.steps ? CurveType.CURVE_STEP : CurveType.LINEAR, - } as RecursivePartial); +const getAreaSeriesStyle = ({ + color, + lines, + points, +}: AreaSeriesComponentProps['visData']): RecursivePartial => ({ + line: { + opacity: isShowLines(lines, points) ? 1 : 0, + stroke: color, + strokeWidth: lines?.lineWidth !== undefined ? Number(lines.lineWidth) : 3, + visible: isShowLines(lines, points), + }, + area: { + fill: color, + opacity: lines?.fill ?? 0, + visible: lines?.show ?? points?.show ?? true, + }, + point: { + fill: getPointFillColor(points, color), + opacity: 1, + radius: points?.radius ?? 3, + stroke: color, + strokeWidth: points?.lineWidth ?? 2, + visible: points?.show ?? false, + shape: points?.symbol === 'cross' ? PointShape.X : points?.symbol, + }, +}); export const AreaSeriesComponent = ({ index, groupId, visData }: AreaSeriesComponentProps) => ( diff --git a/src/plugins/vis_types/timeseries/common/constants.ts b/src/plugins/vis_types/timeseries/common/constants.ts index 4f15cea7faad3..30fb814990925 100644 --- a/src/plugins/vis_types/timeseries/common/constants.ts +++ b/src/plugins/vis_types/timeseries/common/constants.ts @@ -9,6 +9,7 @@ export const UI_SETTINGS = { MAX_BUCKETS_SETTING: 'metrics:max_buckets', ALLOW_STRING_INDICES: 'metrics:allowStringIndices', + ALLOW_CHECKING_FOR_FAILED_SHARDS: 'metrics:allowCheckingForFailedShards', }; export const INDEXES_SEPARATOR = ','; export const AUTO_INTERVAL = 'auto'; diff --git a/src/plugins/vis_types/timeseries/common/types/index.ts b/src/plugins/vis_types/timeseries/common/types/index.ts index 7a35532802678..01b200c6774d1 100644 --- a/src/plugins/vis_types/timeseries/common/types/index.ts +++ b/src/plugins/vis_types/timeseries/common/types/index.ts @@ -11,7 +11,15 @@ import { IndexPattern, Query } from '../../../../data/common'; import { Panel } from './panel_model'; export type { Metric, Series, Panel, MetricType } from './panel_model'; -export type { TimeseriesVisData, PanelData, SeriesData, TableData } from './vis_data'; +export type { + TimeseriesVisData, + PanelData, + SeriesData, + TableData, + DataResponseMeta, + TrackedEsSearches, + PanelSeries, +} from './vis_data'; export interface FetchedIndexPattern { indexPattern: IndexPattern | undefined | null; diff --git a/src/plugins/vis_types/timeseries/common/types/vis_data.ts b/src/plugins/vis_types/timeseries/common/types/vis_data.ts index 1a7be0b467004..07c078a6e8aae 100644 --- a/src/plugins/vis_types/timeseries/common/types/vis_data.ts +++ b/src/plugins/vis_types/timeseries/common/types/vis_data.ts @@ -7,30 +7,38 @@ */ import { PANEL_TYPES } from '../enums'; -import { TimeseriesUIRestrictions } from '../ui_restrictions'; +import type { TimeseriesUIRestrictions } from '../ui_restrictions'; export type TimeseriesVisData = SeriesData | TableData; -export interface TableData { - type: PANEL_TYPES.TABLE; +export type TrackedEsSearches = Record< + string, + { + body: Record; + label?: string; + time: number; + response?: Record; + } +>; + +export interface DataResponseMeta { + type: PANEL_TYPES; uiRestrictions: TimeseriesUIRestrictions; + trackedEsSearches: TrackedEsSearches; +} + +export interface TableData extends DataResponseMeta { series?: PanelData[]; pivot_label?: string; } // series data is not fully typed yet -export type SeriesData = { - type: Exclude; - uiRestrictions: TimeseriesUIRestrictions; +export type SeriesData = DataResponseMeta & { error?: string; -} & { - [key: string]: PanelSeries; -}; +} & Record; export interface PanelSeries { - annotations: { - [key: string]: Annotation[]; - }; + annotations: Record; id: string; series: PanelData[]; error?: string; diff --git a/src/plugins/vis_types/timeseries/kibana.json b/src/plugins/vis_types/timeseries/kibana.json index 40f934e531973..66c5b416a0d96 100644 --- a/src/plugins/vis_types/timeseries/kibana.json +++ b/src/plugins/vis_types/timeseries/kibana.json @@ -4,7 +4,7 @@ "kibanaVersion": "kibana", "server": true, "ui": true, - "requiredPlugins": ["charts", "data", "expressions", "visualizations"], + "requiredPlugins": ["charts", "data", "expressions", "visualizations", "inspector"], "optionalPlugins": ["home","usageCollection"], "requiredBundles": ["kibanaUtils", "kibanaReact", "fieldFormats"], "owner": { diff --git a/src/plugins/vis_types/timeseries/public/application/components/annotation_row.tsx b/src/plugins/vis_types/timeseries/public/application/components/annotation_row.tsx index bc408aef7092a..856948cb7601e 100644 --- a/src/plugins/vis_types/timeseries/public/application/components/annotation_row.tsx +++ b/src/plugins/vis_types/timeseries/public/application/components/annotation_row.tsx @@ -80,7 +80,9 @@ export const AnnotationRow = ({ try { fetchedIndexPattern = index - ? await fetchIndexPattern(index, indexPatterns) + ? await fetchIndexPattern(index, indexPatterns, { + fetchKibanaIndexForStringIndexes: true, + }) : { ...fetchedIndexPattern, defaultIndex: await indexPatterns.getDefault(), diff --git a/src/plugins/vis_types/timeseries/public/application/components/lib/check_if_series_have_same_formatters.test.ts b/src/plugins/vis_types/timeseries/public/application/components/lib/check_if_series_have_same_formatters.test.ts index 71aed8c7315e2..674973b1173f5 100644 --- a/src/plugins/vis_types/timeseries/public/application/components/lib/check_if_series_have_same_formatters.test.ts +++ b/src/plugins/vis_types/timeseries/public/application/components/lib/check_if_series_have_same_formatters.test.ts @@ -26,7 +26,7 @@ describe('checkIfSeriesHaveSameFormatters(seriesModel, fieldFormatMap)', () => { expect(result).toBe(true); }); - it('should return false for the different value_template series formatters', () => { + it('should return true for the different value_template series formatters', () => { const seriesModel = [ { formatter: DATA_FORMATTERS.PERCENT, @@ -39,13 +39,13 @@ describe('checkIfSeriesHaveSameFormatters(seriesModel, fieldFormatMap)', () => { ] as Series[]; const result = checkIfSeriesHaveSameFormatters(seriesModel, fieldFormatMap); - expect(result).toBe(false); + expect(result).toBe(true); }); it('should return true for the same field formatters', () => { const seriesModel = [ - { formatter: DATA_FORMATTERS.DEFAULT, metrics: [{ field: 'someField' }] }, - { formatter: DATA_FORMATTERS.DEFAULT, metrics: [{ field: 'someField' }] }, + { formatter: DATA_FORMATTERS.DEFAULT, metrics: [{ type: 'avg', field: 'someField' }] }, + { formatter: DATA_FORMATTERS.DEFAULT, metrics: [{ type: 'avg', field: 'someField' }] }, ] as Series[]; const result = checkIfSeriesHaveSameFormatters(seriesModel, fieldFormatMap); @@ -54,11 +54,11 @@ describe('checkIfSeriesHaveSameFormatters(seriesModel, fieldFormatMap)', () => { it('should return false for the different field formatters', () => { const seriesModel = [ - { formatter: DATA_FORMATTERS.DEFAULT, metrics: [{ field: 'someField' }] }, + { formatter: DATA_FORMATTERS.DEFAULT, metrics: [{ type: 'avg', field: 'someField' }] }, { formatter: DATA_FORMATTERS.DEFAULT, - metrics: [{ field: 'anotherField' }], + metrics: [{ id: 'avg', field: 'anotherField' }], }, ] as Series[]; const result = checkIfSeriesHaveSameFormatters(seriesModel, fieldFormatMap); @@ -71,9 +71,12 @@ describe('checkIfSeriesHaveSameFormatters(seriesModel, fieldFormatMap)', () => { { formatter: DATA_FORMATTERS.DEFAULT, - metrics: [{ field: 'someField' }, { field: 'field' }], + metrics: [ + { type: 'avg', field: 'someField' }, + { type: 'avg', field: 'field' }, + ], }, - { formatter: DATA_FORMATTERS.DEFAULT, metrics: [{ field: 'someField' }] }, + { formatter: DATA_FORMATTERS.DEFAULT, metrics: [{ type: 'avg', field: 'someField' }] }, ] as Series[]; const result = checkIfSeriesHaveSameFormatters(seriesModel, fieldFormatMap); diff --git a/src/plugins/vis_types/timeseries/public/application/components/lib/check_if_series_have_same_formatters.ts b/src/plugins/vis_types/timeseries/public/application/components/lib/check_if_series_have_same_formatters.ts index 44715d1262d05..c92b7e7aedd3e 100644 --- a/src/plugins/vis_types/timeseries/public/application/components/lib/check_if_series_have_same_formatters.ts +++ b/src/plugins/vis_types/timeseries/public/application/components/lib/check_if_series_have_same_formatters.ts @@ -6,8 +6,8 @@ * Side Public License, v 1. */ -import { last, isEqual } from 'lodash'; import { DATA_FORMATTERS } from '../../../../common/enums'; +import { aggs } from '../../../../common/agg_utils'; import type { Series } from '../../../../common/types'; import type { FieldFormatMap } from '../../../../../../data/common'; @@ -15,19 +15,28 @@ export const checkIfSeriesHaveSameFormatters = ( seriesModel: Series[], fieldFormatMap?: FieldFormatMap ) => { - const allSeriesHaveDefaultFormatting = seriesModel.every( - (seriesGroup) => seriesGroup.formatter === DATA_FORMATTERS.DEFAULT - ); + const uniqFormatters = new Set(); - return allSeriesHaveDefaultFormatting && fieldFormatMap - ? seriesModel - .map(({ metrics }) => fieldFormatMap[last(metrics)?.field ?? '']) - .every((fieldFormat, index, [firstSeriesFieldFormat]) => - isEqual(fieldFormat, firstSeriesFieldFormat) - ) - : seriesModel.every( - (series) => - series.formatter === seriesModel[0].formatter && - series.value_template === seriesModel[0].value_template - ); + seriesModel.forEach((seriesGroup) => { + if (seriesGroup.formatter === DATA_FORMATTERS.DEFAULT) { + const activeMetric = seriesGroup.metrics[seriesGroup.metrics.length - 1]; + const aggMeta = aggs.find((agg) => agg.id === activeMetric.type); + + if ( + activeMetric.field && + aggMeta?.meta.isFieldRequired && + fieldFormatMap?.[activeMetric.field] + ) { + return uniqFormatters.add(JSON.stringify(fieldFormatMap[activeMetric.field])); + } + } + uniqFormatters.add( + JSON.stringify({ + // requirement: in the case of using TSVB formatters, we do not need to check the value_template, just formatter! + formatter: seriesGroup.formatter, + }) + ); + }); + + return uniqFormatters.size === 1; }; diff --git a/src/plugins/vis_types/timeseries/public/application/components/lib/index_pattern_select/field_text_select.tsx b/src/plugins/vis_types/timeseries/public/application/components/lib/index_pattern_select/field_text_select.tsx index 86d1758932301..682279d5639e5 100644 --- a/src/plugins/vis_types/timeseries/public/application/components/lib/index_pattern_select/field_text_select.tsx +++ b/src/plugins/vis_types/timeseries/public/application/components/lib/index_pattern_select/field_text_select.tsx @@ -37,7 +37,7 @@ export const FieldTextSelect = ({ useDebounce( () => { - if (inputValue !== indexPatternString) { + if ((inputValue ?? '') !== (indexPatternString ?? '')) { onIndexChange(inputValue); } }, diff --git a/src/plugins/vis_types/timeseries/public/application/components/lib/index_pattern_select/index_pattern_select.tsx b/src/plugins/vis_types/timeseries/public/application/components/lib/index_pattern_select/index_pattern_select.tsx index 840787e2af1af..6c095a9074bb7 100644 --- a/src/plugins/vis_types/timeseries/public/application/components/lib/index_pattern_select/index_pattern_select.tsx +++ b/src/plugins/vis_types/timeseries/public/application/components/lib/index_pattern_select/index_pattern_select.tsx @@ -111,7 +111,7 @@ export const IndexPatternSelect = ({ label={indexPatternLabel} helpText={fetchedIndex.defaultIndex && getIndexPatternHelpText(useKibanaIndices)} labelAppend={ - fetchedIndex.indexPatternString && !fetchedIndex.indexPattern ? ( + !useKibanaIndices && fetchedIndex.indexPatternString && !fetchedIndex.indexPattern ? ( val; + : createTickFormatter(undefined, undefined, getConfig); TimeseriesVisualization.addYAxis(yAxis, { tickFormatter, diff --git a/src/plugins/vis_types/timeseries/public/application/components/vis_types/timeseries/vis.test.js b/src/plugins/vis_types/timeseries/public/application/components/vis_types/timeseries/vis.test.js index 12ae70cca1036..0fd909af7376d 100644 --- a/src/plugins/vis_types/timeseries/public/application/components/vis_types/timeseries/vis.test.js +++ b/src/plugins/vis_types/timeseries/public/application/components/vis_types/timeseries/vis.test.js @@ -153,7 +153,7 @@ describe('TimeseriesVisualization', () => { const yAxisFormattedValue = timeSeriesProps.yAxis[0].tickFormatter(value); - expect(yAxisFormattedValue).toBe(value); + expect(yAxisFormattedValue).toBe(`${value}`); }); test('should return the same stringified number from yAxis formatter for byte and percent series', () => { @@ -173,18 +173,6 @@ describe('TimeseriesVisualization', () => { expect(yAxis[0].tickFormatter(value)).toBe('500B'); }); - test('should return simple number from yAxis formatter and different values from the same byte formatters, but with different value templates', () => { - const timeSeriesProps = setupTimeSeriesProps( - ['byte', 'byte'], - ['{{value}}', '{{value}} value'] - ); - const { series, yAxis } = timeSeriesProps; - - expect(series[0].tickFormat(value)).toBe('500B'); - expect(series[1].tickFormat(value)).toBe('500B value'); - expect(yAxis[0].tickFormatter(value)).toBe(value); - }); - test('should return percent formatted value from yAxis formatter and three percent formatted series with the same value templates', () => { const timeSeriesProps = setupTimeSeriesProps(['percent', 'percent', 'percent']); @@ -204,7 +192,7 @@ describe('TimeseriesVisualization', () => { expect(series[0].tickFormat(value)).toBe('500 template'); expect(series[1].tickFormat(value)).toBe('500B template'); - expect(yAxis[0].tickFormatter(value)).toBe(value); + expect(yAxis[0].tickFormatter(value)).toBe(`${value}`); }); test('should return field formatted value for yAxis and single series with default formatter', () => { @@ -232,7 +220,7 @@ describe('TimeseriesVisualization', () => { expect(series[1].tickFormat(value)).toBe('500 years'); expect(series[2].tickFormat(value)).toBe('42 years'); expect(series[3].tickFormat(value)).toBe('$500'); - expect(yAxis[0].tickFormatter(value)).toBe(value); + expect(yAxis[0].tickFormatter(value)).toBe(`${value}`); }); test('should return simple number from yAxis formatter and correctly formatted series values', () => { @@ -243,7 +231,7 @@ describe('TimeseriesVisualization', () => { expect(series[1].tickFormat(value)).toBe('500B'); expect(series[2].tickFormat(value)).toBe('50000%'); expect(series[3].tickFormat(value)).toBe('$500'); - expect(yAxis[0].tickFormatter(value)).toBe(value); + expect(yAxis[0].tickFormatter(value)).toBe(`${value}`); }); }); }); diff --git a/src/plugins/vis_types/timeseries/public/application/visualizations/views/timeseries/decorators/__snapshots__/area_decorator.test.js.snap b/src/plugins/vis_types/timeseries/public/application/visualizations/views/timeseries/decorators/__snapshots__/area_decorator.test.js.snap index fceb9c3fdb819..7ded8e2254aa9 100644 --- a/src/plugins/vis_types/timeseries/public/application/visualizations/views/timeseries/decorators/__snapshots__/area_decorator.test.js.snap +++ b/src/plugins/vis_types/timeseries/public/application/visualizations/views/timeseries/decorators/__snapshots__/area_decorator.test.js.snap @@ -1,7 +1,7 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`src/legacy/core_plugins/metrics/public/visualizations/views/timeseries/decorators/area_decorator.js should render and match a snapshot 1`] = ` - should render and match a snapshot 1`] = ` - ({ help: '', }, }, - async fn(input, args, { getSearchSessionId, isSyncColorsEnabled, getExecutionContext }) { + async fn( + input, + args, + { getSearchSessionId, isSyncColorsEnabled, getExecutionContext, inspectorAdapters } + ) { const visParams: TimeseriesVisParams = JSON.parse(args.params); const uiState = JSON.parse(args.uiState); const syncColors = isSyncColorsEnabled?.() ?? false; @@ -65,6 +69,7 @@ export const createMetricsFn = (): TimeseriesExpressionFunctionDefinition => ({ uiState, searchSessionId: getSearchSessionId(), executionContext: getExecutionContext(), + inspectorAdapters, }); return { diff --git a/src/plugins/vis_types/timeseries/public/metrics_type.ts b/src/plugins/vis_types/timeseries/public/metrics_type.ts index a51e0a48c3212..548368b30759a 100644 --- a/src/plugins/vis_types/timeseries/public/metrics_type.ts +++ b/src/plugins/vis_types/timeseries/public/metrics_type.ts @@ -27,6 +27,7 @@ import { import { getDataStart } from './services'; import type { TimeseriesVisDefaultParams, TimeseriesVisParams } from './types'; import type { IndexPatternValue, Panel } from '../common/types'; +import { RequestAdapter } from '../../../inspector/public'; export const withReplacedIds = ( vis: Vis @@ -153,7 +154,9 @@ export const metricsVisDefinition: VisTypeDefinition< } return []; }, - inspectorAdapters: {}, + inspectorAdapters: () => ({ + requests: new RequestAdapter(), + }), requiresSearch: true, getUsedIndexPattern: getUsedIndexPatterns, }; diff --git a/src/plugins/vis_types/timeseries/public/request_handler.ts b/src/plugins/vis_types/timeseries/public/request_handler.ts index e9037c0b84a5e..bb15f32886cdc 100644 --- a/src/plugins/vis_types/timeseries/public/request_handler.ts +++ b/src/plugins/vis_types/timeseries/public/request_handler.ts @@ -6,13 +6,14 @@ * Side Public License, v 1. */ import type { KibanaExecutionContext } from 'src/core/public'; +import type { Adapters } from 'src/plugins/inspector'; import { getTimezone } from './application/lib/get_timezone'; import { getUISettings, getDataStart, getCoreStart } from './services'; -import { ROUTES } from '../common/constants'; +import { ROUTES, UI_SETTINGS } from '../common/constants'; +import { KibanaContext, handleResponse } from '../../../data/public'; import type { TimeseriesVisParams } from './types'; import type { TimeseriesVisData } from '../common/types'; -import type { KibanaContext } from '../../../data/public'; interface MetricsRequestHandlerParams { input: KibanaContext | null; @@ -20,6 +21,7 @@ interface MetricsRequestHandlerParams { visParams: TimeseriesVisParams; searchSessionId?: string; executionContext?: KibanaExecutionContext; + inspectorAdapters?: Adapters; } export const metricsRequestHandler = async ({ @@ -28,9 +30,11 @@ export const metricsRequestHandler = async ({ visParams, searchSessionId, executionContext, + inspectorAdapters, }: MetricsRequestHandlerParams): Promise => { const config = getUISettings(); const data = getDataStart(); + const theme = getCoreStart().theme; const timezone = getTimezone(config); const uiStateObj = uiState[visParams.type] ?? {}; @@ -48,7 +52,8 @@ export const metricsRequestHandler = async ({ try { const searchSessionOptions = dataSearch.session.getSearchOptions(searchSessionId); - return await getCoreStart().http.post(ROUTES.VIS_DATA, { + + const visData: TimeseriesVisData = await getCoreStart().http.post(ROUTES.VIS_DATA, { body: JSON.stringify({ timerange: { timezone, @@ -64,6 +69,21 @@ export const metricsRequestHandler = async ({ }), context: executionContext, }); + + inspectorAdapters?.requests?.reset(); + + Object.entries(visData.trackedEsSearches || {}).forEach(([key, query]) => { + inspectorAdapters?.requests + ?.start(query.label ?? key, { searchSessionId }) + .json(query.body) + .ok({ time: query.time }); + + if (query.response && config.get(UI_SETTINGS.ALLOW_CHECKING_FOR_FAILED_SHARDS)) { + handleResponse({ body: query.body }, { rawResponse: query.response }, theme); + } + }); + + return visData; } finally { if (untrackSearch && dataSearch.session.isCurrentSession(searchSessionId)) { // untrack if this search still belongs to current session diff --git a/src/plugins/vis_types/timeseries/server/lib/search_strategies/index.ts b/src/plugins/vis_types/timeseries/server/lib/search_strategies/index.ts index ca0c50a79564a..721e1dad473f0 100644 --- a/src/plugins/vis_types/timeseries/server/lib/search_strategies/index.ts +++ b/src/plugins/vis_types/timeseries/server/lib/search_strategies/index.ts @@ -11,6 +11,7 @@ import { AbstractSearchStrategy } from './strategies'; export { SearchStrategyRegistry } from './search_strategy_registry'; export { AbstractSearchStrategy, RollupSearchStrategy, DefaultSearchStrategy } from './strategies'; +export type { EsSearchRequest } from './strategies/abstract_search_strategy'; export type SearchCapabilities = DefaultSearchCapabilities; export type SearchStrategy = AbstractSearchStrategy; diff --git a/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.ts b/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.ts index 6216bce00fc7d..1a52132612f71 100644 --- a/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.ts +++ b/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.ts @@ -9,7 +9,7 @@ import { IndexPatternsService } from '../../../../../../data/common'; import { from } from 'rxjs'; -import { AbstractSearchStrategy } from './abstract_search_strategy'; +import { AbstractSearchStrategy, EsSearchRequest } from './abstract_search_strategy'; import type { FieldSpec } from '../../../../../../data/common'; import type { CachedIndexPatternFetcher } from '../lib/cached_index_pattern_fetcher'; import type { @@ -64,7 +64,7 @@ describe('AbstractSearchStrategy', () => { }); test('should return response', async () => { - const searches = [{ body: 'body', index: 'index' }]; + const searches: EsSearchRequest[] = [{ body: {}, index: 'index' }]; const responses = await abstractSearchStrategy.search( requestContext, @@ -84,7 +84,7 @@ describe('AbstractSearchStrategy', () => { expect(requestContext.search.search).toHaveBeenCalledWith( { params: { - body: 'body', + body: {}, index: 'index', }, indexType: undefined, diff --git a/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts b/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts index bce07d2cdb300..1d3650ccedbd3 100644 --- a/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts +++ b/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts @@ -6,40 +6,67 @@ * Side Public License, v 1. */ +import { tap } from 'rxjs/operators'; +import { omit } from 'lodash'; import { IndexPatternsService } from '../../../../../../data/server'; import { toSanitizedFieldType } from '../../../../common/fields_utils'; -import type { FetchedIndexPattern } from '../../../../common/types'; +import type { FetchedIndexPattern, TrackedEsSearches } from '../../../../common/types'; import type { VisTypeTimeseriesRequest, VisTypeTimeseriesRequestHandlerContext, VisTypeTimeseriesVisDataRequest, } from '../../../types'; +export interface EsSearchRequest { + body: Record; + index?: string; + trackingEsSearchMeta?: { + requestId: string; + requestLabel?: string; + }; +} + export abstract class AbstractSearchStrategy { async search( requestContext: VisTypeTimeseriesRequestHandlerContext, req: VisTypeTimeseriesVisDataRequest, - bodies: any[], + esRequests: EsSearchRequest[], + trackedEsSearches?: TrackedEsSearches, indexType?: string ) { const requests: any[] = []; - bodies.forEach((body) => { + esRequests.forEach(({ body, index, trackingEsSearchMeta }) => { + const startTime = Date.now(); requests.push( requestContext.search .search( { indexType, params: { - ...body, + body, + index, }, }, req.body.searchSession ) + .pipe( + tap((data) => { + if (trackingEsSearchMeta?.requestId && trackedEsSearches) { + trackedEsSearches[trackingEsSearchMeta.requestId] = { + body, + time: Date.now() - startTime, + label: trackingEsSearchMeta.requestLabel, + response: omit(data.rawResponse, 'aggregations'), + }; + } + }) + ) .toPromise() ); }); + return Promise.all(requests); } diff --git a/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/rollup_search_strategy.ts b/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/rollup_search_strategy.ts index e3ede57774224..2508c68066017 100644 --- a/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/rollup_search_strategy.ts +++ b/src/plugins/vis_types/timeseries/server/lib/search_strategies/strategies/rollup_search_strategy.ts @@ -10,10 +10,10 @@ import { getCapabilitiesForRollupIndices, IndexPatternsService, } from '../../../../../../data/server'; -import { AbstractSearchStrategy } from './abstract_search_strategy'; +import { AbstractSearchStrategy, EsSearchRequest } from './abstract_search_strategy'; import { RollupSearchCapabilities } from '../capabilities/rollup_search_capabilities'; -import type { FetchedIndexPattern } from '../../../../common/types'; +import type { FetchedIndexPattern, TrackedEsSearches } from '../../../../common/types'; import type { CachedIndexPatternFetcher } from '../lib/cached_index_pattern_fetcher'; import type { VisTypeTimeseriesRequest, @@ -29,9 +29,10 @@ export class RollupSearchStrategy extends AbstractSearchStrategy { async search( requestContext: VisTypeTimeseriesRequestHandlerContext, req: VisTypeTimeseriesVisDataRequest, - bodies: any[] + esRequests: EsSearchRequest[], + trackedEsSearches?: TrackedEsSearches ) { - return super.search(requestContext, req, bodies, 'rollup'); + return super.search(requestContext, req, esRequests, trackedEsSearches, 'rollup'); } async getRollupData( diff --git a/src/plugins/vis_types/timeseries/server/lib/vis_data/annotations/get_request_params.ts b/src/plugins/vis_types/timeseries/server/lib/vis_data/annotations/get_request_params.ts index 1973e3b85b966..41f7e7c86708f 100644 --- a/src/plugins/vis_types/timeseries/server/lib/vis_data/annotations/get_request_params.ts +++ b/src/plugins/vis_types/timeseries/server/lib/vis_data/annotations/get_request_params.ts @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ - +import { i18n } from '@kbn/i18n'; import type { Annotation, Panel } from '../../../../common/types'; import { buildAnnotationRequest } from './build_request_body'; import type { @@ -13,7 +13,7 @@ import type { VisTypeTimeseriesRequestServices, VisTypeTimeseriesVisDataRequest, } from '../../../types'; -import type { SearchStrategy, SearchCapabilities } from '../../search_strategies'; +import type { SearchStrategy, SearchCapabilities, EsSearchRequest } from '../../search_strategies'; export type AnnotationServices = VisTypeTimeseriesRequestServices & { capabilities: SearchCapabilities; @@ -32,7 +32,7 @@ export async function getAnnotationRequestParams( uiSettings, cachedIndexPatternFetcher, }: AnnotationServices -) { +): Promise { const annotationIndex = await cachedIndexPatternFetcher(annotation.index_pattern); const request = await buildAnnotationRequest({ @@ -52,5 +52,14 @@ export async function getAnnotationRequestParams( runtime_mappings: annotationIndex.indexPattern?.getComputedFields().runtimeFields ?? {}, timeout: esShardTimeout > 0 ? `${esShardTimeout}ms` : undefined, }, + trackingEsSearchMeta: { + requestId: annotation.id, + requestLabel: i18n.translate('visTypeTimeseries.annotationRequest.label', { + defaultMessage: 'Annotation: {id}', + values: { + id: annotation.id, + }, + }), + }, }; } diff --git a/src/plugins/vis_types/timeseries/server/lib/vis_data/get_annotations.ts b/src/plugins/vis_types/timeseries/server/lib/vis_data/get_annotations.ts index 8a005deccaea9..481ddc7891817 100644 --- a/src/plugins/vis_types/timeseries/server/lib/vis_data/get_annotations.ts +++ b/src/plugins/vis_types/timeseries/server/lib/vis_data/get_annotations.ts @@ -10,8 +10,7 @@ import { handleAnnotationResponse } from './response_processors/annotations'; import { AnnotationServices, getAnnotationRequestParams } from './annotations/get_request_params'; import { getLastSeriesTimestamp } from './helpers/timestamp'; import type { VisTypeTimeseriesVisDataRequest } from '../../types'; -import type { Annotation, Panel } from '../../../common/types'; -import type { PanelSeries } from '../../../common/types/vis_data'; +import type { Annotation, Panel, TrackedEsSearches, PanelSeries } from '../../../common/types'; function validAnnotation(annotation: Annotation) { return annotation.fields && annotation.icon && annotation.template && !annotation.hidden; @@ -22,26 +21,33 @@ interface GetAnnotationsParams { panel: Panel; series: Array; services: AnnotationServices; + trackedEsSearches: TrackedEsSearches; } -export async function getAnnotations({ req, panel, series, services }: GetAnnotationsParams) { +export async function getAnnotations({ + req, + panel, + series, + services, + trackedEsSearches, +}: GetAnnotationsParams) { const annotations = panel.annotations!.filter(validAnnotation); const lastSeriesTimestamp = getLastSeriesTimestamp(series); const handleAnnotationResponseBy = handleAnnotationResponse(lastSeriesTimestamp); - const bodiesPromises = annotations.map((annotation) => - getAnnotationRequestParams(req, panel, annotation, services) - ); - - const searches = (await Promise.all(bodiesPromises)).reduce( - (acc, items) => acc.concat(items as any), - [] + const searches = await Promise.all( + annotations.map((annotation) => getAnnotationRequestParams(req, panel, annotation, services)) ); if (!searches.length) return { responses: [] }; try { - const data = await services.searchStrategy.search(services.requestContext, req, searches); + const data = await services.searchStrategy.search( + services.requestContext, + req, + searches, + trackedEsSearches + ); return annotations.reduce((acc, annotation, index) => { acc[annotation.id] = handleAnnotationResponseBy(data[index].rawResponse, annotation); diff --git a/src/plugins/vis_types/timeseries/server/lib/vis_data/get_series_data.ts b/src/plugins/vis_types/timeseries/server/lib/vis_data/get_series_data.ts index e67592271728d..9b111b0469d22 100644 --- a/src/plugins/vis_types/timeseries/server/lib/vis_data/get_series_data.ts +++ b/src/plugins/vis_types/timeseries/server/lib/vis_data/get_series_data.ts @@ -20,7 +20,7 @@ import type { VisTypeTimeseriesVisDataRequest, VisTypeTimeseriesRequestServices, } from '../../types'; -import type { Panel } from '../../../common/types'; +import type { Panel, DataResponseMeta } from '../../../common/types'; import { PANEL_TYPES } from '../../../common/enums'; export async function getSeriesData( @@ -52,13 +52,14 @@ export async function getSeriesData( } const { searchStrategy, capabilities } = strategy; - const meta = { + const handleError = handleErrorResponse(panel); + + const meta: DataResponseMeta = { type: panel.type, uiRestrictions: capabilities.uiRestrictions, + trackedEsSearches: {}, }; - const handleError = handleErrorResponse(panel); - try { const bodiesPromises = getActiveSeries(panel).map((series) => { isAggSupported(series.metrics, capabilities); @@ -80,7 +81,7 @@ export async function getSeriesData( ); const searches = await Promise.all(bodiesPromises); - const data = await searchStrategy.search(requestContext, req, searches); + const data = await searchStrategy.search(requestContext, req, searches, meta.trackedEsSearches); const series = await Promise.all( data.map( @@ -101,6 +102,7 @@ export async function getSeriesData( searchStrategy, capabilities, }, + trackedEsSearches: meta.trackedEsSearches, }); } diff --git a/src/plugins/vis_types/timeseries/server/lib/vis_data/get_table_data.ts b/src/plugins/vis_types/timeseries/server/lib/vis_data/get_table_data.ts index d810fba50abce..2b63749fac642 100644 --- a/src/plugins/vis_types/timeseries/server/lib/vis_data/get_table_data.ts +++ b/src/plugins/vis_types/timeseries/server/lib/vis_data/get_table_data.ts @@ -24,7 +24,8 @@ import type { VisTypeTimeseriesRequestServices, VisTypeTimeseriesVisDataRequest, } from '../../types'; -import type { Panel } from '../../../common/types'; +import type { Panel, DataResponseMeta } from '../../../common/types'; +import type { EsSearchRequest } from '../search_strategies'; export async function getTableData( requestContext: VisTypeTimeseriesRequestHandlerContext, @@ -69,11 +70,11 @@ export async function getTableData( return panel.pivot_id; }; - const meta = { + const meta: DataResponseMeta = { type: panel.type, uiRestrictions: capabilities.uiRestrictions, + trackedEsSearches: {}, }; - const handleError = handleErrorResponse(panel); try { @@ -88,29 +89,38 @@ export async function getTableData( throw new PivotNotSelectedForTableError(); } - const body = await buildTableRequest({ - req, - panel, - esQueryConfig: services.esQueryConfig, - seriesIndex: panelIndex, - capabilities, - uiSettings: services.uiSettings, - buildSeriesMetaParams: () => - services.buildSeriesMetaParams(panelIndex, Boolean(panel.use_kibana_indexes)), - }); - - const [resp] = await searchStrategy.search(requestContext, req, [ + const searches: EsSearchRequest[] = [ { + index: panelIndex.indexPatternString, body: { - ...body, + ...(await buildTableRequest({ + req, + panel, + esQueryConfig: services.esQueryConfig, + seriesIndex: panelIndex, + capabilities, + uiSettings: services.uiSettings, + buildSeriesMetaParams: () => + services.buildSeriesMetaParams(panelIndex, Boolean(panel.use_kibana_indexes)), + })), runtime_mappings: panelIndex.indexPattern?.getComputedFields().runtimeFields ?? {}, }, - index: panelIndex.indexPatternString, + trackingEsSearchMeta: { + requestId: panel.id, + requestLabel: i18n.translate('visTypeTimeseries.tableRequest.label', { + defaultMessage: 'Table: {id}', + values: { + id: panel.id, + }, + }), + }, }, - ]); + ]; + + const data = await searchStrategy.search(requestContext, req, searches, meta.trackedEsSearches); const buckets = get( - resp.rawResponse ? resp.rawResponse : resp, + data[0].rawResponse ? data[0].rawResponse : data[0], 'aggregations.pivot.buckets', [] ); diff --git a/src/plugins/vis_types/timeseries/server/lib/vis_data/series/get_request_params.ts b/src/plugins/vis_types/timeseries/server/lib/vis_data/series/get_request_params.ts index 046b207050ca0..d176eb8b99392 100644 --- a/src/plugins/vis_types/timeseries/server/lib/vis_data/series/get_request_params.ts +++ b/src/plugins/vis_types/timeseries/server/lib/vis_data/series/get_request_params.ts @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ - +import { i18n } from '@kbn/i18n'; import { buildRequestBody } from './build_request_body'; import type { FetchedIndexPattern, Panel, Series } from '../../../../common/types'; @@ -13,7 +13,7 @@ import type { VisTypeTimeseriesRequestServices, VisTypeTimeseriesVisDataRequest, } from '../../../types'; -import type { SearchCapabilities } from '../../search_strategies'; +import type { SearchCapabilities, EsSearchRequest } from '../../search_strategies'; export async function getSeriesRequestParams( req: VisTypeTimeseriesVisDataRequest, @@ -28,7 +28,7 @@ export async function getSeriesRequestParams( cachedIndexPatternFetcher, buildSeriesMetaParams, }: VisTypeTimeseriesRequestServices -) { +): Promise { let seriesIndex = panelIndex; if (series.override_index_pattern) { @@ -53,5 +53,14 @@ export async function getSeriesRequestParams( runtime_mappings: seriesIndex.indexPattern?.getComputedFields().runtimeFields ?? {}, timeout: esShardTimeout > 0 ? `${esShardTimeout}ms` : undefined, }, + trackingEsSearchMeta: { + requestId: series.id, + requestLabel: i18n.translate('visTypeTimeseries.seriesRequest.label', { + defaultMessage: 'Series: {id}', + values: { + id: series.id, + }, + }), + }, }; } diff --git a/src/plugins/vis_types/timeseries/server/ui_settings.ts b/src/plugins/vis_types/timeseries/server/ui_settings.ts index 2adbc31482f04..c64d5771479b6 100644 --- a/src/plugins/vis_types/timeseries/server/ui_settings.ts +++ b/src/plugins/vis_types/timeseries/server/ui_settings.ts @@ -36,4 +36,18 @@ export const getUiSettings: () => Record = () => ({ }), schema: schema.boolean(), }, + [UI_SETTINGS.ALLOW_CHECKING_FOR_FAILED_SHARDS]: { + name: i18n.translate('visTypeTimeseries.advancedSettings.allowCheckingForFailedShardsTitle', { + defaultMessage: 'Show TSVB request shard failures', + }), + value: true, + description: i18n.translate( + 'visTypeTimeseries.advancedSettings.allowCheckingForFailedShardsText', + { + defaultMessage: + 'Show warning message for partial data in TSVB charts if the request succeeds for some shards but fails for others.', + } + ), + schema: schema.boolean(), + }, }); diff --git a/src/plugins/vis_types/xy/public/components/xy_settings.tsx b/src/plugins/vis_types/xy/public/components/xy_settings.tsx index 304b0756c30b6..1393fc252fcf6 100644 --- a/src/plugins/vis_types/xy/public/components/xy_settings.tsx +++ b/src/plugins/vis_types/xy/public/components/xy_settings.tsx @@ -11,7 +11,7 @@ import React, { FC } from 'react'; import { Direction, Settings, - SettingsSpecProps, + SettingsProps, DomainRange, Position, PartialTheme, @@ -50,7 +50,7 @@ type XYSettingsProps = Pick< | 'xAxis' | 'orderBucketsBySum' > & { - onPointerUpdate: SettingsSpecProps['onPointerUpdate']; + onPointerUpdate: SettingsProps['onPointerUpdate']; xDomain?: DomainRange; adjustedXDomain?: DomainRange; showLegend: boolean; diff --git a/src/plugins/visualizations/public/visualize_app/components/visualize_top_nav.tsx b/src/plugins/visualizations/public/visualize_app/components/visualize_top_nav.tsx index bc15cbc83ace0..0ef26a8b72f05 100644 --- a/src/plugins/visualizations/public/visualize_app/components/visualize_top_nav.tsx +++ b/src/plugins/visualizations/public/visualize_app/components/visualize_top_nav.tsx @@ -68,8 +68,8 @@ const TopNav = ({ // start a new session to make sure all data is up to date services.data.search.session.start(); - await visInstance.embeddableHandler.reload(); - }, [visInstance.embeddableHandler, services.data.search.session]); + // embeddable handler is subscribed to session service and will refresh + }, [services.data.search.session]); const handleRefresh = useCallback( (_payload: any, isUpdate?: boolean) => { diff --git a/src/plugins/visualizations/public/visualize_app/utils/use/use_editor_updates.ts b/src/plugins/visualizations/public/visualize_app/utils/use/use_editor_updates.ts index fd962f3f1afbd..a8a6fa3eea2a4 100644 --- a/src/plugins/visualizations/public/visualize_app/utils/use/use_editor_updates.ts +++ b/src/plugins/visualizations/public/visualize_app/utils/use/use_editor_updates.ts @@ -60,12 +60,16 @@ export const useEditorUpdates = ( timeRange: timefilter.getTime(), filters: filterManager.getFilters(), query: queryString.getQuery(), + searchSessionId: services.data.search.session.getSessionId(), }); } }; const subscriptions = state$.subscribe({ - next: reloadVisualization, + next: () => { + services.data.search.session.start(); + reloadVisualization(); + }, error: services.fatalErrors.add, }); @@ -97,6 +101,16 @@ export const useEditorUpdates = ( vis.uiState.on('change', updateOnChange); + const sessionSubscription = services.data.search.session + .getSession$() + .subscribe((sessionId) => { + if (embeddableHandler.getInput().searchSessionId !== sessionId) { + embeddableHandler.updateInput({ + searchSessionId: sessionId, + }); + } + }); + const unsubscribeStateUpdates = appState.subscribe((state) => { setCurrentAppState(state); if (savedVis && savedVis.id && !services.history.location.pathname.includes(savedVis.id)) { @@ -152,6 +166,7 @@ export const useEditorUpdates = ( subscriptions.unsubscribe(); vis.uiState.off('change', updateOnChange); unsubscribeStateUpdates(); + sessionSubscription.unsubscribe(); }; } }, [appState, eventEmitter, visInstance, services, setHasUnsavedChanges, visEditorController]); diff --git a/test/functional/apps/visualize/_tsvb_chart.ts b/test/functional/apps/visualize/_tsvb_chart.ts index 1958cc4699f92..4080ca2a0ba75 100644 --- a/test/functional/apps/visualize/_tsvb_chart.ts +++ b/test/functional/apps/visualize/_tsvb_chart.ts @@ -52,8 +52,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await visualBuilder.clickDataTab('metric'); }); - it('should not have inspector enabled', async () => { - await inspector.expectIsNotEnabled(); + it('should have inspector enabled', async () => { + await inspector.expectIsEnabled(); }); it('should show correct data', async () => { diff --git a/test/functional/page_objects/visualize_chart_page.ts b/test/functional/page_objects/visualize_chart_page.ts index dc36197034691..a3fbd631722f5 100644 --- a/test/functional/page_objects/visualize_chart_page.ts +++ b/test/functional/page_objects/visualize_chart_page.ts @@ -255,7 +255,7 @@ export class VisualizeChartPageObject extends FtrService { if (isVisTypeHeatmapChart) { const legendItems = - (await this.getEsChartDebugState(heatmapChartSelector))?.legend!.items ?? []; + (await this.getEsChartDebugState(heatmapChartSelector))?.legend?.items ?? []; return legendItems.map(({ name }) => name); } diff --git a/x-pack/plugins/apm/common/alert_types.ts b/x-pack/plugins/apm/common/alert_types.ts index 68ca22c41ec92..04288fccf0a05 100644 --- a/x-pack/plugins/apm/common/alert_types.ts +++ b/x-pack/plugins/apm/common/alert_types.ts @@ -7,9 +7,14 @@ import { i18n } from '@kbn/i18n'; import type { ValuesType } from 'utility-types'; -import type { AsDuration, AsPercent } from '../../observability/common'; +import type { + AsDuration, + AsPercent, + TimeUnitChar, +} from '../../observability/common'; import type { ActionGroup } from '../../alerting/common'; import { ANOMALY_SEVERITY, ANOMALY_THRESHOLD } from './ml_constants'; +import { formatDurationFromTimeUnitChar } from '../../observability/common'; export const APM_SERVER_FEATURE_ID = 'apm'; @@ -33,17 +38,25 @@ export function formatErrorCountReason({ threshold, measured, serviceName, + windowSize, + windowUnit, }: { threshold: number; measured: number; serviceName: string; + windowSize: number; + windowUnit: string; }) { return i18n.translate('xpack.apm.alertTypes.errorCount.reason', { - defaultMessage: `Error count is greater than {threshold} (current value is {measured}) for {serviceName}`, + defaultMessage: `Error count is {measured} in the last {interval} for {serviceName}. Alert when > {threshold}.`, values: { threshold, measured, serviceName, + interval: formatDurationFromTimeUnitChar( + windowSize, + windowUnit as TimeUnitChar + ), }, }); } @@ -53,18 +66,34 @@ export function formatTransactionDurationReason({ measured, serviceName, asDuration, + aggregationType, + windowSize, + windowUnit, }: { threshold: number; measured: number; serviceName: string; asDuration: AsDuration; + aggregationType: string; + windowSize: number; + windowUnit: string; }) { + let aggregationTypeFormatted = + aggregationType.charAt(0).toUpperCase() + aggregationType.slice(1); + if (aggregationTypeFormatted === 'Avg') + aggregationTypeFormatted = aggregationTypeFormatted + '.'; + return i18n.translate('xpack.apm.alertTypes.transactionDuration.reason', { - defaultMessage: `Latency is above {threshold} (current value is {measured}) for {serviceName}`, + defaultMessage: `{aggregationType} latency is {measured} in the last {interval} for {serviceName}. Alert when > {threshold}.`, values: { threshold: asDuration(threshold), measured: asDuration(measured), serviceName, + aggregationType: aggregationTypeFormatted, + interval: formatDurationFromTimeUnitChar( + windowSize, + windowUnit as TimeUnitChar + ), }, }); } @@ -74,18 +103,26 @@ export function formatTransactionErrorRateReason({ measured, serviceName, asPercent, + windowSize, + windowUnit, }: { threshold: number; measured: number; serviceName: string; asPercent: AsPercent; + windowSize: number; + windowUnit: string; }) { return i18n.translate('xpack.apm.alertTypes.transactionErrorRate.reason', { - defaultMessage: `Failed transactions rate is greater than {threshold} (current value is {measured}) for {serviceName}`, + defaultMessage: `Failed transactions is {measured} in the last {interval} for {serviceName}. Alert when > {threshold}.`, values: { threshold: asPercent(threshold, 100), measured: asPercent(measured, 100), serviceName, + interval: formatDurationFromTimeUnitChar( + windowSize, + windowUnit as TimeUnitChar + ), }, }); } @@ -94,19 +131,27 @@ export function formatTransactionDurationAnomalyReason({ serviceName, severityLevel, measured, + windowSize, + windowUnit, }: { serviceName: string; severityLevel: string; measured: number; + windowSize: number; + windowUnit: string; }) { return i18n.translate( 'xpack.apm.alertTypes.transactionDurationAnomaly.reason', { - defaultMessage: `{severityLevel} anomaly detected for {serviceName} (score was {measured})`, + defaultMessage: `{severityLevel} anomaly with a score of {measured} was detected in the last {interval} for {serviceName}.`, values: { serviceName, severityLevel, measured, + interval: formatDurationFromTimeUnitChar( + windowSize, + windowUnit as TimeUnitChar + ), }, } ); diff --git a/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts b/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts index db50a68aa0018..3be124573728b 100644 --- a/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts +++ b/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts @@ -8,20 +8,10 @@ import { i18n } from '@kbn/i18n'; import { lazy } from 'react'; import { stringify } from 'querystring'; -import { - ALERT_EVALUATION_THRESHOLD, - ALERT_EVALUATION_VALUE, - ALERT_SEVERITY, -} from '@kbn/rule-data-utils'; +import { ALERT_REASON } from '@kbn/rule-data-utils'; import type { ObservabilityRuleTypeRegistry } from '../../../../observability/public'; import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values'; -import { - AlertType, - formatErrorCountReason, - formatTransactionDurationAnomalyReason, - formatTransactionDurationReason, - formatTransactionErrorRateReason, -} from '../../../common/alert_types'; +import { AlertType } from '../../../common/alert_types'; // copied from elasticsearch_fieldnames.ts to limit page load bundle size const SERVICE_ENVIRONMENT = 'service.environment'; @@ -49,11 +39,7 @@ export function registerApmAlerts( }), format: ({ fields }) => { return { - reason: formatErrorCountReason({ - threshold: fields[ALERT_EVALUATION_THRESHOLD]!, - measured: fields[ALERT_EVALUATION_VALUE]!, - serviceName: String(fields[SERVICE_NAME][0]), - }), + reason: fields[ALERT_REASON]!, link: format({ pathname: `/app/apm/services/${String( fields[SERVICE_NAME][0] @@ -98,12 +84,8 @@ export function registerApmAlerts( } ), format: ({ fields, formatters: { asDuration } }) => ({ - reason: formatTransactionDurationReason({ - threshold: fields[ALERT_EVALUATION_THRESHOLD]!, - measured: fields[ALERT_EVALUATION_VALUE]!, - serviceName: String(fields[SERVICE_NAME][0]), - asDuration, - }), + reason: fields[ALERT_REASON]!, + link: format({ pathname: `/app/apm/services/${fields[SERVICE_NAME][0]!}`, query: { @@ -149,12 +131,7 @@ export function registerApmAlerts( } ), format: ({ fields, formatters: { asPercent } }) => ({ - reason: formatTransactionErrorRateReason({ - threshold: fields[ALERT_EVALUATION_THRESHOLD]!, - measured: fields[ALERT_EVALUATION_VALUE]!, - serviceName: String(fields[SERVICE_NAME][0]), - asPercent, - }), + reason: fields[ALERT_REASON]!, link: format({ pathname: `/app/apm/services/${String(fields[SERVICE_NAME][0]!)}`, query: { @@ -199,11 +176,7 @@ export function registerApmAlerts( } ), format: ({ fields }) => ({ - reason: formatTransactionDurationAnomalyReason({ - serviceName: String(fields[SERVICE_NAME][0]), - severityLevel: String(fields[ALERT_SEVERITY]), - measured: Number(fields[ALERT_EVALUATION_VALUE]), - }), + reason: fields[ALERT_REASON]!, link: format({ pathname: `/app/apm/services/${String(fields[SERVICE_NAME][0])}`, query: { diff --git a/x-pack/plugins/apm/public/components/app/rum_dashboard/charts/page_load_dist_chart.tsx b/x-pack/plugins/apm/public/components/app/rum_dashboard/charts/page_load_dist_chart.tsx index 8b34ad8980774..ff9fb97197db0 100644 --- a/x-pack/plugins/apm/public/components/app/rum_dashboard/charts/page_load_dist_chart.tsx +++ b/x-pack/plugins/apm/public/components/app/rum_dashboard/charts/page_load_dist_chart.tsx @@ -121,6 +121,8 @@ export function PageLoadDistChart({ fit={Fit.Linear} id={'PagesPercentage'} name={I18LABELS.overall} + xAccessor="x" + yAccessors={['y']} xScaleType={ScaleType.Linear} yScaleType={ScaleType.Linear} data={data?.pageLoadDistribution ?? []} diff --git a/x-pack/plugins/apm/public/components/app/rum_dashboard/charts/visitor_breakdown_chart.tsx b/x-pack/plugins/apm/public/components/app/rum_dashboard/charts/visitor_breakdown_chart.tsx index 89f49a9669b45..2cdeb7be85ce3 100644 --- a/x-pack/plugins/apm/public/components/app/rum_dashboard/charts/visitor_breakdown_chart.tsx +++ b/x-pack/plugins/apm/public/components/app/rum_dashboard/charts/visitor_breakdown_chart.tsx @@ -38,9 +38,15 @@ interface Props { } const theme: PartialTheme = { + chartMargins: { top: 0, bottom: 0, left: 0, right: 0 }, legend: { verticalWidth: 100, }, + partition: { + linkLabel: { maximumSection: Infinity, maxCount: 0 }, + outerSizeRatio: 1, // - 0.5 * Math.random(), + circlePadding: 4, + }, }; export function VisitorBreakdownChart({ loading, options }: Props) { @@ -64,6 +70,8 @@ export function VisitorBreakdownChart({ loading, options }: Props) { data={ options?.length ? options : [{ count: 1, name: I18LABELS.noData }] } + layout={PartitionLayout.sunburst} + clockwiseSectors={false} valueAccessor={(d: Datum) => d.count as number} valueGetter="percent" percentFormatter={(d: number) => @@ -78,14 +86,6 @@ export function VisitorBreakdownChart({ loading, options }: Props) { }, }, ]} - config={{ - partitionLayout: PartitionLayout.sunburst, - linkLabel: { maximumSection: Infinity, maxCount: 0 }, - margin: { top: 0, bottom: 0, left: 0, right: 0 }, - outerSizeRatio: 1, // - 0.5 * Math.random(), - circlePadding: 4, - clockwiseSectors: false, - }} /> diff --git a/x-pack/plugins/apm/public/components/app/rum_dashboard/page_load_distribution/breakdown_series.tsx b/x-pack/plugins/apm/public/components/app/rum_dashboard/page_load_distribution/breakdown_series.tsx index db5932a96fb12..5e98f36cc0798 100644 --- a/x-pack/plugins/apm/public/components/app/rum_dashboard/page_load_distribution/breakdown_series.tsx +++ b/x-pack/plugins/apm/public/components/app/rum_dashboard/page_load_distribution/breakdown_series.tsx @@ -54,6 +54,8 @@ export function BreakdownSeries({ id={`${field}-${value}-${name}`} key={`${field}-${value}-${name}`} name={name} + xAccessor="x" + yAccessors={['y']} xScaleType={ScaleType.Linear} yScaleType={ScaleType.Linear} curve={CurveType.CURVE_CATMULL_ROM} diff --git a/x-pack/plugins/apm/public/components/app/service_profiling/service_profiling_flamegraph.tsx b/x-pack/plugins/apm/public/components/app/service_profiling/service_profiling_flamegraph.tsx index 6f8c1d685ba2b..e0c2483b70f88 100644 --- a/x-pack/plugins/apm/public/components/app/service_profiling/service_profiling_flamegraph.tsx +++ b/x-pack/plugins/apm/public/components/app/service_profiling/service_profiling_flamegraph.tsx @@ -12,6 +12,7 @@ import { PrimitiveValue, Settings, TooltipInfo, + PartialTheme, } from '@elastic/charts'; import { EuiCheckbox, @@ -286,6 +287,18 @@ export function ServiceProfilingFlamegraph({ }, [points, highlightFilter, data]); const chartTheme = useChartTheme(); + const themeOverrides: PartialTheme = { + chartMargins: { top: 0, bottom: 0, left: 0, right: 0 }, + partition: { + fillLabel: { + fontFamily: theme.eui.euiCodeFontFamily, + clipText: true, + }, + fontFamily: theme.eui.euiCodeFontFamily, + minFontSize: 9, + maxFontSize: 9, + }, + }; const chartSize = { height: layers.length * 20, @@ -305,7 +318,7 @@ export function ServiceProfilingFlamegraph({ ( d.value as number} valueFormatter={() => ''} - config={{ - fillLabel: { - fontFamily: theme.eui.euiCodeFontFamily, - clipText: true, - }, - drilldown: true, - fontFamily: theme.eui.euiCodeFontFamily, - minFontSize: 9, - maxFontSize: 9, - maxRowCount: 1, - partitionLayout: PartitionLayout.icicle, - }} /> diff --git a/x-pack/plugins/apm/public/components/app/service_profiling/service_profiling_timeline.tsx b/x-pack/plugins/apm/public/components/app/service_profiling/service_profiling_timeline.tsx index d5dc2f5d56afc..a625d87f05d9c 100644 --- a/x-pack/plugins/apm/public/components/app/service_profiling/service_profiling_timeline.tsx +++ b/x-pack/plugins/apm/public/components/app/service_profiling/service_profiling_timeline.tsx @@ -98,6 +98,7 @@ export function ServiceProfilingTimeline({ xScaleType={ScaleType.Time} yScaleType={ScaleType.Linear} xAccessor="x" + yAccessors={['y']} stackAccessors={['x']} /> ))} diff --git a/x-pack/plugins/apm/public/components/shared/charts/breakdown_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/breakdown_chart/index.tsx index 78bfd8911c351..a6989897641bd 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/breakdown_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/breakdown_chart/index.tsx @@ -171,7 +171,12 @@ export function BreakdownChart({ }) ) : ( // When timeseries is empty, loads an AreaSeries chart to show the default empty message. - + )} diff --git a/x-pack/plugins/apm/public/components/shared/charts/spark_plot/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/spark_plot/index.tsx index 2f38ab9cdeb4b..0843fafe0f92f 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/spark_plot/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/spark_plot/index.tsx @@ -10,11 +10,11 @@ import { Chart, CurveType, LineSeries, + PartialTheme, ScaleType, Settings, } from '@elastic/charts'; import { EuiFlexGroup, EuiFlexItem, EuiIcon } from '@elastic/eui'; -import { merge } from 'lodash'; import React from 'react'; import { useChartTheme } from '../../../../../../observability/public'; import { Coordinate } from '../../../../../typings/timeseries'; @@ -60,7 +60,7 @@ export function SparkPlot({ const comparisonChartTheme = getComparisonChartTheme(theme); const hasComparisonSeries = !!comparisonSeries?.length; - const sparkplotChartTheme = merge({}, defaultChartTheme, { + const sparkplotChartTheme: PartialTheme = { chartMargins: { left: 0, right: 0, top: 0, bottom: 0 }, lineSeriesStyle: { point: { opacity: 0 }, @@ -69,7 +69,7 @@ export function SparkPlot({ point: { opacity: 0 }, }, ...(hasComparisonSeries ? comparisonChartTheme : {}), - }); + }; const colorValue = theme.eui[color]; @@ -95,7 +95,7 @@ export function SparkPlot({ {hasValidTimeseries(series) ? ( diff --git a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx index 1cb8a4facfd69..64c070c25f94b 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/timeseries_chart.tsx @@ -126,13 +126,15 @@ export function TimeseriesChart({ onBrushEnd={(event) => onBrushEnd({ x: (event as XYBrushEvent).x, history }) } - theme={{ - ...chartTheme, - areaSeriesStyle: { - line: { visible: false }, + theme={[ + customTheme, + { + areaSeriesStyle: { + line: { visible: false }, + }, }, - ...customTheme, - }} + ...chartTheme, + ]} onPointerUpdate={setPointerEvent} externalPointerEvents={{ tooltip: { visible: true }, diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.tsx index b33f152a63016..91d3c0a727877 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_distribution_chart/index.tsx @@ -156,28 +156,31 @@ export function TransactionDistributionChart({ = ({ yScaleType={ScaleType.Linear} xAccessor="x" yAccessors={['y']} - data={chartData.length > 0 ? chartData : [{ x: 0, y: 0 }]} + data={ + chartData.length > 0 ? chartData : [{ x: 0, y: 0, dataMin: 0, dataMax: 0, percent: 0 }] + } curve={CurveType.CURVE_STEP_AFTER} /> diff --git a/x-pack/plugins/infra/server/lib/adapters/metrics/kibana_metrics_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/metrics/kibana_metrics_adapter.ts index e05a5b647ad2b..2a87c9cbca994 100644 --- a/x-pack/plugins/infra/server/lib/adapters/metrics/kibana_metrics_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/metrics/kibana_metrics_adapter.ts @@ -6,8 +6,8 @@ */ import { i18n } from '@kbn/i18n'; -import { flatten, get } from 'lodash'; import { KibanaRequest } from 'src/core/server'; +import { flatten, get } from 'lodash'; import { TIMESTAMP_FIELD } from '../../../../common/constants'; import { NodeDetailsMetricData } from '../../../../common/http_api/node_details_api'; import { KibanaFramework } from '../framework/kibana_framework_adapter'; @@ -63,7 +63,7 @@ export class KibanaMetricsAdapter implements InfraMetricsAdapter { .then((results) => { return results.filter(isVisSeriesData).map((result) => { const metricIds = Object.keys(result).filter( - (k) => !['type', 'uiRestrictions'].includes(k) + (k) => !['type', 'uiRestrictions', 'trackedEsSearches'].includes(k) ); return metricIds.map((id: string) => { diff --git a/x-pack/plugins/lens/kibana.json b/x-pack/plugins/lens/kibana.json index 884b17a085ad6..1debe6e6141b2 100644 --- a/x-pack/plugins/lens/kibana.json +++ b/x-pack/plugins/lens/kibana.json @@ -6,6 +6,7 @@ "ui": true, "requiredPlugins": [ "data", + "dataViews", "charts", "expressions", "fieldFormats", diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx index ef160b1dd682b..8cd8e4f50d625 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx @@ -14,6 +14,7 @@ import { ShapeTreeNode, HierarchyOfArrays, Chart, + PartialTheme, } from '@elastic/charts'; import { shallow } from 'enzyme'; import type { LensMultiTable } from '../../common'; @@ -110,7 +111,7 @@ describe('PieVisualization component', () => { test('it sets the correct lines per legend item', () => { const component = shallow(); - expect(component.find(Settings).prop('theme')).toEqual({ + expect(component.find(Settings).prop('theme')[0]).toMatchObject({ background: { color: undefined, }, @@ -395,7 +396,9 @@ describe('PieVisualization component', () => { const component = shallow( ); - expect(component.find(Partition).prop('config')?.outerSizeRatio).toBeCloseTo(1 / 1.05); + expect( + component.find(Settings).prop('theme')[0].partition?.outerSizeRatio + ).toBeCloseTo(1 / 1.05); }); test('it should bound the shrink the chart area to ~20% when some small slices are detected', () => { @@ -419,7 +422,9 @@ describe('PieVisualization component', () => { const component = shallow( ); - expect(component.find(Partition).prop('config')?.outerSizeRatio).toBeCloseTo(1 / 1.2); + expect( + component.find(Settings).prop('theme')[0].partition?.outerSizeRatio + ).toBeCloseTo(1 / 1.2); }); }); }); diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx index 9fb016dc0a2d7..008ab9a9cae9e 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx @@ -8,19 +8,18 @@ import { uniq } from 'lodash'; import React from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; +import { Required } from '@kbn/utility-types'; import { EuiText } from '@elastic/eui'; import { Chart, Datum, LayerValue, Partition, - PartitionConfig, PartitionLayer, - PartitionFillLabel, - RecursivePartial, Position, Settings, ElementClickListener, + PartialTheme, } from '@elastic/charts'; import { RenderMode } from 'src/plugins/expressions'; import type { LensFilterEvent } from '../types'; @@ -99,7 +98,7 @@ export function PieComponent( }); } - const fillLabel: Partial = { + const fillLabel: PartitionLayer['fillLabel'] = { valueFont: { fontWeight: 700, }, @@ -202,42 +201,52 @@ export function PieComponent( }; }); - const { legend, partitionType: partitionLayout, label: chartType } = PartitionChartsMeta[shape]; + const { legend, partitionType, label: chartType } = PartitionChartsMeta[shape]; - const config: RecursivePartial = { - partitionLayout, - fontFamily: chartTheme.barSeriesStyle?.displayValue?.fontFamily, - outerSizeRatio: 1, - specialFirstInnermostSector: true, - minFontSize: 10, - maxFontSize: 16, - // Labels are added outside the outer ring when the slice is too small - linkLabel: { - maxCount: 5, - fontSize: 11, - // Dashboard background color is affected by dark mode, which we need - // to account for in outer labels - // This does not handle non-dashboard embeddables, which are allowed to - // have different backgrounds. - textColor: chartTheme.axes?.axisTitle?.fill, + const themeOverrides: Required = { + chartMargins: { top: 0, bottom: 0, left: 0, right: 0 }, + background: { + color: undefined, // removes background for embeddables + }, + legend: { + labelOptions: { maxLines: truncateLegend ? legendMaxLines ?? 1 : 0 }, + }, + partition: { + fontFamily: chartTheme.barSeriesStyle?.displayValue?.fontFamily, + outerSizeRatio: 1, + minFontSize: 10, + maxFontSize: 16, + // Labels are added outside the outer ring when the slice is too small + linkLabel: { + maxCount: 5, + fontSize: 11, + // Dashboard background color is affected by dark mode, which we need + // to account for in outer labels + // This does not handle non-dashboard embeddables, which are allowed to + // have different backgrounds. + textColor: chartTheme.axes?.axisTitle?.fill, + }, + sectorLineStroke: chartTheme.lineSeriesStyle?.point?.fill, + sectorLineWidth: 1.5, + circlePadding: 4, }, - sectorLineStroke: chartTheme.lineSeriesStyle?.point?.fill, - sectorLineWidth: 1.5, - circlePadding: 4, }; if (isTreemapOrMosaicShape(shape)) { if (hideLabels || categoryDisplay === 'hide') { - config.fillLabel = { textColor: 'rgba(0,0,0,0)' }; + themeOverrides.partition.fillLabel = { textColor: 'rgba(0,0,0,0)' }; } } else { - config.emptySizeRatio = shape === 'donut' ? emptySizeRatio : 0; + themeOverrides.partition.emptySizeRatio = shape === 'donut' ? emptySizeRatio : 0; if (hideLabels || categoryDisplay === 'hide') { // Force all labels to be linked, then prevent links from showing - config.linkLabel = { maxCount: 0, maximumSection: Number.POSITIVE_INFINITY }; + themeOverrides.partition.linkLabel = { + maxCount: 0, + maximumSection: Number.POSITIVE_INFINITY, + }; } else if (categoryDisplay === 'inside') { // Prevent links from showing - config.linkLabel = { maxCount: 0 }; + themeOverrides.partition.linkLabel = { maxCount: 0 }; } else { // if it contains any slice below 2% reduce the ratio // first step: sum it up the overall sum @@ -246,7 +255,7 @@ export function PieComponent( const smallSlices = slices.filter((value) => value < 0.02).length; if (smallSlices) { // shrink up to 20% to give some room for the linked values - config.outerSizeRatio = 1 / (1 + Math.min(smallSlices * 0.05, 0.2)); + themeOverrides.partition.outerSizeRatio = 1 / (1 + Math.min(smallSlices * 0.05, 0.2)); } } } @@ -322,27 +331,19 @@ export function PieComponent( legendMaxDepth={nestedLegend ? undefined : 1 /* Color is based only on first layer */} onElementClick={props.interactive ?? true ? onElementClickHandler : undefined} legendAction={props.interactive ? getLegendAction(firstTable, onClickValue) : undefined} - theme={{ - ...chartTheme, - background: { - ...chartTheme.background, - color: undefined, // removes background for embeddables - }, - legend: { - labelOptions: { maxLines: truncateLegend ? legendMaxLines ?? 1 : 0 }, - }, - }} + theme={[themeOverrides, chartTheme]} baseTheme={chartBaseTheme} /> getSliceValue(d, metricColumn)} percentFormatter={(d: number) => percentFormatter.convert(d / 100)} valueGetter={hideLabels || numberDisplay === 'value' ? undefined : 'percent'} valueFormatter={(d: number) => (hideLabels ? '' : formatters[metricColumn.id].convert(d))} layers={layers} - config={config} topGroove={hideLabels || categoryDisplay === 'hide' ? 0 : undefined} /> diff --git a/x-pack/plugins/lens/public/xy_visualization/__snapshots__/expression.test.tsx.snap b/x-pack/plugins/lens/public/xy_visualization/__snapshots__/expression.test.tsx.snap index e2566aa22ce9e..1402cd715283a 100644 --- a/x-pack/plugins/lens/public/xy_visualization/__snapshots__/expression.test.tsx.snap +++ b/x-pack/plugins/lens/public/xy_visualization/__snapshots__/expression.test.tsx.snap @@ -4,7 +4,7 @@ exports[`xy_expression XYChart component it renders area 1`] = ` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - = T extends React.FunctionComponent ? P : T; -type SeriesSpec = InferPropType & - InferPropType & - InferPropType; +type SeriesSpec = LineSeriesProps & BarSeriesProps & AreaSeriesProps; export type XYChartRenderProps = XYChartProps & { chartsThemeService: ChartsPluginSetup['theme']; diff --git a/x-pack/plugins/lens/server/plugin.tsx b/x-pack/plugins/lens/server/plugin.tsx index 3f0a41efc21c7..5f8f15b21ff94 100644 --- a/x-pack/plugins/lens/server/plugin.tsx +++ b/x-pack/plugins/lens/server/plugin.tsx @@ -7,6 +7,7 @@ import { Plugin, CoreSetup, CoreStart, PluginInitializerContext, Logger } from 'src/core/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { PluginStart as DataViewsServerPluginStart } from 'src/plugins/data_views/server'; import { PluginStart as DataPluginStart, PluginSetup as DataPluginSetup, @@ -15,6 +16,7 @@ import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; import { FieldFormatsStart } from 'src/plugins/field_formats/server'; import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; import { setupRoutes } from './routes'; +import { getUiSettings } from './ui_settings'; import { registerLensUsageCollector, initializeLensTelemetry, @@ -37,6 +39,7 @@ export interface PluginStartContract { taskManager?: TaskManagerStartContract; fieldFormats: FieldFormatsStart; data: DataPluginStart; + dataViews: DataViewsServerPluginStart; } export interface LensServerPluginSetup { @@ -55,6 +58,7 @@ export class LensServerPlugin implements Plugin { + it('should remove missing fields by matching names', () => { + expect( + existingFields( + [ + { name: 'a', aggregatable: true, searchable: true, type: 'string' }, + { name: 'b', aggregatable: true, searchable: true, type: 'string' }, + ], + [ + { name: 'a', isScript: false, isMeta: false }, + { name: 'b', isScript: false, isMeta: true }, + { name: 'c', isScript: false, isMeta: false }, + ] + ) + ).toEqual(['a', 'b']); + }); + + it('should keep scripted and runtime fields', () => { + expect( + existingFields( + [{ name: 'a', aggregatable: true, searchable: true, type: 'string' }], + [ + { name: 'a', isScript: false, isMeta: false }, + { name: 'b', isScript: true, isMeta: false }, + { name: 'c', runtimeField: { type: 'keyword' }, isMeta: false, isScript: false }, + { name: 'd', isMeta: true, isScript: false }, + ] + ) + ).toEqual(['a', 'b', 'c']); + }); +}); + +describe('legacyExistingFields', () => { function field(opts: string | Partial): Field { const obj = typeof opts === 'object' ? opts : {}; const name = (typeof opts === 'string' ? opts : opts.name) || 'test'; @@ -26,7 +58,7 @@ describe('existingFields', () => { } it('should handle root level fields', () => { - const result = existingFields( + const result = legacyExistingFields( [searchResults({ foo: ['bar'] }), searchResults({ baz: [0] })], [field('foo'), field('bar'), field('baz')] ); @@ -35,7 +67,7 @@ describe('existingFields', () => { }); it('should handle basic arrays, ignoring empty ones', () => { - const result = existingFields( + const result = legacyExistingFields( [searchResults({ stuff: ['heyo', 'there'], empty: [] })], [field('stuff'), field('empty')] ); @@ -44,7 +76,7 @@ describe('existingFields', () => { }); it('should handle objects with dotted fields', () => { - const result = existingFields( + const result = legacyExistingFields( [searchResults({ 'geo.country_name': ['US'] })], [field('geo.country_name')] ); @@ -53,7 +85,7 @@ describe('existingFields', () => { }); it('supports scripted fields', () => { - const result = existingFields( + const result = legacyExistingFields( [searchResults({ bar: ['scriptvalue'] })], [field({ name: 'bar', isScript: true })] ); @@ -62,7 +94,7 @@ describe('existingFields', () => { }); it('supports runtime fields', () => { - const result = existingFields( + const result = legacyExistingFields( [searchResults({ runtime_foo: ['scriptvalue'] })], [ field({ @@ -76,7 +108,7 @@ describe('existingFields', () => { }); it('supports meta fields', () => { - const result = existingFields( + const result = legacyExistingFields( [ { // @ts-expect-error _mymeta is not defined on estypes.SearchHit diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index 3a57a77a97726..0ee1d92d1b4ec 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -11,9 +11,11 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { schema } from '@kbn/config-schema'; import { RequestHandlerContext, ElasticsearchClient } from 'src/core/server'; import { CoreSetup, Logger } from 'src/core/server'; -import { IndexPattern, IndexPatternsService, RuntimeField } from 'src/plugins/data/common'; +import { RuntimeField } from 'src/plugins/data/common'; +import { DataViewsService, DataView, FieldSpec } from 'src/plugins/data_views/common'; import { BASE_API_URL } from '../../common'; import { UI_SETTINGS } from '../../../../../src/plugins/data/server'; +import { FIELD_EXISTENCE_SETTING } from '../ui_settings'; import { PluginStartContract } from '../plugin'; export function isBoomError(error: { isBoom?: boolean }): error is Boom.Boom { @@ -53,24 +55,24 @@ export async function existingFieldsRoute(setup: CoreSetup, }, }, async (context, req, res) => { - const [{ savedObjects, elasticsearch, uiSettings }, { data }] = + const [{ savedObjects, elasticsearch, uiSettings }, { dataViews }] = await setup.getStartServices(); const savedObjectsClient = savedObjects.getScopedClient(req); - const includeFrozen: boolean = await uiSettings - .asScopedToClient(savedObjectsClient) - .get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN); + const uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient); + const [includeFrozen, useSampling]: boolean[] = await Promise.all([ + uiSettingsClient.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN), + uiSettingsClient.get(FIELD_EXISTENCE_SETTING), + ]); const esClient = elasticsearch.client.asScoped(req).asCurrentUser; try { return res.ok({ body: await fetchFieldExistence({ ...req.params, ...req.body, - indexPatternsService: await data.indexPatterns.indexPatternsServiceFactory( - savedObjectsClient, - esClient - ), + dataViewsService: await dataViews.dataViewsServiceFactory(savedObjectsClient, esClient), context, includeFrozen, + useSampling, }), }); } catch (e) { @@ -103,16 +105,64 @@ export async function existingFieldsRoute(setup: CoreSetup, async function fetchFieldExistence({ context, indexPatternId, - indexPatternsService, + dataViewsService, dslQuery = { match_all: {} }, fromDate, toDate, timeFieldName, includeFrozen, + useSampling, }: { indexPatternId: string; context: RequestHandlerContext; - indexPatternsService: IndexPatternsService; + dataViewsService: DataViewsService; + dslQuery: object; + fromDate?: string; + toDate?: string; + timeFieldName?: string; + includeFrozen: boolean; + useSampling: boolean; +}) { + if (useSampling) { + return legacyFetchFieldExistenceSampling({ + context, + indexPatternId, + dataViewsService, + dslQuery, + fromDate, + toDate, + timeFieldName, + includeFrozen, + }); + } + + const metaFields: string[] = await context.core.uiSettings.client.get(UI_SETTINGS.META_FIELDS); + const dataView = await dataViewsService.get(indexPatternId); + const allFields = buildFieldList(dataView, metaFields); + const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, { + // filled in by data views service + pattern: '', + filter: toQuery(timeFieldName, fromDate, toDate, dslQuery), + }); + return { + indexPatternTitle: dataView.title, + existingFieldNames: existingFields(existingFieldList, allFields), + }; +} + +async function legacyFetchFieldExistenceSampling({ + context, + indexPatternId, + dataViewsService, + dslQuery, + fromDate, + toDate, + timeFieldName, + includeFrozen, +}: { + indexPatternId: string; + context: RequestHandlerContext; + dataViewsService: DataViewsService; dslQuery: object; fromDate?: string; toDate?: string; @@ -120,7 +170,7 @@ async function fetchFieldExistence({ includeFrozen: boolean; }) { const metaFields: string[] = await context.core.uiSettings.client.get(UI_SETTINGS.META_FIELDS); - const indexPattern = await indexPatternsService.get(indexPatternId); + const indexPattern = await dataViewsService.get(indexPatternId); const fields = buildFieldList(indexPattern, metaFields); const docs = await fetchIndexPatternStats({ @@ -136,14 +186,14 @@ async function fetchFieldExistence({ return { indexPatternTitle: indexPattern.title, - existingFieldNames: existingFields(docs, fields), + existingFieldNames: legacyExistingFields(docs, fields), }; } /** * Exported only for unit tests. */ -export function buildFieldList(indexPattern: IndexPattern, metaFields: string[]): Field[] { +export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] { return indexPattern.fields.map((field) => { return { name: field.name, @@ -177,27 +227,7 @@ async function fetchIndexPatternStats({ fields: Field[]; includeFrozen: boolean; }) { - const filter = - timeFieldName && fromDate && toDate - ? [ - { - range: { - [timeFieldName]: { - format: 'strict_date_optional_time', - gte: fromDate, - lte: toDate, - }, - }, - }, - dslQuery, - ] - : [dslQuery]; - - const query = { - bool: { - filter, - }, - }; + const query = toQuery(timeFieldName, fromDate, toDate, dslQuery); const scriptedFields = fields.filter((f) => f.isScript); const runtimeFields = fields.filter((f) => f.runtimeField); @@ -242,10 +272,51 @@ async function fetchIndexPatternStats({ return result.hits.hits; } +function toQuery( + timeFieldName: string | undefined, + fromDate: string | undefined, + toDate: string | undefined, + dslQuery: object +) { + const filter = + timeFieldName && fromDate && toDate + ? [ + { + range: { + [timeFieldName]: { + format: 'strict_date_optional_time', + gte: fromDate, + lte: toDate, + }, + }, + }, + dslQuery, + ] + : [dslQuery]; + + const query = { + bool: { + filter, + }, + }; + return query; +} + +/** + * Exported only for unit tests. + */ +export function existingFields(filteredFields: FieldSpec[], allFields: Field[]): string[] { + const filteredFieldsSet = new Set(filteredFields.map((f) => f.name)); + + return allFields + .filter((field) => field.isScript || field.runtimeField || filteredFieldsSet.has(field.name)) + .map((f) => f.name); +} + /** * Exported only for unit tests. */ -export function existingFields(docs: estypes.SearchHit[], fields: Field[]): string[] { +export function legacyExistingFields(docs: estypes.SearchHit[], fields: Field[]): string[] { const missingFields = new Set(fields); for (const doc of docs) { diff --git a/x-pack/plugins/lens/server/ui_settings.ts b/x-pack/plugins/lens/server/ui_settings.ts new file mode 100644 index 0000000000000..63f16f3aeebb7 --- /dev/null +++ b/x-pack/plugins/lens/server/ui_settings.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { schema } from '@kbn/config-schema'; + +import { UiSettingsParams } from 'kibana/server'; + +export const FIELD_EXISTENCE_SETTING = 'lens:useFieldExistenceSampling'; + +export const getUiSettings: () => Record = () => ({ + [FIELD_EXISTENCE_SETTING]: { + name: i18n.translate('xpack.lens.advancedSettings.useFieldExistenceSampling.title', { + defaultMessage: 'Use field existence sampling', + }), + value: false, + description: i18n.translate( + 'xpack.lens.advancedSettings.useFieldExistenceSampling.description', + { + defaultMessage: + 'If enabled, document sampling is used to determine field existence (available or empty) for the Lens field list instead of relying on index mappings.', + } + ), + category: ['visualization'], + schema: schema.boolean(), + }, +}); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/total_feature_importance_summary/feature_importance_summary.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/total_feature_importance_summary/feature_importance_summary.tsx index 8d5d4c5e4ca23..7d80b91f94c77 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/total_feature_importance_summary/feature_importance_summary.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/total_feature_importance_summary/feature_importance_summary.tsx @@ -18,7 +18,7 @@ import { RecursivePartial, AxisStyle, PartialTheme, - BarSeriesSpec, + BarSeriesProps, } from '@elastic/charts'; import { i18n } from '@kbn/i18n'; import { euiLightVars as euiVars } from '@kbn/ui-theme'; @@ -100,13 +100,18 @@ export const FeatureImportanceSummaryPanel: FC { - let sortedData: Array<{ - featureName: string; - meanImportance: number; - className?: FeatureImportanceClassName; - }> = []; - let _barSeriesSpec: Partial = { + interface Datum { + featureName: string; + meanImportance: number; + className?: FeatureImportanceClassName; + } + type PlotData = Datum[]; + type SeriesProps = Omit; + const [plotData, barSeriesSpec, showLegend, chartHeight] = useMemo< + [plotData: PlotData, barSeriesSpec: SeriesProps, showLegend?: boolean, chartHeight?: number] + >(() => { + let sortedData: PlotData = []; + let _barSeriesSpec: SeriesProps = { xAccessor: 'featureName', yAccessors: ['meanImportance'], name: i18n.translate( @@ -122,7 +127,7 @@ export const FeatureImportanceSummaryPanel: FC = ({ const showBrush = !!onCellsSelection; - const swimLaneConfig = useMemo(() => { + const themeOverrides = useMemo(() => { if (!showSwimlane) return {}; - const config: HeatmapSpec['config'] = { - grid: { - cellHeight: { - min: CELL_HEIGHT, - max: CELL_HEIGHT, + const theme: PartialTheme = { + heatmap: { + grid: { + cellHeight: { + min: CELL_HEIGHT, + max: CELL_HEIGHT, + }, + stroke: { + width: 1, + color: euiTheme.euiBorderColor, + }, }, - stroke: { - width: 1, - color: euiTheme.euiBorderColor, + cell: { + maxWidth: 'fill', + maxHeight: 'fill', + label: { + visible: false, + }, + border: { + stroke: euiTheme.euiBorderColor, + strokeWidth: 0, + }, }, - }, - cell: { - maxWidth: 'fill', - maxHeight: 'fill', - label: { - visible: false, + yAxisLabel: { + visible: showYAxis, + width: Y_AXIS_LABEL_WIDTH, + textColor: euiTheme.euiTextSubduedColor, + padding: Y_AXIS_LABEL_PADDING, + fontSize: parseInt(euiTheme.euiFontSizeXS, 10), }, - border: { - stroke: euiTheme.euiBorderColor, - strokeWidth: 0, + xAxisLabel: { + visible: showTimeline, + textColor: euiTheme.euiTextSubduedColor, + fontSize: parseInt(euiTheme.euiFontSizeXS, 10), + // Required to calculate where the swimlane ends + width: X_AXIS_RIGHT_OVERFLOW * 2, }, - }, - yAxisLabel: { - visible: showYAxis, - width: Y_AXIS_LABEL_WIDTH, - textColor: euiTheme.euiTextSubduedColor, - padding: Y_AXIS_LABEL_PADDING, - formatter: (laneLabel: string) => { - return laneLabel === '' ? EMPTY_FIELD_VALUE_LABEL : laneLabel; + brushMask: { + visible: showBrush, + fill: isDarkTheme ? 'rgb(30,31,35,80%)' : 'rgb(247,247,247,50%)', }, - fontSize: parseInt(euiTheme.euiFontSizeXS, 10), - }, - xAxisLabel: { - visible: showTimeline, - textColor: euiTheme.euiTextSubduedColor, - formatter: (v: number) => { - timeBuckets.setInterval(`${swimlaneData.interval}s`); - const scaledDateFormat = timeBuckets.getScaledDateFormat(); - return moment(v).format(scaledDateFormat); + brushArea: { + visible: showBrush, + stroke: isDarkTheme ? 'rgb(255, 255, 255)' : 'rgb(105, 112, 125)', }, - fontSize: parseInt(euiTheme.euiFontSizeXS, 10), - // Required to calculate where the swimlane ends - width: X_AXIS_RIGHT_OVERFLOW * 2, - }, - brushMask: { - visible: showBrush, - fill: isDarkTheme ? 'rgb(30,31,35,80%)' : 'rgb(247,247,247,50%)', - }, - brushArea: { - visible: showBrush, - stroke: isDarkTheme ? 'rgb(255, 255, 255)' : 'rgb(105, 112, 125)', + ...(showLegend ? { maxLegendHeight: LEGEND_HEIGHT } : {}), }, - ...(showLegend ? { maxLegendHeight: LEGEND_HEIGHT } : {}), - timeZone: 'UTC', }; - return config; + return theme; }, [ showSwimlane, swimlaneType, @@ -427,6 +421,7 @@ export const SwimlaneContainer: FC = ({ {showSwimlane && !isLoading && ( = ({ = ({ }, }} ySortPredicate="dataIndex" - config={swimLaneConfig} + yAxisLabelFormatter={(laneLabel) => { + return laneLabel === '' ? EMPTY_FIELD_VALUE_LABEL : String(laneLabel); + }} + xAxisLabelFormatter={(v) => { + timeBuckets.setInterval(`${swimlaneData.interval}s`); + const scaledDateFormat = timeBuckets.getScaledDateFormat(); + return moment(v).format(scaledDateFormat); + }} /> )} diff --git a/x-pack/plugins/observability/public/hooks/use_chart_theme.tsx b/x-pack/plugins/observability/public/hooks/use_chart_theme.tsx index 02fc0ef32dde9..42ff021679ce0 100644 --- a/x-pack/plugins/observability/public/hooks/use_chart_theme.tsx +++ b/x-pack/plugins/observability/public/hooks/use_chart_theme.tsx @@ -5,34 +5,34 @@ * 2.0. */ +import { PartialTheme } from '@elastic/charts'; import { EUI_CHARTS_THEME_DARK, EUI_CHARTS_THEME_LIGHT } from '@elastic/eui/dist/eui_charts_theme'; import { useTheme } from './use_theme'; -export function useChartTheme() { +export function useChartTheme(): PartialTheme[] { const theme = useTheme(); const baseChartTheme = theme.darkMode ? EUI_CHARTS_THEME_DARK.theme : EUI_CHARTS_THEME_LIGHT.theme; - return { - ...baseChartTheme, - chartMargins: { - left: 10, - right: 10, - top: 10, - bottom: 10, + return [ + { + chartMargins: { + left: 10, + right: 10, + top: 10, + bottom: 10, + }, + background: { + color: 'transparent', + }, + lineSeriesStyle: { + point: { visible: false }, + }, + areaSeriesStyle: { + point: { visible: false }, + }, }, - background: { - ...baseChartTheme.background, - color: 'transparent', - }, - lineSeriesStyle: { - ...baseChartTheme.lineSeriesStyle, - point: { visible: false }, - }, - areaSeriesStyle: { - ...baseChartTheme.areaSeriesStyle, - point: { visible: false }, - }, - }; + baseChartTheme, + ]; } diff --git a/x-pack/plugins/security_solution/public/common/components/charts/common.tsx b/x-pack/plugins/security_solution/public/common/components/charts/common.tsx index ee292a66702e5..d7bafffec9a8f 100644 --- a/x-pack/plugins/security_solution/public/common/components/charts/common.tsx +++ b/x-pack/plugins/security_solution/public/common/components/charts/common.tsx @@ -13,7 +13,7 @@ import { Rendering, Rotation, ScaleType, - SettingsSpecProps, + SettingsProps, TickFormatter, Position, BrushEndListener, @@ -52,7 +52,7 @@ export interface ChartSeriesConfigs { tickSize?: number | undefined; }; yAxisTitle?: string | undefined; - settings?: Partial; + settings?: SettingsProps; } export interface ChartSeriesData { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index 4717dd6f92104..a2018280bebc6 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -16,7 +16,11 @@ import { containsInvalidItems, customValidators, } from '../../../../common/components/threat_match/helpers'; -import { isThreatMatchRule, isThresholdRule } from '../../../../../common/detection_engine/utils'; +import { + isEqlRule, + isThreatMatchRule, + isThresholdRule, +} from '../../../../../common/detection_engine/utils'; import { isMlRule } from '../../../../../common/machine_learning/helpers'; import { FieldValueQueryBar } from '../query_bar'; import { @@ -30,6 +34,7 @@ import { DefineStepRule } from '../../../pages/detection_engine/rules/types'; import { debounceAsync, eqlValidator } from '../eql_query_bar/validators'; import { CUSTOM_QUERY_REQUIRED, + EQL_QUERY_REQUIRED, INVALID_CUSTOM_QUERY, INDEX_HELPER_TEXT, THREAT_MATCH_INDEX_HELPER_TEXT, @@ -82,16 +87,14 @@ export const schema: FormSchema = { const { query, filters } = value as FieldValueQueryBar; const needsValidation = !isMlRule(formData.ruleType); if (!needsValidation) { - return; + return undefined; } - - return isEmpty(query.query as string) && isEmpty(filters) - ? { - code: 'ERR_FIELD_MISSING', - path, - message: CUSTOM_QUERY_REQUIRED, - } - : undefined; + const isFieldEmpty = isEmpty(query.query as string) && isEmpty(filters); + if (!isFieldEmpty) { + return undefined; + } + const message = isEqlRule(formData.ruleType) ? EQL_QUERY_REQUIRED : CUSTOM_QUERY_REQUIRED; + return { code: 'ERR_FIELD_MISSING', path, message }; }, }, { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx index d41d36813dee2..a2b01ba87dd69 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx @@ -14,6 +14,13 @@ export const CUSTOM_QUERY_REQUIRED = i18n.translate( } ); +export const EQL_QUERY_REQUIRED = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError', + { + defaultMessage: 'An EQL query is required.', + } +); + export const INVALID_CUSTOM_QUERY = i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customQueryFieldInvalidError', { diff --git a/x-pack/plugins/transform/common/types/pivot_aggs.ts b/x-pack/plugins/transform/common/types/pivot_aggs.ts index ced4d0a9bce0c..44308940a5870 100644 --- a/x-pack/plugins/transform/common/types/pivot_aggs.ts +++ b/x-pack/plugins/transform/common/types/pivot_aggs.ts @@ -18,6 +18,7 @@ export const PIVOT_SUPPORTED_AGGS = { VALUE_COUNT: 'value_count', FILTER: 'filter', TOP_METRICS: 'top_metrics', + TERMS: 'terms', } as const; export type PivotSupportedAggs = typeof PIVOT_SUPPORTED_AGGS[keyof typeof PIVOT_SUPPORTED_AGGS]; diff --git a/x-pack/plugins/transform/public/app/common/index.ts b/x-pack/plugins/transform/public/app/common/index.ts index 7081b6db2fe40..7a84ef9c8baa3 100644 --- a/x-pack/plugins/transform/public/app/common/index.ts +++ b/x-pack/plugins/transform/public/app/common/index.ts @@ -39,7 +39,9 @@ export { getEsAggFromAggConfig, isPivotAggsConfigWithUiSupport, isPivotAggsConfigPercentiles, + isPivotAggsConfigTerms, PERCENTILES_AGG_DEFAULT_PERCENTS, + TERMS_AGG_DEFAULT_SIZE, pivotAggsFieldSupport, } from './pivot_aggs'; export type { diff --git a/x-pack/plugins/transform/public/app/common/pivot_aggs.ts b/x-pack/plugins/transform/public/app/common/pivot_aggs.ts index 6f3d3de79c391..42ce5744bcd26 100644 --- a/x-pack/plugins/transform/public/app/common/pivot_aggs.ts +++ b/x-pack/plugins/transform/public/app/common/pivot_aggs.ts @@ -28,6 +28,7 @@ export function isPivotSupportedAggs(arg: unknown): arg is PivotSupportedAggs { } export const PERCENTILES_AGG_DEFAULT_PERCENTS = [1, 5, 25, 50, 75, 95, 99]; +export const TERMS_AGG_DEFAULT_SIZE = 10; export const pivotAggsFieldSupport = { [KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER], @@ -45,6 +46,7 @@ export const pivotAggsFieldSupport = { PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER, PIVOT_SUPPORTED_AGGS.TOP_METRICS, + PIVOT_SUPPORTED_AGGS.TERMS, ], [KBN_FIELD_TYPES.MURMUR3]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER], [KBN_FIELD_TYPES.NUMBER]: [ @@ -63,6 +65,7 @@ export const pivotAggsFieldSupport = { PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER, PIVOT_SUPPORTED_AGGS.TOP_METRICS, + PIVOT_SUPPORTED_AGGS.TERMS, ], [KBN_FIELD_TYPES._SOURCE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER], [KBN_FIELD_TYPES.UNKNOWN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER], @@ -226,9 +229,15 @@ interface PivotAggsConfigPercentiles extends PivotAggsConfigWithUiBase { percents: number[]; } +interface PivotAggsConfigTerms extends PivotAggsConfigWithUiBase { + agg: typeof PIVOT_SUPPORTED_AGGS.TERMS; + size: number; +} + export type PivotAggsConfigWithUiSupport = | PivotAggsConfigWithUiBase | PivotAggsConfigPercentiles + | PivotAggsConfigTerms | PivotAggsConfigWithExtendedForm; export function isPivotAggsConfigWithUiSupport(arg: unknown): arg is PivotAggsConfigWithUiSupport { @@ -258,6 +267,10 @@ export function isPivotAggsConfigPercentiles(arg: unknown): arg is PivotAggsConf ); } +export function isPivotAggsConfigTerms(arg: unknown): arg is PivotAggsConfigTerms { + return isPopulatedObject(arg, ['agg', 'field', 'size']) && arg.agg === PIVOT_SUPPORTED_AGGS.TERMS; +} + export type PivotAggsConfig = PivotAggsConfigBase | PivotAggsConfigWithUiSupport; export type PivotAggsConfigWithUiSupportDict = Dictionary; diff --git a/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts b/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts index a79b96acd5d7d..300c9c84993a1 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts @@ -13,7 +13,7 @@ import { EuiDataGridColumn } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { getFlattenedObject } from '@kbn/std'; -import { sample, difference } from 'lodash'; +import { difference } from 'lodash'; import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/common'; import type { PreviewMappingsProperties } from '../../../common/api_schemas/transforms'; @@ -79,12 +79,16 @@ export function getCombinedProperties( populatedProperties: PreviewMappingsProperties, docs: Array> ): PreviewMappingsProperties { - // Take a sample from docs and resolve missing mappings - const sampleDoc = sample(docs) ?? {}; - const missingMappings = difference(Object.keys(sampleDoc), Object.keys(populatedProperties)); + // Identify missing mappings + const missingMappings = difference( + // Create an array of unique flattened field names across all docs + [...new Set(docs.flatMap(Object.keys))], + Object.keys(populatedProperties) + ); return { ...populatedProperties, ...missingMappings.reduce((acc, curr) => { + const sampleDoc = docs.find((d) => typeof d[curr] !== 'undefined') ?? {}; acc[curr] = { type: typeof sampleDoc[curr] as ES_FIELD_TYPES }; return acc; }, {} as PreviewMappingsProperties), diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx index 53f2716551289..7370a4fd9287d 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx @@ -13,6 +13,7 @@ import { EuiButton, EuiCodeBlock, EuiComboBox, + EuiFieldNumber, EuiFieldText, EuiForm, EuiFormRow, @@ -32,9 +33,11 @@ import { import { isAggName, isPivotAggsConfigPercentiles, + isPivotAggsConfigTerms, isPivotAggsConfigWithUiSupport, getEsAggFromAggConfig, PERCENTILES_AGG_DEFAULT_PERCENTS, + TERMS_AGG_DEFAULT_SIZE, PivotAggsConfig, PivotAggsConfigWithUiSupportDict, } from '../../../../common'; @@ -75,6 +78,30 @@ function parsePercentsInput(inputValue: string | undefined) { return []; } +// Input string should only include comma separated numbers +function isValidPercentsInput(inputValue: string) { + return /^[0-9]+(,[0-9]+)*$/.test(inputValue); +} + +function getDefaultSize(defaultData: PivotAggsConfig): number | undefined { + if (isPivotAggsConfigTerms(defaultData)) { + return defaultData.size; + } +} + +function parseSizeInput(inputValue: string | undefined) { + if (inputValue !== undefined && isValidSizeInput(inputValue)) { + return parseInt(inputValue, 10); + } + + return TERMS_AGG_DEFAULT_SIZE; +} + +// Input string should only include numbers +function isValidSizeInput(inputValue: string) { + return /^\d+$/.test(inputValue); +} + export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onChange, options }) => { const [aggConfigDef, setAggConfigDef] = useState(cloneDeep(defaultData)); @@ -85,6 +112,9 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha ); const [percents, setPercents] = useState(getDefaultPercents(defaultData)); + const [validPercents, setValidPercents] = useState(agg === PIVOT_SUPPORTED_AGGS.PERCENTILES); + const [size, setSize] = useState(getDefaultSize(defaultData)); + const [validSize, setValidSize] = useState(agg === PIVOT_SUPPORTED_AGGS.TERMS); const isUnsupportedAgg = !isPivotAggsConfigWithUiSupport(defaultData); @@ -118,10 +148,19 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha if (aggVal === PIVOT_SUPPORTED_AGGS.PERCENTILES && percents === undefined) { setPercents(PERCENTILES_AGG_DEFAULT_PERCENTS); } + if (aggVal === PIVOT_SUPPORTED_AGGS.TERMS && size === undefined) { + setSize(TERMS_AGG_DEFAULT_SIZE); + } } function updatePercents(inputValue: string) { setPercents(parsePercentsInput(inputValue)); + setValidPercents(isValidPercentsInput(inputValue)); + } + + function updateSize(inputValue: string) { + setSize(parseSizeInput(inputValue)); + setValidSize(isValidSizeInput(inputValue)); } function getUpdatedItem(): PivotAggsConfig { @@ -137,21 +176,29 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha resultField = field[0]; } - if (agg !== PIVOT_SUPPORTED_AGGS.PERCENTILES) { + if (agg === PIVOT_SUPPORTED_AGGS.PERCENTILES) { updatedItem = { - ...aggConfigDef, agg, aggName, field: resultField, dropDownName: defaultData.dropDownName, + percents, + }; + } else if (agg === PIVOT_SUPPORTED_AGGS.TERMS) { + updatedItem = { + agg, + aggName, + field: resultField, + dropDownName: defaultData.dropDownName, + size, }; } else { updatedItem = { + ...aggConfigDef, agg, aggName, field: resultField, dropDownName: defaultData.dropDownName, - percents, }; } @@ -202,13 +249,18 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha percentsText = percents.toString(); } - const validPercents = - agg === PIVOT_SUPPORTED_AGGS.PERCENTILES && parsePercentsInput(percentsText).length > 0; + let sizeText; + if (size !== undefined) { + sizeText = size.toString(); + } let formValid = validAggName; if (formValid && agg === PIVOT_SUPPORTED_AGGS.PERCENTILES) { formValid = validPercents; } + if (formValid && agg === PIVOT_SUPPORTED_AGGS.TERMS) { + formValid = validSize; + } if (isPivotAggsWithExtendedForm(aggConfigDef)) { formValid = validAggName && aggConfigDef.isValid(); } @@ -325,6 +377,23 @@ export const PopoverForm: React.FC = ({ defaultData, otherAggNames, onCha /> )} + {agg === PIVOT_SUPPORTED_AGGS.TERMS && ( + + updateSize(e.target.value)} /> + + )} {isUnsupportedAgg && ( + - - + diff --git a/x-pack/plugins/uptime/public/components/common/charts/donut_chart.tsx b/x-pack/plugins/uptime/public/components/common/charts/donut_chart.tsx index 007ec8f737852..3638b52e39783 100644 --- a/x-pack/plugins/uptime/public/components/common/charts/donut_chart.tsx +++ b/x-pack/plugins/uptime/public/components/common/charts/donut_chart.tsx @@ -9,7 +9,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiIcon } from '@elastic/eui'; import React, { useContext } from 'react'; import { i18n } from '@kbn/i18n'; import styled from 'styled-components'; -import { Chart, Datum, Partition, Settings, PartitionLayout } from '@elastic/charts'; +import { Chart, Datum, Partition, Settings, PartitionLayout, PartialTheme } from '@elastic/charts'; import { DonutChartLegend } from './donut_chart_legend'; import { UptimeThemeContext } from '../../../contexts'; @@ -28,6 +28,19 @@ export const GreenCheckIcon = styled(EuiIcon)` position: absolute; `; +const themeOverrides: PartialTheme = { + chartMargins: { top: 0, bottom: 0, left: 0, right: 0 }, + partition: { + linkLabel: { + maximumSection: Infinity, + }, + idealFontSizeJump: 1.1, + outerSizeRatio: 0.9, + emptySizeRatio: 0.4, + circlePadding: 4, + }, +}; + export const DonutChart = ({ height, down, up }: DonutChartProps) => { const { colors: { danger, gray }, @@ -44,15 +57,18 @@ export const DonutChart = ({ height, down, up }: DonutChartProps) => { 'Pie chart showing the current status. {down} of {total} monitors are down.', values: { down, total: up + down }, })} - {...chartTheme} > - + d.value as number} layers={[ { @@ -65,17 +81,6 @@ export const DonutChart = ({ height, down, up }: DonutChartProps) => { }, }, ]} - config={{ - partitionLayout: PartitionLayout.sunburst, - linkLabel: { - maximumSection: Infinity, - }, - margin: { top: 0, bottom: 0, left: 0, right: 0 }, - idealFontSizeJump: 1.1, - outerSizeRatio: 0.9, - emptySizeRatio: 0.4, - circlePadding: 4, - }} /> {down === 0 && } diff --git a/x-pack/test/api_integration/apis/lens/existing_fields.ts b/x-pack/test/api_integration/apis/lens/existing_fields.ts index 952659c2960d4..e51980e47fd06 100644 --- a/x-pack/test/api_integration/apis/lens/existing_fields.ts +++ b/x-pack/test/api_integration/apis/lens/existing_fields.ts @@ -9,171 +9,46 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; -const TEST_START_TIME = '2015-09-19T06:31:44.000'; -const TEST_END_TIME = '2015-09-23T18:31:44.000'; +const TEST_START_TIME = '2010-09-19T06:31:44.000'; +const TEST_END_TIME = '2023-09-23T18:31:44.000'; const COMMON_HEADERS = { 'kbn-xsrf': 'some-xsrf-token', }; +const metaFields = ['_id', '_index', '_score', '_source', '_type']; const fieldsWithData = [ - '@message', - '@message.raw', - '@tags', - '@tags.raw', - '@timestamp', - '_id', - '_index', - 'agent', - 'agent.raw', - 'bytes', - 'clientip', - 'extension', - 'extension.raw', - 'geo.coordinates', - 'geo.dest', - 'geo.src', - 'geo.srcdest', - 'headings', - 'headings.raw', - 'host', - 'host.raw', - 'index', - 'index.raw', - 'ip', - 'links', - 'links.raw', - 'machine.os', - 'machine.os.raw', - 'machine.ram', - 'machine.ram_range', - 'memory', - 'phpmemory', - 'referer', - 'request', - 'request.raw', - 'response', - 'response.raw', - 'spaces', - 'spaces.raw', - 'type', - 'url', - 'url.raw', - 'utc_time', - 'xss', - 'xss.raw', - 'runtime_number', - - 'relatedContent.article:modified_time', - 'relatedContent.article:published_time', - 'relatedContent.article:section', - 'relatedContent.article:section.raw', - 'relatedContent.article:tag', - 'relatedContent.article:tag.raw', - 'relatedContent.og:description', - 'relatedContent.og:description.raw', - 'relatedContent.og:image', - 'relatedContent.og:image.raw', - 'relatedContent.og:image:height', - 'relatedContent.og:image:height.raw', - 'relatedContent.og:image:width', - 'relatedContent.og:image:width.raw', - 'relatedContent.og:site_name', - 'relatedContent.og:site_name.raw', - 'relatedContent.og:title', - 'relatedContent.og:title.raw', - 'relatedContent.og:type', - 'relatedContent.og:type.raw', - 'relatedContent.og:url', - 'relatedContent.og:url.raw', - 'relatedContent.twitter:card', - 'relatedContent.twitter:card.raw', - 'relatedContent.twitter:description', - 'relatedContent.twitter:description.raw', - 'relatedContent.twitter:image', - 'relatedContent.twitter:image.raw', - 'relatedContent.twitter:site', - 'relatedContent.twitter:site.raw', - 'relatedContent.twitter:title', - 'relatedContent.twitter:title.raw', - 'relatedContent.url', - 'relatedContent.url.raw', -]; - -const metricBeatData = [ - '@timestamp', - '_id', - '_index', - 'agent.ephemeral_id', - 'agent.ephemeral_id.keyword', - 'agent.hostname', - 'agent.hostname.keyword', - 'agent.id', - 'agent.id.keyword', - 'agent.type', - 'agent.type.keyword', - 'agent.version', - 'agent.version.keyword', - 'ecs.version', - 'ecs.version.keyword', - 'event.dataset', - 'event.dataset.keyword', - 'event.duration', - 'event.module', - 'event.module.keyword', - 'host.architecture', - 'host.architecture.keyword', - 'host.hostname', - 'host.hostname.keyword', - 'host.id', - 'host.id.keyword', - 'host.name', - 'host.name.keyword', - 'host.os.build', - 'host.os.build.keyword', - 'host.os.family', - 'host.os.family.keyword', - 'host.os.kernel', - 'host.os.kernel.keyword', - 'host.os.name', - 'host.os.name.keyword', - 'host.os.platform', - 'host.os.platform.keyword', - 'host.os.version', - 'host.os.version.keyword', - 'metricset.name', - 'metricset.name.keyword', - 'service.type', - 'service.type.keyword', - 'system.cpu.cores', - 'system.cpu.idle.pct', - 'system.cpu.iowait.pct', - 'system.cpu.irq.pct', - 'system.cpu.nice.pct', - 'system.cpu.softirq.pct', - 'system.cpu.steal.pct', - 'system.cpu.system.pct', - 'system.cpu.total.pct', - 'system.cpu.user.pct', + 'ts', + 'filter_field', + 'textfield1', + 'textfield2', + 'mapping_runtime_field', + 'data_view_runtime_field', ]; export default ({ getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); + const kibanaServer = getService('kibanaServer'); describe('existing_fields apis', () => { before(async () => { - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional'); - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/visualize/default'); + await esArchiver.load('x-pack/test/api_integration/es_archives/lens/constant_keyword'); + await kibanaServer.importExport.load( + 'x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json' + ); }); + after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional'); - await esArchiver.unload('x-pack/test/functional/es_archives/visualize/default'); + await esArchiver.unload('x-pack/test/api_integration/es_archives/lens/constant_keyword'); + await kibanaServer.importExport.unload( + 'x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json' + ); }); describe('existence', () => { it('should find which fields exist in the sample documents', async () => { const { body } = await supertest - .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`) + .post(`/api/lens/existing_fields/existence_index`) .set(COMMON_HEADERS) .send({ dslQuery: { @@ -186,76 +61,89 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - expect(body.indexPatternTitle).to.eql('logstash-*'); - expect(body.existingFieldNames.sort()).to.eql(fieldsWithData.sort()); + expect(body.indexPatternTitle).to.eql('existence_index_*'); + expect(body.existingFieldNames.sort()).to.eql([...metaFields, ...fieldsWithData].sort()); }); - it('should succeed for thousands of fields', async () => { + it('should return fields filtered by term query', async () => { + const expectedFieldNames = [ + 'ts', + 'filter_field', + 'textfield1', + // textfield2 and mapping_runtime_field are defined on the other index + 'data_view_runtime_field', + ]; + const { body } = await supertest - .post(`/api/lens/existing_fields/${encodeURIComponent('metricbeat-*')}`) + .post(`/api/lens/existing_fields/existence_index`) .set(COMMON_HEADERS) .send({ - dslQuery: { match_all: {} }, + dslQuery: { + bool: { + filter: [{ term: { filter_field: 'a' } }], + }, + }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, }) .expect(200); - - expect(body.indexPatternTitle).to.eql('metricbeat-*'); - expect(body.existingFieldNames.sort()).to.eql(metricBeatData.sort()); + expect(body.existingFieldNames.sort()).to.eql( + [...metaFields, ...expectedFieldNames].sort() + ); }); - it('should return fields filtered by query and filters', async () => { + it('should return fields filtered by match_phrase query', async () => { const expectedFieldNames = [ - '@message', - '@message.raw', - '@tags', - '@tags.raw', - '@timestamp', - '_id', - '_index', - 'agent', - 'agent.raw', - 'bytes', - 'clientip', - 'extension', - 'extension.raw', - 'headings', - 'headings.raw', - 'host', - 'host.raw', - 'index', - 'index.raw', - 'referer', - 'request', - 'request.raw', - 'response', - 'response.raw', - 'runtime_number', - 'spaces', - 'spaces.raw', - 'type', - 'url', - 'url.raw', - 'utc_time', - 'xss', - 'xss.raw', + 'ts', + 'filter_field', + 'textfield1', + // textfield2 and mapping_runtime_field are defined on the other index + 'data_view_runtime_field', ]; const { body } = await supertest - .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`) + .post(`/api/lens/existing_fields/existence_index`) .set(COMMON_HEADERS) .send({ dslQuery: { bool: { - filter: [{ match: { referer: 'https://www.taylorswift.com/' } }], + filter: [{ match_phrase: { filter_field: 'a' } }], }, }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, }) .expect(200); - expect(body.existingFieldNames.sort()).to.eql(expectedFieldNames.sort()); + expect(body.existingFieldNames.sort()).to.eql( + [...metaFields, ...expectedFieldNames].sort() + ); + }); + + it('should return fields filtered by time range', async () => { + const expectedFieldNames = [ + 'ts', + 'filter_field', + 'textfield1', + // textfield2 and mapping_runtime_field are defined on the other index + 'data_view_runtime_field', + ]; + + const { body } = await supertest + .post(`/api/lens/existing_fields/existence_index`) + .set(COMMON_HEADERS) + .send({ + dslQuery: { + bool: { + filter: [{ term: { filter_field: 'a' } }], + }, + }, + fromDate: TEST_START_TIME, + toDate: '2021-12-12', + }) + .expect(200); + expect(body.existingFieldNames.sort()).to.eql( + [...metaFields, ...expectedFieldNames].sort() + ); }); }); }); diff --git a/x-pack/test/api_integration/apis/lens/index.ts b/x-pack/test/api_integration/apis/lens/index.ts index 5b51f2dbd94e3..04508f011158a 100644 --- a/x-pack/test/api_integration/apis/lens/index.ts +++ b/x-pack/test/api_integration/apis/lens/index.ts @@ -10,6 +10,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function lensApiIntegrationTests({ loadTestFile }: FtrProviderContext) { describe('Lens', () => { loadTestFile(require.resolve('./existing_fields')); + loadTestFile(require.resolve('./legacy_existing_fields')); loadTestFile(require.resolve('./field_stats')); loadTestFile(require.resolve('./telemetry')); }); diff --git a/x-pack/test/api_integration/apis/lens/legacy_existing_fields.ts b/x-pack/test/api_integration/apis/lens/legacy_existing_fields.ts new file mode 100644 index 0000000000000..370807c99d806 --- /dev/null +++ b/x-pack/test/api_integration/apis/lens/legacy_existing_fields.ts @@ -0,0 +1,269 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +const TEST_START_TIME = '2015-09-19T06:31:44.000'; +const TEST_END_TIME = '2015-09-23T18:31:44.000'; +const COMMON_HEADERS = { + 'kbn-xsrf': 'some-xsrf-token', +}; + +const fieldsWithData = [ + '@message', + '@message.raw', + '@tags', + '@tags.raw', + '@timestamp', + '_id', + '_index', + 'agent', + 'agent.raw', + 'bytes', + 'clientip', + 'extension', + 'extension.raw', + 'geo.coordinates', + 'geo.dest', + 'geo.src', + 'geo.srcdest', + 'headings', + 'headings.raw', + 'host', + 'host.raw', + 'index', + 'index.raw', + 'ip', + 'links', + 'links.raw', + 'machine.os', + 'machine.os.raw', + 'machine.ram', + 'machine.ram_range', + 'memory', + 'phpmemory', + 'referer', + 'request', + 'request.raw', + 'response', + 'response.raw', + 'spaces', + 'spaces.raw', + 'type', + 'url', + 'url.raw', + 'utc_time', + 'xss', + 'xss.raw', + 'runtime_number', + + 'relatedContent.article:modified_time', + 'relatedContent.article:published_time', + 'relatedContent.article:section', + 'relatedContent.article:section.raw', + 'relatedContent.article:tag', + 'relatedContent.article:tag.raw', + 'relatedContent.og:description', + 'relatedContent.og:description.raw', + 'relatedContent.og:image', + 'relatedContent.og:image.raw', + 'relatedContent.og:image:height', + 'relatedContent.og:image:height.raw', + 'relatedContent.og:image:width', + 'relatedContent.og:image:width.raw', + 'relatedContent.og:site_name', + 'relatedContent.og:site_name.raw', + 'relatedContent.og:title', + 'relatedContent.og:title.raw', + 'relatedContent.og:type', + 'relatedContent.og:type.raw', + 'relatedContent.og:url', + 'relatedContent.og:url.raw', + 'relatedContent.twitter:card', + 'relatedContent.twitter:card.raw', + 'relatedContent.twitter:description', + 'relatedContent.twitter:description.raw', + 'relatedContent.twitter:image', + 'relatedContent.twitter:image.raw', + 'relatedContent.twitter:site', + 'relatedContent.twitter:site.raw', + 'relatedContent.twitter:title', + 'relatedContent.twitter:title.raw', + 'relatedContent.url', + 'relatedContent.url.raw', +]; + +const metricBeatData = [ + '@timestamp', + '_id', + '_index', + 'agent.ephemeral_id', + 'agent.ephemeral_id.keyword', + 'agent.hostname', + 'agent.hostname.keyword', + 'agent.id', + 'agent.id.keyword', + 'agent.type', + 'agent.type.keyword', + 'agent.version', + 'agent.version.keyword', + 'ecs.version', + 'ecs.version.keyword', + 'event.dataset', + 'event.dataset.keyword', + 'event.duration', + 'event.module', + 'event.module.keyword', + 'host.architecture', + 'host.architecture.keyword', + 'host.hostname', + 'host.hostname.keyword', + 'host.id', + 'host.id.keyword', + 'host.name', + 'host.name.keyword', + 'host.os.build', + 'host.os.build.keyword', + 'host.os.family', + 'host.os.family.keyword', + 'host.os.kernel', + 'host.os.kernel.keyword', + 'host.os.name', + 'host.os.name.keyword', + 'host.os.platform', + 'host.os.platform.keyword', + 'host.os.version', + 'host.os.version.keyword', + 'metricset.name', + 'metricset.name.keyword', + 'service.type', + 'service.type.keyword', + 'system.cpu.cores', + 'system.cpu.idle.pct', + 'system.cpu.iowait.pct', + 'system.cpu.irq.pct', + 'system.cpu.nice.pct', + 'system.cpu.softirq.pct', + 'system.cpu.steal.pct', + 'system.cpu.system.pct', + 'system.cpu.total.pct', + 'system.cpu.user.pct', +]; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + const kibanaServer = getService('kibanaServer'); + + describe('existing_fields apis legacy', () => { + before(async () => { + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional'); + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/visualize/default'); + await kibanaServer.uiSettings.update({ + 'lens:useFieldExistenceSampling': true, + }); + }); + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional'); + await esArchiver.unload('x-pack/test/functional/es_archives/visualize/default'); + await kibanaServer.uiSettings.update({ + 'lens:useFieldExistenceSampling': false, + }); + }); + + describe('existence', () => { + it('should find which fields exist in the sample documents', async () => { + const { body } = await supertest + .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`) + .set(COMMON_HEADERS) + .send({ + dslQuery: { + bool: { + filter: [{ match_all: {} }], + }, + }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + }) + .expect(200); + + expect(body.indexPatternTitle).to.eql('logstash-*'); + expect(body.existingFieldNames.sort()).to.eql(fieldsWithData.sort()); + }); + + it('should succeed for thousands of fields', async () => { + const { body } = await supertest + .post(`/api/lens/existing_fields/${encodeURIComponent('metricbeat-*')}`) + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + }) + .expect(200); + + expect(body.indexPatternTitle).to.eql('metricbeat-*'); + expect(body.existingFieldNames.sort()).to.eql(metricBeatData.sort()); + }); + + it('should return fields filtered by query and filters', async () => { + const expectedFieldNames = [ + '@message', + '@message.raw', + '@tags', + '@tags.raw', + '@timestamp', + '_id', + '_index', + 'agent', + 'agent.raw', + 'bytes', + 'clientip', + 'extension', + 'extension.raw', + 'headings', + 'headings.raw', + 'host', + 'host.raw', + 'index', + 'index.raw', + 'referer', + 'request', + 'request.raw', + 'response', + 'response.raw', + 'runtime_number', + 'spaces', + 'spaces.raw', + 'type', + 'url', + 'url.raw', + 'utc_time', + 'xss', + 'xss.raw', + ]; + + const { body } = await supertest + .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`) + .set(COMMON_HEADERS) + .send({ + dslQuery: { + bool: { + filter: [{ match: { referer: 'https://www.taylorswift.com/' } }], + }, + }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + }) + .expect(200); + expect(body.existingFieldNames.sort()).to.eql(expectedFieldNames.sort()); + }); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/upgrade_assistant/es_deprecations.ts b/x-pack/test/api_integration/apis/upgrade_assistant/es_deprecations.ts index aea003a317963..b05cf8b901b5b 100644 --- a/x-pack/test/api_integration/apis/upgrade_assistant/es_deprecations.ts +++ b/x-pack/test/api_integration/apis/upgrade_assistant/es_deprecations.ts @@ -5,36 +5,101 @@ * 2.0. */ +import type { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import expect from '@kbn/expect'; + import { FtrProviderContext } from '../../ftr_provider_context'; +import { + API_BASE_PATH, + indexSettingDeprecations, +} from '../../../../plugins/upgrade_assistant/common/constants'; +import { EnrichedDeprecationInfo } from '../../../../plugins/upgrade_assistant/common/types'; + +const translogSettingsIndexDeprecation: IndicesCreateRequest = { + index: 'deprecated_settings', + body: { + settings: { + // @ts-expect-error setting is removed in 8.0 + 'translog.retention.size': '1b', + 'translog.retention.age': '5m', + 'index.soft_deletes.enabled': true, + }, + }, +}; export default function ({ getService }: FtrProviderContext) { const supertestWithoutAuth = getService('supertestWithoutAuth'); + const supertest = getService('supertest'); const security = getService('security'); + const es = getService('es'); + const log = getService('log'); describe('Elasticsearch deprecations', () => { describe('GET /api/upgrade_assistant/es_deprecations', () => { - it('handles auth error', async () => { - const ROLE_NAME = 'authErrorRole'; - const USER_NAME = 'authErrorUser'; - const USER_PASSWORD = 'password'; - - try { - await security.role.create(ROLE_NAME, {}); - await security.user.create(USER_NAME, { - password: USER_PASSWORD, - roles: [ROLE_NAME], - }); - - await supertestWithoutAuth - .get('/api/upgrade_assistant/es_deprecations') - .auth(USER_NAME, USER_PASSWORD) - .set('kbn-xsrf', 'kibana') - .send() - .expect(403); - } finally { - await security.role.delete(ROLE_NAME); - await security.user.delete(USER_NAME); - } + describe('error handling', () => { + it('handles auth error', async () => { + const ROLE_NAME = 'authErrorRole'; + const USER_NAME = 'authErrorUser'; + const USER_PASSWORD = 'password'; + + try { + await security.role.create(ROLE_NAME, {}); + await security.user.create(USER_NAME, { + password: USER_PASSWORD, + roles: [ROLE_NAME], + }); + + await supertestWithoutAuth + .get(`${API_BASE_PATH}/es_deprecations`) + .auth(USER_NAME, USER_PASSWORD) + .set('kbn-xsrf', 'kibana') + .send() + .expect(403); + } finally { + await security.role.delete(ROLE_NAME); + await security.user.delete(USER_NAME); + } + }); + }); + + // Only applicable on 7.x + describe.skip('index setting deprecation', () => { + before(async () => { + try { + // Create index that will trigger deprecation warning + await es.indices.create(translogSettingsIndexDeprecation); + } catch (e) { + log.debug('Error creating test index'); + throw e; + } + }); + + after(async () => { + try { + await es.indices.delete({ + index: [translogSettingsIndexDeprecation.index], + }); + } catch (e) { + log.debug('Error deleting text index'); + throw e; + } + }); + + it('returns the expected deprecation message for deprecated translog index settings', async () => { + const { body: apiRequestResponse } = await supertest + .get(`${API_BASE_PATH}/es_deprecations`) + .set('kbn-xsrf', 'xxx') + .expect(200); + + const indexSettingDeprecation = apiRequestResponse.deprecations.find( + (deprecation: EnrichedDeprecationInfo) => + deprecation.index === translogSettingsIndexDeprecation.index + ); + + expect(indexSettingDeprecation.message).to.equal( + indexSettingDeprecations.translog.deprecationMessage + ); + }); }); }); }); diff --git a/x-pack/test/api_integration/es_archives/lens/constant_keyword/data.json b/x-pack/test/api_integration/es_archives/lens/constant_keyword/data.json new file mode 100644 index 0000000000000..8ef482e7b40c3 --- /dev/null +++ b/x-pack/test/api_integration/es_archives/lens/constant_keyword/data.json @@ -0,0 +1,25 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "existence_index_1", + "source": { + "filter_field": "a", + "textfield1": "test", + "ts": "2021-01-02" + } + } +} + +{ + "type": "doc", + "value": { + "id": "1", + "index": "existence_index_2", + "source": { + "filter_field": "b", + "textfield2": "test", + "ts": "2022-01-02" + } + } +} diff --git a/x-pack/test/api_integration/es_archives/lens/constant_keyword/mappings.json b/x-pack/test/api_integration/es_archives/lens/constant_keyword/mappings.json new file mode 100644 index 0000000000000..af5dc3a651e96 --- /dev/null +++ b/x-pack/test/api_integration/es_archives/lens/constant_keyword/mappings.json @@ -0,0 +1,59 @@ +{ + "type": "index", + "value": { + "index": "existence_index_1", + "mappings": { + "properties": { + "filter_field": { + "type": "constant_keyword", + "value": "a" + }, + "textfield1": { + "type": "keyword" + }, + "ts": { + "type": "date" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} + +{ + "type": "index", + "value": { + "index": "existence_index_2", + "mappings": { + "runtime": { + "mapping_runtime_field": { + "type": "keyword", + "script" : { "source" : "emit('abc')" } + } + }, + "properties": { + "filter_field": { + "type": "constant_keyword", + "value": "b" + }, + "textfield2": { + "type": "keyword" + }, + "ts": { + "type": "date" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} \ No newline at end of file diff --git a/x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json b/x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json new file mode 100644 index 0000000000000..fb7c105ec462b --- /dev/null +++ b/x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json @@ -0,0 +1,16 @@ +{ + "attributes": { + "timeFieldName": "ts", + "title": "existence_index_*", + "runtimeFieldMap":"{\"data_view_runtime_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit('a')\"}}}" + }, + "coreMigrationVersion": "8.0.0", + "id": "existence_index", + "migrationVersion": { + "index-pattern": "7.11.0" + }, + "references": [], + "type": "index-pattern", + "updated_at": "2018-12-21T00:43:07.096Z", + "version": "WzEzLDJd" +} diff --git a/x-pack/test/functional/apps/lens/drag_and_drop.ts b/x-pack/test/functional/apps/lens/drag_and_drop.ts index 2858ff1588f7c..27e336a1cbc12 100644 --- a/x-pack/test/functional/apps/lens/drag_and_drop.ts +++ b/x-pack/test/functional/apps/lens/drag_and_drop.ts @@ -328,8 +328,10 @@ export default function ({ getPageObjects }: FtrProviderContext) { }); it('overwrite existing time dimension if one exists already', async () => { + await PageObjects.lens.searchField('utc'); await PageObjects.lens.dragFieldToWorkspace('utc_time'); await PageObjects.lens.waitForVisualization(); + await PageObjects.lens.searchField('client'); await PageObjects.lens.dragFieldToWorkspace('clientip'); await PageObjects.lens.waitForVisualization(); expect(await PageObjects.lens.getDimensionTriggersTexts('lnsXY_xDimensionPanel')).to.eql([ diff --git a/x-pack/test/functional/apps/lens/epoch_millis.ts b/x-pack/test/functional/apps/lens/epoch_millis.ts index 9ff418c8e5ce8..deaa3e720101e 100644 --- a/x-pack/test/functional/apps/lens/epoch_millis.ts +++ b/x-pack/test/functional/apps/lens/epoch_millis.ts @@ -30,13 +30,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('should show field list', async () => { await PageObjects.visualize.navigateToNewVisualization(); await PageObjects.visualize.clickVisType('lens'); - await PageObjects.lens.switchDataPanelIndexPattern('epoch-millis'); + await PageObjects.lens.switchDataPanelIndexPattern('epoch-millis*'); await PageObjects.lens.goToTimeRange(); await PageObjects.lens.switchToVisualization('lnsDatatable'); const fieldList = await PageObjects.lens.findAllFields(); expect(fieldList).to.contain('@timestamp'); - // not defined for document in time range, only out of time range - expect(fieldList).not.to.contain('agent.raw'); }); it('should able to configure a regular metric', async () => { diff --git a/x-pack/test/functional/apps/transform/creation_index_pattern.ts b/x-pack/test/functional/apps/transform/creation_index_pattern.ts index 21aa191b8480d..d93087e630ac4 100644 --- a/x-pack/test/functional/apps/transform/creation_index_pattern.ts +++ b/x-pack/test/functional/apps/transform/creation_index_pattern.ts @@ -337,6 +337,64 @@ export default function ({ getService }: FtrProviderContext) { discoverQueryHits: '10', }, } as PivotTransformTestData, + { + type: 'pivot', + suiteTitle: 'batch transform with terms group and terms agg', + source: 'ft_ecommerce', + groupByEntries: [ + { + identifier: 'terms(customer_gender)', + label: 'customer_gender', + } as GroupByEntry, + ], + aggregationEntries: [ + { + identifier: 'terms(geoip.city_name)', + label: 'geoip.city_name.terms', + }, + ], + transformId: `ec_3_${Date.now()}`, + transformDescription: + 'ecommerce batch transform with group by terms(customer_gender) and aggregation terms(geoip.city_name)', + get destinationIndex(): string { + return `user-${this.transformId}`; + }, + discoverAdjustSuperDatePicker: false, + expected: { + pivotAdvancedEditorValueArr: ['{', ' "group_by": {', ' "customer_gender": {'], + pivotAdvancedEditorValue: { + group_by: { + customer_gender: { + terms: { + field: 'customer_gender', + }, + }, + }, + aggregations: { + 'geoip.city_name': { + terms: { + field: 'geoip.city_name', + size: 3, + }, + }, + }, + }, + transformPreview: { + column: 0, + values: ['FEMALE', 'MALE'], + }, + row: { + status: TRANSFORM_STATE.STOPPED, + mode: 'batch', + progress: '100', + }, + indexPreview: { + columns: 10, + rows: 5, + }, + discoverQueryHits: '2', + }, + } as PivotTransformTestData, { type: 'latest', suiteTitle: 'batch transform with the latest function', @@ -351,7 +409,7 @@ export default function ({ getService }: FtrProviderContext) { identifier: 'order_date', label: 'order_date', }, - transformId: `ec_3_${Date.now()}`, + transformId: `ec_4_${Date.now()}`, transformDescription: 'ecommerce batch transform with the latest function config, sort by order_data, country code as unique key', diff --git a/x-pack/test/functional/es_archives/lens/epoch_millis/data.json b/x-pack/test/functional/es_archives/lens/epoch_millis/data.json index db9d5ccc379d7..a15bc25f56802 100644 --- a/x-pack/test/functional/es_archives/lens/epoch_millis/data.json +++ b/x-pack/test/functional/es_archives/lens/epoch_millis/data.json @@ -2,7 +2,7 @@ "type": "doc", "value": { "id": "AU_x4-TaGFA8no6QjiSJ", - "index": "epoch-millis", + "index": "epoch-millis1", "source": { "@message": "212.113.62.183 - - [2015-09-21T06:09:20.045Z] \"GET /uploads/dafydd-williams.jpg HTTP/1.1\" 200 3182 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1\"", "@tags": [ @@ -75,7 +75,7 @@ "type": "doc", "value": { "id": "AU_x4-TaGFA8no6QjiSL", - "index": "epoch-millis", + "index": "epoch-millis2", "source": { "@message": "156.252.112.76 - - [2015-09-21T21:13:02.070Z] \"GET /uploads/aleksandr-samokutyayev.jpg HTTP/1.1\" 200 6176 \"-\" \"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24\"", "@tags": [ diff --git a/x-pack/test/functional/es_archives/lens/epoch_millis/mappings.json b/x-pack/test/functional/es_archives/lens/epoch_millis/mappings.json index ee1c8dce8219d..ae803d98870d7 100644 --- a/x-pack/test/functional/es_archives/lens/epoch_millis/mappings.json +++ b/x-pack/test/functional/es_archives/lens/epoch_millis/mappings.json @@ -1,7 +1,382 @@ { "type": "index", "value": { - "index": "epoch-millis", + "index": "epoch-millis1", + "mappings": { + "dynamic_templates": [ + { + "string_fields": { + "mapping": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "match": "*", + "match_mapping_type": "string" + } + } + ], + "runtime": { + "runtime_number": { + "type": "long", + "script" : { "source" : "emit(doc['bytes'].value)" } + } + }, + "properties": { + "@message": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "@tags": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "@timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "agent": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "bytes": { + "type": "long" + }, + "clientip": { + "type": "ip" + }, + "extension": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "geo": { + "properties": { + "coordinates": { + "type": "geo_point" + }, + "dest": { + "type": "keyword" + }, + "src": { + "type": "keyword" + }, + "srcdest": { + "type": "keyword" + } + } + }, + "headings": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "host": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "id": { + "type": "integer" + }, + "index": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "ip": { + "type": "ip" + }, + "links": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "machine": { + "properties": { + "os": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "ram": { + "type": "long" + }, + "ram_range": { + "type": "long_range" + } + } + }, + "memory": { + "type": "double" + }, + "meta": { + "properties": { + "char": { + "type": "keyword" + }, + "related": { + "type": "text" + }, + "user": { + "properties": { + "firstname": { + "type": "text" + }, + "lastname": { + "type": "integer" + } + } + } + } + }, + "phpmemory": { + "type": "long" + }, + "referer": { + "type": "keyword" + }, + "relatedContent": { + "properties": { + "article:modified_time": { + "type": "date" + }, + "article:published_time": { + "type": "date" + }, + "article:section": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "article:tag": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "og:description": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "og:image": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "og:image:height": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "og:image:width": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "og:site_name": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "og:title": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "og:type": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "og:url": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "twitter:card": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "twitter:description": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "twitter:image": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "twitter:site": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "twitter:title": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "url": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "request": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "response": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "spaces": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "type": { + "type": "keyword" + }, + "url": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + }, + "utc_time": { + "type": "date" + }, + "xss": { + "fields": { + "raw": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "settings": { + "index": { + "analysis": { + "analyzer": { + "url": { + "max_token_length": "1000", + "tokenizer": "uax_url_email", + "type": "standard" + } + } + }, + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} + +{ + "type": "index", + "value": { + "index": "epoch-millis2", "mappings": { "dynamic_templates": [ { diff --git a/x-pack/test/functional/fixtures/kbn_archiver/lens/epoch_millis.json b/x-pack/test/functional/fixtures/kbn_archiver/lens/epoch_millis.json index fc7deabc0ead1..bd4a9ed17cc6e 100644 --- a/x-pack/test/functional/fixtures/kbn_archiver/lens/epoch_millis.json +++ b/x-pack/test/functional/fixtures/kbn_archiver/lens/epoch_millis.json @@ -1,7 +1,7 @@ { "attributes": { "timeFieldName": "@timestamp", - "title": "epoch-millis" + "title": "epoch-millis*" }, "coreMigrationVersion": "8.0.0", "id": "epoch-millis", diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap b/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap index 7e74063480e90..a080cbf7247a6 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap @@ -21,7 +21,7 @@ Object { "apm.transaction_error_rate_opbeans-go_request_ENVIRONMENT_NOT_DEFINED", ], "kibana.alert.reason": Array [ - "Failed transactions rate is greater than 30% (current value is 50%) for opbeans-go", + "Failed transactions is 50% in the last 5 mins for opbeans-go. Alert when > 30%.", ], "kibana.alert.rule.category": Array [ "Failed transaction rate threshold", @@ -85,7 +85,7 @@ Object { "apm.transaction_error_rate_opbeans-go_request_ENVIRONMENT_NOT_DEFINED", ], "kibana.alert.reason": Array [ - "Failed transactions rate is greater than 30% (current value is 50%) for opbeans-go", + "Failed transactions is 50% in the last 5 mins for opbeans-go. Alert when > 30%.", ], "kibana.alert.rule.category": Array [ "Failed transaction rate threshold", diff --git a/x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js b/x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js index 6a326840bc551..635cb8e288ae1 100644 --- a/x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js +++ b/x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js @@ -83,6 +83,32 @@ export default function ({ getService }) { }); }); + it('can resume after reindexing was stopped right after creating the new index', async () => { + await esArchiver.load('x-pack/test/functional/es_archives/upgrade_assistant/reindex'); + + // This new index is the new soon to be created reindexed index. We create it + // upfront to simulate a situation in which the user restarted kibana half + // way through the reindex process and ended up with an extra index. + await es.indices.create({ index: 'reindexed-v7-dummydata' }); + + const { body } = await supertest + .post(`/api/upgrade_assistant/reindex/dummydata`) + .set('kbn-xsrf', 'xxx') + .expect(200); + + expect(body.indexName).to.equal('dummydata'); + expect(body.status).to.equal(ReindexStatus.inProgress); + + const lastState = await waitForReindexToComplete('dummydata'); + expect(lastState.errorMessage).to.equal(null); + expect(lastState.status).to.equal(ReindexStatus.completed); + + // Cleanup newly created index + await es.indices.delete({ + index: lastState.newIndexName, + }); + }); + it('should update any aliases', async () => { await esArchiver.load('x-pack/test/functional/es_archives/upgrade_assistant/reindex'); diff --git a/yarn.lock b/yarn.lock index 50a4bdea73c76..c92b32c7d0451 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1431,10 +1431,10 @@ dependencies: object-hash "^1.3.0" -"@elastic/charts@40.2.0": - version "40.2.0" - resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-40.2.0.tgz#2e329ce4f495731f478cbaf2f8f3b89b5167a65b" - integrity sha512-N0t7YK58Kce/s9LEgaocrD75NYuFMwrcI1QNIPcwZ9IAOHY8/9yRHD5Ipoz0caGibAgOE8OunGkpyPY/NHKB5Q== +"@elastic/charts@43.1.1": + version "43.1.1" + resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-43.1.1.tgz#2a9cd4bbde9397b86a45d8aa604a1950ae0997c0" + integrity sha512-lYTdwpARIDXD15iC4cujKplBhGXb3zriBATp0wFsqgT9XE9TMOzlQ9dgylWQ+2x6OlataZLrOMnWXiFQ3uJqqQ== dependencies: "@popperjs/core" "^2.4.0" chroma-js "^2.1.0"