diff --git a/src/plugins/data/common/search/aggs/agg_types.ts b/src/plugins/data/common/search/aggs/agg_types.ts index 2e65152314cf7..0cbc3664a8659 100644 --- a/src/plugins/data/common/search/aggs/agg_types.ts +++ b/src/plugins/data/common/search/aggs/agg_types.ts @@ -29,6 +29,7 @@ export const getAggTypes = () => ({ { name: METRIC_TYPES.SUM, fn: metrics.getSumMetricAgg }, { name: METRIC_TYPES.MEDIAN, fn: metrics.getMedianMetricAgg }, { name: METRIC_TYPES.SINGLE_PERCENTILE, fn: metrics.getSinglePercentileMetricAgg }, + { name: METRIC_TYPES.SINGLE_PERCENTILE_RANK, fn: metrics.getSinglePercentileRankMetricAgg }, { name: METRIC_TYPES.MIN, fn: metrics.getMinMetricAgg }, { name: METRIC_TYPES.MAX, fn: metrics.getMaxMetricAgg }, { name: METRIC_TYPES.STD_DEV, fn: metrics.getStdDeviationMetricAgg }, @@ -102,6 +103,7 @@ export const getAggTypesFunctions = () => [ metrics.aggMax, metrics.aggMedian, metrics.aggSinglePercentile, + metrics.aggSinglePercentileRank, metrics.aggMin, metrics.aggMovingAvg, metrics.aggPercentileRanks, diff --git a/src/plugins/data/common/search/aggs/aggs_service.test.ts b/src/plugins/data/common/search/aggs/aggs_service.test.ts index 6090e965489e7..46d0921426de0 100644 --- a/src/plugins/data/common/search/aggs/aggs_service.test.ts +++ b/src/plugins/data/common/search/aggs/aggs_service.test.ts @@ -88,6 +88,7 @@ describe('Aggs service', () => { "sum", "median", "single_percentile", + "single_percentile_rank", "min", "max", "std_dev", @@ -141,6 +142,7 @@ describe('Aggs service', () => { "sum", "median", "single_percentile", + "single_percentile_rank", "min", "max", "std_dev", diff --git a/src/plugins/data/common/search/aggs/metrics/index.ts b/src/plugins/data/common/search/aggs/metrics/index.ts index 4d80e36325100..55af141b8fcb7 100644 --- a/src/plugins/data/common/search/aggs/metrics/index.ts +++ b/src/plugins/data/common/search/aggs/metrics/index.ts @@ -48,6 +48,8 @@ export * from './percentile_ranks_fn'; export * from './percentile_ranks'; export * from './percentiles_fn'; export * from './percentiles'; +export * from './single_percentile_rank_fn'; +export * from './single_percentile_rank'; export * from './serial_diff_fn'; export * from './serial_diff'; export * from './std_deviation_fn'; diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_types.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_types.ts index eed6d0a378fc2..4174808892a16 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_types.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_types.ts @@ -21,6 +21,7 @@ export enum METRIC_TYPES { GEO_CENTROID = 'geo_centroid', MEDIAN = 'median', SINGLE_PERCENTILE = 'single_percentile', + SINGLE_PERCENTILE_RANK = 'single_percentile_rank', MIN = 'min', MAX = 'max', MOVING_FN = 'moving_avg', diff --git a/src/plugins/data/common/search/aggs/metrics/percentiles_get_value.test.ts b/src/plugins/data/common/search/aggs/metrics/percentiles_get_value.test.ts new file mode 100644 index 0000000000000..06fdb8f0133ea --- /dev/null +++ b/src/plugins/data/common/search/aggs/metrics/percentiles_get_value.test.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { getPercentileValue } from './percentiles_get_value'; +import type { IResponseAggConfig } from './lib/get_response_agg_config_class'; + +describe('getPercentileValue', () => { + test('should return the correct value for an IResponseAggConfig', () => { + const agg = { + id: '0.400', + key: 400, + parentId: '0', + } as IResponseAggConfig; + const bucket = { + '0': { + values: [ + { + key: 400, + value: 24.21909648206358, + }, + ], + }, + doc_count: 2356, + }; + const value = getPercentileValue(agg, bucket); + expect(value).toEqual(24.21909648206358); + }); +}); diff --git a/src/plugins/data/common/search/aggs/metrics/single_percentile_rank.test.ts b/src/plugins/data/common/search/aggs/metrics/single_percentile_rank.test.ts new file mode 100644 index 0000000000000..030b45422e7da --- /dev/null +++ b/src/plugins/data/common/search/aggs/metrics/single_percentile_rank.test.ts @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { AggConfigs, IAggConfigs } from '../agg_configs'; +import { mockAggTypesRegistry } from '../test_helpers'; +import { METRIC_TYPES } from './metric_agg_types'; + +describe('AggTypeMetricSinglePercentileRankProvider class', () => { + let aggConfigs: IAggConfigs; + + beforeEach(() => { + const typesRegistry = mockAggTypesRegistry(); + const field = { + name: 'bytes', + }; + const indexPattern = { + id: '1234', + title: 'logstash-*', + fields: { + getByName: () => field, + filter: () => [field], + }, + } as any; + + aggConfigs = new AggConfigs( + indexPattern, + [ + { + id: METRIC_TYPES.SINGLE_PERCENTILE_RANK, + type: METRIC_TYPES.SINGLE_PERCENTILE_RANK, + schema: 'metric', + params: { + field: 'bytes', + value: 1024, + }, + }, + ], + { + typesRegistry, + } + ); + }); + + it('requests the percentile ranks aggregation in the Elasticsearch query DSL', () => { + const dsl: Record = aggConfigs.toDsl(); + + expect(dsl.single_percentile_rank.percentile_ranks.field).toEqual('bytes'); + expect(dsl.single_percentile_rank.percentile_ranks.values).toEqual([1024]); + }); + + it('points to right value within multi metric for value bucket path', () => { + expect(aggConfigs.byId(METRIC_TYPES.SINGLE_PERCENTILE_RANK)!.getValueBucketPath()).toEqual( + `${METRIC_TYPES.SINGLE_PERCENTILE_RANK}.1024` + ); + }); + + it('converts the response', () => { + const agg = aggConfigs.getResponseAggs()[0]; + + expect( + agg.getValue({ + [agg.id]: { + values: { + '1024.0': 123, + }, + }, + }) + ).toEqual(1.23); + }); + + it('should not throw error for empty buckets', () => { + const agg = aggConfigs.getResponseAggs()[0]; + expect(agg.getValue({})).toEqual(NaN); + }); + + it('produces the expected expression ast', () => { + const agg = aggConfigs.getResponseAggs()[0]; + expect(agg.toExpressionAst()).toMatchInlineSnapshot(` + Object { + "chain": Array [ + Object { + "arguments": Object { + "enabled": Array [ + true, + ], + "field": Array [ + "bytes", + ], + "id": Array [ + "single_percentile_rank", + ], + "schema": Array [ + "metric", + ], + "value": Array [ + 1024, + ], + }, + "function": "aggSinglePercentileRank", + "type": "function", + }, + ], + "type": "expression", + } + `); + }); + + it('supports scripted fields', () => { + const typesRegistry = mockAggTypesRegistry(); + const field = { + name: 'bytes', + scripted: true, + language: 'painless', + script: 'return 456', + }; + const indexPattern = { + id: '1234', + title: 'logstash-*', + fields: { + getByName: () => field, + filter: () => [field], + }, + } as any; + + aggConfigs = new AggConfigs( + indexPattern, + [ + { + id: METRIC_TYPES.SINGLE_PERCENTILE_RANK, + type: METRIC_TYPES.SINGLE_PERCENTILE_RANK, + schema: 'metric', + params: { + field: 'bytes', + value: 1024, + }, + }, + ], + { + typesRegistry, + } + ); + + expect(aggConfigs.toDsl().single_percentile_rank.percentile_ranks.script.source).toEqual( + 'return 456' + ); + expect(aggConfigs.toDsl().single_percentile_rank.percentile_ranks.values).toEqual([1024]); + }); +}); diff --git a/src/plugins/data/common/search/aggs/metrics/single_percentile_rank.ts b/src/plugins/data/common/search/aggs/metrics/single_percentile_rank.ts new file mode 100644 index 0000000000000..a5cb3d787341f --- /dev/null +++ b/src/plugins/data/common/search/aggs/metrics/single_percentile_rank.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { aggSinglePercentileRankFnName } from './single_percentile_rank_fn'; +import { MetricAggType } from './metric_agg_type'; +import { METRIC_TYPES } from './metric_agg_types'; +import type { IResponseAggConfig } from './lib/get_response_agg_config_class'; +import { KBN_FIELD_TYPES } from '../../..'; +import { BaseAggParams } from '../types'; + +const singlePercentileTitle = i18n.translate('data.search.aggs.metrics.singlePercentileRankTitle', { + defaultMessage: 'Percentile rank', +}); + +export interface AggParamsSinglePercentileRank extends BaseAggParams { + field: string; + value: number; +} + +export const getSinglePercentileRankMetricAgg = () => { + return new MetricAggType({ + name: METRIC_TYPES.SINGLE_PERCENTILE_RANK, + expressionName: aggSinglePercentileRankFnName, + dslName: 'percentile_ranks', + title: singlePercentileTitle, + valueType: 'number', + makeLabel(aggConfig) { + return i18n.translate('data.search.aggs.metrics.singlePercentileRankLabel', { + defaultMessage: 'Percentile rank of {field}', + values: { field: aggConfig.getFieldDisplayName() }, + }); + }, + getValueBucketPath(aggConfig) { + return `${aggConfig.id}.${aggConfig.params.value}`; + }, + getSerializedFormat(agg) { + return { + id: 'percent', + }; + }, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.HISTOGRAM], + }, + { + name: 'value', + default: 0, + write: (agg, output) => { + output.params.values = [agg.params.value]; + }, + }, + ], + getValue(agg, bucket) { + let valueKey = String(agg.params.value); + if (Number.isInteger(agg.params.value)) { + valueKey += '.0'; + } + const { values } = bucket[agg.id] ?? {}; + return values ? values[valueKey] / 100 : NaN; + }, + }); +}; diff --git a/src/plugins/data/common/search/aggs/metrics/single_percentile_rank_fn.ts b/src/plugins/data/common/search/aggs/metrics/single_percentile_rank_fn.ts new file mode 100644 index 0000000000000..92c88a8bcd909 --- /dev/null +++ b/src/plugins/data/common/search/aggs/metrics/single_percentile_rank_fn.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common'; +import { AggExpressionType, AggExpressionFunctionArgs, METRIC_TYPES } from '..'; + +export const aggSinglePercentileRankFnName = 'aggSinglePercentileRank'; + +type Input = any; +type AggArgs = AggExpressionFunctionArgs; +type Output = AggExpressionType; +type FunctionDefinition = ExpressionFunctionDefinition< + typeof aggSinglePercentileRankFnName, + Input, + AggArgs, + Output +>; + +export const aggSinglePercentileRank = (): FunctionDefinition => ({ + name: aggSinglePercentileRankFnName, + help: i18n.translate('data.search.aggs.function.metrics.singlePercentileRank.help', { + defaultMessage: 'Generates a serialized agg config for a single percentile rank agg', + }), + type: 'agg_type', + args: { + id: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.singlePercentileRank.id.help', { + defaultMessage: 'ID for this aggregation', + }), + }, + enabled: { + types: ['boolean'], + default: true, + help: i18n.translate('data.search.aggs.metrics.singlePercentileRank.enabled.help', { + defaultMessage: 'Specifies whether this aggregation should be enabled', + }), + }, + schema: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.singlePercentileRank.schema.help', { + defaultMessage: 'Schema to use for this aggregation', + }), + }, + field: { + types: ['string'], + required: true, + help: i18n.translate('data.search.aggs.metrics.singlePercentileRank.field.help', { + defaultMessage: 'Field to use for this aggregation', + }), + }, + value: { + types: ['number'], + required: true, + help: i18n.translate('data.search.aggs.metrics.singlePercentileRank.value.help', { + defaultMessage: 'Percentile rank value to fetch', + }), + }, + json: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.singlePercentileRank.json.help', { + defaultMessage: 'Advanced json to include when the agg is sent to Elasticsearch', + }), + }, + customLabel: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.singlePercentileRank.customLabel.help', { + defaultMessage: 'Represents a custom label for this aggregation', + }), + }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', + }), + }, + }, + fn: (input, args) => { + const { id, enabled, schema, ...rest } = args; + + return { + type: 'agg_type', + value: { + id, + enabled, + schema, + type: METRIC_TYPES.SINGLE_PERCENTILE_RANK, + params: { + ...rest, + }, + }, + }; + }, +}); diff --git a/src/plugins/data/common/search/aggs/types.ts b/src/plugins/data/common/search/aggs/types.ts index 0d5a51eaef151..942bdc492f6f7 100644 --- a/src/plugins/data/common/search/aggs/types.ts +++ b/src/plugins/data/common/search/aggs/types.ts @@ -56,6 +56,7 @@ import { AggParamsMax, AggParamsMedian, AggParamsSinglePercentile, + AggParamsSinglePercentileRank, AggParamsMin, AggParamsMovingAvg, AggParamsPercentileRanks, @@ -89,6 +90,7 @@ import { METRIC_TYPES, aggFilteredMetric, aggSinglePercentile, + aggSinglePercentileRank, } from '.'; import { AggParamsSampler } from './buckets/sampler'; import { AggParamsDiversifiedSampler } from './buckets/diversified_sampler'; @@ -175,6 +177,7 @@ export interface AggParamsMapping { [METRIC_TYPES.MAX]: AggParamsMax; [METRIC_TYPES.MEDIAN]: AggParamsMedian; [METRIC_TYPES.SINGLE_PERCENTILE]: AggParamsSinglePercentile; + [METRIC_TYPES.SINGLE_PERCENTILE_RANK]: AggParamsSinglePercentileRank; [METRIC_TYPES.MIN]: AggParamsMin; [METRIC_TYPES.STD_DEV]: AggParamsStdDeviation; [METRIC_TYPES.SUM]: AggParamsSum; @@ -225,6 +228,7 @@ export interface AggFunctionsMapping { aggMax: ReturnType; aggMedian: ReturnType; aggSinglePercentile: ReturnType; + aggSinglePercentileRank: ReturnType; aggMin: ReturnType; aggMovingAvg: ReturnType; aggPercentileRanks: ReturnType; diff --git a/src/plugins/data/public/search/aggs/aggs_service.test.ts b/src/plugins/data/public/search/aggs/aggs_service.test.ts index 4309c68d639d8..cc193b9cef15a 100644 --- a/src/plugins/data/public/search/aggs/aggs_service.test.ts +++ b/src/plugins/data/public/search/aggs/aggs_service.test.ts @@ -54,7 +54,7 @@ describe('AggsService - public', () => { service.setup(setupDeps); const start = service.start(startDeps); expect(start.types.getAll().buckets.length).toBe(16); - expect(start.types.getAll().metrics.length).toBe(24); + expect(start.types.getAll().metrics.length).toBe(25); }); test('registers custom agg types', () => { @@ -71,7 +71,7 @@ describe('AggsService - public', () => { const start = service.start(startDeps); expect(start.types.getAll().buckets.length).toBe(17); expect(start.types.getAll().buckets.some(({ name }) => name === 'foo')).toBe(true); - expect(start.types.getAll().metrics.length).toBe(25); + expect(start.types.getAll().metrics.length).toBe(26); expect(start.types.getAll().metrics.some(({ name }) => name === 'bar')).toBe(true); }); }); diff --git a/src/plugins/vis_types/gauge/public/vis_type/gauge.tsx b/src/plugins/vis_types/gauge/public/vis_type/gauge.tsx index 214a71616f427..3de528bfe5e6c 100644 --- a/src/plugins/vis_types/gauge/public/vis_type/gauge.tsx +++ b/src/plugins/vis_types/gauge/public/vis_type/gauge.tsx @@ -98,6 +98,7 @@ export const getGaugeVisTypeDefinition = ( '!geo_bounds', '!filtered_metric', '!single_percentile', + '!single_percentile_rank', ], defaults: [{ schema: 'metric', type: 'count' }], }, diff --git a/src/plugins/vis_types/gauge/public/vis_type/goal.tsx b/src/plugins/vis_types/gauge/public/vis_type/goal.tsx index 0d356a5f2c721..c953cd3e5dfe2 100644 --- a/src/plugins/vis_types/gauge/public/vis_type/goal.tsx +++ b/src/plugins/vis_types/gauge/public/vis_type/goal.tsx @@ -90,6 +90,7 @@ export const getGoalVisTypeDefinition = ( '!geo_bounds', '!filtered_metric', '!single_percentile', + '!single_percentile_rank', ], defaults: [{ schema: 'metric', type: 'count' }], }, diff --git a/src/plugins/vis_types/heatmap/public/vis_type/heatmap.tsx b/src/plugins/vis_types/heatmap/public/vis_type/heatmap.tsx index 4c62af96c9b40..6f711eb2667df 100644 --- a/src/plugins/vis_types/heatmap/public/vis_type/heatmap.tsx +++ b/src/plugins/vis_types/heatmap/public/vis_type/heatmap.tsx @@ -88,6 +88,7 @@ export const getHeatmapVisTypeDefinition = ({ 'top_hits', '!filtered_metric', '!single_percentile', + '!single_percentile_rank', ], defaults: [{ schema: 'metric', type: 'count' }], }, diff --git a/src/plugins/vis_types/metric/public/metric_vis_type.ts b/src/plugins/vis_types/metric/public/metric_vis_type.ts index 7c141131fa8e5..15ec40d3bd612 100644 --- a/src/plugins/vis_types/metric/public/metric_vis_type.ts +++ b/src/plugins/vis_types/metric/public/metric_vis_type.ts @@ -66,6 +66,7 @@ export const createMetricVisTypeDefinition = (): VisTypeDefinition => '!geo_bounds', '!filtered_metric', '!single_percentile', + '!single_percentile_rank', ], aggSettings: { top_hits: { diff --git a/src/plugins/vis_types/table/public/table_vis_type.ts b/src/plugins/vis_types/table/public/table_vis_type.ts index 80c9d7bb9f00d..b4e7a2274852e 100644 --- a/src/plugins/vis_types/table/public/table_vis_type.ts +++ b/src/plugins/vis_types/table/public/table_vis_type.ts @@ -48,7 +48,13 @@ export const tableVisTypeDefinition: VisTypeDefinition = { title: i18n.translate('visTypeTable.tableVisEditorConfig.schemas.metricTitle', { defaultMessage: 'Metric', }), - aggFilter: ['!geo_centroid', '!geo_bounds', '!filtered_metric', '!single_percentile'], + aggFilter: [ + '!geo_centroid', + '!geo_bounds', + '!filtered_metric', + '!single_percentile', + '!single_percentile_rank', + ], aggSettings: { top_hits: { allowStrings: true, diff --git a/src/plugins/vis_types/tagcloud/public/tag_cloud_type.ts b/src/plugins/vis_types/tagcloud/public/tag_cloud_type.ts index c278c47dd9022..35b7845ec515f 100644 --- a/src/plugins/vis_types/tagcloud/public/tag_cloud_type.ts +++ b/src/plugins/vis_types/tagcloud/public/tag_cloud_type.ts @@ -62,6 +62,7 @@ export const getTagCloudVisTypeDefinition = ({ palettes }: TagCloudVisDependenci '!geo_centroid', '!filtered_metric', '!single_percentile', + '!single_percentile_rank', ], defaults: [{ schema: 'metric', type: 'count' }], }, diff --git a/src/plugins/vis_types/timeseries/public/trigger_action/get_series.test.ts b/src/plugins/vis_types/timeseries/public/trigger_action/get_series.test.ts index 31c8072486615..cfd858a345669 100644 --- a/src/plugins/vis_types/timeseries/public/trigger_action/get_series.test.ts +++ b/src/plugins/vis_types/timeseries/public/trigger_action/get_series.test.ts @@ -325,6 +325,48 @@ describe('getSeries', () => { ]); }); + test('should return the correct config for the percentile ranks aggregation', () => { + const metric = [ + { + field: 'day_of_week_i', + id: 'id1', + type: 'percentile_rank', + values: ['1', '5', '7'], + colors: ['rgba(211,96,134,1)', 'rgba(155,33,230,1)', '#68BC00'], + }, + ] as Metric[]; + const config = getSeries(metric, 1); + expect(config).toStrictEqual([ + { + agg: 'percentile_rank', + color: 'rgba(211,96,134,1)', + fieldName: 'day_of_week_i', + isFullReference: false, + params: { + value: '1', + }, + }, + { + agg: 'percentile_rank', + color: 'rgba(155,33,230,1)', + fieldName: 'day_of_week_i', + isFullReference: false, + params: { + value: '5', + }, + }, + { + agg: 'percentile_rank', + color: '#68BC00', + fieldName: 'day_of_week_i', + isFullReference: false, + params: { + value: '7', + }, + }, + ]); + }); + test('should return the correct formula config for a top_hit size 1 aggregation', () => { const metric = [ { @@ -461,4 +503,55 @@ describe('getSeries', () => { }, ]); }); + + test('should return the correct formula for the math aggregation with percentile ranks as variables', () => { + const metric = [ + { + field: 'day_of_week_i', + id: 'e72265d2-2106-4af9-b646-33afd9cddcad', + values: ['1', '5', '7'], + colors: ['rgba(211,96,134,1)', 'rgba(155,33,230,1)', '#68BC00'], + type: 'percentile_rank', + }, + { + field: 'day_of_week_i', + id: '6280b080-7d1c-11ec-bfa7-3798d98f8341', + type: 'avg', + }, + { + id: '23a05540-7d18-11ec-a589-45a3784fc1ce', + script: 'params.perc1 + params.perc5 + params.avg', + type: 'math', + variables: [ + { + field: 'e72265d2-2106-4af9-b646-33afd9cddcad[1]', + id: '25840960-7d18-11ec-a589-45a3784fc1ce', + name: 'perc1', + }, + { + field: 'e72265d2-2106-4af9-b646-33afd9cddcad[5]', + id: '2a440270-7d18-11ec-a589-45a3784fc1ce', + name: 'perc5', + }, + { + field: '6280b080-7d1c-11ec-bfa7-3798d98f8341', + id: '64c82f80-7d1c-11ec-bfa7-3798d98f8341', + name: 'avg', + }, + ], + }, + ] as Metric[]; + const config = getSeries(metric, 1); + expect(config).toStrictEqual([ + { + agg: 'formula', + fieldName: 'document', + isFullReference: true, + params: { + formula: + 'percentile_rank(day_of_week_i, value=1) + percentile_rank(day_of_week_i, value=5) + average(day_of_week_i)', + }, + }, + ]); + }); }); diff --git a/src/plugins/vis_types/timeseries/public/trigger_action/get_series.ts b/src/plugins/vis_types/timeseries/public/trigger_action/get_series.ts index fef35f229359f..e180df5b4c1a9 100644 --- a/src/plugins/vis_types/timeseries/public/trigger_action/get_series.ts +++ b/src/plugins/vis_types/timeseries/public/trigger_action/get_series.ts @@ -10,6 +10,7 @@ import type { Metric } from '../../common/types'; import { SUPPORTED_METRICS } from './supported_metrics'; import { getPercentilesSeries, + getPercentileRankSeries, getFormulaSeries, getParentPipelineSeries, getSiblingPipelineSeriesFormula, @@ -45,6 +46,19 @@ export const getSeries = ( } break; } + case 'percentile_rank': { + const values = metrics[metricIdx].values; + const colors = metrics[metricIdx].colors; + if (values?.length) { + const percentileRanksSeries = getPercentileRankSeries( + values, + colors, + fieldName + ) as VisualizeEditorLayersContext['metrics']; + metricsArray = [...metricsArray, ...percentileRanksSeries]; + } + break; + } case 'math': { // find the metric idx that has math expression const mathMetricIdx = metrics.findIndex((metric) => metric.type === 'math'); @@ -69,7 +83,7 @@ export const getSeries = ( } // should treat percentiles differently - if (currentMetric.type === 'percentile') { + if (currentMetric.type === 'percentile' || currentMetric.type === 'percentile_rank') { variables.forEach((variable) => { const [_, meta] = variable?.field?.split('[') ?? []; const metaValue = Number(meta?.replace(']', '')); @@ -100,7 +114,7 @@ export const getSeries = ( break; } case 'cumulative_sum': { - // percentile value is derived from the field Id. It has the format xxx-xxx-xxx-xxx[percentile] + // percentile and percentile_rank value is derived from the field Id. It has the format xxx-xxx-xxx-xxx[percentile] const [fieldId, meta] = metrics[metricIdx]?.field?.split('[') ?? []; const subFunctionMetric = metrics.find((metric) => metric.id === fieldId); if (!subFunctionMetric || subFunctionMetric.type === 'static') { diff --git a/src/plugins/vis_types/timeseries/public/trigger_action/index.test.ts b/src/plugins/vis_types/timeseries/public/trigger_action/index.test.ts index 0ce0d0883b670..25edcb389932e 100644 --- a/src/plugins/vis_types/timeseries/public/trigger_action/index.test.ts +++ b/src/plugins/vis_types/timeseries/public/trigger_action/index.test.ts @@ -87,7 +87,7 @@ describe('triggerTSVBtoLensConfiguration', () => { ...model.series[0], metrics: [ { - type: 'percentile_rank', + type: 'std_deviation', }, ] as Series['metrics'], }, diff --git a/src/plugins/vis_types/timeseries/public/trigger_action/metrics_helpers.test.ts b/src/plugins/vis_types/timeseries/public/trigger_action/metrics_helpers.test.ts index 3abb325b4c75a..292515adf21e6 100644 --- a/src/plugins/vis_types/timeseries/public/trigger_action/metrics_helpers.test.ts +++ b/src/plugins/vis_types/timeseries/public/trigger_action/metrics_helpers.test.ts @@ -7,7 +7,11 @@ */ import { METRIC_TYPES } from '@kbn/data-plugin/public'; import type { Metric, MetricType } from '../../common/types'; -import { getPercentilesSeries, getParentPipelineSeries } from './metrics_helpers'; +import { + getPercentilesSeries, + getPercentileRankSeries, + getParentPipelineSeries, +} from './metrics_helpers'; describe('getPercentilesSeries', () => { test('should return correct config for multiple percentiles', () => { @@ -78,6 +82,37 @@ describe('getPercentilesSeries', () => { }); }); +describe('getPercentileRankSeries', () => { + test('should return correct config for multiple percentile ranks', () => { + const values = ['1', '5', '7'] as Metric['values']; + const colors = ['#68BC00', 'rgba(0,63,188,1)', 'rgba(188,38,0,1)'] as Metric['colors']; + const config = getPercentileRankSeries(values, colors, 'day_of_week_i'); + expect(config).toStrictEqual([ + { + agg: 'percentile_rank', + color: '#68BC00', + fieldName: 'day_of_week_i', + isFullReference: false, + params: { value: '1' }, + }, + { + agg: 'percentile_rank', + color: 'rgba(0,63,188,1)', + fieldName: 'day_of_week_i', + isFullReference: false, + params: { value: '5' }, + }, + { + agg: 'percentile_rank', + color: 'rgba(188,38,0,1)', + fieldName: 'day_of_week_i', + isFullReference: false, + params: { value: '7' }, + }, + ]); + }); +}); + describe('getParentPipelineSeries', () => { test('should return correct config for pipeline agg on percentiles', () => { const metrics = [ @@ -124,6 +159,36 @@ describe('getParentPipelineSeries', () => { ]); }); + test('should return correct config for pipeline agg on percentile ranks', () => { + const metrics = [ + { + field: 'AvgTicketPrice', + id: '04558549-f19f-4a87-9923-27df8b81af3e', + values: ['400', '500', '700'], + colors: ['rgba(211,96,134,1)', 'rgba(155,33,230,1)', '#68BC00'], + type: 'percentile_rank', + }, + { + field: '04558549-f19f-4a87-9923-27df8b81af3e[400.0]', + id: '764f4110-7db9-11ec-9fdf-91a8881dd06b', + type: 'derivative', + unit: '', + }, + ] as Metric[]; + const config = getParentPipelineSeries(METRIC_TYPES.DERIVATIVE, 1, metrics); + expect(config).toStrictEqual([ + { + agg: 'differences', + fieldName: 'AvgTicketPrice', + isFullReference: true, + params: { + value: 400, + }, + pipelineAggType: 'percentile_rank', + }, + ]); + }); + test('should return null config for pipeline agg on non-supported sub-aggregation', () => { const metrics = [ { diff --git a/src/plugins/vis_types/timeseries/public/trigger_action/metrics_helpers.ts b/src/plugins/vis_types/timeseries/public/trigger_action/metrics_helpers.ts index 9f6c7dd4236b1..e3d8fa0434cbd 100644 --- a/src/plugins/vis_types/timeseries/public/trigger_action/metrics_helpers.ts +++ b/src/plugins/vis_types/timeseries/public/trigger_action/metrics_helpers.ts @@ -21,6 +21,22 @@ export const getPercentilesSeries = (percentiles: Metric['percentiles'], fieldNa }); }; +export const getPercentileRankSeries = ( + values: Metric['values'], + colors: Metric['colors'], + fieldName?: string +) => { + return values?.map((value, index) => { + return { + agg: 'percentile_rank', + isFullReference: false, + color: colors?.[index], + fieldName: fieldName ?? 'document', + params: { value }, + }; + }); +}; + export const getFormulaSeries = (script: string) => { return [ { @@ -77,6 +93,7 @@ export const computeParentSeries = ( ...(currentMetric.window && { window: currentMetric.window }), ...(timeScale && { timeScale }), ...(pipelineAgg === 'percentile' && meta && { percentile: meta }), + ...(pipelineAgg === 'percentile_rank' && meta && { value: meta }), }, }, ]; @@ -151,6 +168,9 @@ export const getParentPipelineSeriesFormula = ( if (additionalPipelineAggMap.name === 'percentile' && nestedMetaValue) { additionalFunctionArgs = `, percentile=${nestedMetaValue}`; } + if (additionalPipelineAggMap.name === 'percentile_rank' && nestedMetaValue) { + additionalFunctionArgs = `, value=${nestedMetaValue}`; + } formula = `${aggMap.name}(${pipelineAgg}(${additionalPipelineAggMap.name}(${ additionalSubFunction.field ?? '' }${additionalFunctionArgs ?? ''})))`; @@ -159,6 +179,9 @@ export const getParentPipelineSeriesFormula = ( if (pipelineAgg === 'percentile' && percentileValue) { additionalFunctionArgs = `, percentile=${percentileValue}`; } + if (pipelineAgg === 'percentile_rank' && percentileValue) { + additionalFunctionArgs = `, value=${percentileValue}`; + } if (pipelineAgg === 'filter_ratio') { const script = getFilterRatioFormula(subFunctionMetric); if (!script) { @@ -183,7 +206,8 @@ export const getSiblingPipelineSeriesFormula = ( currentMetric: Metric, metrics: Metric[] ) => { - const subFunctionMetric = metrics.find((metric) => metric.id === currentMetric.field); + const [nestedFieldId, nestedMeta] = currentMetric.field?.split('[') ?? []; + const subFunctionMetric = metrics.find((metric) => metric.id === nestedFieldId); if (!subFunctionMetric || subFunctionMetric.type === 'static') { return null; } @@ -213,10 +237,23 @@ export const getSiblingPipelineSeriesFormula = ( additionalSubFunctionField ?? '' }))${minMax})`; } else { + let additionalFunctionArgs; + // handle percentile and percentile_rank + const nestedMetaValue = Number(nestedMeta?.replace(']', '')); + if (pipelineAggMap.name === 'percentile' && nestedMetaValue) { + additionalFunctionArgs = `, percentile=${nestedMetaValue}`; + } + if (pipelineAggMap.name === 'percentile_rank' && nestedMetaValue) { + additionalFunctionArgs = `, value=${nestedMetaValue}`; + } if (currentMetric.type === 'positive_only') { - minMax = `, 0, ${pipelineAggMap.name}(${subMetricField ?? ''})`; + minMax = `, 0, ${pipelineAggMap.name}(${subMetricField ?? ''}${ + additionalFunctionArgs ? `${additionalFunctionArgs}` : '' + })`; } - formula += `${pipelineAggMap.name}(${subMetricField ?? ''})${minMax})`; + formula += `${pipelineAggMap.name}(${subMetricField ?? ''}${ + additionalFunctionArgs ? `${additionalFunctionArgs}` : '' + })${minMax})`; } return formula; }; @@ -285,6 +322,9 @@ export const getFormulaEquivalent = ( metaValue ? `, percentile=${metaValue}` : '' })`; } + case 'percentile_rank': { + return `${aggregation}(${currentMetric.field}${metaValue ? `, value=${metaValue}` : ''})`; + } case 'cumulative_sum': case 'derivative': case 'moving_average': { diff --git a/src/plugins/vis_types/timeseries/public/trigger_action/supported_metrics.ts b/src/plugins/vis_types/timeseries/public/trigger_action/supported_metrics.ts index 3a304590b642d..30b6f47da5f7e 100644 --- a/src/plugins/vis_types/timeseries/public/trigger_action/supported_metrics.ts +++ b/src/plugins/vis_types/timeseries/public/trigger_action/supported_metrics.ts @@ -72,6 +72,10 @@ export const SUPPORTED_METRICS: { [key: string]: AggOptions } = { name: 'percentile', isFullReference: false, }, + percentile_rank: { + name: 'percentile_rank', + isFullReference: false, + }, sum: { name: 'sum', isFullReference: false, diff --git a/src/plugins/vis_types/xy/public/vis_types/area.ts b/src/plugins/vis_types/xy/public/vis_types/area.ts index 01668680ac24c..84a6a65d2753a 100644 --- a/src/plugins/vis_types/xy/public/vis_types/area.ts +++ b/src/plugins/vis_types/xy/public/vis_types/area.ts @@ -136,7 +136,13 @@ export const areaVisTypeDefinition = { title: i18n.translate('visTypeXy.area.metricsTitle', { defaultMessage: 'Y-axis', }), - aggFilter: ['!geo_centroid', '!geo_bounds', '!filtered_metric', '!single_percentile'], + aggFilter: [ + '!geo_centroid', + '!geo_bounds', + '!filtered_metric', + '!single_percentile', + '!single_percentile_rank', + ], min: 1, defaults: [{ schema: 'metric', type: 'count' }], }, diff --git a/src/plugins/vis_types/xy/public/vis_types/histogram.ts b/src/plugins/vis_types/xy/public/vis_types/histogram.ts index 5405ac31eba42..dd1ee2836b10f 100644 --- a/src/plugins/vis_types/xy/public/vis_types/histogram.ts +++ b/src/plugins/vis_types/xy/public/vis_types/histogram.ts @@ -140,7 +140,13 @@ export const histogramVisTypeDefinition = { defaultMessage: 'Y-axis', }), min: 1, - aggFilter: ['!geo_centroid', '!geo_bounds', '!filtered_metric', '!single_percentile'], + aggFilter: [ + '!geo_centroid', + '!geo_bounds', + '!filtered_metric', + '!single_percentile', + '!single_percentile_rank', + ], defaults: [{ schema: 'metric', type: 'count' }], }, { diff --git a/src/plugins/vis_types/xy/public/vis_types/horizontal_bar.ts b/src/plugins/vis_types/xy/public/vis_types/horizontal_bar.ts index aaf4ef2e2d51b..dda1ead899faf 100644 --- a/src/plugins/vis_types/xy/public/vis_types/horizontal_bar.ts +++ b/src/plugins/vis_types/xy/public/vis_types/horizontal_bar.ts @@ -139,7 +139,13 @@ export const horizontalBarVisTypeDefinition = { defaultMessage: 'Y-axis', }), min: 1, - aggFilter: ['!geo_centroid', '!geo_bounds', '!filtered_metric', '!single_percentile'], + aggFilter: [ + '!geo_centroid', + '!geo_bounds', + '!filtered_metric', + '!single_percentile', + '!single_percentile_rank', + ], defaults: [{ schema: 'metric', type: 'count' }], }, { diff --git a/src/plugins/vis_types/xy/public/vis_types/line.ts b/src/plugins/vis_types/xy/public/vis_types/line.ts index bd528b6565ab2..a4ad14d7f5442 100644 --- a/src/plugins/vis_types/xy/public/vis_types/line.ts +++ b/src/plugins/vis_types/xy/public/vis_types/line.ts @@ -135,7 +135,13 @@ export const lineVisTypeDefinition = { name: 'metric', title: i18n.translate('visTypeXy.line.metricTitle', { defaultMessage: 'Y-axis' }), min: 1, - aggFilter: ['!geo_centroid', '!geo_bounds', '!filtered_metric', '!single_percentile'], + aggFilter: [ + '!geo_centroid', + '!geo_bounds', + '!filtered_metric', + '!single_percentile', + '!single_percentile_rank', + ], defaults: [{ schema: 'metric', type: 'count' }], }, { diff --git a/x-pack/plugins/lens/public/index.ts b/x-pack/plugins/lens/public/index.ts index 1fb09f989fb6a..405563a0a0ce2 100644 --- a/x-pack/plugins/lens/public/index.ts +++ b/x-pack/plugins/lens/public/index.ts @@ -57,6 +57,7 @@ export type { SumIndexPatternColumn, MedianIndexPatternColumn, PercentileIndexPatternColumn, + PercentileRanksIndexPatternColumn, CountIndexPatternColumn, LastValueIndexPatternColumn, CumulativeSumIndexPatternColumn, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index 7f21cf21000b1..25ad7dfb97b4c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -1942,9 +1942,9 @@ describe('IndexPatternDimensionEditorPanel', () => { 'Minimum', 'Moving average', 'Percentile', + 'Percentile rank', 'Sum', 'Unique count', - '\u00a0', ]); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md index ae244109ed53e..b2620ce9b06af 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md @@ -11,6 +11,7 @@ differences(count()) differences(sum(bytes), normalize_unit='1s') last_value(bytes, sort=timestamp) percentile(bytes, percent=95) +percentile_rank(bytes, value=1024) Adding features beyond what we already support. New features are: diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 36039a058908b..efddd9d533f62 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -19,6 +19,7 @@ import { termsOperation } from './terms'; import { filtersOperation } from './filters'; import { cardinalityOperation } from './cardinality'; import { percentileOperation } from './percentile'; +import { percentileRanksOperation } from './percentile_ranks'; import { minOperation, averageOperation, @@ -65,6 +66,7 @@ export type { TermsIndexPatternColumn } from './terms'; export type { FiltersIndexPatternColumn, Filter } from './filters'; export type { CardinalityIndexPatternColumn } from './cardinality'; export type { PercentileIndexPatternColumn } from './percentile'; +export type { PercentileRanksIndexPatternColumn } from './percentile_ranks'; export type { MinIndexPatternColumn, AvgIndexPatternColumn, @@ -104,6 +106,7 @@ const internalOperationDefinitions = [ sumOperation, medianOperation, percentileOperation, + percentileRanksOperation, lastValueOperation, countOperation, rangeOperation, @@ -127,6 +130,7 @@ export { filtersOperation } from './filters'; export { dateHistogramOperation } from './date_histogram'; export { minOperation, averageOperation, sumOperation, maxOperation } from './metrics'; export { percentileOperation } from './percentile'; +export { percentileRanksOperation } from './percentile_ranks'; export { countOperation } from './count'; export { lastValueOperation } from './last_value'; export { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile_ranks.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile_ranks.test.tsx new file mode 100644 index 0000000000000..4812c782a5f67 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile_ranks.test.tsx @@ -0,0 +1,365 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { act } from 'react-dom/test-utils'; +import { EuiFieldNumber } from '@elastic/eui'; +import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from '@kbn/core/public'; +import { EuiFormRow } from '@elastic/eui'; +import { shallow, mount } from 'enzyme'; +import { unifiedSearchPluginMock } from '@kbn/unified-search-plugin/public/mocks'; +import { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; +import { dataPluginMock } from '@kbn/data-plugin/public/mocks'; +import { createMockedIndexPattern } from '../../mocks'; +import { percentileRanksOperation } from '.'; +import { IndexPattern, IndexPatternLayer } from '../../types'; +import type { PercentileRanksIndexPatternColumn } from './percentile_ranks'; +import { TermsIndexPatternColumn } from './terms'; + +jest.mock('lodash', () => { + const original = jest.requireActual('lodash'); + + return { + ...original, + debounce: (fn: unknown) => fn, + }; +}); + +const uiSettingsMock = {} as IUiSettingsClient; + +const defaultProps = { + storage: {} as IStorageWrapper, + uiSettings: uiSettingsMock, + savedObjectsClient: {} as SavedObjectsClientContract, + dateRange: { fromDate: 'now-1d', toDate: 'now' }, + data: dataPluginMock.createStartContract(), + unifiedSearch: unifiedSearchPluginMock.createStartContract(), + http: {} as HttpSetup, + indexPattern: { + ...createMockedIndexPattern(), + hasRestrictions: false, + } as IndexPattern, + operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), + layerId: '1', +}; + +describe('percentile ranks', () => { + let layer: IndexPatternLayer; + const InlineOptions = percentileRanksOperation.paramEditor!; + + beforeEach(() => { + layer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + } as TermsIndexPatternColumn, + col2: { + label: 'Percentile (100) of a', + dataType: 'number', + isBucketed: false, + sourceField: 'a', + operationType: 'percentile_rank', + params: { + value: 100, + }, + } as PercentileRanksIndexPatternColumn, + }, + }; + }); + + describe('getPossibleOperationForField', () => { + it('should accept number', () => { + expect( + percentileRanksOperation.getPossibleOperationForField({ + name: 'bytes', + displayName: 'bytes', + type: 'number', + esTypes: ['long'], + searchable: true, + aggregatable: true, + }) + ).toEqual({ + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }); + }); + + it('should accept histogram', () => { + expect( + percentileRanksOperation.getPossibleOperationForField({ + name: 'response_time', + displayName: 'response_time', + type: 'histogram', + esTypes: ['histogram'], + searchable: true, + aggregatable: true, + }) + ).toEqual({ + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }); + }); + + it('should reject keywords', () => { + expect( + percentileRanksOperation.getPossibleOperationForField({ + name: 'origin', + displayName: 'origin', + type: 'string', + esTypes: ['keyword'], + searchable: true, + aggregatable: true, + }) + ).toBeUndefined(); + }); + }); + + describe('toEsAggsFn', () => { + it('should reflect params correctly', () => { + const percentileRanksColumn = layer.columns.col2 as PercentileRanksIndexPatternColumn; + const esAggsFn = percentileRanksOperation.toEsAggsFn( + percentileRanksColumn, + 'col1', + {} as IndexPattern, + layer, + uiSettingsMock, + [] + ); + expect(esAggsFn).toEqual( + expect.objectContaining({ + arguments: expect.objectContaining({ + value: [100], + field: ['a'], + }), + }) + ); + }); + }); + + describe('onFieldChange', () => { + it('should change correctly to new field', () => { + const oldColumn: PercentileRanksIndexPatternColumn = { + operationType: 'percentile_rank', + sourceField: 'bytes', + label: 'Percentile rank (100) of bytes', + isBucketed: true, + dataType: 'number', + params: { + value: 100, + }, + }; + const indexPattern = createMockedIndexPattern(); + const newNumberField = indexPattern.getFieldByName('memory')!; + const column = percentileRanksOperation.onFieldChange(oldColumn, newNumberField); + + expect(column).toEqual( + expect.objectContaining({ + dataType: 'number', + sourceField: 'memory', + params: expect.objectContaining({ + value: 100, + }), + }) + ); + expect(column.label).toContain('memory'); + }); + }); + + describe('buildColumn', () => { + it('should set default percentile rank', () => { + const indexPattern = createMockedIndexPattern(); + const bytesField = indexPattern.fields.find(({ name }) => name === 'bytes')!; + bytesField.displayName = 'test'; + const percentileRanksColumn = percentileRanksOperation.buildColumn({ + indexPattern, + field: bytesField, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, + }); + expect(percentileRanksColumn.dataType).toEqual('number'); + expect(percentileRanksColumn.params.value).toEqual(0); + expect(percentileRanksColumn.label).toEqual('Percentile rank (0) of test'); + }); + + it('should create a percentile rank from formula', () => { + const indexPattern = createMockedIndexPattern(); + const bytesField = indexPattern.fields.find(({ name }) => name === 'bytes')!; + bytesField.displayName = 'test'; + const percentileRanksColumn = percentileRanksOperation.buildColumn( + { + indexPattern, + field: bytesField, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, + }, + { value: 1024 } + ); + expect(percentileRanksColumn.dataType).toEqual('number'); + expect(percentileRanksColumn.params.value).toEqual(1024); + expect(percentileRanksColumn.label).toEqual('Percentile rank (1024) of test'); + }); + + it('should create a percentile rank from formula with filter', () => { + const indexPattern = createMockedIndexPattern(); + const bytesField = indexPattern.fields.find(({ name }) => name === 'bytes')!; + bytesField.displayName = 'test'; + const percentileRanksColumn = percentileRanksOperation.buildColumn( + { + indexPattern, + field: bytesField, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, + }, + { value: 1024, kql: 'bytes > 100' } + ); + expect(percentileRanksColumn.dataType).toEqual('number'); + expect(percentileRanksColumn.params.value).toEqual(1024); + expect(percentileRanksColumn.filter).toEqual({ language: 'kuery', query: 'bytes > 100' }); + expect(percentileRanksColumn.label).toEqual('Percentile rank (1024) of test'); + }); + }); + + describe('isTransferable', () => { + it('should transfer from number to histogram', () => { + const indexPattern = createMockedIndexPattern(); + indexPattern.getFieldByName = jest.fn().mockReturnValue({ + name: 'response_time', + displayName: 'response_time', + type: 'histogram', + esTypes: ['histogram'], + aggregatable: true, + }); + expect( + percentileRanksOperation.isTransferable( + { + label: '', + sourceField: 'response_time', + isBucketed: false, + dataType: 'number', + operationType: 'percentile_rank', + params: { + value: 10, + }, + }, + indexPattern, + {} + ) + ).toBeTruthy(); + }); + }); + + describe('param editor', () => { + it('should render current percentile rank', () => { + const updateLayerSpy = jest.fn(); + const instance = shallow( + + ); + + const input = instance.find('[data-test-subj="lns-indexPattern-percentile_ranks-input"]'); + + expect(input.prop('value')).toEqual('100'); + }); + + it('should update state on change', () => { + const updateLayerSpy = jest.fn(); + const instance = mount( + + ); + + const input = instance + .find('[data-test-subj="lns-indexPattern-percentile_ranks-input"]') + .find(EuiFieldNumber); + + act(() => { + input.prop('onChange')!({ + currentTarget: { value: '103' }, + } as React.ChangeEvent); + }); + + instance.update(); + + expect(updateLayerSpy).toHaveBeenCalledWith({ + ...layer, + columns: { + ...layer.columns, + col2: { + ...layer.columns.col2, + params: { + value: 103, + }, + label: 'Percentile rank (103) of a', + }, + }, + }); + }); + + it('should not update on invalid input, but show invalid value locally', () => { + const updateLayerSpy = jest.fn(); + const instance = mount( + + ); + + const input = instance + .find('[data-test-subj="lns-indexPattern-percentile_ranks-input"]') + .find(EuiFieldNumber); + + act(() => { + input.prop('onChange')!({ + currentTarget: { value: 'miaou' }, + } as React.ChangeEvent); + }); + + instance.update(); + + expect(updateLayerSpy).not.toHaveBeenCalled(); + + expect( + instance + .find('[data-test-subj="lns-indexPattern-percentile_ranks-form"]') + .find(EuiFormRow) + .prop('isInvalid') + ).toEqual(true); + expect( + instance + .find('[data-test-subj="lns-indexPattern-percentile_ranks-input"]') + .find(EuiFieldNumber) + .prop('value') + ).toEqual('miaou'); + }); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile_ranks.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile_ranks.tsx new file mode 100644 index 0000000000000..f153b2aca669b --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile_ranks.tsx @@ -0,0 +1,242 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFormRow, EuiFieldNumberProps, EuiFieldNumber } from '@elastic/eui'; +import React, { useCallback } from 'react'; +import { i18n } from '@kbn/i18n'; +import { AggFunctionsMapping } from '@kbn/data-plugin/public'; +import { buildExpressionFunction } from '@kbn/expressions-plugin/public'; +import { OperationDefinition } from '.'; +import { + getFormatFromPreviousColumn, + getInvalidFieldMessage, + getSafeName, + isValidNumber, + getFilter, + isColumnOfType, + combineErrorMessages, +} from './helpers'; +import { FieldBasedIndexPatternColumn } from './column_types'; +import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; +import { useDebouncedValue } from '../../../shared_components'; +import { getDisallowedPreviousShiftMessage } from '../../time_shift_utils'; + +export interface PercentileRanksIndexPatternColumn extends FieldBasedIndexPatternColumn { + operationType: 'percentile_rank'; + params: { + value: number; + }; +} + +function ofName(name: string, value: number, timeShift: string | undefined) { + return adjustTimeScaleLabelSuffix( + i18n.translate('xpack.lens.indexPattern.percentileRanksOf', { + defaultMessage: 'Percentile rank ({value}) of {name}', + values: { name, value }, + }), + undefined, + undefined, + undefined, + timeShift + ); +} + +const DEFAULT_PERCENTILE_RANKS_VALUE = 0; + +const supportedFieldTypes = ['number', 'histogram']; + +export const percentileRanksOperation: OperationDefinition< + PercentileRanksIndexPatternColumn, + 'field', + { value: number } +> = { + type: 'percentile_rank', + displayName: i18n.translate('xpack.lens.indexPattern.percentileRank', { + defaultMessage: 'Percentile rank', + }), + input: 'field', + operationParams: [ + { + name: 'value', + type: 'number', + required: false, + defaultValue: DEFAULT_PERCENTILE_RANKS_VALUE, + }, + ], + filterable: true, + shiftable: true, + getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => { + if (supportedFieldTypes.includes(fieldType) && aggregatable && !aggregationRestrictions) { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + } + }, + isTransferable: (column, newIndexPattern) => { + const newField = newIndexPattern.getFieldByName(column.sourceField); + + return Boolean( + newField && + supportedFieldTypes.includes(newField.type) && + newField.aggregatable && + !newField.aggregationRestrictions + ); + }, + getDefaultLabel: (column, indexPattern, columns) => + ofName(getSafeName(column.sourceField, indexPattern), column.params.value, column.timeShift), + buildColumn: ({ field, previousColumn, indexPattern }, columnParams) => { + const existingPercentileRanksParam = + previousColumn && + isColumnOfType('percentile_rank', previousColumn) && + previousColumn.params.value; + const newPercentileRanksParam = + columnParams?.value ?? (existingPercentileRanksParam || DEFAULT_PERCENTILE_RANKS_VALUE); + return { + label: ofName( + getSafeName(field.name, indexPattern), + newPercentileRanksParam, + previousColumn?.timeShift + ), + dataType: 'number', + operationType: 'percentile_rank', + sourceField: field.name, + isBucketed: false, + scale: 'ratio', + filter: getFilter(previousColumn, columnParams), + timeShift: columnParams?.shift || previousColumn?.timeShift, + params: { + value: newPercentileRanksParam, + ...getFormatFromPreviousColumn(previousColumn), + }, + }; + }, + onFieldChange: (oldColumn, field) => { + return { + ...oldColumn, + label: ofName(field.displayName, oldColumn.params.value, oldColumn.timeShift), + sourceField: field.name, + }; + }, + toEsAggsFn: (column, columnId, _indexPattern) => { + return buildExpressionFunction( + 'aggSinglePercentileRank', + { + id: columnId, + enabled: true, + schema: 'metric', + field: column.sourceField, + value: column.params.value, + // time shift is added to wrapping aggFilteredMetric if filter is set + timeShift: column.filter ? undefined : column.timeShift, + } + ).toAst(); + }, + getErrorMessage: (layer, columnId, indexPattern) => + combineErrorMessages([ + getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), + getDisallowedPreviousShiftMessage(layer, columnId), + ]), + paramEditor: function PercentileParamEditor({ + layer, + updateLayer, + currentColumn, + columnId, + indexPattern, + }) { + const onChange = useCallback( + (value) => { + if (!isValidNumber(value) || Number(value) === currentColumn.params.value) { + return; + } + updateLayer({ + ...layer, + columns: { + ...layer.columns, + [columnId]: { + ...currentColumn, + label: currentColumn.customLabel + ? currentColumn.label + : ofName( + indexPattern.getFieldByName(currentColumn.sourceField)?.displayName || + currentColumn.sourceField, + Number(value), + currentColumn.timeShift + ), + params: { + ...currentColumn.params, + value: Number(value), + }, + } as PercentileRanksIndexPatternColumn, + }, + }); + }, + [updateLayer, layer, columnId, currentColumn, indexPattern] + ); + const { inputValue, handleInputChange: handleInputChangeWithoutValidation } = useDebouncedValue< + string | undefined + >( + { + onChange, + value: String(currentColumn.params.value), + }, + { allowFalsyValue: true } + ); + const inputValueIsValid = isValidNumber(inputValue); + + const handleInputChange: EuiFieldNumberProps['onChange'] = useCallback( + (e) => { + handleInputChangeWithoutValidation(e.currentTarget.value); + }, + [handleInputChangeWithoutValidation] + ); + + return ( + + + + ); + }, + documentation: { + section: 'elasticsearch', + signature: i18n.translate('xpack.lens.indexPattern.percentileRanks.signature', { + defaultMessage: 'field: string, [value]: number', + }), + description: i18n.translate('xpack.lens.indexPattern.percentileRanks.documentation.markdown', { + defaultMessage: ` +Returns the percentage of values which are below a certain value. For example, if a value is greater than or equal to 95% of the observed values it is said to be at the 95th percentile rank + +Example: Get the percentage of values which are below of 100: +\`percentile_rank(bytes, value=100)\` + `, + }), + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.test.ts index 438af2bfc7ab2..3998836074f6b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.test.ts @@ -17,6 +17,7 @@ import { isSortableByColumn, } from './helpers'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import type { PercentileRanksIndexPatternColumn } from '../percentile_ranks'; import { MULTI_KEY_VISUAL_SEPARATOR } from './constants'; const indexPattern = createMockedIndexPattern(); @@ -452,6 +453,44 @@ describe('isSortableByColumn()', () => { ).toBeFalsy(); }); + it('should not be sortable by percentile_rank column with non integer value', () => { + expect( + isSortableByColumn( + getLayer(getStringBasedOperationColumn(), [ + { + label: 'Percentile rank (1024.5) of bytes', + dataType: 'number', + operationType: 'percentile_rank', + sourceField: 'bytes', + isBucketed: false, + scale: 'ratio', + params: { value: 1024.5 }, + } as PercentileRanksIndexPatternColumn, + ]), + 'col2' + ) + ).toBeFalsy(); + }); + + it('should be sortable by percentile_rank column with integer value', () => { + expect( + isSortableByColumn( + getLayer(getStringBasedOperationColumn(), [ + { + label: 'Percentile rank (1024) of bytes', + dataType: 'number', + operationType: 'percentile_rank', + sourceField: 'bytes', + isBucketed: false, + scale: 'ratio', + params: { value: 1024 }, + } as PercentileRanksIndexPatternColumn, + ]), + 'col2' + ) + ).toBeTruthy(); + }); + describe('last_value operation', () => { it('should NOT be sortable when using top-hit agg', () => { expect( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts index 3f4bd781e8ff8..a3d749f4308cf 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/helpers.ts @@ -19,6 +19,7 @@ import type { FrameDatasourceAPI } from '../../../../types'; import type { FiltersIndexPatternColumn } from '..'; import type { TermsIndexPatternColumn } from './types'; import { LastValueIndexPatternColumn } from '../last_value'; +import type { PercentileRanksIndexPatternColumn } from '../percentile_ranks'; import type { IndexPatternLayer, IndexPattern, IndexPatternField } from '../../../types'; import { MULTI_KEY_VISUAL_SEPARATOR, supportedTypes } from './constants'; @@ -213,12 +214,23 @@ function checkLastValue(column: GenericIndexPatternColumn) { ); } +export function isPercentileRankSortable(column: GenericIndexPatternColumn) { + // allow the rank by metric only if the percentile rank value is integer + // https://github.com/elastic/elasticsearch/issues/66677 + return ( + column.operationType !== 'percentile_rank' || + (column.operationType === 'percentile_rank' && + Number.isInteger((column as PercentileRanksIndexPatternColumn).params.value)) + ); +} + export function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { const column = layer.columns[columnId]; return ( column && !column.isBucketed && checkLastValue(column) && + isPercentileRankSortable(column) && !('references' in column) && !isReferenced(layer, columnId) ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 2021f6132c7f9..419e087411810 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -39,6 +39,7 @@ import { getMultiTermsScriptedFieldErrorMessage, getFieldsByValidationState, isSortableByColumn, + isPercentileRankSortable, } from './helpers'; import { DEFAULT_MAX_DOC_COUNT, @@ -235,6 +236,16 @@ export const termsOperation: OperationDefinition { ); }); + it('should default percentile rank with non integer value to alphabetical sort', () => { + const newLayer = { + ...layer, + columns: { + ...layer.columns, + col2: { + ...layer.columns.col2, + operationType: 'percentile_rank', + params: { + value: 100.2, + }, + }, + }, + }; + const termsColumn = layer.columns.col1 as TermsIndexPatternColumn; + const esAggsFn = termsOperation.toEsAggsFn( + { + ...termsColumn, + params: { ...termsColumn.params, orderBy: { type: 'column', columnId: 'col2' } }, + }, + 'col1', + {} as IndexPattern, + newLayer, + uiSettingsMock, + ['col1', 'col2'] + ); + expect(esAggsFn).toEqual( + expect.objectContaining({ + function: 'aggTerms', + arguments: expect.objectContaining({ + orderBy: ['_key'], + }), + }) + ); + }); + it('should not enable missing bucket if other bucket is not set', () => { const termsColumn = layer.columns.col1 as TermsIndexPatternColumn; const esAggsFn = termsOperation.toEsAggsFn( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts index 53f69816b4add..5fec40f5e6175 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts @@ -28,6 +28,7 @@ export type { SumIndexPatternColumn, MedianIndexPatternColumn, PercentileIndexPatternColumn, + PercentileRanksIndexPatternColumn, CountIndexPatternColumn, LastValueIndexPatternColumn, CumulativeSumIndexPatternColumn, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index aff48326fb92e..8fb89d46cee2c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -93,6 +93,7 @@ describe('getOperationTypesForField', () => { 'max', 'unique_count', 'percentile', + 'percentile_rank', 'last_value', ]); }); @@ -117,6 +118,7 @@ describe('getOperationTypesForField', () => { 'max', 'unique_count', 'percentile', + 'percentile_rank', 'last_value', ]); }); @@ -369,6 +371,11 @@ describe('getOperationTypesForField', () => { "operationType": "percentile", "type": "field", }, + Object { + "field": "bytes", + "operationType": "percentile_rank", + "type": "field", + }, Object { "field": "bytes", "operationType": "last_value", diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts index 6800a3a3ecb27..4c12702615f86 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts @@ -27,6 +27,7 @@ export type { SumIndexPatternColumn, MedianIndexPatternColumn, PercentileIndexPatternColumn, + PercentileRanksIndexPatternColumn, CountIndexPatternColumn, LastValueIndexPatternColumn, CumulativeSumIndexPatternColumn,