From 08d67080e81d5963de5c0c50a22ed3f77943bce6 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Thu, 22 Dec 2022 11:31:15 +0100 Subject: [PATCH 01/23] time_series and tsdb rate support in lens --- src/plugins/data/common/search/aggs/types.ts | 2 + .../operations/definitions/cardinality.tsx | 8 +- .../operations/definitions/count.tsx | 8 +- .../operations/definitions/index.ts | 10 +- .../operations/definitions/last_value.tsx | 4 +- .../operations/definitions/metrics.tsx | 8 +- .../operations/definitions/percentile.tsx | 3 +- .../definitions/percentile_ranks.tsx | 8 +- .../operations/definitions/rate.tsx | 236 ++++++++++++++++++ .../operations/definitions/time_series.tsx | 80 ++++++ .../form_based/operations/layer_helpers.ts | 4 +- .../form_based/operations/operations.ts | 17 +- .../public/datasources/form_based/utils.tsx | 59 ++++- 13 files changed, 427 insertions(+), 20 deletions(-) create mode 100644 x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx create mode 100644 x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx diff --git a/src/plugins/data/common/search/aggs/types.ts b/src/plugins/data/common/search/aggs/types.ts index b7105baa2bd99..4bd20b63945a2 100644 --- a/src/plugins/data/common/search/aggs/types.ts +++ b/src/plugins/data/common/search/aggs/types.ts @@ -36,6 +36,7 @@ import { aggMedian, aggMin, aggMovingAvg, + aggRate, AggParamsAvg, AggParamsBucketAvg, AggParamsBucketAvgSerialized, @@ -309,4 +310,5 @@ export interface AggFunctionsMapping { aggSum: ReturnType; aggTopHit: ReturnType; aggTopMetrics: ReturnType; + aggRate: ReturnType; } diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/cardinality.tsx index 5537d2c6ee5fc..4290b968d2ca5 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/cardinality.tsx @@ -80,10 +80,16 @@ export const cardinalityOperation: OperationDefinition< }), allowAsReference: true, input: 'field', - getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type }) => { + getPossibleOperationForField: ({ + aggregationRestrictions, + aggregatable, + type, + timeSeriesMetric, + }) => { if ( supportedTypes.has(type) && aggregatable && + timeSeriesMetric !== 'counter' && (!aggregationRestrictions || aggregationRestrictions.cardinality) ) { return { dataType: 'number', isBucketed: IS_BUCKETED, scale: SCALE }; diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/count.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/count.tsx index 2295442f35ad5..174567b371ef7 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/count.tsx @@ -101,10 +101,16 @@ export const countOperation: OperationDefinition { + getPossibleOperationForField: ({ + aggregationRestrictions, + aggregatable, + type, + timeSeriesMetric, + }) => { if ( type === 'document' || (aggregatable && + timeSeriesMetric !== 'counter' && (!aggregationRestrictions || aggregationRestrictions.value_count) && supportedTypes.has(type)) ) { diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index 4ab267d36d2f2..eb506b037299b 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -12,7 +12,7 @@ import { CoreStart, } from '@kbn/core/public'; import type { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; -import type { +import { ExpressionAstExpressionBuilder, ExpressionAstFunction, } from '@kbn/expressions-plugin/public'; @@ -25,6 +25,8 @@ import { filtersOperation } from './filters'; import { cardinalityOperation } from './cardinality'; import { percentileOperation } from './percentile'; import { percentileRanksOperation } from './percentile_ranks'; +import { rateOperation } from './rate'; +import { timeSeriesOperation } from './time_series'; import { minOperation, averageOperation, @@ -115,6 +117,7 @@ const internalOperationDefinitions = [ filtersOperation, termsOperation, dateHistogramOperation, + timeSeriesOperation, minOperation, maxOperation, averageOperation, @@ -130,6 +133,7 @@ const internalOperationDefinitions = [ cumulativeSumOperation, counterRateOperation, derivativeOperation, + rateOperation, movingAverageOperation, mathOperation, formulaOperation, @@ -169,6 +173,8 @@ export { } from './calculations'; export { formulaOperation } from './formula/formula'; export { staticValueOperation } from './static_value'; +export { rateOperation } from './rate'; +export { timeSeriesOperation } from './time_series'; /** * Properties passed to the operation-specific part of the popover editor @@ -489,7 +495,7 @@ interface FieldlessOperationDefinition * Returns the meta data of the operation if applied. Undefined * if the field is not applicable. */ - getPossibleOperation: () => OperationMetadata; + getPossibleOperation: (index: IndexPattern) => OperationMetadata | undefined; /** * Function turning a column into an agg config passed to the `esaggs` function * together with the agg configs returned from other columns. diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.tsx index 1f0e50ecb2ac6..f5abbc74683c6 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.tsx @@ -199,8 +199,8 @@ export const lastValueOperation: OperationDefinition< : oldColumn.filter, }; }, - getPossibleOperationForField: ({ aggregationRestrictions, type }) => { - if (supportedTypes.has(type) && !aggregationRestrictions) { + getPossibleOperationForField: ({ aggregationRestrictions, type, timeSeriesMetric }) => { + if (supportedTypes.has(type) && !aggregationRestrictions && timeSeriesMetric !== 'counter') { return { dataType: type as DataType, isBucketed: false, diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx index 5477879a9a3fa..1815d26a01229 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx @@ -94,10 +94,16 @@ function buildMetricOperation>({ description, input: 'field', timeScalingMode: optionalTimeScaling ? 'optional' : undefined, - getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => { + getPossibleOperationForField: ({ + aggregationRestrictions, + aggregatable, + type: fieldType, + timeSeriesMetric, + }) => { if ( (supportedTypes.includes(fieldType) || (supportsDate && fieldType === 'date')) && aggregatable && + timeSeriesMetric !== 'counter' && (!aggregationRestrictions || aggregationRestrictions[type]) ) { return { diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx index a42268276613f..df9f6b768981b 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx @@ -88,10 +88,11 @@ export const percentileOperation: OperationDefinition< filterable: true, shiftable: true, canReduceTimeRange: true, - getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => { + getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType , timeSeriesMetric}) => { if ( supportedFieldTypes.includes(fieldType) && aggregatable && + timeSeriesMetric !== 'counter' && (!aggregationRestrictions || aggregationRestrictions.percentiles) ) { return { diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile_ranks.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile_ranks.tsx index 15275c81ff9c3..0f44acb92de49 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile_ranks.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile_ranks.tsx @@ -80,10 +80,16 @@ export const percentileRanksOperation: OperationDefinition< filterable: true, shiftable: true, canReduceTimeRange: true, - getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => { + getPossibleOperationForField: ({ + aggregationRestrictions, + aggregatable, + type: fieldType, + timeSeriesMetric, + }) => { if ( supportedFieldTypes.includes(fieldType) && aggregatable && + timeSeriesMetric !== 'counter' && (!aggregationRestrictions || !aggregationRestrictions.percentile_ranks) ) { return { diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx new file mode 100644 index 0000000000000..750f6abd93827 --- /dev/null +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx @@ -0,0 +1,236 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiFormRow, EuiComboBox } from '@elastic/eui'; +import { AggFunctionsMapping } from '@kbn/data-plugin/public'; +import { buildExpression, buildExpressionFunction } from '@kbn/expressions-plugin/public'; +import { useCallback } from 'react'; +import { useDebouncedValue } from '../../../../shared_components'; +import { OperationDefinition } from '.'; +import { FieldBasedIndexPatternColumn, ValueFormatConfig } from './column_types'; + +import { + getFormatFromPreviousColumn, + getInvalidFieldMessage, + getSafeName, + getFilter, + combineErrorMessages, +} from './helpers'; +import { isMetricCounterField } from '../../utils'; + +const supportedTypes = new Set(['number']); + +const SCALE = 'ratio'; +const OPERATION_TYPE = 'rate'; +const IS_BUCKETED = false; + +function ofName(name: string) { + return i18n.translate('xpack.lens.indexPattern.cardinalityOf', { + defaultMessage: 'Rate of {name}', + values: { + name, + }, + }); +} + +type AggregateFnTypes = 'sum' | 'max' | 'min' | 'avg'; + +export interface RateIndexPatternColumn extends FieldBasedIndexPatternColumn { + operationType: typeof OPERATION_TYPE; + params?: { + format?: ValueFormatConfig; + aggregateFn: AggregateFnTypes; + }; +} + +export const rateOperation: OperationDefinition = { + type: OPERATION_TYPE, + displayName: i18n.translate('xpack.lens.indexPattern.rate', { + defaultMessage: 'Rate', + }), + allowAsReference: true, + input: 'field', + operationParams: [{ name: 'aggregation', type: 'string', required: false, defaultValue: 'max' }], + getPossibleOperationForField: ({ + aggregationRestrictions, + aggregatable, + type, + timeSeriesMetric, + }) => { + if (supportedTypes.has(type) && aggregatable && timeSeriesMetric === 'counter') { + return { dataType: 'number', isBucketed: IS_BUCKETED, scale: SCALE }; + } + }, + getErrorMessage: (layer, columnId, indexPattern) => + combineErrorMessages([ + getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), + // getDisallowedPreviousShiftMessage(layer, columnId), + // getColumnReducedTimeRangeError(layer, columnId, indexPattern), + ]), + isTransferable: (column, newIndexPattern) => { + const newField = newIndexPattern.getFieldByName(column.sourceField); + + return Boolean( + newField && + supportedTypes.has(newField.type) && + newField.aggregatable && + isMetricCounterField(newField) && + (!newField.aggregationRestrictions || newField.aggregationRestrictions.cardinality) + ); + }, + filterable: true, + shiftable: true, + canReduceTimeRange: true, + getDefaultLabel: (column, indexPattern) => ofName(getSafeName(column.sourceField, indexPattern)), + buildColumn({ field, previousColumn }, columnParams) { + return { + label: ofName(field.displayName), + dataType: 'number', + operationType: OPERATION_TYPE, + scale: SCALE, + sourceField: field.name, + isBucketed: IS_BUCKETED, + filter: getFilter(previousColumn, columnParams), + timeShift: columnParams?.shift || previousColumn?.timeShift, + reducedTimeRange: columnParams?.reducedTimeRange || previousColumn?.reducedTimeRange, + params: { + aggregateFn: 'sum', + ...getFormatFromPreviousColumn(previousColumn), + }, + }; + }, + + toEsAggsFn: (column, columnId) => { + if (true) { + // if wrap in timeseries and bucket agg + return buildExpressionFunction('aggBucketMax', { + id: columnId, + enabled: true, + schema: 'metric', + customBucket: buildExpression([ + buildExpressionFunction('aggTimeSeries', { + id: `-timeseries`, + enabled: true, + schema: 'bucket', + }), + ]), + customMetric: buildExpression([ + buildExpressionFunction('aggRate', { + id: '-metric', + enabled: true, + schema: 'metric', + field: column.sourceField, + // time shift is added to wrapping aggFilteredMetric if filter is set + timeShift: column.filter ? undefined : column.timeShift, + unit: 'hour', + }), + ]), + }).toAst(); + } + + // if we are already wrapped in timeseries (user defined a breakdown) + // this is optimization, we get the results running above code as well but is more resource intensive + return buildExpressionFunction('aggRate', { + id: columnId, + enabled: true, + schema: 'metric', + field: column.sourceField, + // time shift is added to wrapping aggFilteredMetric if filter is set + timeShift: column.filter ? undefined : column.timeShift, + unit: 'hour', + }).toAst(); + }, + onFieldChange: (oldColumn, field) => { + return { + ...oldColumn, + label: ofName(field.displayName), + sourceField: field.name, + }; + }, + documentation: { + section: 'elasticsearch', + signature: i18n.translate('xpack.lens.indexPattern.rate.signature', { + defaultMessage: 'field: string', + }), + description: i18n.translate('xpack.lens.indexPattern.rate.documentation.markdown', { + defaultMessage: ` +Calculates the number of unique values of a specified field. Works for number, string, date and boolean values. + +Example: Calculate the number of different products: +\`unique_count(product.name)\` + +Example: Calculate the number of different products from the "clothes" group: +\`unique_count(product.name, kql='product.group=clothes')\` + `, + }), + }, + quickFunctionDocumentation: i18n.translate( + 'xpack.lens.indexPattern.cardinality.documentation.quick', + { + defaultMessage: ` +The number of unique values for a specified number, string, date, or boolean field. + `, + } + ), + paramEditor: function RateParamEditor({ + paramEditorUpdater, + currentColumn, + indexPattern, + paramEditorCustomProps, + }) { + const onChange = useCallback( + (newAggFn: AggregateFnTypes) => + paramEditorUpdater({ + ...currentColumn, + params: { + ...currentColumn.params, + aggregateFn: newAggFn, + }, + } as RateIndexPatternColumn), + [currentColumn, paramEditorUpdater] + ); + const { inputValue, handleInputChange } = useDebouncedValue({ + value: currentColumn?.params?.aggregateFn ?? 'sum', + defaultValue: 'sum', + onChange, + }); + + const label = i18n.translate('xpack.lens.indexPattern.rate.aggregateFn', { + defaultMessage: 'Aggregate Function', + }); + + const options = [ + { value: 'sum', label: 'Sum' }, + { value: 'avg', label: 'Average' }, + { value: 'max', label: 'Max' }, + { value: 'min', label: 'Min' }, + ]; + + return ( + + value === inputValue) || options[0]]} + onChange={(choices) => { + if (!choices.length || choices[0].value == null) { + return; + } + handleInputChange(choices[0].value as AggregateFnTypes); + }} + /> + + ); + }, +}; diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx new file mode 100644 index 0000000000000..aaeddea1cf04e --- /dev/null +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; + +import { AggFunctionsMapping } from '@kbn/data-plugin/public'; +import { buildExpressionFunction } from '@kbn/expressions-plugin/public'; +import { BaseIndexPatternColumn, OperationDefinition } from '.'; + +export interface TimeSeriesIndexPatternColumn extends BaseIndexPatternColumn { + operationType: 'time_series'; + params: {}; +} + +export type TimeSeriesColumnParams = TimeSeriesIndexPatternColumn['params']; +export type UpdateParamsFnType = ( + paramName: K, + value: TimeSeriesColumnParams[K] +) => void; + +export const timeSeriesOperation: OperationDefinition< + TimeSeriesIndexPatternColumn, + 'none', + TimeSeriesColumnParams +> = { + type: 'time_series', + displayName: i18n.translate('xpack.lens.indexPattern.timeSeries', { + defaultMessage: 'All Series', + }), + priority: 4, // Higher than terms, so numbers get histogram + input: 'none', + getPossibleOperation: (index) => { + if (index.fields.find((f) => f.timeSeriesDimension || f.timeSeriesMetric)) { + return { + dataType: 'string', + isBucketed: true, + scale: 'ordinal', + }; + } + }, + getDefaultLabel: () => 'Time Series', + buildColumn() { + return { + label: 'series', + dataType: 'string', + operationType: 'time_series', + isBucketed: true, + scale: 'ordinal', // ordinal for Range + params: { + includeEmptyRows: true, + }, + }; + }, + isTransferable: (column, newIndexPattern) => { + return true; // newIndexPattern.(column.sourceField); + }, + toEsAggsFn: (column, columnId) => { + return buildExpressionFunction('aggTimeSeries', { + id: columnId, + enabled: true, + schema: 'bucket', + }).toAst(); + }, + paramEditor: () => { + return <>; + }, + quickFunctionDocumentation: i18n.translate( + 'xpack.lens.indexPattern.time_series.documentation.quick', + { + defaultMessage: ` + Buckets by all time series. + `, + } + ), +}; diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/layer_helpers.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/layer_helpers.ts index f1b93b33467a5..23938baa83365 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/layer_helpers.ts @@ -360,7 +360,7 @@ export function insertNewColumn({ // TODO: need to create on the fly the new columns for Formula, // like we do for fullReferences to show a seamless transition } - const possibleOperation = operationDefinition.getPossibleOperation(); + const possibleOperation = operationDefinition.getPossibleOperation(indexPattern); const isBucketed = Boolean(possibleOperation?.isBucketed); const addOperationFn = isBucketed ? addBucket : addMetric; const buildColumnFn = columnParams @@ -1689,7 +1689,7 @@ export function isOperationAllowedAsReference({ hasValidMetadata = Boolean(metadata) && validation.validateMetadata(metadata!, operationType, field.name); } else if (operationDefinition.input === 'none') { - const metadata = operationDefinition.getPossibleOperation(); + const metadata = operationDefinition.getPossibleOperation(indexPattern); hasValidMetadata = Boolean(metadata) && validation.validateMetadata(metadata!, operationType); } else if (operationDefinition.input === 'fullReference') { const metadata = operationDefinition.getPossibleOperation(indexPattern); diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/operations.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/operations.ts index b44c0a0e4cc55..151081e190a68 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/operations.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/operations.ts @@ -241,13 +241,16 @@ export function getAvailableOperationsByMetadata( ); }); } else if (operationDefinition.input === 'none') { - addToMap( - { - type: 'none', - operationType: operationDefinition.type, - }, - operationDefinition.getPossibleOperation() - ); + const validOperation = operationDefinition.getPossibleOperation(indexPattern); + if (validOperation) { + addToMap( + { + type: 'none', + operationType: operationDefinition.type, + }, + validOperation + ); + } } else if (operationDefinition.input === 'fullReference') { const validOperation = operationDefinition.getPossibleOperation(indexPattern); if (validOperation) { diff --git a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx index 5837d77c2a92d..61837cfa30829 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx @@ -26,10 +26,19 @@ import { import { estypes } from '@elastic/elasticsearch'; import type { DateRange } from '../../../common/types'; -import type { FramePublicAPI, IndexPattern, StateSetter, UserMessage } from '../../types'; +import type { + FramePublicAPI, + IndexPattern, + IndexPatternField, + StateSetter, + UserMessage, +} from '../../types'; import { renewIDs } from '../../utils'; import type { FormBasedLayer, FormBasedPersistedState, FormBasedPrivateState } from './types'; -import type { ReferenceBasedIndexPatternColumn } from './operations/definitions/column_types'; +import type { + FieldBasedIndexPatternColumn, + ReferenceBasedIndexPatternColumn, +} from './operations/definitions/column_types'; import { operationDefinitionMap, @@ -51,6 +60,10 @@ import { supportsRarityRanking } from './operations/definitions/terms'; import { DEFAULT_MAX_DOC_COUNT } from './operations/definitions/terms/constants'; import { getOriginalId } from '../../../common/expressions/datatable/transpose_helpers'; import { isQueryValid } from '../../shared_components'; +import { + checkForDateHistogram, + checkReferences, +} from './operations/definitions/calculations/utils'; export function isColumnInvalid( layer: FormBasedLayer, @@ -121,6 +134,48 @@ export function fieldIsInvalid( return !!getInvalidFieldMessage(layer, columnId, indexPattern)?.length; } +export function isMetricCounterField(field?: IndexPatternField) { + return field?.timeSeriesMetric === 'counter'; +} + +function checkReferencedColumnMetric( + layer: FormBasedLayer, + columnId: string, + indexPattern: IndexPattern +) { + const column = layer.columns[columnId] as ReferenceBasedIndexPatternColumn; + return column.references + .filter((referencedId) => 'sourceField' in layer.columns[referencedId]) + .map((referencedId) => { + const fieldName = (layer.columns[referencedId] as FieldBasedIndexPatternColumn).sourceField; + if (!isMetricCounterField(indexPattern.getFieldByName(fieldName))) { + return i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { + defaultMessage: 'Dimension "{dimensionLabel}" is configured incorrectly', + values: { + dimensionLabel: layer.columns[referencedId].label, + }, + }); + } + }); +} + +export function getErrorForRateReference( + layer: FormBasedLayer, + columnId: string, + name: string, + indexPattern: IndexPattern +) { + const dateErrors = checkForDateHistogram(layer, name) ?? []; + const referenceErrors = checkReferences(layer, columnId) ?? []; + const metricCounterErrors = checkReferencedColumnMetric(layer, columnId, indexPattern) ?? []; + if (metricCounterErrors.length) { + return metricCounterErrors.concat(referenceErrors); + } + if (dateErrors.length) { + return dateErrors.concat(referenceErrors); + } +} + const accuracyModeDisabledWarning = ( columnName: string, columnId: string, From a2c9574d8b6a4f6eb51810a266c1e02b4e999396 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Tue, 27 Dec 2022 14:25:09 +0100 Subject: [PATCH 02/23] removing time_series breakdown --- .../operations/definitions/index.ts | 3 - .../operations/definitions/rate.tsx | 12 --- .../operations/definitions/time_series.tsx | 80 ------------------- 3 files changed, 95 deletions(-) delete mode 100644 x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index eb506b037299b..158d6187f26bc 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -26,7 +26,6 @@ import { cardinalityOperation } from './cardinality'; import { percentileOperation } from './percentile'; import { percentileRanksOperation } from './percentile_ranks'; import { rateOperation } from './rate'; -import { timeSeriesOperation } from './time_series'; import { minOperation, averageOperation, @@ -117,7 +116,6 @@ const internalOperationDefinitions = [ filtersOperation, termsOperation, dateHistogramOperation, - timeSeriesOperation, minOperation, maxOperation, averageOperation, @@ -174,7 +172,6 @@ export { export { formulaOperation } from './formula/formula'; export { staticValueOperation } from './static_value'; export { rateOperation } from './rate'; -export { timeSeriesOperation } from './time_series'; /** * Properties passed to the operation-specific part of the popover editor diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx index 750f6abd93827..dface2e5917e6 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx @@ -133,18 +133,6 @@ export const rateOperation: OperationDefinition('aggRate', { - id: columnId, - enabled: true, - schema: 'metric', - field: column.sourceField, - // time shift is added to wrapping aggFilteredMetric if filter is set - timeShift: column.filter ? undefined : column.timeShift, - unit: 'hour', - }).toAst(); }, onFieldChange: (oldColumn, field) => { return { diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx deleted file mode 100644 index aaeddea1cf04e..0000000000000 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; -import { i18n } from '@kbn/i18n'; - -import { AggFunctionsMapping } from '@kbn/data-plugin/public'; -import { buildExpressionFunction } from '@kbn/expressions-plugin/public'; -import { BaseIndexPatternColumn, OperationDefinition } from '.'; - -export interface TimeSeriesIndexPatternColumn extends BaseIndexPatternColumn { - operationType: 'time_series'; - params: {}; -} - -export type TimeSeriesColumnParams = TimeSeriesIndexPatternColumn['params']; -export type UpdateParamsFnType = ( - paramName: K, - value: TimeSeriesColumnParams[K] -) => void; - -export const timeSeriesOperation: OperationDefinition< - TimeSeriesIndexPatternColumn, - 'none', - TimeSeriesColumnParams -> = { - type: 'time_series', - displayName: i18n.translate('xpack.lens.indexPattern.timeSeries', { - defaultMessage: 'All Series', - }), - priority: 4, // Higher than terms, so numbers get histogram - input: 'none', - getPossibleOperation: (index) => { - if (index.fields.find((f) => f.timeSeriesDimension || f.timeSeriesMetric)) { - return { - dataType: 'string', - isBucketed: true, - scale: 'ordinal', - }; - } - }, - getDefaultLabel: () => 'Time Series', - buildColumn() { - return { - label: 'series', - dataType: 'string', - operationType: 'time_series', - isBucketed: true, - scale: 'ordinal', // ordinal for Range - params: { - includeEmptyRows: true, - }, - }; - }, - isTransferable: (column, newIndexPattern) => { - return true; // newIndexPattern.(column.sourceField); - }, - toEsAggsFn: (column, columnId) => { - return buildExpressionFunction('aggTimeSeries', { - id: columnId, - enabled: true, - schema: 'bucket', - }).toAst(); - }, - paramEditor: () => { - return <>; - }, - quickFunctionDocumentation: i18n.translate( - 'xpack.lens.indexPattern.time_series.documentation.quick', - { - defaultMessage: ` - Buckets by all time series. - `, - } - ), -}; From 7f72062f1cda2cef6845fd1e36e5812cdc083c7c Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 15 Feb 2023 14:08:22 +0100 Subject: [PATCH 03/23] tests and cleanup --- .../operations/definitions/index.ts | 3 +- .../operations/definitions/rate.test.ts | 290 ++++++++++++++++++ .../operations/definitions/rate.tsx | 16 +- 3 files changed, 298 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.test.ts diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index 158d6187f26bc..284aa2c5efd1b 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -12,7 +12,7 @@ import { CoreStart, } from '@kbn/core/public'; import type { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; -import { +import type { ExpressionAstExpressionBuilder, ExpressionAstFunction, } from '@kbn/expressions-plugin/public'; @@ -105,6 +105,7 @@ export type { } from './calculations'; export type { CountIndexPatternColumn } from './count'; export type { LastValueIndexPatternColumn } from './last_value'; +export type { RateIndexPatternColumn } from './rate'; export type { RangeIndexPatternColumn } from './ranges'; export type { FormulaIndexPatternColumn, MathIndexPatternColumn } from './formula'; export type { StaticValueIndexPatternColumn } from './static_value'; diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.test.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.test.ts new file mode 100644 index 0000000000000..423afaa40e890 --- /dev/null +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.test.ts @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TermsIndexPatternColumn, RateIndexPatternColumn, rateOperation } from '.'; +import { FormBasedLayer } from '../../../..'; +import { IndexPattern } from '../../../../types'; +import { IUiSettingsClient } from '@kbn/core-ui-settings-browser'; +import { createMockedIndexPattern } from '../../mocks'; + +const uiSettingsMock = {} as IUiSettingsClient; + +describe('rate function', () => { + let layer: FormBasedLayer; + + beforeEach(() => { + layer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + } as TermsIndexPatternColumn, + col2: { + label: 'rate of a', + dataType: 'number', + isBucketed: false, + sourceField: 'a', + operationType: 'rate', + params: { + aggregateFn: 'max', + }, + } as RateIndexPatternColumn, + }, + }; + }); + + describe('toEsAggsFn', () => { + it('should reflect params correctly', () => { + const column = layer.columns.col2 as RateIndexPatternColumn; + const result = rateOperation.toEsAggsFn( + { + ...column, + params: { ...column.params! }, + }, + 'col1', + {} as IndexPattern, + layer, + uiSettingsMock, + [] + ); + expect(result).toMatchInlineSnapshot(` + Object { + "arguments": Object { + "customBucket": Array [ + Object { + "chain": Array [ + Object { + "arguments": Object { + "enabled": Array [ + true, + ], + "id": Array [ + "-timeseries", + ], + "schema": Array [ + "bucket", + ], + }, + "function": "aggTimeSeries", + "type": "function", + }, + ], + "type": "expression", + }, + ], + "customMetric": Array [ + Object { + "chain": Array [ + Object { + "arguments": Object { + "enabled": Array [ + true, + ], + "field": Array [ + "a", + ], + "id": Array [ + "-metric", + ], + "schema": Array [ + "metric", + ], + "unit": Array [ + "hour", + ], + }, + "function": "aggRate", + "type": "function", + }, + ], + "type": "expression", + }, + ], + "enabled": Array [ + true, + ], + "id": Array [ + "col1", + ], + "schema": Array [ + "metric", + ], + }, + "function": "aggBucketMax", + "type": "function", + } + `); + }); + }); + + describe('onFieldChange', () => { + it('should change correctly to new field', () => { + const oldColumn: RateIndexPatternColumn = { + operationType: 'rate', + sourceField: 'source', + label: 'Last value of source', + isBucketed: true, + dataType: 'string', + params: { + aggregateFn: 'sum', + }, + }; + const indexPattern = createMockedIndexPattern(); + const newNumberField = indexPattern.getFieldByName('bytes')!; + const column = rateOperation.onFieldChange(oldColumn, newNumberField); + + expect(column).toEqual( + expect.objectContaining({ + dataType: 'string', + sourceField: 'bytes', + params: { + aggregateFn: 'sum', + }, + }) + ); + expect(column.label).toContain('bytes'); + }); + }); + + describe('getPossibleOperationForField', () => { + it('should return operation with the right type', () => { + expect( + rateOperation.getPossibleOperationForField({ + aggregatable: true, + searchable: true, + name: 'test', + displayName: 'test', + type: 'boolean', + }) + ).toEqual(undefined); + + expect( + rateOperation.getPossibleOperationForField({ + aggregatable: true, + searchable: true, + name: 'test', + displayName: 'test', + type: 'ip', + }) + ).toEqual(undefined); + }); + + it('should not return an operation if field is not tsdb counter metric', () => { + expect( + rateOperation.getPossibleOperationForField({ + aggregatable: true, + searchable: true, + name: 'test', + displayName: 'test', + type: 'string', + }) + ).toEqual(undefined); + + expect( + rateOperation.getPossibleOperationForField({ + aggregatable: true, + aggregationRestrictions: {}, + searchable: true, + name: 'test', + displayName: 'test', + type: 'string', + }) + ).toEqual(undefined); + // does it have to be aggregatable? + expect( + rateOperation.getPossibleOperationForField({ + aggregatable: false, + searchable: true, + name: 'test', + displayName: 'test', + type: 'string', + }) + ).toEqual(undefined); + }); + }); + + describe('buildColumn', () => { + it('should return number type', () => { + const lastValueColumn = rateOperation.buildColumn({ + indexPattern: createMockedIndexPattern(), + field: { + aggregatable: true, + searchable: true, + type: 'string', + name: 'test', + displayName: 'test', + }, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, + }); + expect(lastValueColumn.dataType).toEqual('number'); + }); + }); + + it('should pick the previous format configuration if set', () => { + const indexPattern = createMockedIndexPattern(); + expect( + rateOperation.buildColumn({ + indexPattern, + layer: { + columns: { + col1: { + label: 'Count', + dataType: 'number', + isBucketed: false, + sourceField: '___records___', + operationType: 'count', + }, + }, + columnOrder: [], + indexPatternId: '', + }, + + field: { + aggregatable: true, + searchable: true, + type: 'boolean', + name: 'test', + displayName: 'test', + }, + previousColumn: { + label: 'Count', + dataType: 'number', + isBucketed: false, + sourceField: '___records___', + operationType: 'count', + params: { + format: { + id: 'number', + params: { + decimals: 2, + }, + }, + }, + }, + }).params + ).toEqual( + expect.objectContaining({ + format: { + id: 'number', + params: { + decimals: 2, + }, + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx index dface2e5917e6..211901a31a5b0 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx @@ -31,7 +31,7 @@ const OPERATION_TYPE = 'rate'; const IS_BUCKETED = false; function ofName(name: string) { - return i18n.translate('xpack.lens.indexPattern.cardinalityOf', { + return i18n.translate('xpack.lens.indexPattern.rateOf', { defaultMessage: 'Rate of {name}', values: { name, @@ -69,7 +69,7 @@ export const rateOperation: OperationDefinition combineErrorMessages([ - getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), + getInvalidFieldMessage(layer, columnId, indexPattern), // getDisallowedPreviousShiftMessage(layer, columnId), // getColumnReducedTimeRangeError(layer, columnId, indexPattern), ]), @@ -81,7 +81,7 @@ export const rateOperation: OperationDefinition { if (true) { // if wrap in timeseries and bucket agg @@ -158,14 +157,11 @@ Example: Calculate the number of different products from the "clothes" group: `, }), }, - quickFunctionDocumentation: i18n.translate( - 'xpack.lens.indexPattern.cardinality.documentation.quick', - { - defaultMessage: ` + quickFunctionDocumentation: i18n.translate('xpack.lens.indexPattern.rate.documentation.quick', { + defaultMessage: ` The number of unique values for a specified number, string, date, or boolean field. `, - } - ), + }), paramEditor: function RateParamEditor({ paramEditorUpdater, currentColumn, From c9a8322383a19e16cd1940672e2f35f928715b5f Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 15 Feb 2023 15:31:26 +0100 Subject: [PATCH 04/23] fix types --- .../datasources/form_based/operations/definitions/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index 284aa2c5efd1b..acfdad8b21e46 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -634,7 +634,7 @@ interface FullReferenceOperationDefinition { * Returns the meta data of the operation if applied. Undefined * if the operation can't be added with these fields. */ - getPossibleOperation: (indexPattern: IndexPattern) => OperationMetadata | undefined; + getPossibleOperation: (indexPattern?: IndexPattern) => OperationMetadata | undefined; /** * A chain of expression functions which will transform the table */ From 7856042b1f141454f51c14d676f53f5e6c6ffa11 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 15 Feb 2023 16:50:25 +0100 Subject: [PATCH 05/23] fix types --- .../datasources/form_based/operations/definitions/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index acfdad8b21e46..04b88a18f8b02 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -493,7 +493,7 @@ interface FieldlessOperationDefinition * Returns the meta data of the operation if applied. Undefined * if the field is not applicable. */ - getPossibleOperation: (index: IndexPattern) => OperationMetadata | undefined; + getPossibleOperation: (index?: IndexPattern) => OperationMetadata | undefined; /** * Function turning a column into an agg config passed to the `esaggs` function * together with the agg configs returned from other columns. @@ -634,7 +634,7 @@ interface FullReferenceOperationDefinition { * Returns the meta data of the operation if applied. Undefined * if the operation can't be added with these fields. */ - getPossibleOperation: (indexPattern?: IndexPattern) => OperationMetadata | undefined; + getPossibleOperation: (indexPattern: IndexPattern) => OperationMetadata | undefined; /** * A chain of expression functions which will transform the table */ From 988c91568554945142d54ebb90264dcc14cf17ad Mon Sep 17 00:00:00 2001 From: ppisljar Date: Thu, 16 Feb 2023 11:04:50 +0100 Subject: [PATCH 06/23] moving helpers --- .../operations/definitions/helpers.tsx | 46 +++++++++++++++ .../operations/definitions/rate.tsx | 6 +- .../public/datasources/form_based/utils.tsx | 59 +------------------ 3 files changed, 51 insertions(+), 60 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx index 69b442ae97d48..7d2bcc767ea80 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx @@ -21,6 +21,10 @@ import { } from './column_types'; import type { FormBasedLayer } from '../../types'; import { hasField } from '../../pure_utils'; +import { + checkForDateHistogram, + checkReferences +} from "@kbn/lens-plugin/public/datasources/form_based/operations/definitions/calculations/utils"; export function getInvalidFieldMessage( layer: FormBasedLayer, @@ -204,3 +208,45 @@ export function getFilter( } return filter; } + +export function isMetricCounterField(field?: IndexPatternField) { + return field?.timeSeriesMetric === 'counter'; +} + +function checkReferencedColumnMetric( + layer: FormBasedLayer, + columnId: string, + indexPattern: IndexPattern +) { + const column = layer.columns[columnId] as ReferenceBasedIndexPatternColumn; + return column.references + .filter((referencedId) => 'sourceField' in layer.columns[referencedId]) + .map((referencedId) => { + const fieldName = (layer.columns[referencedId] as FieldBasedIndexPatternColumn).sourceField; + if (!isMetricCounterField(indexPattern.getFieldByName(fieldName))) { + return i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { + defaultMessage: 'Dimension "{dimensionLabel}" is configured incorrectly', + values: { + dimensionLabel: layer.columns[referencedId].label, + }, + }); + } + }); +} + +export function getErrorForRateReference( + layer: FormBasedLayer, + columnId: string, + name: string, + indexPattern: IndexPattern +) { + const dateErrors = checkForDateHistogram(layer, name) ?? []; + const referenceErrors = checkReferences(layer, columnId) ?? []; + const metricCounterErrors = checkReferencedColumnMetric(layer, columnId, indexPattern) ?? []; + if (metricCounterErrors.length) { + return metricCounterErrors.concat(referenceErrors); + } + if (dateErrors.length) { + return dateErrors.concat(referenceErrors); + } +} diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx index 211901a31a5b0..18787b7f97d3a 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx @@ -21,8 +21,9 @@ import { getSafeName, getFilter, combineErrorMessages, + isMetricCounterField, + getErrorForRateReference, } from './helpers'; -import { isMetricCounterField } from '../../utils'; const supportedTypes = new Set(['number']); @@ -70,8 +71,7 @@ export const rateOperation: OperationDefinition combineErrorMessages([ getInvalidFieldMessage(layer, columnId, indexPattern), - // getDisallowedPreviousShiftMessage(layer, columnId), - // getColumnReducedTimeRangeError(layer, columnId, indexPattern), + getErrorForRateReference(layer, columnId, 'rate', indexPattern), ]), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx index 61837cfa30829..5837d77c2a92d 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx @@ -26,19 +26,10 @@ import { import { estypes } from '@elastic/elasticsearch'; import type { DateRange } from '../../../common/types'; -import type { - FramePublicAPI, - IndexPattern, - IndexPatternField, - StateSetter, - UserMessage, -} from '../../types'; +import type { FramePublicAPI, IndexPattern, StateSetter, UserMessage } from '../../types'; import { renewIDs } from '../../utils'; import type { FormBasedLayer, FormBasedPersistedState, FormBasedPrivateState } from './types'; -import type { - FieldBasedIndexPatternColumn, - ReferenceBasedIndexPatternColumn, -} from './operations/definitions/column_types'; +import type { ReferenceBasedIndexPatternColumn } from './operations/definitions/column_types'; import { operationDefinitionMap, @@ -60,10 +51,6 @@ import { supportsRarityRanking } from './operations/definitions/terms'; import { DEFAULT_MAX_DOC_COUNT } from './operations/definitions/terms/constants'; import { getOriginalId } from '../../../common/expressions/datatable/transpose_helpers'; import { isQueryValid } from '../../shared_components'; -import { - checkForDateHistogram, - checkReferences, -} from './operations/definitions/calculations/utils'; export function isColumnInvalid( layer: FormBasedLayer, @@ -134,48 +121,6 @@ export function fieldIsInvalid( return !!getInvalidFieldMessage(layer, columnId, indexPattern)?.length; } -export function isMetricCounterField(field?: IndexPatternField) { - return field?.timeSeriesMetric === 'counter'; -} - -function checkReferencedColumnMetric( - layer: FormBasedLayer, - columnId: string, - indexPattern: IndexPattern -) { - const column = layer.columns[columnId] as ReferenceBasedIndexPatternColumn; - return column.references - .filter((referencedId) => 'sourceField' in layer.columns[referencedId]) - .map((referencedId) => { - const fieldName = (layer.columns[referencedId] as FieldBasedIndexPatternColumn).sourceField; - if (!isMetricCounterField(indexPattern.getFieldByName(fieldName))) { - return i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { - defaultMessage: 'Dimension "{dimensionLabel}" is configured incorrectly', - values: { - dimensionLabel: layer.columns[referencedId].label, - }, - }); - } - }); -} - -export function getErrorForRateReference( - layer: FormBasedLayer, - columnId: string, - name: string, - indexPattern: IndexPattern -) { - const dateErrors = checkForDateHistogram(layer, name) ?? []; - const referenceErrors = checkReferences(layer, columnId) ?? []; - const metricCounterErrors = checkReferencedColumnMetric(layer, columnId, indexPattern) ?? []; - if (metricCounterErrors.length) { - return metricCounterErrors.concat(referenceErrors); - } - if (dateErrors.length) { - return dateErrors.concat(referenceErrors); - } -} - const accuracyModeDisabledWarning = ( columnName: string, columnId: string, From b8784217e1b5cab026b450e3449b127b63525442 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Thu, 22 Dec 2022 11:31:15 +0100 Subject: [PATCH 07/23] time_series and tsdb rate support in lens --- .../operations/definitions/index.ts | 5 +- .../operations/definitions/time_series.tsx | 80 +++++++++++++++++++ .../public/datasources/form_based/utils.tsx | 59 +++++++++++++- 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index 04b88a18f8b02..f23deb920854c 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -12,7 +12,7 @@ import { CoreStart, } from '@kbn/core/public'; import type { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; -import type { +import { ExpressionAstExpressionBuilder, ExpressionAstFunction, } from '@kbn/expressions-plugin/public'; @@ -26,6 +26,7 @@ import { cardinalityOperation } from './cardinality'; import { percentileOperation } from './percentile'; import { percentileRanksOperation } from './percentile_ranks'; import { rateOperation } from './rate'; +import { timeSeriesOperation } from './time_series'; import { minOperation, averageOperation, @@ -117,6 +118,7 @@ const internalOperationDefinitions = [ filtersOperation, termsOperation, dateHistogramOperation, + timeSeriesOperation, minOperation, maxOperation, averageOperation, @@ -173,6 +175,7 @@ export { export { formulaOperation } from './formula/formula'; export { staticValueOperation } from './static_value'; export { rateOperation } from './rate'; +export { timeSeriesOperation } from './time_series'; /** * Properties passed to the operation-specific part of the popover editor diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx new file mode 100644 index 0000000000000..aaeddea1cf04e --- /dev/null +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; + +import { AggFunctionsMapping } from '@kbn/data-plugin/public'; +import { buildExpressionFunction } from '@kbn/expressions-plugin/public'; +import { BaseIndexPatternColumn, OperationDefinition } from '.'; + +export interface TimeSeriesIndexPatternColumn extends BaseIndexPatternColumn { + operationType: 'time_series'; + params: {}; +} + +export type TimeSeriesColumnParams = TimeSeriesIndexPatternColumn['params']; +export type UpdateParamsFnType = ( + paramName: K, + value: TimeSeriesColumnParams[K] +) => void; + +export const timeSeriesOperation: OperationDefinition< + TimeSeriesIndexPatternColumn, + 'none', + TimeSeriesColumnParams +> = { + type: 'time_series', + displayName: i18n.translate('xpack.lens.indexPattern.timeSeries', { + defaultMessage: 'All Series', + }), + priority: 4, // Higher than terms, so numbers get histogram + input: 'none', + getPossibleOperation: (index) => { + if (index.fields.find((f) => f.timeSeriesDimension || f.timeSeriesMetric)) { + return { + dataType: 'string', + isBucketed: true, + scale: 'ordinal', + }; + } + }, + getDefaultLabel: () => 'Time Series', + buildColumn() { + return { + label: 'series', + dataType: 'string', + operationType: 'time_series', + isBucketed: true, + scale: 'ordinal', // ordinal for Range + params: { + includeEmptyRows: true, + }, + }; + }, + isTransferable: (column, newIndexPattern) => { + return true; // newIndexPattern.(column.sourceField); + }, + toEsAggsFn: (column, columnId) => { + return buildExpressionFunction('aggTimeSeries', { + id: columnId, + enabled: true, + schema: 'bucket', + }).toAst(); + }, + paramEditor: () => { + return <>; + }, + quickFunctionDocumentation: i18n.translate( + 'xpack.lens.indexPattern.time_series.documentation.quick', + { + defaultMessage: ` + Buckets by all time series. + `, + } + ), +}; diff --git a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx index 5837d77c2a92d..61837cfa30829 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx @@ -26,10 +26,19 @@ import { import { estypes } from '@elastic/elasticsearch'; import type { DateRange } from '../../../common/types'; -import type { FramePublicAPI, IndexPattern, StateSetter, UserMessage } from '../../types'; +import type { + FramePublicAPI, + IndexPattern, + IndexPatternField, + StateSetter, + UserMessage, +} from '../../types'; import { renewIDs } from '../../utils'; import type { FormBasedLayer, FormBasedPersistedState, FormBasedPrivateState } from './types'; -import type { ReferenceBasedIndexPatternColumn } from './operations/definitions/column_types'; +import type { + FieldBasedIndexPatternColumn, + ReferenceBasedIndexPatternColumn, +} from './operations/definitions/column_types'; import { operationDefinitionMap, @@ -51,6 +60,10 @@ import { supportsRarityRanking } from './operations/definitions/terms'; import { DEFAULT_MAX_DOC_COUNT } from './operations/definitions/terms/constants'; import { getOriginalId } from '../../../common/expressions/datatable/transpose_helpers'; import { isQueryValid } from '../../shared_components'; +import { + checkForDateHistogram, + checkReferences, +} from './operations/definitions/calculations/utils'; export function isColumnInvalid( layer: FormBasedLayer, @@ -121,6 +134,48 @@ export function fieldIsInvalid( return !!getInvalidFieldMessage(layer, columnId, indexPattern)?.length; } +export function isMetricCounterField(field?: IndexPatternField) { + return field?.timeSeriesMetric === 'counter'; +} + +function checkReferencedColumnMetric( + layer: FormBasedLayer, + columnId: string, + indexPattern: IndexPattern +) { + const column = layer.columns[columnId] as ReferenceBasedIndexPatternColumn; + return column.references + .filter((referencedId) => 'sourceField' in layer.columns[referencedId]) + .map((referencedId) => { + const fieldName = (layer.columns[referencedId] as FieldBasedIndexPatternColumn).sourceField; + if (!isMetricCounterField(indexPattern.getFieldByName(fieldName))) { + return i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { + defaultMessage: 'Dimension "{dimensionLabel}" is configured incorrectly', + values: { + dimensionLabel: layer.columns[referencedId].label, + }, + }); + } + }); +} + +export function getErrorForRateReference( + layer: FormBasedLayer, + columnId: string, + name: string, + indexPattern: IndexPattern +) { + const dateErrors = checkForDateHistogram(layer, name) ?? []; + const referenceErrors = checkReferences(layer, columnId) ?? []; + const metricCounterErrors = checkReferencedColumnMetric(layer, columnId, indexPattern) ?? []; + if (metricCounterErrors.length) { + return metricCounterErrors.concat(referenceErrors); + } + if (dateErrors.length) { + return dateErrors.concat(referenceErrors); + } +} + const accuracyModeDisabledWarning = ( columnName: string, columnId: string, From 8bb84578c7fec34399ebd4bc35ad1ca8eb5ed153 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Tue, 27 Dec 2022 14:25:09 +0100 Subject: [PATCH 08/23] removing time_series breakdown --- .../operations/definitions/index.ts | 3 - .../operations/definitions/time_series.tsx | 80 ------------------- 2 files changed, 83 deletions(-) delete mode 100644 x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index f23deb920854c..dd90fb5145a57 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -26,7 +26,6 @@ import { cardinalityOperation } from './cardinality'; import { percentileOperation } from './percentile'; import { percentileRanksOperation } from './percentile_ranks'; import { rateOperation } from './rate'; -import { timeSeriesOperation } from './time_series'; import { minOperation, averageOperation, @@ -118,7 +117,6 @@ const internalOperationDefinitions = [ filtersOperation, termsOperation, dateHistogramOperation, - timeSeriesOperation, minOperation, maxOperation, averageOperation, @@ -175,7 +173,6 @@ export { export { formulaOperation } from './formula/formula'; export { staticValueOperation } from './static_value'; export { rateOperation } from './rate'; -export { timeSeriesOperation } from './time_series'; /** * Properties passed to the operation-specific part of the popover editor diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx deleted file mode 100644 index aaeddea1cf04e..0000000000000 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/time_series.tsx +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; -import { i18n } from '@kbn/i18n'; - -import { AggFunctionsMapping } from '@kbn/data-plugin/public'; -import { buildExpressionFunction } from '@kbn/expressions-plugin/public'; -import { BaseIndexPatternColumn, OperationDefinition } from '.'; - -export interface TimeSeriesIndexPatternColumn extends BaseIndexPatternColumn { - operationType: 'time_series'; - params: {}; -} - -export type TimeSeriesColumnParams = TimeSeriesIndexPatternColumn['params']; -export type UpdateParamsFnType = ( - paramName: K, - value: TimeSeriesColumnParams[K] -) => void; - -export const timeSeriesOperation: OperationDefinition< - TimeSeriesIndexPatternColumn, - 'none', - TimeSeriesColumnParams -> = { - type: 'time_series', - displayName: i18n.translate('xpack.lens.indexPattern.timeSeries', { - defaultMessage: 'All Series', - }), - priority: 4, // Higher than terms, so numbers get histogram - input: 'none', - getPossibleOperation: (index) => { - if (index.fields.find((f) => f.timeSeriesDimension || f.timeSeriesMetric)) { - return { - dataType: 'string', - isBucketed: true, - scale: 'ordinal', - }; - } - }, - getDefaultLabel: () => 'Time Series', - buildColumn() { - return { - label: 'series', - dataType: 'string', - operationType: 'time_series', - isBucketed: true, - scale: 'ordinal', // ordinal for Range - params: { - includeEmptyRows: true, - }, - }; - }, - isTransferable: (column, newIndexPattern) => { - return true; // newIndexPattern.(column.sourceField); - }, - toEsAggsFn: (column, columnId) => { - return buildExpressionFunction('aggTimeSeries', { - id: columnId, - enabled: true, - schema: 'bucket', - }).toAst(); - }, - paramEditor: () => { - return <>; - }, - quickFunctionDocumentation: i18n.translate( - 'xpack.lens.indexPattern.time_series.documentation.quick', - { - defaultMessage: ` - Buckets by all time series. - `, - } - ), -}; From ce9dd68128b00e7a2e7a6f5a3a2a7e85a1972f7c Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 15 Feb 2023 14:08:22 +0100 Subject: [PATCH 09/23] tests and cleanup --- .../datasources/form_based/operations/definitions/index.ts | 2 +- .../datasources/form_based/operations/definitions/rate.tsx | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index dd90fb5145a57..04b88a18f8b02 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -12,7 +12,7 @@ import { CoreStart, } from '@kbn/core/public'; import type { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; -import { +import type { ExpressionAstExpressionBuilder, ExpressionAstFunction, } from '@kbn/expressions-plugin/public'; diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx index 18787b7f97d3a..76f8f097e393d 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx @@ -71,7 +71,8 @@ export const rateOperation: OperationDefinition combineErrorMessages([ getInvalidFieldMessage(layer, columnId, indexPattern), - getErrorForRateReference(layer, columnId, 'rate', indexPattern), + // getDisallowedPreviousShiftMessage(layer, columnId), + // getColumnReducedTimeRangeError(layer, columnId, indexPattern), ]), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); From 7383642193dd5e6330d99bd698719aa44ee2a9d4 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 15 Feb 2023 15:31:26 +0100 Subject: [PATCH 10/23] fix types --- .../datasources/form_based/operations/definitions/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index 04b88a18f8b02..f87472d0c75f2 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -634,7 +634,7 @@ interface FullReferenceOperationDefinition { * Returns the meta data of the operation if applied. Undefined * if the operation can't be added with these fields. */ - getPossibleOperation: (indexPattern: IndexPattern) => OperationMetadata | undefined; + getPossibleOperation: (indexPattern?: IndexPattern) => OperationMetadata | undefined; /** * A chain of expression functions which will transform the table */ From 09c0844c0fd126fd503523aebaa2df956f50cadd Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 15 Feb 2023 16:50:25 +0100 Subject: [PATCH 11/23] fix types --- .../datasources/form_based/operations/definitions/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts index f87472d0c75f2..04b88a18f8b02 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts @@ -634,7 +634,7 @@ interface FullReferenceOperationDefinition { * Returns the meta data of the operation if applied. Undefined * if the operation can't be added with these fields. */ - getPossibleOperation: (indexPattern?: IndexPattern) => OperationMetadata | undefined; + getPossibleOperation: (indexPattern: IndexPattern) => OperationMetadata | undefined; /** * A chain of expression functions which will transform the table */ From 87fbd578e854f71a95271a9273b49b54514d5454 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Thu, 16 Feb 2023 11:04:50 +0100 Subject: [PATCH 12/23] moving helpers --- .../operations/definitions/rate.tsx | 3 +- .../public/datasources/form_based/utils.tsx | 59 +------------------ 2 files changed, 3 insertions(+), 59 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx index 76f8f097e393d..18787b7f97d3a 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx @@ -71,8 +71,7 @@ export const rateOperation: OperationDefinition combineErrorMessages([ getInvalidFieldMessage(layer, columnId, indexPattern), - // getDisallowedPreviousShiftMessage(layer, columnId), - // getColumnReducedTimeRangeError(layer, columnId, indexPattern), + getErrorForRateReference(layer, columnId, 'rate', indexPattern), ]), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx index 61837cfa30829..5837d77c2a92d 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx @@ -26,19 +26,10 @@ import { import { estypes } from '@elastic/elasticsearch'; import type { DateRange } from '../../../common/types'; -import type { - FramePublicAPI, - IndexPattern, - IndexPatternField, - StateSetter, - UserMessage, -} from '../../types'; +import type { FramePublicAPI, IndexPattern, StateSetter, UserMessage } from '../../types'; import { renewIDs } from '../../utils'; import type { FormBasedLayer, FormBasedPersistedState, FormBasedPrivateState } from './types'; -import type { - FieldBasedIndexPatternColumn, - ReferenceBasedIndexPatternColumn, -} from './operations/definitions/column_types'; +import type { ReferenceBasedIndexPatternColumn } from './operations/definitions/column_types'; import { operationDefinitionMap, @@ -60,10 +51,6 @@ import { supportsRarityRanking } from './operations/definitions/terms'; import { DEFAULT_MAX_DOC_COUNT } from './operations/definitions/terms/constants'; import { getOriginalId } from '../../../common/expressions/datatable/transpose_helpers'; import { isQueryValid } from '../../shared_components'; -import { - checkForDateHistogram, - checkReferences, -} from './operations/definitions/calculations/utils'; export function isColumnInvalid( layer: FormBasedLayer, @@ -134,48 +121,6 @@ export function fieldIsInvalid( return !!getInvalidFieldMessage(layer, columnId, indexPattern)?.length; } -export function isMetricCounterField(field?: IndexPatternField) { - return field?.timeSeriesMetric === 'counter'; -} - -function checkReferencedColumnMetric( - layer: FormBasedLayer, - columnId: string, - indexPattern: IndexPattern -) { - const column = layer.columns[columnId] as ReferenceBasedIndexPatternColumn; - return column.references - .filter((referencedId) => 'sourceField' in layer.columns[referencedId]) - .map((referencedId) => { - const fieldName = (layer.columns[referencedId] as FieldBasedIndexPatternColumn).sourceField; - if (!isMetricCounterField(indexPattern.getFieldByName(fieldName))) { - return i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { - defaultMessage: 'Dimension "{dimensionLabel}" is configured incorrectly', - values: { - dimensionLabel: layer.columns[referencedId].label, - }, - }); - } - }); -} - -export function getErrorForRateReference( - layer: FormBasedLayer, - columnId: string, - name: string, - indexPattern: IndexPattern -) { - const dateErrors = checkForDateHistogram(layer, name) ?? []; - const referenceErrors = checkReferences(layer, columnId) ?? []; - const metricCounterErrors = checkReferencedColumnMetric(layer, columnId, indexPattern) ?? []; - if (metricCounterErrors.length) { - return metricCounterErrors.concat(referenceErrors); - } - if (dateErrors.length) { - return dateErrors.concat(referenceErrors); - } -} - const accuracyModeDisabledWarning = ( columnName: string, columnId: string, From 4994a7347ed07c7ea119b2b7e844a056d7972d93 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 16 Feb 2023 10:10:46 +0000 Subject: [PATCH 13/23] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../form_based/operations/definitions/helpers.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx index 7d2bcc767ea80..dcbe8b723dd73 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx @@ -8,6 +8,7 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; +import { checkForDateHistogram, checkReferences } from './calculations/utils'; import type { IndexPattern, IndexPatternField } from '../../../../types'; import { type FieldBasedOperationErrorMessage, @@ -21,10 +22,6 @@ import { } from './column_types'; import type { FormBasedLayer } from '../../types'; import { hasField } from '../../pure_utils'; -import { - checkForDateHistogram, - checkReferences -} from "@kbn/lens-plugin/public/datasources/form_based/operations/definitions/calculations/utils"; export function getInvalidFieldMessage( layer: FormBasedLayer, From e555e5e67a755e05b296724e0ab2841fc922d05f Mon Sep 17 00:00:00 2001 From: ppisljar Date: Thu, 16 Feb 2023 11:32:36 +0100 Subject: [PATCH 14/23] fixing types --- .../form_based/operations/definitions/helpers.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx index dcbe8b723dd73..495339bb7d045 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx @@ -236,9 +236,9 @@ export function getErrorForRateReference( columnId: string, name: string, indexPattern: IndexPattern -) { - const dateErrors = checkForDateHistogram(layer, name) ?? []; - const referenceErrors = checkReferences(layer, columnId) ?? []; +): string[] | undefined { + const dateErrors: string[] = checkForDateHistogram(layer, name) ?? []; + const referenceErrors: string[] = checkReferences(layer, columnId) ?? []; const metricCounterErrors = checkReferencedColumnMetric(layer, columnId, indexPattern) ?? []; if (metricCounterErrors.length) { return metricCounterErrors.concat(referenceErrors); From 23182f09de82e2cbffde3359a4e8a552d7d0a77d Mon Sep 17 00:00:00 2001 From: ppisljar Date: Thu, 16 Feb 2023 12:38:28 +0100 Subject: [PATCH 15/23] fixing --- .../form_based/operations/definitions/calculations/utils.ts | 2 +- .../datasources/form_based/operations/definitions/helpers.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts index bac4a8940c689..d4b13abdebe0b 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts @@ -90,7 +90,7 @@ export function checkReferences(layer: FormBasedLayer, columnId: string) { const errors: string[] = []; - column.references.forEach((referenceId, index) => { + column.references?.forEach((referenceId, index) => { if (!layer.columns[referenceId]) { errors.push( i18n.translate('xpack.lens.indexPattern.missingReferenceError', { diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx index 495339bb7d045..0035665486470 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx @@ -217,7 +217,7 @@ function checkReferencedColumnMetric( ) { const column = layer.columns[columnId] as ReferenceBasedIndexPatternColumn; return column.references - .filter((referencedId) => 'sourceField' in layer.columns[referencedId]) + ?.filter((referencedId) => 'sourceField' in layer.columns[referencedId]) .map((referencedId) => { const fieldName = (layer.columns[referencedId] as FieldBasedIndexPatternColumn).sourceField; if (!isMetricCounterField(indexPattern.getFieldByName(fieldName))) { From 79acb02bc8f17e42589a990201838b887d63e9b2 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 16 Feb 2023 11:07:42 +0000 Subject: [PATCH 16/23] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../form_based/operations/definitions/percentile.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx index df9f6b768981b..2c507cee5ffcd 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx @@ -88,7 +88,12 @@ export const percentileOperation: OperationDefinition< filterable: true, shiftable: true, canReduceTimeRange: true, - getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType , timeSeriesMetric}) => { + getPossibleOperationForField: ({ + aggregationRestrictions, + aggregatable, + type: fieldType, + timeSeriesMetric, + }) => { if ( supportedFieldTypes.includes(fieldType) && aggregatable && From 9404f36b65df3ec60c20a74b97f63ff6fc84c2f8 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Thu, 16 Feb 2023 14:57:52 +0100 Subject: [PATCH 17/23] more fixes --- .../operations/definitions/helpers.tsx | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx index 0035665486470..c723ff03c3012 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx @@ -8,7 +8,6 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; -import { checkForDateHistogram, checkReferences } from './calculations/utils'; import type { IndexPattern, IndexPatternField } from '../../../../types'; import { type FieldBasedOperationErrorMessage, @@ -209,41 +208,3 @@ export function getFilter( export function isMetricCounterField(field?: IndexPatternField) { return field?.timeSeriesMetric === 'counter'; } - -function checkReferencedColumnMetric( - layer: FormBasedLayer, - columnId: string, - indexPattern: IndexPattern -) { - const column = layer.columns[columnId] as ReferenceBasedIndexPatternColumn; - return column.references - ?.filter((referencedId) => 'sourceField' in layer.columns[referencedId]) - .map((referencedId) => { - const fieldName = (layer.columns[referencedId] as FieldBasedIndexPatternColumn).sourceField; - if (!isMetricCounterField(indexPattern.getFieldByName(fieldName))) { - return i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { - defaultMessage: 'Dimension "{dimensionLabel}" is configured incorrectly', - values: { - dimensionLabel: layer.columns[referencedId].label, - }, - }); - } - }); -} - -export function getErrorForRateReference( - layer: FormBasedLayer, - columnId: string, - name: string, - indexPattern: IndexPattern -): string[] | undefined { - const dateErrors: string[] = checkForDateHistogram(layer, name) ?? []; - const referenceErrors: string[] = checkReferences(layer, columnId) ?? []; - const metricCounterErrors = checkReferencedColumnMetric(layer, columnId, indexPattern) ?? []; - if (metricCounterErrors.length) { - return metricCounterErrors.concat(referenceErrors); - } - if (dateErrors.length) { - return dateErrors.concat(referenceErrors); - } -} From 58b57d9f61a16fa0d02176bd7ecd30308c6a9d04 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Thu, 16 Feb 2023 14:58:19 +0100 Subject: [PATCH 18/23] more fixes --- .../definitions/calculations/utils.ts | 46 ++++++++++++++++++- .../operations/definitions/rate.tsx | 2 - 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts index d4b13abdebe0b..397403c3d1277 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts @@ -17,6 +17,8 @@ import { adjustTimeScaleLabelSuffix } from '../../time_scale_utils'; import type { ReferenceBasedIndexPatternColumn } from '../column_types'; import { getManagedColumnsFrom, isColumnValidAsReference } from '../../layer_helpers'; import { operationDefinitionMap } from '..'; +import { FieldBasedIndexPatternColumn } from '../../../types'; +import {IndexPatternField} from "../../../../../types"; export const buildLabelFunction = (ofName: (name?: string) => string) => @@ -90,7 +92,7 @@ export function checkReferences(layer: FormBasedLayer, columnId: string) { const errors: string[] = []; - column.references?.forEach((referenceId, index) => { + column.references.forEach((referenceId, index) => { if (!layer.columns[referenceId]) { errors.push( i18n.translate('xpack.lens.indexPattern.missingReferenceError', { @@ -203,3 +205,45 @@ export function optionallHistogramBasedOperationToExpression( }, ]; } + +function isMetricCounterField(field?: IndexPatternField) { + return field?.timeSeriesMetric === 'counter'; +} + +function checkReferencedColumnMetric( + layer: FormBasedLayer, + columnId: string, + indexPattern: IndexPattern +) { + const column = layer.columns[columnId] as ReferenceBasedIndexPatternColumn; + return column.references + .filter((referencedId) => 'sourceField' in layer.columns[referencedId]) + .map((referencedId) => { + const fieldName = (layer.columns[referencedId] as FieldBasedIndexPatternColumn).sourceField; + if (!isMetricCounterField(indexPattern.getFieldByName(fieldName))) { + return i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { + defaultMessage: 'Dimension "{dimensionLabel}" is configured incorrectly', + values: { + dimensionLabel: layer.columns[referencedId].label, + }, + }); + } + }); +} + +export function getErrorForRateReference( + layer: FormBasedLayer, + columnId: string, + name: string, + indexPattern: IndexPattern +) { + const dateErrors = checkForDateHistogram(layer, name) ?? []; + const referenceErrors = checkReferences(layer, columnId) ?? []; + const metricCounterErrors = checkReferencedColumnMetric(layer, columnId, indexPattern) ?? []; + if (metricCounterErrors.length) { + return metricCounterErrors.concat(referenceErrors); + } + if (dateErrors.length) { + return dateErrors.concat(referenceErrors); + } +} diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx index 18787b7f97d3a..2c4af717c9066 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx @@ -22,7 +22,6 @@ import { getFilter, combineErrorMessages, isMetricCounterField, - getErrorForRateReference, } from './helpers'; const supportedTypes = new Set(['number']); @@ -71,7 +70,6 @@ export const rateOperation: OperationDefinition combineErrorMessages([ getInvalidFieldMessage(layer, columnId, indexPattern), - getErrorForRateReference(layer, columnId, 'rate', indexPattern), ]), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); From a927ada92d8b4186ec738c59bbb268ff41d58884 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 1 Mar 2023 12:50:06 +0100 Subject: [PATCH 19/23] revert field caps changes --- .../field_capabilities/field_caps_response.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_caps_response.ts b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_caps_response.ts index 843849a0ddeca..e188d97ade589 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_caps_response.ts +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_caps_response.ts @@ -72,7 +72,6 @@ export function readFieldCapsResponse( fieldCapsResponse: estypes.FieldCapsResponse ): FieldDescriptor[] { const capsByNameThenType = fieldCapsResponse.fields; - const kibanaFormattedCaps = Object.keys(capsByNameThenType).reduce<{ array: FieldDescriptor[]; hash: Record; @@ -91,16 +90,14 @@ export function readFieldCapsResponse( }); const timeSeriesMetricProp = uniq(types.map((t) => capsByType[t].time_series_metric)); - const isTimeSeriesCounter = !!timeSeriesMetricProp.find((item) => item === 'counter'); - const isAggregatable = - types.some((type) => { - return ( - !!capsByType[type].aggregatable || - (!!capsByType[type].non_aggregatable_indices && - capsByType[type].non_aggregatable_indices!.length > 0) - ); - }) && !isTimeSeriesCounter; + const isAggregatable = types.some((type) => { + return ( + capsByType[type].aggregatable || + (!!capsByType[type].non_aggregatable_indices && + capsByType[type].non_aggregatable_indices!.length > 0) + ); + }); // If there are multiple types but they all resolve to the same kibana type // ignore the conflict and carry on (my wayward son) From dc9a021ce15b8ec0524faf82cbe1cc5ff7407945 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 1 Mar 2023 12:52:15 +0100 Subject: [PATCH 20/23] enable min and max on counter fields --- .../form_based/operations/definitions/calculations/utils.ts | 2 +- .../datasources/form_based/operations/definitions/metrics.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts index 397403c3d1277..a90fb9ce24ae9 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts @@ -18,7 +18,7 @@ import type { ReferenceBasedIndexPatternColumn } from '../column_types'; import { getManagedColumnsFrom, isColumnValidAsReference } from '../../layer_helpers'; import { operationDefinitionMap } from '..'; import { FieldBasedIndexPatternColumn } from '../../../types'; -import {IndexPatternField} from "../../../../../types"; +import { IndexPatternField } from '../../../../../types'; export const buildLabelFunction = (ofName: (name?: string) => string) => diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx index 1815d26a01229..fd47521aec5f0 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx @@ -103,7 +103,7 @@ function buildMetricOperation>({ if ( (supportedTypes.includes(fieldType) || (supportsDate && fieldType === 'date')) && aggregatable && - timeSeriesMetric !== 'counter' && + (timeSeriesMetric !== 'counter' || ['min', 'max'].includes(type)) && (!aggregationRestrictions || aggregationRestrictions[type]) ) { return { From c475c2b308a578d13356043dca546f740e818d4a Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 1 Mar 2023 12:54:31 +0100 Subject: [PATCH 21/23] updates to rate operation --- .../operations/definitions/rate.tsx | 61 +++++++++++++------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx index 2c4af717c9066..c6c23e387d946 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/rate.tsx @@ -39,7 +39,7 @@ function ofName(name: string) { }); } -type AggregateFnTypes = 'sum' | 'max' | 'min' | 'avg'; +type AggregateFnTypes = 'aggBucketSum' | 'aggBucketMax' | 'aggBucketMin' | 'aggBucketAvg'; export interface RateIndexPatternColumn extends FieldBasedIndexPatternColumn { operationType: typeof OPERATION_TYPE; @@ -56,21 +56,21 @@ export const rateOperation: OperationDefinition { - if (supportedTypes.has(type) && aggregatable && timeSeriesMetric === 'counter') { + if (supportedTypes.has(type) && aggregatable) { return { dataType: 'number', isBucketed: IS_BUCKETED, scale: SCALE }; } }, getErrorMessage: (layer, columnId, indexPattern) => - combineErrorMessages([ - getInvalidFieldMessage(layer, columnId, indexPattern), - ]), + combineErrorMessages([getInvalidFieldMessage(layer, columnId, indexPattern)]), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); @@ -98,15 +98,16 @@ export const rateOperation: OperationDefinition { - if (true) { + toEsAggsFn: (column, columnId, indexPattern) => { + const aggFn = column.params?.aggregateFn || 'aggBucketMax'; + if (indexPattern.getFieldByName(column.sourceField)?.timeSeriesMetric === 'counter') { // if wrap in timeseries and bucket agg - return buildExpressionFunction('aggBucketMax', { + return buildExpressionFunction(aggFn, { id: columnId, enabled: true, schema: 'metric', @@ -125,7 +126,33 @@ export const rateOperation: OperationDefinition('aggDateHistogram', { + id: `-datehistogram`, + enabled: true, + schema: 'bucket', + field: '@timestamp', + interval: '1d', + }), + ]), + customMetric: buildExpression([ + buildExpressionFunction('aggRate', { + id: '-metric', + enabled: true, + schema: 'metric', + field: column.sourceField, + // time shift is added to wrapping aggFilteredMetric if filter is set + timeShift: column.filter ? undefined : column.timeShift, + unit: 'second', }), ]), }).toAst(); @@ -178,8 +205,8 @@ The number of unique values for a specified number, string, date, or boolean fie [currentColumn, paramEditorUpdater] ); const { inputValue, handleInputChange } = useDebouncedValue({ - value: currentColumn?.params?.aggregateFn ?? 'sum', - defaultValue: 'sum', + value: currentColumn?.params?.aggregateFn ?? 'aggBucketMax', + defaultValue: 'aggBucketMax', onChange, }); @@ -188,10 +215,10 @@ The number of unique values for a specified number, string, date, or boolean fie }); const options = [ - { value: 'sum', label: 'Sum' }, - { value: 'avg', label: 'Average' }, - { value: 'max', label: 'Max' }, - { value: 'min', label: 'Min' }, + { value: 'aggBucketSum', label: 'Sum' }, + { value: 'aggBucketAvg', label: 'Average' }, + { value: 'aggBucketMax', label: 'Max' }, + { value: 'aggBucketMin', label: 'Min' }, ]; return ( From 75b787f7f2cca753afe3c89afbf9da9bf9f0d319 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Wed, 1 Mar 2023 12:55:04 +0100 Subject: [PATCH 22/23] sample data change --- .../data_sets/logs/field_mappings.ts | 15 ++------ .../sample_data/data_sets/logs/logs.json.gz | Bin 1533472 -> 1533472 bytes .../sample_data/sample_data_installer.ts | 34 +++++++++++++++--- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/plugins/home/server/services/sample_data/data_sets/logs/field_mappings.ts b/src/plugins/home/server/services/sample_data/data_sets/logs/field_mappings.ts index 7b094acbf3a4a..2f00f14345669 100644 --- a/src/plugins/home/server/services/sample_data/data_sets/logs/field_mappings.ts +++ b/src/plugins/home/server/services/sample_data/data_sets/logs/field_mappings.ts @@ -8,13 +8,8 @@ export const fieldMappings = { request: { - type: 'text', - fields: { - keyword: { - type: 'keyword', - time_series_dimension: true, - }, - }, + type: 'keyword', + time_series_dimension: true, }, geo: { properties: { @@ -117,9 +112,6 @@ export const fieldMappings = { ip: { type: 'ip', }, - ip_range: { - type: 'ip_range', - }, '@timestamp': { type: 'date', }, @@ -127,9 +119,6 @@ export const fieldMappings = { type: 'alias', path: '@timestamp', }, - timestamp_range: { - type: 'date_range', - }, phpmemory: { type: 'long', }, diff --git a/src/plugins/home/server/services/sample_data/data_sets/logs/logs.json.gz b/src/plugins/home/server/services/sample_data/data_sets/logs/logs.json.gz index 59047cfa411cafa298b465d7ceb9d1ddec4de3d8..cba6e5189a9b89da25ad8b55ac60e014377b4189 100644 GIT binary patch delta 87 zcmWN_xfOyy06@Vfh<}yK6CM#&P|l|IVeliG*e2{^i?IW6!bv%%9GTY;e#`bj4pK-d gN2%l_{|s`Li`3FcE1mRmm7Cn{I$6xQHgc2_jnfqHx1aIi?)>Z6O>nt;j_RxynsS dspOwQ?(&dYp3+DwFX^P0LEiF_uZ%m>yqpXL6zc#0 diff --git a/src/plugins/home/server/services/sample_data/sample_data_installer.ts b/src/plugins/home/server/services/sample_data/sample_data_installer.ts index 443f2b5635e33..f2087b7c3f964 100644 --- a/src/plugins/home/server/services/sample_data/sample_data_installer.ts +++ b/src/plugins/home/server/services/sample_data/sample_data_installer.ts @@ -157,22 +157,48 @@ export class SampleDataInstaller { const index = createIndexName(dataset.id, dataIndex.id); try { if (dataIndex.isDataStream) { - const request = { + /*const request = { name: index, body: { template: { - settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + 'index.mode': 'time_series', + 'index.routing_path': 'request', + 'index.time_series.start_time': '2022-11-01T00:00:00.000000Z', + 'index.time_series.end_time': '2023-05-01T00:00:00.000000Z', + }, mappings: { properties: dataIndex.fields }, }, index_patterns: [index], data_stream: {}, }, + };*/ + + const request = { + index, + body: { + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + 'index.mode': 'time_series', + 'index.routing_path': 'request', + 'index.time_series.start_time': '2022-11-01T00:00:00.000000Z', + 'index.time_series.end_time': '2023-05-01T00:00:00.000000Z', + }, + mappings: { properties: dataIndex.fields }, + }, }; - await this.esClient.asCurrentUser.indices.putIndexTemplate(request); + + console.log(JSON.stringify(request)); + + await this.esClient.asCurrentUser.indices.create(request); + /*await this.esClient.asCurrentUser.indices.putIndexTemplate(request); await this.esClient.asCurrentUser.indices.createDataStream({ name: index, - }); + });*/ } else { await this.esClient.asCurrentUser.indices.create({ index, From 9318dfd1057b46a01669b95962fc979e1b102bec Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 1 Mar 2023 12:35:02 +0000 Subject: [PATCH 23/23] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../home/server/services/sample_data/sample_data_installer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/home/server/services/sample_data/sample_data_installer.ts b/src/plugins/home/server/services/sample_data/sample_data_installer.ts index f2087b7c3f964..7eaaed08639aa 100644 --- a/src/plugins/home/server/services/sample_data/sample_data_installer.ts +++ b/src/plugins/home/server/services/sample_data/sample_data_installer.ts @@ -157,7 +157,7 @@ export class SampleDataInstaller { const index = createIndexName(dataset.id, dataIndex.id); try { if (dataIndex.isDataStream) { - /*const request = { + /* const request = { name: index, body: { template: { @@ -194,7 +194,7 @@ export class SampleDataInstaller { console.log(JSON.stringify(request)); await this.esClient.asCurrentUser.indices.create(request); - /*await this.esClient.asCurrentUser.indices.putIndexTemplate(request); + /* await this.esClient.asCurrentUser.indices.putIndexTemplate(request); await this.esClient.asCurrentUser.indices.createDataStream({ name: index,