Lens tsdb rate support

src/plugins/data/common/search/aggs/types.ts

@@ -36,6 +36,7 @@ import {
   aggMedian,
   aggMin,
   aggMovingAvg,
+  aggRate,
   AggParamsAvg,
   AggParamsBucketAvg,
   AggParamsBucketAvgSerialized,
@@ -309,4 +310,5 @@ export interface AggFunctionsMapping {
   aggSum: ReturnType<typeof aggSum>;
   aggTopHit: ReturnType<typeof aggTopHit>;
   aggTopMetrics: ReturnType<typeof aggTopMetrics>;
+  aggRate: ReturnType<typeof aggRate>;
 }

src/plugins/data_views/server/fetcher/lib/field_capabilities/field_caps_response.ts

@@ -72,7 +72,6 @@ export function readFieldCapsResponse(
   fieldCapsResponse: estypes.FieldCapsResponse
 ): FieldDescriptor[] {
   const capsByNameThenType = fieldCapsResponse.fields;
-
   const kibanaFormattedCaps = Object.keys(capsByNameThenType).reduce<{
     array: FieldDescriptor[];
     hash: Record<string, FieldDescriptor>;
@@ -91,16 +90,14 @@ export function readFieldCapsResponse(
       });
 
       const timeSeriesMetricProp = uniq(types.map((t) => capsByType[t].time_series_metric));
-      const isTimeSeriesCounter = !!timeSeriesMetricProp.find((item) => item === 'counter');
 
-      const isAggregatable =
-        types.some((type) => {
-          return (
-            !!capsByType[type].aggregatable ||
-            (!!capsByType[type].non_aggregatable_indices &&
-              capsByType[type].non_aggregatable_indices!.length > 0)
-          );
-        }) && !isTimeSeriesCounter;
+      const isAggregatable = types.some((type) => {
+        return (
+          capsByType[type].aggregatable ||
+          (!!capsByType[type].non_aggregatable_indices &&
+            capsByType[type].non_aggregatable_indices!.length > 0)
+        );
+      });
 
       // If there are multiple types but they all resolve to the same kibana type
       // ignore the conflict and carry on (my wayward son)

src/plugins/home/server/services/sample_data/data_sets/logs/field_mappings.ts

@@ -8,13 +8,8 @@
 
 export const fieldMappings = {
   request: {
-    type: 'text',
-    fields: {
-      keyword: {
-        type: 'keyword',
-        time_series_dimension: true,
-      },
-    },
+    type: 'keyword',
+    time_series_dimension: true,
   },
   geo: {
     properties: {
@@ -117,19 +112,13 @@ export const fieldMappings = {
   ip: {
     type: 'ip',
   },
-  ip_range: {
-    type: 'ip_range',
-  },
   '@timestamp': {
     type: 'date',
   },
   timestamp: {
     type: 'alias',
     path: '@timestamp',
   },
-  timestamp_range: {
-    type: 'date_range',
-  },
   phpmemory: {
     type: 'long',
   },

src/plugins/home/server/services/sample_data/sample_data_installer.ts

@@ -157,22 +157,48 @@ export class SampleDataInstaller {
     const index = createIndexName(dataset.id, dataIndex.id);
     try {
       if (dataIndex.isDataStream) {
-        const request = {
+        /* const request = {
           name: index,
           body: {
             template: {
-              settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+              settings: {
+                number_of_shards: 1,
+                auto_expand_replicas: '0-1',
+                'index.mode': 'time_series',
+                'index.routing_path': 'request',
+                'index.time_series.start_time': '2022-11-01T00:00:00.000000Z',
+                'index.time_series.end_time': '2023-05-01T00:00:00.000000Z',
+              },
               mappings: { properties: dataIndex.fields },
             },
             index_patterns: [index],
             data_stream: {},
           },
+        };*/
+
+        const request = {
+          index,
+          body: {
+            settings: {
+              number_of_shards: 1,
+              auto_expand_replicas: '0-1',
+              'index.mode': 'time_series',
+              'index.routing_path': 'request',
+              'index.time_series.start_time': '2022-11-01T00:00:00.000000Z',
+              'index.time_series.end_time': '2023-05-01T00:00:00.000000Z',
+            },
+            mappings: { properties: dataIndex.fields },
+          },
         };
-        await this.esClient.asCurrentUser.indices.putIndexTemplate(request);
+
+        console.log(JSON.stringify(request));
+
+        await this.esClient.asCurrentUser.indices.create(request);
+        /* await this.esClient.asCurrentUser.indices.putIndexTemplate(request);
 
         await this.esClient.asCurrentUser.indices.createDataStream({
           name: index,
-        });
+        });*/
       } else {
         await this.esClient.asCurrentUser.indices.create({
           index,

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/calculations/utils.ts

@@ -17,6 +17,8 @@ import { adjustTimeScaleLabelSuffix } from '../../time_scale_utils';
 import type { ReferenceBasedIndexPatternColumn } from '../column_types';
 import { getManagedColumnsFrom, isColumnValidAsReference } from '../../layer_helpers';
 import { operationDefinitionMap } from '..';
+import { FieldBasedIndexPatternColumn } from '../../../types';
+import { IndexPatternField } from '../../../../../types';
 
 export const buildLabelFunction =
   (ofName: (name?: string) => string) =>
@@ -203,3 +205,45 @@ export function optionallHistogramBasedOperationToExpression(
     },
   ];
 }
+
+function isMetricCounterField(field?: IndexPatternField) {
+  return field?.timeSeriesMetric === 'counter';
+}
+
+function checkReferencedColumnMetric(
+  layer: FormBasedLayer,
+  columnId: string,
+  indexPattern: IndexPattern
+) {
+  const column = layer.columns[columnId] as ReferenceBasedIndexPatternColumn;
+  return column.references
+    .filter((referencedId) => 'sourceField' in layer.columns[referencedId])
+    .map((referencedId) => {
+      const fieldName = (layer.columns[referencedId] as FieldBasedIndexPatternColumn).sourceField;
+      if (!isMetricCounterField(indexPattern.getFieldByName(fieldName))) {
+        return i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', {
+          defaultMessage: 'Dimension "{dimensionLabel}" is configured incorrectly',
+          values: {
+            dimensionLabel: layer.columns[referencedId].label,
+          },
+        });
+      }
+    });
+}
+
+export function getErrorForRateReference(
+  layer: FormBasedLayer,
+  columnId: string,
+  name: string,
+  indexPattern: IndexPattern
+) {
+  const dateErrors = checkForDateHistogram(layer, name) ?? [];
+  const referenceErrors = checkReferences(layer, columnId) ?? [];
+  const metricCounterErrors = checkReferencedColumnMetric(layer, columnId, indexPattern) ?? [];
+  if (metricCounterErrors.length) {
+    return metricCounterErrors.concat(referenceErrors);
+  }
+  if (dateErrors.length) {
+    return dateErrors.concat(referenceErrors);
+  }
+}

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/cardinality.tsx

@@ -80,10 +80,16 @@ export const cardinalityOperation: OperationDefinition<
   }),
   allowAsReference: true,
   input: 'field',
-  getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type }) => {
+  getPossibleOperationForField: ({
+    aggregationRestrictions,
+    aggregatable,
+    type,
+    timeSeriesMetric,
+  }) => {
     if (
       supportedTypes.has(type) &&
       aggregatable &&
+      timeSeriesMetric !== 'counter' &&
       (!aggregationRestrictions || aggregationRestrictions.cardinality)
     ) {
       return { dataType: 'number', isBucketed: IS_BUCKETED, scale: SCALE };

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/count.tsx

@@ -101,10 +101,16 @@ export const countOperation: OperationDefinition<CountIndexPatternColumn, 'field
       sourceField: field.name,
     };
   },
-  getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type }) => {
+  getPossibleOperationForField: ({
+    aggregationRestrictions,
+    aggregatable,
+    type,
+    timeSeriesMetric,
+  }) => {
     if (
       type === 'document' ||
       (aggregatable &&
+        timeSeriesMetric !== 'counter' &&
         (!aggregationRestrictions || aggregationRestrictions.value_count) &&
         supportedTypes.has(type))
     ) {

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx

@@ -204,3 +204,7 @@ export function getFilter(
   }
   return filter;
 }
+
+export function isMetricCounterField(field?: IndexPatternField) {
+  return field?.timeSeriesMetric === 'counter';
+}

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/index.ts

@@ -25,6 +25,7 @@ import { filtersOperation } from './filters';
 import { cardinalityOperation } from './cardinality';
 import { percentileOperation } from './percentile';
 import { percentileRanksOperation } from './percentile_ranks';
+import { rateOperation } from './rate';
 import {
   minOperation,
   averageOperation,
@@ -104,6 +105,7 @@ export type {
 } from './calculations';
 export type { CountIndexPatternColumn } from './count';
 export type { LastValueIndexPatternColumn } from './last_value';
+export type { RateIndexPatternColumn } from './rate';
 export type { RangeIndexPatternColumn } from './ranges';
 export type { FormulaIndexPatternColumn, MathIndexPatternColumn } from './formula';
 export type { StaticValueIndexPatternColumn } from './static_value';
@@ -130,6 +132,7 @@ const internalOperationDefinitions = [
   cumulativeSumOperation,
   counterRateOperation,
   derivativeOperation,
+  rateOperation,
   movingAverageOperation,
   mathOperation,
   formulaOperation,
@@ -169,6 +172,7 @@ export {
 } from './calculations';
 export { formulaOperation } from './formula/formula';
 export { staticValueOperation } from './static_value';
+export { rateOperation } from './rate';
 
 /**
  * Properties passed to the operation-specific part of the popover editor
@@ -489,7 +493,7 @@ interface FieldlessOperationDefinition<C extends BaseIndexPatternColumn, P = {}>
    * Returns the meta data of the operation if applied. Undefined
    * if the field is not applicable.
    */
-  getPossibleOperation: () => OperationMetadata;
+  getPossibleOperation: (index?: IndexPattern) => OperationMetadata | undefined;
   /**
    * Function turning a column into an agg config passed to the `esaggs` function
    * together with the agg configs returned from other columns.

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.tsx

@@ -199,8 +199,8 @@ export const lastValueOperation: OperationDefinition<
           : oldColumn.filter,
     };
   },
-  getPossibleOperationForField: ({ aggregationRestrictions, type }) => {
-    if (supportedTypes.has(type) && !aggregationRestrictions) {
+  getPossibleOperationForField: ({ aggregationRestrictions, type, timeSeriesMetric }) => {
+    if (supportedTypes.has(type) && !aggregationRestrictions && timeSeriesMetric !== 'counter') {
       return {
         dataType: type as DataType,
         isBucketed: false,

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx

@@ -94,10 +94,16 @@ function buildMetricOperation<T extends MetricColumn<string>>({
     description,
     input: 'field',
     timeScalingMode: optionalTimeScaling ? 'optional' : undefined,
-    getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => {
+    getPossibleOperationForField: ({
+      aggregationRestrictions,
+      aggregatable,
+      type: fieldType,
+      timeSeriesMetric,
+    }) => {
       if (
         (supportedTypes.includes(fieldType) || (supportsDate && fieldType === 'date')) &&
         aggregatable &&
+        (timeSeriesMetric !== 'counter' || ['min', 'max'].includes(type)) &&
         (!aggregationRestrictions || aggregationRestrictions[type])
       ) {
         return {

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx

@@ -88,10 +88,16 @@ export const percentileOperation: OperationDefinition<
   filterable: true,
   shiftable: true,
   canReduceTimeRange: true,
-  getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => {
+  getPossibleOperationForField: ({
+    aggregationRestrictions,
+    aggregatable,
+    type: fieldType,
+    timeSeriesMetric,
+  }) => {
     if (
       supportedFieldTypes.includes(fieldType) &&
       aggregatable &&
+      timeSeriesMetric !== 'counter' &&
       (!aggregationRestrictions || aggregationRestrictions.percentiles)
     ) {
       return {

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile_ranks.tsx

@@ -80,10 +80,16 @@ export const percentileRanksOperation: OperationDefinition<
   filterable: true,
   shiftable: true,
   canReduceTimeRange: true,
-  getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => {
+  getPossibleOperationForField: ({
+    aggregationRestrictions,
+    aggregatable,
+    type: fieldType,
+    timeSeriesMetric,
+  }) => {
     if (
       supportedFieldTypes.includes(fieldType) &&
       aggregatable &&
+      timeSeriesMetric !== 'counter' &&
       (!aggregationRestrictions || !aggregationRestrictions.percentile_ranks)
     ) {
       return {