Merge branch 'main' into elastic#194932

packages/kbn-esql-utils/index.ts

@@ -30,6 +30,7 @@ export {
   retrieveMetadataColumns,
   getQueryColumnsFromESQLQuery,
   isESQLColumnSortable,
+  isESQLColumnGroupable,
   TextBasedLanguages,
 } from './src';
 

packages/kbn-esql-utils/src/index.ts

@@ -31,4 +31,4 @@ export {
   getStartEndParams,
   hasStartEndParams,
 } from './utils/run_query';
-export { isESQLColumnSortable } from './utils/esql_fields_utils';
+export { isESQLColumnSortable, isESQLColumnGroupable } from './utils/esql_fields_utils';

packages/kbn-esql-utils/src/utils/esql_fields_utils.test.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 import type { DatatableColumn } from '@kbn/expressions-plugin/common';
-import { isESQLColumnSortable } from './esql_fields_utils';
+import { isESQLColumnSortable, isESQLColumnGroupable } from './esql_fields_utils';
 
 describe('esql fields helpers', () => {
   describe('isESQLColumnSortable', () => {
@@ -63,4 +63,45 @@ describe('esql fields helpers', () => {
       expect(isESQLColumnSortable(keywordField)).toBeTruthy();
     });
   });
+
+  describe('isESQLColumnGroupable', () => {
+    it('returns false for unsupported fields', () => {
+      const unsupportedField = {
+        id: 'unsupported',
+        name: 'unsupported',
+        meta: {
+          type: 'unknown',
+          esType: 'unknown',
+        },
+        isNull: false,
+      } as DatatableColumn;
+      expect(isESQLColumnGroupable(unsupportedField)).toBeFalsy();
+    });
+
+    it('returns false for counter fields', () => {
+      const tsdbField = {
+        id: 'tsbd_counter',
+        name: 'tsbd_counter',
+        meta: {
+          type: 'number',
+          esType: 'counter_long',
+        },
+        isNull: false,
+      } as DatatableColumn;
+      expect(isESQLColumnGroupable(tsdbField)).toBeFalsy();
+    });
+
+    it('returns true for everything else', () => {
+      const keywordField = {
+        id: 'sortable',
+        name: 'sortable',
+        meta: {
+          type: 'string',
+          esType: 'keyword',
+        },
+        isNull: false,
+      } as DatatableColumn;
+      expect(isESQLColumnGroupable(keywordField)).toBeTruthy();
+    });
+  });
 });

packages/kbn-esql-utils/src/utils/esql_fields_utils.ts

@@ -12,6 +12,7 @@ import type { DatatableColumn } from '@kbn/expressions-plugin/common';
 const SPATIAL_FIELDS = ['geo_point', 'geo_shape', 'point', 'shape'];
 const SOURCE_FIELD = '_source';
 const TSDB_COUNTER_FIELDS_PREFIX = 'counter_';
+const UNKNOWN_FIELD = 'unknown';
 
 /**
  * Check if a column is sortable.
@@ -38,3 +39,23 @@ export const isESQLColumnSortable = (column: DatatableColumn): boolean => {
 
   return true;
 };
+
+/**
+ * Check if a column is groupable (| STATS ... BY <column>).
+ *
+ * @param column The DatatableColumn of the field.
+ * @returns True if the column is groupable, false otherwise.
+ */
+
+export const isESQLColumnGroupable = (column: DatatableColumn): boolean => {
+  // we don't allow grouping on the unknown field types
+  if (column.meta?.type === UNKNOWN_FIELD) {
+    return false;
+  }
+  // we don't allow grouping on tsdb counter fields
+  if (column.meta?.esType && column.meta?.esType?.indexOf(TSDB_COUNTER_FIELDS_PREFIX) !== -1) {
+    return false;
+  }
+
+  return true;
+};

src/plugins/unified_histogram/public/chart/breakdown_field_selector.tsx

@@ -11,6 +11,7 @@ import React, { useCallback, useMemo } from 'react';
 import { EuiSelectableOption } from '@elastic/eui';
 import { FieldIcon, getFieldIconProps, comboBoxFieldOptionMatcher } from '@kbn/field-utils';
 import { css } from '@emotion/react';
+import { isESQLColumnGroupable } from '@kbn/esql-utils';
 import { type DataView, DataViewField } from '@kbn/data-views-plugin/common';
 import type { DatatableColumn } from '@kbn/expressions-plugin/common';
 import { convertDatatableColumnToDataViewFieldSpec } from '@kbn/data-view-utils';
@@ -34,10 +35,10 @@ export interface BreakdownFieldSelectorProps {
 const mapToDropdownFields = (dataView: DataView, esqlColumns?: DatatableColumn[]) => {
   if (esqlColumns) {
     return (
+      // filter out unsupported field types and counter time series metrics
       esqlColumns
+        .filter(isESQLColumnGroupable)
         .map((column) => new DataViewField(convertDatatableColumnToDataViewFieldSpec(column)))
-        // filter out unsupported field types
-        .filter((field) => field.type !== 'unknown')
     );
   }
 

src/plugins/unified_histogram/public/services/lens_vis_service.attributes.test.ts

@@ -674,7 +674,8 @@ describe('LensVisService attributes', () => {
               },
             ],
             "query": Object {
-              "esql": "from logstash-* | limit 10",
+              "esql": "from logstash-* | limit 10
+      | EVAL timestamp=DATE_TRUNC(10 minute, timestamp) | stats results = count(*) by timestamp | rename timestamp as \`timestamp every 10 minute\`",
             },
             "visualization": Object {
               "gridConfig": Object {
@@ -706,7 +707,7 @@ describe('LensVisService attributes', () => {
           "timeField": "timestamp",
           "timeInterval": undefined,
         },
-        "suggestionType": "lensSuggestion",
+        "suggestionType": "histogramForESQL",
       }
     `);
   });

src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts

@@ -254,6 +254,37 @@ describe('LensVisService suggestions', () => {
     expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery);
   });
 
+  test('should return histogramSuggestion even if suggestions returned by the api', async () => {
+    const lensVis = await getLensVisMock({
+      filters: [],
+      query: { esql: 'from the-data-view | limit 100' },
+      dataView: dataViewMock,
+      timeInterval: 'auto',
+      timeRange: {
+        from: '2023-09-03T08:00:00.000Z',
+        to: '2023-09-04T08:56:28.274Z',
+      },
+      breakdownField: undefined,
+      columns: [
+        {
+          id: 'var0',
+          name: 'var0',
+          meta: {
+            type: 'number',
+          },
+        },
+      ],
+      isPlainRecord: true,
+      allSuggestions: allSuggestionsMock,
+      hasHistogramSuggestionForESQL: true,
+    });
+
+    expect(lensVis.currentSuggestionContext?.type).toBe(
+      UnifiedHistogramSuggestionType.histogramForESQL
+    );
+    expect(lensVis.currentSuggestionContext?.suggestion).toBeDefined();
+  });
+
   test('should return histogramSuggestion if no suggestions returned by the api with a geo point breakdown field correctly', async () => {
     const lensVis = await getLensVisMock({
       filters: [],

src/plugins/unified_histogram/public/services/lens_vis_service.ts

@@ -235,7 +235,7 @@ export class LensVisService {
     let currentSuggestion: Suggestion | undefined;
 
     // takes lens suggestions if provided
-    const availableSuggestionsWithType: Array<{
+    let availableSuggestionsWithType: Array<{
       suggestion: UnifiedHistogramSuggestionContext['suggestion'];
       type: UnifiedHistogramSuggestionType;
     }> = [];
@@ -254,6 +254,9 @@ export class LensVisService {
         breakdownField,
       });
       if (histogramSuggestionForESQL) {
+        // In case if histogram suggestion, we want to empty the array and push the new suggestion
+        // to ensure that only the histogram suggestion is available
+        availableSuggestionsWithType = [];
         availableSuggestionsWithType.push({
           suggestion: histogramSuggestionForESQL,
           type: UnifiedHistogramSuggestionType.histogramForESQL,

x-pack/plugins/data_usage/common/rest_types/data_streams.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { schema } from '@kbn/config-schema';
+import { schema, TypeOf } from '@kbn/config-schema';
 
 export const DataStreamsResponseSchema = {
   body: () =>
@@ -16,3 +16,5 @@ export const DataStreamsResponseSchema = {
       })
     ),
 };
+
+export type DataStreamsResponseBodySchemaBody = TypeOf<typeof DataStreamsResponseSchema.body>;

x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts

@@ -41,15 +41,15 @@ describe('usage_metrics schemas', () => {
     ).not.toThrow();
   });
 
-  it('should error if `dataStream` list is empty', () => {
+  it('should not error if `dataStream` list is empty', () => {
     expect(() =>
       UsageMetricsRequestSchema.validate({
         from: new Date().toISOString(),
         to: new Date().toISOString(),
         metricTypes: ['storage_retained'],
         dataStreams: [],
       })
-    ).toThrowError('[dataStreams]: array size is [0], but cannot be smaller than [1]');
+    ).not.toThrow();
   });
 
   it('should error if `dataStream` is given type not array', () => {
@@ -71,7 +71,7 @@ describe('usage_metrics schemas', () => {
         metricTypes: ['storage_retained'],
         dataStreams: ['ds_1', '  '],
       })
-    ).toThrow('[dataStreams]: [dataStreams] list cannot contain empty values');
+    ).toThrow('[dataStreams]: list cannot contain empty values');
   });
 
   it('should error if `metricTypes` is empty string', () => {
@@ -82,7 +82,7 @@ describe('usage_metrics schemas', () => {
         dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
         metricTypes: ' ',
       })
-    ).toThrow();
+    ).toThrow('[metricTypes]: could not parse array value from json input');
   });
 
   it('should error if `metricTypes` contains an empty item', () => {
@@ -93,7 +93,7 @@ describe('usage_metrics schemas', () => {
         dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
         metricTypes: [' ', 'storage_retained'], // First item is invalid
       })
-    ).toThrowError(/list cannot contain empty values/);
+    ).toThrow('list cannot contain empty values');
   });
 
   it('should error if `metricTypes` is not a valid type', () => {
@@ -116,7 +116,7 @@ describe('usage_metrics schemas', () => {
         metricTypes: ['storage_retained', 'foo'],
       })
     ).toThrow(
-      '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate'
+      '[metricTypes]: must be one of ingest_rate, storage_retained, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate'
     );
   });
 

x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts

@@ -7,9 +7,11 @@
 
 import { schema, type TypeOf } from '@kbn/config-schema';
 
-const METRIC_TYPE_VALUES = [
-  'storage_retained',
-  'ingest_rate',
+// note these should be sorted alphabetically as we sort the URL params on the browser side
+// before making the request, else the cache key will be different and that would invoke a new request
+export const DEFAULT_METRIC_TYPES = ['ingest_rate', 'storage_retained'] as const;
+export const METRIC_TYPE_VALUES = [
+  ...DEFAULT_METRIC_TYPES,
   'search_vcu',
   'ingest_vcu',
   'ml_vcu',
@@ -21,6 +23,22 @@ const METRIC_TYPE_VALUES = [
 
 export type MetricTypes = (typeof METRIC_TYPE_VALUES)[number];
 
+export const isDefaultMetricType = (metricType: string) =>
+  // @ts-ignore
+  DEFAULT_METRIC_TYPES.includes(metricType);
+
+export const METRIC_TYPE_API_VALUES_TO_UI_OPTIONS_MAP = Object.freeze<Record<MetricTypes, string>>({
+  storage_retained: 'Data Retained in Storage',
+  ingest_rate: 'Data Ingested',
+  search_vcu: 'Search VCU',
+  ingest_vcu: 'Ingest VCU',
+  ml_vcu: 'ML VCU',
+  index_latency: 'Index Latency',
+  index_rate: 'Index Rate',
+  search_latency: 'Search Latency',
+  search_rate: 'Search Rate',
+});
+
 // type guard for MetricTypes
 export const isMetricType = (type: string): type is MetricTypes =>
   METRIC_TYPE_VALUES.includes(type as MetricTypes);
@@ -47,21 +65,20 @@ export const UsageMetricsRequestSchema = schema.object({
       if (trimmedValues.some((v) => !v.length)) {
         return '[metricTypes] list cannot contain empty values';
       } else if (trimmedValues.some((v) => !isValidMetricType(v))) {
-        return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`;
+        return `must be one of ${METRIC_TYPE_VALUES.join(', ')}`;
       }
     },
   }),
   dataStreams: schema.arrayOf(schema.string(), {
-    minSize: 1,
     validate: (values) => {
       if (values.map((v) => v.trim()).some((v) => !v.length)) {
-        return '[dataStreams] list cannot contain empty values';
+        return 'list cannot contain empty values';
       }
     },
   }),
 });
 
-export type UsageMetricsRequestSchemaQueryParams = TypeOf<typeof UsageMetricsRequestSchema>;
+export type UsageMetricsRequestBody = TypeOf<typeof UsageMetricsRequestSchema>;
 
 export const UsageMetricsResponseSchema = {
   body: () =>