[8.x] [ES|QL] Supports custom formatters in charts (#201540) (#203717)

# Backport This will backport the following commits from `main` to `8.x`: - [[ES|QL] Supports custom formatters in charts (#201540)](#201540)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Peter Pisljar <[email protected]>
elastic · Dec 11, 2024 · ab026f8 · ab026f8
1 parent d2c551a
commit ab026f8
Show file tree

Hide file tree

Showing 18 changed files with 275 additions and 56 deletions.
diff --git a/src/platform/packages/private/kbn-esql-editor/src/fetch_fields_from_esql.ts b/src/platform/packages/private/kbn-esql-editor/src/fetch_fields_from_esql.ts
@@ -12,7 +12,7 @@ import { lastValueFrom } from 'rxjs';
 import { Query, AggregateQuery, TimeRange } from '@kbn/es-query';
 import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
 import type { Datatable } from '@kbn/expressions-plugin/public';
-import { type DataView, textBasedQueryStateToAstWithValidation } from '@kbn/data-plugin/common';
+import { textBasedQueryStateToAstWithValidation } from '@kbn/data-plugin/common';
 
 interface TextBasedLanguagesErrorResponse {
   error: {
@@ -26,16 +26,20 @@ export function fetchFieldsFromESQL(
   expressions: ExpressionsStart,
   time?: TimeRange,
   abortController?: AbortController,
-  dataView?: DataView
+  timeFieldName?: string
 ) {
   return textBasedQueryStateToAstWithValidation({
     query,
     time,
-    dataView,
+    timeFieldName,
   })
     .then((ast) => {
       if (ast) {
-        const executionContract = expressions.execute(ast, null);
+        const executionContract = expressions.execute(ast, null, {
+          searchContext: {
+            timeRange: time,
+          },
+        });
 
         if (abortController) {
           abortController.signal.onabort = () => {

diff --git a/src/plugins/data/common/datatable_utilities/datatable_utilities_service.ts b/src/plugins/data/common/datatable_utilities/datatable_utilities_service.ts
@@ -74,10 +74,7 @@ export class DatatableUtilitiesService {
       timeZone: string;
     }> = {}
   ): DateHistogramMeta | undefined {
-    if (column.meta.source !== 'esaggs') {
-      return;
-    }
-    if (column.meta.sourceParams?.type !== BUCKET_TYPES.DATE_HISTOGRAM) {
+    if (!column.meta.sourceParams || !column.meta.sourceParams.params) {
       return;
     }
 

diff --git a/src/plugins/data/common/query/text_based_query_state_to_ast_with_validation.test.ts b/src/plugins/data/common/query/text_based_query_state_to_ast_with_validation.test.ts
@@ -7,7 +7,6 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { createStubDataView } from '@kbn/data-views-plugin/common/mocks';
 import { textBasedQueryStateToAstWithValidation } from './text_based_query_state_to_ast_with_validation';
 
 describe('textBasedQueryStateToAstWithValidation', () => {
@@ -25,21 +24,14 @@ describe('textBasedQueryStateToAstWithValidation', () => {
   });
 
   it('returns an object with the correct structure for an SQL query with existing dataview', async () => {
-    const dataView = createStubDataView({
-      spec: {
-        id: 'foo',
-        title: 'foo',
-        timeFieldName: '@timestamp',
-      },
-    });
     const actual = await textBasedQueryStateToAstWithValidation({
       filters: [],
       query: { esql: 'FROM foo' },
       time: {
         from: 'now',
         to: 'now+7d',
       },
-      dataView,
+      timeFieldName: '@timestamp',
     });
 
     expect(actual).toHaveProperty(
@@ -76,21 +68,14 @@ describe('textBasedQueryStateToAstWithValidation', () => {
   });
 
   it('returns an object with the correct structure for ES|QL', async () => {
-    const dataView = createStubDataView({
-      spec: {
-        id: 'foo',
-        title: 'foo',
-        timeFieldName: '@timestamp',
-      },
-    });
     const actual = await textBasedQueryStateToAstWithValidation({
       filters: [],
       query: { esql: 'from logs*' },
       time: {
         from: 'now',
         to: 'now+7d',
       },
-      dataView,
+      timeFieldName: '@timestamp',
       titleForInspector: 'Custom title',
       descriptionForInspector: 'Custom desc',
     });

diff --git a/src/plugins/data/common/query/text_based_query_state_to_ast_with_validation.ts b/src/plugins/data/common/query/text_based_query_state_to_ast_with_validation.ts
@@ -8,12 +8,10 @@
  */
 
 import { isOfAggregateQueryType, Query } from '@kbn/es-query';
-import type { DataView } from '@kbn/data-views-plugin/common';
 import type { QueryState } from '..';
 import { textBasedQueryStateToExpressionAst } from './text_based_query_state_to_ast';
 
 interface Args extends QueryState {
-  dataView?: DataView;
   inputQuery?: Query;
   timeFieldName?: string;
   titleForInspector?: string;
@@ -26,7 +24,7 @@ interface Args extends QueryState {
  * @param query kibana query or aggregate query
  * @param inputQuery
  * @param time kibana time range
- * @param dataView
+ * @param timeFieldName
  * @param titleForInspector
  * @param descriptionForInspector
  */
@@ -35,7 +33,7 @@ export async function textBasedQueryStateToAstWithValidation({
   query,
   inputQuery,
   time,
-  dataView,
+  timeFieldName,
   titleForInspector,
   descriptionForInspector,
 }: Args) {
@@ -46,7 +44,7 @@ export async function textBasedQueryStateToAstWithValidation({
       query,
       inputQuery,
       time,
-      timeFieldName: dataView?.timeFieldName,
+      timeFieldName,
       titleForInspector,
       descriptionForInspector,
     });

diff --git a/src/plugins/data/common/search/expressions/esql.ts b/src/plugins/data/common/search/expressions/esql.ts
@@ -315,7 +315,17 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
             (body.all_columns ?? body.columns)?.map(({ name, type }) => ({
               id: name,
               name,
-              meta: { type: esFieldTypeToKibanaFieldType(type), esType: type },
+              meta: {
+                type: esFieldTypeToKibanaFieldType(type),
+                esType: type,
+                sourceParams:
+                  type === 'date'
+                    ? {
+                        appliedTimeRange: input?.timeRange,
+                        params: {},
+                      }
+                    : {},
+              },
               isNull: hasEmptyColumns ? !lookup.has(name) : false,
             })) ?? [];
 

diff --git a/src/plugins/discover/public/application/main/data_fetching/fetch_esql.ts b/src/plugins/discover/public/application/main/data_fetching/fetch_esql.ts
@@ -56,7 +56,7 @@ export function fetchEsql({
     filters,
     query,
     time: timeRange,
-    dataView,
+    timeFieldName: dataView.timeFieldName,
     inputQuery,
     titleForInspector: i18n.translate('discover.inspectorEsqlRequestTitle', {
       defaultMessage: 'Table',

diff --git a/x-pack/plugins/lens/common/expressions/map_to_columns/map_to_columns.test.ts b/x-pack/plugins/lens/common/expressions/map_to_columns/map_to_columns.test.ts
@@ -319,8 +319,8 @@ describe('map_to_columns', () => {
       );
 
       expect(result.columns).toStrictEqual([
-        { id: 'a', name: 'A', meta: { type: 'number' } },
-        { id: 'b', name: 'B', meta: { type: 'number' } },
+        { id: 'a', name: 'A', meta: { type: 'number', field: undefined, params: undefined } },
+        { id: 'b', name: 'B', meta: { type: 'number', field: undefined, params: undefined } },
       ]);
 
       expect(result.rows).toStrictEqual([

diff --git a/x-pack/plugins/lens/common/expressions/map_to_columns/map_to_columns_fn_textbased.ts b/x-pack/plugins/lens/common/expressions/map_to_columns/map_to_columns_fn_textbased.ts
@@ -32,7 +32,16 @@ export const mapToOriginalColumnsTextBased: MapToColumnsExpressionFunction['fn']
       if (!(column.id in idMap)) {
         return [];
       }
-      return idMap[column.id].map((originalColumn) => ({ ...column, id: originalColumn.id }));
+      return idMap[column.id].map((originalColumn) => ({
+        ...column,
+        id: originalColumn.id,
+        name: originalColumn.label,
+        meta: {
+          ...column.meta,
+          field: originalColumn.sourceField,
+          params: originalColumn.format,
+        },
+      }));
     }),
   };
 };
diff --git a/x-pack/plugins/lens/common/expressions/map_to_columns/types.ts b/x-pack/plugins/lens/common/expressions/map_to_columns/types.ts
@@ -6,8 +6,9 @@
  */
 
 import { Datatable, ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common';
+import { SerializedFieldFormat } from '@kbn/field-formats-plugin/common';
 
-export type OriginalColumn = { id: string; label: string } & (
+export type OriginalColumn = { id: string; label: string; format?: SerializedFieldFormat } & (
   | { operationType: 'date_histogram'; sourceField: string }
   | { operationType: string; sourceField: never }
 );

diff --git a/x-pack/plugins/lens/public/datasources/form_based/dimension_panel/format_selector.tsx b/x-pack/plugins/lens/public/datasources/form_based/dimension_panel/format_selector.tsx
@@ -28,6 +28,7 @@ import {
 } from '@kbn/field-formats-plugin/common';
 import { css } from '@emotion/react';
 import type { DocLinksStart } from '@kbn/core/public';
+import { TextBasedLayerColumn } from '../../text_based/types';
 import { LensAppServices } from '../../../app_plugin/types';
 import { GenericIndexPatternColumn } from '../form_based';
 import { isColumnFormatted } from '../operations/definitions/helpers';
@@ -127,7 +128,7 @@ type FormatParams = NonNullable<ValueFormatConfig['params']>;
 type FormatParamsKeys = keyof FormatParams;
 
 export interface FormatSelectorProps {
-  selectedColumn: GenericIndexPatternColumn;
+  selectedColumn: GenericIndexPatternColumn | TextBasedLayerColumn;
   onChange: (newFormat?: { id: string; params?: FormatParams }) => void;
   docLinks: DocLinksStart;
 }

diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx
@@ -10,6 +10,7 @@ import React, { Fragment } from 'react';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { isEqual } from 'lodash';
 import { Query } from '@kbn/es-query';
+import { TextBasedLayerColumn } from '../../../text_based/types';
 import type { IndexPattern, IndexPatternField } from '../../../../types';
 import {
   type FieldBasedOperationErrorMessage,
@@ -178,8 +179,8 @@ export const isColumn = (
 };
 
 export function isColumnFormatted(
-  column: GenericIndexPatternColumn
-): column is FormattedIndexPatternColumn {
+  column: GenericIndexPatternColumn | TextBasedLayerColumn
+): column is FormattedIndexPatternColumn | TextBasedLayerColumn {
   return Boolean(
     'params' in column &&
       (column as FormattedIndexPatternColumn).params &&

diff --git a/x-pack/plugins/lens/public/datasources/text_based/components/dimension_editor.tsx b/x-pack/plugins/lens/public/datasources/text_based/components/dimension_editor.tsx
@@ -5,16 +5,24 @@
  * 2.0.
  */
 
-import React, { useEffect, useState, useMemo } from 'react';
+import React, { useEffect, useState, useMemo, useCallback } from 'react';
 import { i18n } from '@kbn/i18n';
-import { EuiFormRow } from '@elastic/eui';
+import { EuiFormRow, useEuiTheme, EuiText } from '@elastic/eui';
 import { euiThemeVars } from '@kbn/ui-theme';
 import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
 import { fetchFieldsFromESQL } from '@kbn/esql-editor';
+import { NameInput } from '@kbn/visualization-ui-components';
+import { css } from '@emotion/react';
+import { mergeLayer, updateColumnFormat, updateColumnLabel } from '../utils';
+import {
+  FormatSelector,
+  FormatSelectorProps,
+} from '../../form_based/dimension_panel/format_selector';
 import type { DatasourceDimensionEditorProps, DataType } from '../../../types';
 import { FieldSelect, type FieldOptionCompatible } from './field_select';
 import type { TextBasedPrivateState } from '../types';
 import { isNotNumeric, isNumeric } from '../utils';
+import { TextBasedLayer } from '../types';
 
 export type TextBasedDimensionEditorProps =
   DatasourceDimensionEditorProps<TextBasedPrivateState> & {
@@ -24,14 +32,30 @@ export type TextBasedDimensionEditorProps =
 export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
   const [allColumns, setAllColumns] = useState<FieldOptionCompatible[]>([]);
   const query = props.state.layers[props.layerId]?.query;
+  const { euiTheme } = useEuiTheme();
+  const {
+    isFullscreen,
+    columnId,
+    layerId,
+    state,
+    setState,
+    indexPatterns,
+    dateRange,
+    expressions,
+  } = props;
 
   useEffect(() => {
     // in case the columns are not in the cache, I refetch them
     async function fetchColumns() {
       if (query) {
         const table = await fetchFieldsFromESQL(
           { esql: `${query.esql} | limit 0` },
-          props.expressions
+          expressions,
+          { from: dateRange.fromDate, to: dateRange.toDate },
+          undefined,
+          Object.values(indexPatterns).length
+            ? Object.values(indexPatterns)[0].timeFieldName
+            : undefined
         );
         if (table) {
           const hasNumberTypeColumns = table.columns?.some(isNumeric);
@@ -55,13 +79,40 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
       }
     }
     fetchColumns();
-  }, [props, props.expressions, query]);
+  }, [
+    dateRange.fromDate,
+    dateRange.toDate,
+    expressions,
+    indexPatterns,
+    props,
+    props.expressions,
+    query,
+  ]);
 
   const selectedField = useMemo(() => {
     const layerColumns = props.state.layers[props.layerId].columns;
     return layerColumns?.find((column) => column.columnId === props.columnId);
   }, [props.columnId, props.layerId, props.state.layers]);
 
+  const updateLayer = useCallback(
+    (newLayer: Partial<TextBasedLayer>) =>
+      setState((prevState) => mergeLayer({ state: prevState, layerId, newLayer })),
+    [layerId, setState]
+  );
+
+  const onFormatChange = useCallback<FormatSelectorProps['onChange']>(
+    (newFormat) => {
+      updateLayer(
+        updateColumnFormat({
+          layer: state.layers[layerId],
+          columnId,
+          value: newFormat,
+        })
+      );
+    },
+    [columnId, layerId, state.layers, updateLayer]
+  );
+
   return (
     <>
       <EuiFormRow
@@ -80,6 +131,7 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
             const newColumn = {
               columnId: props.columnId,
               fieldName: choice.field,
+              label: choice.field,
               meta,
             };
             return props.setState(
@@ -122,6 +174,44 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
           {props.dataSectionExtra}
         </div>
       )}
+      {!isFullscreen && selectedField && (
+        <div className="lnsIndexPatternDimensionEditor--padded lnsIndexPatternDimensionEditor--collapseNext">
+          <EuiText
+            size="s"
+            css={css`
+              margin-bottom: ${euiTheme.size.base};
+            `}
+          >
+            <h4>
+              {i18n.translate('xpack.lens.indexPattern.dimensionEditor.headingAppearance', {
+                defaultMessage: 'Appearance',
+              })}
+            </h4>
+          </EuiText>
+
+          <NameInput
+            value={selectedField.label || ''}
+            defaultValue={''}
+            onChange={(value) => {
+              updateLayer(
+                updateColumnLabel({
+                  layer: state.layers[layerId],
+                  columnId,
+                  value,
+                })
+              );
+            }}
+          />
+
+          {selectedField.meta?.type === 'number' ? (
+            <FormatSelector
+              selectedColumn={selectedField}
+              onChange={onFormatChange}
+              docLinks={props.core.docLinks}
+            />
+          ) : null}
+        </div>
+      )}
     </>
   );
 }