[TSVB] Support custom field format (elastic#101245)

* [TSVB] Support custom field format

Add format_label response processor for series vis data and bucket key formatting to process_bucket for table vis data

* Add ignore_field_formatting for series to support value formatting for all panel types except markdown

* Fix type issue for visData and rename getCustomFieldFormatter to createCustomFieldFormatter

* Update vis.test to cover custom field formats logic and add a migration script to set ignore_field_formatting to true for the series

* Move createCustomFieldFormatter to a separate file, make formatting respect only active metrics field name, refactor vis files and fix label formatting only for grouped by terms series

* Remove services, add getFieldFormatsService  to use it in format_label and get_table_data, replace getCustomFieldFormatter with createCustomFieldFormatter

* Update plugin.ts

* Update start for plugin.ts

* Add formatting for annotations and markdown values

* Refactor some code

* Update some naming and conditions

* Fix formatting of data type labels

* In process_bucket fix case for no getFieldFormatByName

* Add field formatting functional tests for all panel types

* Update tests to make them run correctly for firefox

* Update _tsvb_markdown test setup

* Move series ignoreFieldFormatting check to a separate file, change convertSeriesToVars signature, update migration script and refactor a bit functional tests

* Fix type check for timeseries_visualization.tsx

* Update migrations.js test expected version to 7.15

* Fix tests in _tsvb_chart.ts

* Fix merge conflict remove process_bucket.js

* Update process_bucket.test.ts

* Fix markdown labels formatting

* Add ignore_field_formatting for annotations, enhanced migration script to set that flag to true, refactor data_format_picker

* Fix migration script and add disabling for ignore component when string index pattern is used

* Add supporting URL and color formatters in tsvb table

* Fix eslint

* Remove ignore formatting component, add field formatting option to TSVB data format picker and make it default, remove migration script, update tests and refactor some files

* Fix failing tests, refactor create_field_formatter and add test to it, update some other files

* Fix series formatting for top hit when it has not numeric result

* Handle no fieldFormatMap case for table/vis.js

* Remove "Default" option form DataFormatPicker when index pattern is string, fix markdown variables issue and refactor some code

* Chore(TSVB): Replace aggregations lookup with map

* Fix types, update test expected data and remove unused translations

* Fix i18 check and useEffect in agg.tsx

* Handle aggregations field formatting case

* Fix agg_utils, vis and check_if_numeric_metric tests

* Correct typo and refactor condition in std_metric

* Fix type check

* Get rid of IFieldType

* Add URL and color formatting for topN and metric tabs, fix setting initial custom formatter and switching formatter in agg.tsx

* Update tsvb.asciidoc

* Remove link icon from Date format field help text, update click logic for top N in case of custom field format and fix CI

* Remove unused import

* Revert top N bar extra logic for click

* Refactor some code in agg.tsx

* Add URL and color formatting to Gauge

* Fix bug with terms formatting, refactor some code, update create_field_formatter

* Add comments to _gauge.scss

* Remove unnecessary await

Co-authored-by: Kibana Machine <[email protected]>
Co-authored-by: Uladzislau Lasitsa <[email protected]>

docs/user/dashboard/tsvb.asciidoc

@@ -56,6 +56,8 @@ The index pattern mode unlocks many new features, such as:
 
 * Interactive filters for time series visualizations
 
+* Custom field formats
+
 * Better performance
 
 [float]

src/plugins/vis_type_timeseries/common/agg_utils.test.ts

@@ -60,6 +60,7 @@ describe('agg utils', () => {
         isFieldRequired: true,
         isFilterRatioSupported: false,
         isHistogramSupported: false,
+        isFieldFormattingDisabled: false,
         hasExtendedStats: true,
       };
       const expected = [
@@ -95,6 +96,7 @@ describe('agg utils', () => {
         isFieldRequired: false,
         isFilterRatioSupported: false,
         isHistogramSupported: false,
+        isFieldFormattingDisabled: false,
         hasExtendedStats: false,
       };
       const expected = [

src/plugins/vis_type_timeseries/common/agg_utils.ts

@@ -28,6 +28,7 @@ export interface Agg {
     isFieldRequired: boolean;
     isFilterRatioSupported: boolean;
     isHistogramSupported: boolean;
+    isFieldFormattingDisabled: boolean;
     hasExtendedStats: boolean;
   };
 }
@@ -37,6 +38,7 @@ const aggDefaultMeta = {
   isFieldRequired: true,
   isFilterRatioSupported: false,
   isHistogramSupported: false,
+  isFieldFormattingDisabled: false,
   hasExtendedStats: false,
 };
 
@@ -201,6 +203,7 @@ export const aggs: Agg[] = [
     id: TSVB_METRIC_TYPES.CALCULATION,
     meta: {
       ...aggDefaultMeta,
+      isFieldFormattingDisabled: true,
       type: AGG_TYPE.PARENT_PIPELINE,
       label: i18n.translate('visTypeTimeseries.aggUtils.bucketScriptLabel', {
         defaultMessage: 'Bucket Script',
@@ -342,6 +345,7 @@ export const aggs: Agg[] = [
     id: TSVB_METRIC_TYPES.MATH,
     meta: {
       ...aggDefaultMeta,
+      isFieldFormattingDisabled: true,
       type: AGG_TYPE.SPECIAL,
       label: i18n.translate('visTypeTimeseries.aggUtils.mathLabel', { defaultMessage: 'Math' }),
     },

src/plugins/vis_type_timeseries/common/calculate_label.ts

@@ -82,7 +82,7 @@ export const calculateLabel = (
 
   if (includes(paths, metric.type)) {
     const targetMetric = metrics.find((m) => startsWith(metric.field!, m.id));
-    const targetLabel = calculateLabel(targetMetric!, metrics, fields);
+    const targetLabel = calculateLabel(targetMetric!, metrics, fields, isThrowErrorOnFieldNotFound);
 
     // For percentiles we need to parse the field id to extract the percentile
     // the user configured in the percentile aggregation and specified in the

src/plugins/vis_type_timeseries/common/enums/index.ts

@@ -20,3 +20,12 @@ export enum TOOLTIP_MODES {
   SHOW_ALL = 'show_all',
   SHOW_FOCUSED = 'show_focused',
 }
+
+export enum DATA_FORMATTERS {
+  BYTES = 'bytes',
+  CUSTOM = 'custom',
+  DEFAULT = 'default',
+  DURATION = 'duration',
+  NUMBER = 'number',
+  PERCENT = 'percent',
+}

src/plugins/vis_type_timeseries/common/types/vis_data.ts

@@ -39,6 +39,7 @@ export interface PanelSeries {
 export interface PanelData {
   id: string;
   label: string;
+  labelFormatted?: string;
   data: PanelDataArray[];
   seriesId: string;
   splitByLabel: string;

src/plugins/vis_type_timeseries/public/application/components/aggs/agg.tsx

@@ -6,33 +6,39 @@
  * Side Public License, v 1.
  */
 
-import React, { HTMLAttributes } from 'react';
+import React, { useMemo, useEffect, HTMLAttributes } from 'react';
 // @ts-ignore
 import { aggToComponent } from '../lib/agg_to_component';
 // @ts-ignore
 import { isMetricEnabled } from '../../lib/check_ui_restrictions';
+// @ts-expect-error not typed yet
+import { seriesChangeHandler } from '../lib/series_change_handler';
+import { checkIfNumericMetric } from '../lib/check_if_numeric_metric';
+import { getFormatterType } from '../lib/get_formatter_type';
 import { UnsupportedAgg } from './unsupported_agg';
 import { TemporaryUnsupportedAgg } from './temporary_unsupported_agg';
+import { DATA_FORMATTERS } from '../../../../common/enums';
 import type { Metric, Panel, Series, SanitizedFieldType } from '../../../../common/types';
-import { DragHandleProps } from '../../../types';
-import { TimeseriesUIRestrictions } from '../../../../common/ui_restrictions';
+import type { DragHandleProps } from '../../../types';
+import type { TimeseriesUIRestrictions } from '../../../../common/ui_restrictions';
 
 interface AggProps extends HTMLAttributes<HTMLElement> {
   disableDelete: boolean;
   fields: Record<string, SanitizedFieldType[]>;
+  name: string;
   model: Metric;
   panel: Panel;
   series: Series;
   siblings: Metric[];
   uiRestrictions: TimeseriesUIRestrictions;
   dragHandleProps: DragHandleProps;
+  onChange: (part: Partial<Series>) => void;
   onAdd: () => void;
-  onChange: () => void;
   onDelete: () => void;
 }
 
 export function Agg(props: AggProps) {
-  const { model, uiRestrictions } = props;
+  const { model, uiRestrictions, series, name, onChange, fields, siblings } = props;
 
   let Component = aggToComponent[model.type];
 
@@ -50,6 +56,34 @@ export function Agg(props: AggProps) {
   const indexPattern = props.series.override_index_pattern
     ? props.series.series_index_pattern
     : props.panel.index_pattern;
+  const isKibanaIndexPattern = props.panel.use_kibana_indexes || indexPattern === '';
+
+  const onAggChange = useMemo(
+    () => seriesChangeHandler({ name, model: series, onChange }, siblings),
+    [name, onChange, siblings, series]
+  );
+
+  useEffect(() => {
+    // formatter is based on the last agg, i.e. active or resulting one as pipeline
+    if (siblings[siblings.length - 1]?.id === model.id) {
+      const formatterType = getFormatterType(series.formatter);
+      const isNumericMetric = checkIfNumericMetric(model, fields, indexPattern);
+      const isNumberFormatter = ![DATA_FORMATTERS.DEFAULT, DATA_FORMATTERS.CUSTOM].includes(
+        formatterType
+      );
+
+      if (isNumberFormatter && !isNumericMetric) {
+        onChange({ formatter: DATA_FORMATTERS.DEFAULT });
+      }
+      // in case of string index pattern mode, change default formatter depending on metric type
+      // "number" formatter for numeric metric and "" as custom formatter for any other type
+      if (formatterType === DATA_FORMATTERS.DEFAULT && !isKibanaIndexPattern) {
+        onChange({
+          formatter: isNumericMetric ? DATA_FORMATTERS.NUMBER : '',
+        });
+      }
+    }
+  }, [indexPattern, model, onChange, fields, series.formatter, isKibanaIndexPattern, siblings]);
 
   return (
     <div className={props.className} style={style}>
@@ -58,7 +92,7 @@ export function Agg(props: AggProps) {
         disableDelete={props.disableDelete}
         model={props.model}
         onAdd={props.onAdd}
-        onChange={props.onChange}
+        onChange={onAggChange}
         onDelete={props.onDelete}
         panel={props.panel}
         series={props.series}

src/plugins/vis_type_timeseries/public/application/components/aggs/aggs.tsx

@@ -12,7 +12,6 @@ import { EuiDraggable, EuiDroppable } from '@elastic/eui';
 
 import { Agg } from './agg';
 // @ts-ignore
-import { seriesChangeHandler } from '../lib/series_change_handler';
 import { handleAdd, handleDelete } from '../lib/collection_actions';
 import { newMetricAggFn } from '../lib/new_metric_agg_fn';
 import type { Panel, Series, SanitizedFieldType } from '../../../../common/types';
@@ -26,16 +25,14 @@ export interface AggsProps {
   model: Series;
   fields: Record<string, SanitizedFieldType[]>;
   uiRestrictions: TimeseriesUIRestrictions;
-  onChange(): void;
+  onChange(part: Partial<Series>): void;
 }
 
 export class Aggs extends PureComponent<AggsProps> {
   render() {
-    const { panel, model, fields, uiRestrictions } = this.props;
+    const { panel, model, fields, name, uiRestrictions, onChange } = this.props;
     const list = model.metrics;
 
-    const onChange = seriesChangeHandler(this.props, list);
-
     return (
       <EuiDroppable droppableId={`${DROPPABLE_ID}:${model.id}`} type="MICRO" spacing="s">
         {list.map((row, idx) => (
@@ -51,6 +48,7 @@ export class Aggs extends PureComponent<AggsProps> {
                 key={row.id}
                 disableDelete={list.length < 2}
                 fields={fields}
+                name={name}
                 model={row}
                 onAdd={() => handleAdd(this.props, newMetricAggFn)}
                 onChange={onChange}