[ML] Add support for percentiles aggregation to Transform wizard (#60763

) (#61044) * [ML] Add support for percentiles aggregation to Transform wizard * [ML] Fix type error and comments from review * [ML] Remove unused function Co-authored-by: Elastic Machine <[email protected]> Co-authored-by: Elastic Machine <[email protected]>
elastic · Mar 24, 2020 · 4789f47 · 4789f47
1 parent 192d348
commit 4789f47
Show file tree

Hide file tree

Showing 6 changed files with 217 additions and 8 deletions.
diff --git a/x-pack/plugins/transform/public/app/common/index.ts b/x-pack/plugins/transform/public/app/common/index.ts
@@ -40,6 +40,8 @@ export { GetTransformsResponse, PreviewData, PreviewMappings } from './pivot_pre
 export {
   getEsAggFromAggConfig,
   isPivotAggsConfigWithUiSupport,
+  isPivotAggsConfigPercentiles,
+  PERCENTILES_AGG_DEFAULT_PERCENTS,
   PivotAgg,
   PivotAggDict,
   PivotAggsConfig,

diff --git a/x-pack/plugins/transform/public/app/common/pivot_aggs.ts b/x-pack/plugins/transform/public/app/common/pivot_aggs.ts
@@ -15,10 +15,13 @@ export enum PIVOT_SUPPORTED_AGGS {
   CARDINALITY = 'cardinality',
   MAX = 'max',
   MIN = 'min',
+  PERCENTILES = 'percentiles',
   SUM = 'sum',
   VALUE_COUNT = 'value_count',
 }
 
+export const PERCENTILES_AGG_DEFAULT_PERCENTS = [1, 5, 25, 50, 75, 95, 99];
+
 export const pivotAggsFieldSupport = {
   [KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
   [KBN_FIELD_TYPES.BOOLEAN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
@@ -36,6 +39,7 @@ export const pivotAggsFieldSupport = {
     PIVOT_SUPPORTED_AGGS.CARDINALITY,
     PIVOT_SUPPORTED_AGGS.MAX,
     PIVOT_SUPPORTED_AGGS.MIN,
+    PIVOT_SUPPORTED_AGGS.PERCENTILES,
     PIVOT_SUPPORTED_AGGS.SUM,
     PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
   ],
@@ -60,10 +64,17 @@ export interface PivotAggsConfigBase {
   dropDownName: string;
 }
 
-export interface PivotAggsConfigWithUiSupport extends PivotAggsConfigBase {
+interface PivotAggsConfigWithUiBase extends PivotAggsConfigBase {
   field: EsFieldName;
 }
 
+interface PivotAggsConfigPercentiles extends PivotAggsConfigWithUiBase {
+  agg: PIVOT_SUPPORTED_AGGS.PERCENTILES;
+  percents: number[];
+}
+
+export type PivotAggsConfigWithUiSupport = PivotAggsConfigWithUiBase | PivotAggsConfigPercentiles;
+
 export function isPivotAggsConfigWithUiSupport(arg: any): arg is PivotAggsConfigWithUiSupport {
   return (
     arg.hasOwnProperty('agg') &&
@@ -74,6 +85,15 @@ export function isPivotAggsConfigWithUiSupport(arg: any): arg is PivotAggsConfig
   );
 }
 
+export function isPivotAggsConfigPercentiles(arg: any): arg is PivotAggsConfigPercentiles {
+  return (
+    arg.hasOwnProperty('agg') &&
+    arg.hasOwnProperty('field') &&
+    arg.hasOwnProperty('percents') &&
+    arg.agg === PIVOT_SUPPORTED_AGGS.PERCENTILES
+  );
+}
+
 export type PivotAggsConfig = PivotAggsConfigBase | PivotAggsConfigWithUiSupport;
 
 export type PivotAggsConfigWithUiSupportDict = Dictionary<PivotAggsConfigWithUiSupport>;

diff --git a/...ansform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx b/...ansform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx
@@ -22,8 +22,10 @@ import { dictionaryToArray } from '../../../../../../common/types/common';
 import {
   AggName,
   isAggName,
+  isPivotAggsConfigPercentiles,
   isPivotAggsConfigWithUiSupport,
   getEsAggFromAggConfig,
+  PERCENTILES_AGG_DEFAULT_PERCENTS,
   PivotAggsConfig,
   PivotAggsConfigWithUiSupportDict,
   PIVOT_SUPPORTED_AGGS,
@@ -40,6 +42,33 @@ interface Props {
   onChange(d: PivotAggsConfig): void;
 }
 
+function getDefaultPercents(defaultData: PivotAggsConfig): number[] | undefined {
+  if (isPivotAggsConfigPercentiles(defaultData)) {
+    return defaultData.percents;
+  }
+}
+
+function parsePercentsInput(inputValue: string | undefined) {
+  if (inputValue !== undefined) {
+    const strVals: string[] = inputValue.split(',');
+    const percents: number[] = [];
+    for (const str of strVals) {
+      if (str.trim().length > 0 && isNaN(str as any) === false) {
+        const val = Number(str);
+        if (val >= 0 && val <= 100) {
+          percents.push(val);
+        } else {
+          return [];
+        }
+      }
+    }
+
+    return percents;
+  }
+
+  return [];
+}
+
 export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onChange, options }) => {
   const isUnsupportedAgg = !isPivotAggsConfigWithUiSupport(defaultData);
 
@@ -48,10 +77,45 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
   const [field, setField] = useState(
     isPivotAggsConfigWithUiSupport(defaultData) ? defaultData.field : ''
   );
+  const [percents, setPercents] = useState(getDefaultPercents(defaultData));
 
   const availableFields: SelectOption[] = [];
   const availableAggs: SelectOption[] = [];
 
+  function updateAgg(aggVal: PIVOT_SUPPORTED_AGGS) {
+    setAgg(aggVal);
+    if (aggVal === PIVOT_SUPPORTED_AGGS.PERCENTILES && percents === undefined) {
+      setPercents(PERCENTILES_AGG_DEFAULT_PERCENTS);
+    }
+  }
+
+  function updatePercents(inputValue: string) {
+    setPercents(parsePercentsInput(inputValue));
+  }
+
+  function getUpdatedItem(): PivotAggsConfig {
+    let updatedItem: PivotAggsConfig;
+
+    if (agg !== PIVOT_SUPPORTED_AGGS.PERCENTILES) {
+      updatedItem = {
+        agg,
+        aggName,
+        field,
+        dropDownName: defaultData.dropDownName,
+      };
+    } else {
+      updatedItem = {
+        agg,
+        aggName,
+        field,
+        dropDownName: defaultData.dropDownName,
+        percents,
+      };
+    }
+
+    return updatedItem;
+  }
+
   if (!isUnsupportedAgg) {
     const optionsArr = dictionaryToArray(options);
     optionsArr
@@ -83,7 +147,18 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
     });
   }
 
-  const formValid = validAggName;
+  let percentsText;
+  if (percents !== undefined) {
+    percentsText = percents.toString();
+  }
+
+  const validPercents =
+    agg === PIVOT_SUPPORTED_AGGS.PERCENTILES && parsePercentsInput(percentsText).length > 0;
+
+  let formValid = validAggName;
+  if (formValid && agg === PIVOT_SUPPORTED_AGGS.PERCENTILES) {
+    formValid = validPercents;
+  }
 
   return (
     <EuiForm style={{ width: '300px' }}>
@@ -117,7 +192,7 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
           <EuiSelect
             options={availableAggs}
             value={agg}
-            onChange={e => setAgg(e.target.value as PIVOT_SUPPORTED_AGGS)}
+            onChange={e => updateAgg(e.target.value as PIVOT_SUPPORTED_AGGS)}
           />
         </EuiFormRow>
       )}
@@ -134,6 +209,26 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
           />
         </EuiFormRow>
       )}
+      {agg === PIVOT_SUPPORTED_AGGS.PERCENTILES && (
+        <EuiFormRow
+          label={i18n.translate('xpack.transform.agg.popoverForm.percentsLabel', {
+            defaultMessage: 'Percents',
+          })}
+          error={
+            !validPercents && [
+              i18n.translate('xpack.transform.groupBy.popoverForm.intervalPercents', {
+                defaultMessage: 'Enter a comma-separated list of percentiles',
+              }),
+            ]
+          }
+          isInvalid={!validPercents}
+        >
+          <EuiFieldText
+            defaultValue={percentsText}
+            onChange={e => updatePercents(e.target.value)}
+          />
+        </EuiFormRow>
+      )}
       {isUnsupportedAgg && (
         <EuiCodeEditor
           mode="json"
@@ -147,10 +242,7 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
         />
       )}
       <EuiFormRow hasEmptyLabelSpace>
-        <EuiButton
-          isDisabled={!formValid}
-          onClick={() => onChange({ ...defaultData, aggName, agg, field })}
-        >
+        <EuiButton isDisabled={!formValid} onClick={() => onChange(getUpdatedItem())}>
           {i18n.translate('xpack.transform.agg.popoverForm.submitButtonLabel', {
             defaultMessage: 'Apply',
           })}

diff --git a/...gins/transform/public/app/sections/create_transform/components/step_define/common.test.ts b/...gins/transform/public/app/sections/create_transform/components/step_define/common.test.ts
@@ -37,6 +37,7 @@ describe('Transform: Define Pivot Common', () => {
             { label: 'cardinality( the-f[i]e>ld )' },
             { label: 'max( the-f[i]e>ld )' },
             { label: 'min( the-f[i]e>ld )' },
+            { label: 'percentiles( the-f[i]e>ld )' },
             { label: 'sum( the-f[i]e>ld )' },
             { label: 'value_count( the-f[i]e>ld )' },
           ],
@@ -67,6 +68,13 @@ describe('Transform: Define Pivot Common', () => {
           aggName: 'the-field.min',
           dropDownName: 'min( the-f[i]e>ld )',
         },
+        'percentiles( the-f[i]e>ld )': {
+          agg: 'percentiles',
+          field: ' the-f[i]e>ld ',
+          aggName: 'the-field.percentiles',
+          dropDownName: 'percentiles( the-f[i]e>ld )',
+          percents: [1, 5, 25, 50, 75, 95, 99],
+        },
         'sum( the-f[i]e>ld )': {
           agg: 'sum',
           field: ' the-f[i]e>ld ',

diff --git a/...k/plugins/transform/public/app/sections/create_transform/components/step_define/common.ts b/...k/plugins/transform/public/app/sections/create_transform/components/step_define/common.ts
@@ -12,10 +12,13 @@ import {
   DropDownOption,
   EsFieldName,
   GroupByConfigWithUiSupport,
+  PERCENTILES_AGG_DEFAULT_PERCENTS,
+  PivotAggsConfigWithUiSupport,
   PivotAggsConfigWithUiSupportDict,
   pivotAggsFieldSupport,
   PivotGroupByConfigWithUiSupportDict,
   pivotGroupByFieldSupport,
+  PIVOT_SUPPORTED_AGGS,
   PIVOT_SUPPORTED_GROUP_BY_AGGS,
 } from '../../../../common';
 
@@ -57,6 +60,31 @@ function getDefaultGroupByConfig(
   }
 }
 
+function getDefaultAggregationConfig(
+  aggName: string,
+  dropDownName: string,
+  fieldName: EsFieldName,
+  agg: PIVOT_SUPPORTED_AGGS
+): PivotAggsConfigWithUiSupport {
+  switch (agg) {
+    case PIVOT_SUPPORTED_AGGS.PERCENTILES:
+      return {
+        agg,
+        aggName,
+        dropDownName,
+        field: fieldName,
+        percents: PERCENTILES_AGG_DEFAULT_PERCENTS,
+      };
+    default:
+      return {
+        agg,
+        aggName,
+        dropDownName,
+        field: fieldName,
+      };
+  }
+}
+
 const illegalEsAggNameChars = /[[\]>]/g;
 
 export function getPivotDropdownOptions(indexPattern: IndexPattern) {
@@ -105,7 +133,12 @@ export function getPivotDropdownOptions(indexPattern: IndexPattern) {
         // Option name in the dropdown for the aggregation is in the form of `sum(fieldname)`.
         const dropDownName = `${agg}(${field.name})`;
         aggOption.options.push({ label: dropDownName });
-        aggOptionsData[dropDownName] = { agg, field: field.name, aggName, dropDownName };
+        aggOptionsData[dropDownName] = getDefaultAggregationConfig(
+          aggName,
+          dropDownName,
+          field.name,
+          agg
+        );
       });
     }
     aggOptions.push(aggOption);

diff --git a/x-pack/test/functional/apps/transform/creation_index_pattern.ts b/x-pack/test/functional/apps/transform/creation_index_pattern.ts
@@ -94,6 +94,60 @@ export default function({ getService }: FtrProviderContext) {
           },
         },
       },
+      {
+        suiteTitle: 'batch transform with terms group and percentiles agg',
+        source: 'ecommerce',
+        groupByEntries: [
+          {
+            identifier: 'terms(geoip.country_iso_code)',
+            label: 'geoip.country_iso_code',
+          } as GroupByEntry,
+        ],
+        aggregationEntries: [
+          {
+            identifier: 'percentiles(products.base_price)',
+            label: 'products.base_price.percentiles',
+          },
+        ],
+        transformId: `ec_2_${Date.now()}`,
+        transformDescription:
+          'ecommerce batch transform with group by terms(geoip.country_iso_code) and aggregation percentiles(products.base_price)',
+        get destinationIndex(): string {
+          return `user-${this.transformId}`;
+        },
+        expected: {
+          pivotAdvancedEditorValue: {
+            group_by: {
+              'geoip.country_iso_code': {
+                terms: {
+                  field: 'geoip.country_iso_code',
+                },
+              },
+            },
+            aggregations: {
+              'products.base_price.percentiles': {
+                percentiles: {
+                  field: 'products.base_price',
+                  percents: [1, 5, 25, 50, 75, 95, 99],
+                },
+              },
+            },
+          },
+          pivotPreview: {
+            column: 0,
+            values: ['AE', 'CO', 'EG', 'FR', 'GB'],
+          },
+          row: {
+            status: 'stopped',
+            mode: 'batch',
+            progress: '100',
+          },
+          sourcePreview: {
+            columns: 45,
+            rows: 5,
+          },
+        },
+      },
     ];
 
     for (const testData of testDataList) {