Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML] Add support for percentiles aggregation to Transform wizard #60763

Merged
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[ML] Add support for percentiles aggregation to Transform wizard
  • Loading branch information
peteharverson committed Mar 23, 2020
commit 4fd072b3e11ccd6f16c0cdf23e9978bd524580d8
2 changes: 2 additions & 0 deletions x-pack/plugins/transform/public/app/common/index.ts
Original file line number Diff line number Diff line change
@@ -40,6 +40,8 @@ export { GetTransformsResponse, PreviewData, PreviewMappings } from './pivot_pre
export {
getEsAggFromAggConfig,
isPivotAggsConfigWithUiSupport,
isPivotAggsConfigPercentiles,
PERCENTILES_AGG_DEFAULT_PERCENTS,
PivotAgg,
PivotAggDict,
PivotAggsConfig,
22 changes: 21 additions & 1 deletion x-pack/plugins/transform/public/app/common/pivot_aggs.ts
Original file line number Diff line number Diff line change
@@ -15,10 +15,13 @@ export enum PIVOT_SUPPORTED_AGGS {
CARDINALITY = 'cardinality',
MAX = 'max',
MIN = 'min',
PERCENTILES = 'percentiles',
SUM = 'sum',
VALUE_COUNT = 'value_count',
}

export const PERCENTILES_AGG_DEFAULT_PERCENTS = [1, 5, 25, 50, 75, 95, 99];

export const pivotAggsFieldSupport = {
[KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
[KBN_FIELD_TYPES.BOOLEAN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
@@ -36,6 +39,7 @@ export const pivotAggsFieldSupport = {
PIVOT_SUPPORTED_AGGS.CARDINALITY,
PIVOT_SUPPORTED_AGGS.MAX,
PIVOT_SUPPORTED_AGGS.MIN,
PIVOT_SUPPORTED_AGGS.PERCENTILES,
PIVOT_SUPPORTED_AGGS.SUM,
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
],
@@ -60,10 +64,17 @@ export interface PivotAggsConfigBase {
dropDownName: string;
}

export interface PivotAggsConfigWithUiSupport extends PivotAggsConfigBase {
interface PivotAggsConfigWithUiBase extends PivotAggsConfigBase {
field: EsFieldName;
}

interface PivotAggsConfigPercentiles extends PivotAggsConfigWithUiBase {
agg: PIVOT_SUPPORTED_AGGS.PERCENTILES;
percents: number[];
}

export type PivotAggsConfigWithUiSupport = PivotAggsConfigWithUiBase | PivotAggsConfigPercentiles;

export function isPivotAggsConfigWithUiSupport(arg: any): arg is PivotAggsConfigWithUiSupport {
return (
arg.hasOwnProperty('agg') &&
@@ -74,6 +85,15 @@ export function isPivotAggsConfigWithUiSupport(arg: any): arg is PivotAggsConfig
);
}

export function isPivotAggsConfigPercentiles(arg: any): arg is PivotAggsConfigPercentiles {
return (
arg.hasOwnProperty('agg') &&
arg.hasOwnProperty('field') &&
arg.hasOwnProperty('percents') &&
arg.agg === PIVOT_SUPPORTED_AGGS.PERCENTILES
);
}

export type PivotAggsConfig = PivotAggsConfigBase | PivotAggsConfigWithUiSupport;

export type PivotAggsConfigWithUiSupportDict = Dictionary<PivotAggsConfigWithUiSupport>;
Original file line number Diff line number Diff line change
@@ -10,7 +10,12 @@ import { i18n } from '@kbn/i18n';

import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiPopover } from '@elastic/eui';

import { AggName, PivotAggsConfig, PivotAggsConfigWithUiSupportDict } from '../../../../common';
import {
AggName,
isPivotAggsConfigPercentiles,
PivotAggsConfig,
PivotAggsConfigWithUiSupportDict,
} from '../../../../common';

import { PopoverForm } from './popover_form';

Original file line number Diff line number Diff line change
@@ -22,8 +22,10 @@ import { dictionaryToArray } from '../../../../../../common/types/common';
import {
AggName,
isAggName,
isPivotAggsConfigPercentiles,
isPivotAggsConfigWithUiSupport,
getEsAggFromAggConfig,
PERCENTILES_AGG_DEFAULT_PERCENTS,
PivotAggsConfig,
PivotAggsConfigWithUiSupportDict,
PIVOT_SUPPORTED_AGGS,
@@ -40,6 +42,28 @@ interface Props {
onChange(d: PivotAggsConfig): void;
}

function getDefaultPercents(defaultData: PivotAggsConfig): number[] | undefined {
if (isPivotAggsConfigPercentiles(defaultData)) {
return defaultData.percents;
}

return undefined;
darnautov marked this conversation as resolved.
Show resolved Hide resolved
}

function isPercentsInputValid(inputValue: string | undefined) {
if (inputValue === undefined) {
return false;
}
const numberListRegex = /^(\s*\d+(\.\d+)?)(\s*,\s*\d+(\.\d+)?)*$/;
let isValid = numberListRegex.test(inputValue);
if (isValid === true) {
// Check each percent is no greater 100 (negative values checked by regex).
const values: number[] = inputValue.split(',').map(Number);
isValid = values.find(value => value > 100) === undefined;
}
darnautov marked this conversation as resolved.
Show resolved Hide resolved
return isValid;
}

export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onChange, options }) => {
const isUnsupportedAgg = !isPivotAggsConfigWithUiSupport(defaultData);

@@ -48,10 +72,50 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
const [field, setField] = useState(
isPivotAggsConfigWithUiSupport(defaultData) ? defaultData.field : ''
);
const [percents, setPercents] = useState(getDefaultPercents(defaultData));

const availableFields: SelectOption[] = [];
const availableAggs: SelectOption[] = [];

function updateAgg(aggVal: PIVOT_SUPPORTED_AGGS) {
setAgg(aggVal);
if (aggVal === PIVOT_SUPPORTED_AGGS.PERCENTILES && percents === undefined) {
setPercents(PERCENTILES_AGG_DEFAULT_PERCENTS);
}
}

function updatePercents(inputValue: string) {
const isInputValid = isPercentsInputValid(inputValue);
if (isInputValid === true) {
setPercents(inputValue.split(',').map(Number));
darnautov marked this conversation as resolved.
Show resolved Hide resolved
} else {
setPercents([]);
}
}

function getUpdatedItem(): PivotAggsConfig {
let updatedItem: PivotAggsConfig;

if (agg !== PIVOT_SUPPORTED_AGGS.PERCENTILES) {
updatedItem = {
agg,
aggName,
field,
dropDownName: defaultData.dropDownName,
};
} else {
updatedItem = {
agg,
aggName,
field,
dropDownName: defaultData.dropDownName,
percents,
};
}

return updatedItem;
}

if (!isUnsupportedAgg) {
const optionsArr = dictionaryToArray(options);
optionsArr
@@ -83,7 +147,18 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
});
}

const formValid = validAggName;
let percentsText;
if (percents !== undefined) {
percentsText = percents.toString();
}

const validPercents =
agg === PIVOT_SUPPORTED_AGGS.PERCENTILES && isPercentsInputValid(percentsText);

let formValid = validAggName;
if (formValid && agg === PIVOT_SUPPORTED_AGGS.PERCENTILES) {
formValid = validPercents;
}

return (
<EuiForm style={{ width: '300px' }}>
@@ -117,7 +192,7 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
<EuiSelect
options={availableAggs}
value={agg}
onChange={e => setAgg(e.target.value as PIVOT_SUPPORTED_AGGS)}
onChange={e => updateAgg(e.target.value as PIVOT_SUPPORTED_AGGS)}
/>
</EuiFormRow>
)}
@@ -134,6 +209,26 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
/>
</EuiFormRow>
)}
{agg === PIVOT_SUPPORTED_AGGS.PERCENTILES && (
<EuiFormRow
label={i18n.translate('xpack.transform.agg.popoverForm.percentsLabel', {
defaultMessage: 'Percents',
})}
error={
!validPercents && [
i18n.translate('xpack.transform.groupBy.popoverForm.intervalPercents', {
defaultMessage: 'Enter a comma-separated list of percentiles',
}),
]
}
isInvalid={!validPercents}
>
<EuiFieldText
defaultValue={percentsText}
onChange={e => updatePercents(e.target.value)}
/>
</EuiFormRow>
)}
{isUnsupportedAgg && (
<EuiCodeEditor
mode="json"
@@ -147,10 +242,7 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
/>
)}
<EuiFormRow hasEmptyLabelSpace>
<EuiButton
isDisabled={!formValid}
onClick={() => onChange({ ...defaultData, aggName, agg, field })}
>
<EuiButton isDisabled={!formValid} onClick={() => onChange(getUpdatedItem())}>
{i18n.translate('xpack.transform.agg.popoverForm.submitButtonLabel', {
defaultMessage: 'Apply',
})}
Original file line number Diff line number Diff line change
@@ -37,6 +37,7 @@ describe('Transform: Define Pivot Common', () => {
{ label: 'cardinality( the-f[i]e>ld )' },
{ label: 'max( the-f[i]e>ld )' },
{ label: 'min( the-f[i]e>ld )' },
{ label: 'percentiles( the-f[i]e>ld )' },
{ label: 'sum( the-f[i]e>ld )' },
{ label: 'value_count( the-f[i]e>ld )' },
],
@@ -67,6 +68,13 @@ describe('Transform: Define Pivot Common', () => {
aggName: 'the-field.min',
dropDownName: 'min( the-f[i]e>ld )',
},
'percentiles( the-f[i]e>ld )': {
agg: 'percentiles',
field: ' the-f[i]e>ld ',
aggName: 'the-field.percentiles',
dropDownName: 'percentiles( the-f[i]e>ld )',
percents: [1, 5, 25, 50, 75, 95, 99],
},
'sum( the-f[i]e>ld )': {
agg: 'sum',
field: ' the-f[i]e>ld ',
Original file line number Diff line number Diff line change
@@ -12,10 +12,13 @@ import {
DropDownOption,
EsFieldName,
GroupByConfigWithUiSupport,
PERCENTILES_AGG_DEFAULT_PERCENTS,
PivotAggsConfigWithUiSupport,
PivotAggsConfigWithUiSupportDict,
pivotAggsFieldSupport,
PivotGroupByConfigWithUiSupportDict,
pivotGroupByFieldSupport,
PIVOT_SUPPORTED_AGGS,
PIVOT_SUPPORTED_GROUP_BY_AGGS,
} from '../../../../common';

@@ -57,6 +60,31 @@ function getDefaultGroupByConfig(
}
}

function getDefaultAggregationConfig(
aggName: string,
dropDownName: string,
fieldName: EsFieldName,
agg: PIVOT_SUPPORTED_AGGS
): PivotAggsConfigWithUiSupport {
switch (agg) {
case PIVOT_SUPPORTED_AGGS.PERCENTILES:
return {
agg,
aggName,
dropDownName,
field: fieldName,
percents: PERCENTILES_AGG_DEFAULT_PERCENTS,
};
default:
return {
agg,
aggName,
dropDownName,
field: fieldName,
};
}
}

const illegalEsAggNameChars = /[[\]>]/g;

export function getPivotDropdownOptions(indexPattern: IndexPattern) {
@@ -105,7 +133,12 @@ export function getPivotDropdownOptions(indexPattern: IndexPattern) {
// Option name in the dropdown for the aggregation is in the form of `sum(fieldname)`.
const dropDownName = `${agg}(${field.name})`;
aggOption.options.push({ label: dropDownName });
aggOptionsData[dropDownName] = { agg, field: field.name, aggName, dropDownName };
aggOptionsData[dropDownName] = getDefaultAggregationConfig(
aggName,
dropDownName,
field.name,
agg
);
});
}
aggOptions.push(aggOption);
54 changes: 54 additions & 0 deletions x-pack/test/functional/apps/transform/creation_index_pattern.ts
Original file line number Diff line number Diff line change
@@ -94,6 +94,60 @@ export default function({ getService }: FtrProviderContext) {
},
},
},
{
suiteTitle: 'batch transform with terms group and percentiles agg',
source: 'ecommerce',
groupByEntries: [
{
identifier: 'terms(geoip.country_iso_code)',
label: 'geoip.country_iso_code',
} as GroupByEntry,
],
aggregationEntries: [
{
identifier: 'percentiles(products.base_price)',
label: 'products.base_price.percentiles',
},
],
transformId: `ec_2_${Date.now()}`,
transformDescription:
'ecommerce batch transform with group by terms(geoip.country_iso_code) and aggregation percentiles(products.base_price)',
get destinationIndex(): string {
return `user-${this.transformId}`;
},
expected: {
pivotAdvancedEditorValue: {
group_by: {
'geoip.country_iso_code': {
terms: {
field: 'geoip.country_iso_code',
},
},
},
aggregations: {
'products.base_price.percentiles': {
percentiles: {
field: 'products.base_price',
percents: [1, 5, 25, 50, 75, 95, 99],
},
},
},
},
pivotPreview: {
column: 0,
values: ['AE', 'CO', 'EG', 'FR', 'GB'],
},
row: {
status: 'stopped',
mode: 'batch',
progress: '100',
},
sourcePreview: {
columns: 45,
rows: 5,
},
},
},
];

for (const testData of testDataList) {