Skip to content

Commit

Permalink
[ML] Add better UI support for runtime fields Transforms (#90363)
Browse files Browse the repository at this point in the history
* [ML] Add RT support for transforms from index pattern

* [ML] Add support for cloned transform from api

* [ML] Add support for runtime pivot

* [ML] Add support for api created runtime

* [ML] Add preview for expanded row

* [ML] Add runtime fields to dropdown options

* [ML] Add runtime fields to latest

* [ML] Fix duplicate columns

* [ML] Update types and test

* [ML] Add runtime mappings to index pattern on creation

* [ML] Add callout to show unsupported fields in dfa

* [ML] Update types to RuntimeField

* [ML] Fix runtime fields, remove runtime mappings, fix copy to console

* [ML] Fix incompatible kbn field type

* [ML] Add advanced mappings editor

* [ML] Add support for filter terms agg control

* [ML] Fix jest tests hanging

* [ML] Fix translations

* [ML] Fix over-sized buttons for filter range

* [ML] Update runtime mappings schema

* [ML] Update runtime mappings schema

* [ML] Use isRecord for object checks

* [ML] Fix and more message

* [ML] Update schema to correctly match types

* [ML] Update schema to correctly match types

* [ML] Fix pivot duplicates

* [ML] Rename isRecord to isPopulatedObject

* [ML] Remove fit-content

* [ML] Update runtime field type to prevent potential conflicts

* Revert "[ML] Remove fit-content"

This reverts commit 76c9c79

* [ML] Remove misc comment

* [ML] Fix missing typeof

* [ML] Add sorts and constants

* [ML] Add i18n to includedFields description

* [ML] fix imports

* [ML] Only pass runtime mappings if it's latest

* [ML] Fix functional tests
  • Loading branch information
qn895 authored Feb 17, 2021
1 parent 540b1d3 commit 63ac0f7
Show file tree
Hide file tree
Showing 48 changed files with 1,224 additions and 145 deletions.
6 changes: 4 additions & 2 deletions x-pack/plugins/ml/common/types/feature_importance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* 2.0.
*/

import { isPopulatedObject } from '../util/object_utils';

export type FeatureImportanceClassName = string | number | boolean;

export interface ClassFeatureImportance {
Expand Down Expand Up @@ -87,7 +89,7 @@ export function isClassificationFeatureImportanceBaseline(
baselineData: any
): baselineData is ClassificationFeatureImportanceBaseline {
return (
typeof baselineData === 'object' &&
isPopulatedObject(baselineData) &&
baselineData.hasOwnProperty('classes') &&
Array.isArray(baselineData.classes)
);
Expand All @@ -96,5 +98,5 @@ export function isClassificationFeatureImportanceBaseline(
export function isRegressionFeatureImportanceBaseline(
baselineData: any
): baselineData is RegressionFeatureImportanceBaseline {
return typeof baselineData === 'object' && baselineData.hasOwnProperty('baseline');
return isPopulatedObject(baselineData) && baselineData.hasOwnProperty('baseline');
}
16 changes: 15 additions & 1 deletion x-pack/plugins/ml/common/types/fields.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { ES_FIELD_TYPES, RuntimeField } from '../../../../../src/plugins/data/common';
import { ES_FIELD_TYPES } from '../../../../../src/plugins/data/common';
import {
ML_JOB_AGGREGATION,
KIBANA_AGGREGATION,
Expand Down Expand Up @@ -106,4 +106,18 @@ export interface AggCardinality {
}

export type RollupFields = Record<FieldId, [Record<'agg', ES_AGGREGATION>]>;

// Replace this with import once #88995 is merged
const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const;
type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];

export interface RuntimeField {
type: RuntimeType;
script:
| string
| {
source: string;
};
}

export type RuntimeMappings = Record<string, RuntimeField>;
2 changes: 1 addition & 1 deletion x-pack/plugins/ml/common/util/datafeed_utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export const getDatafeedAggregations = (
};

export const getAggregationBucketsName = (aggregations: any): string | undefined => {
if (typeof aggregations === 'object') {
if (aggregations !== null && typeof aggregations === 'object') {
const keys = Object.keys(aggregations);
return keys.length > 0 ? keys[0] : undefined;
}
Expand Down
22 changes: 9 additions & 13 deletions x-pack/plugins/ml/common/util/job_utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
getDatafeedAggregations,
} from './datafeed_utils';
import { findAggField } from './validation_utils';
import { isPopulatedObject } from './object_utils';

export interface ValidationResults {
valid: boolean;
Expand All @@ -51,17 +52,9 @@ export function calculateDatafeedFrequencyDefaultSeconds(bucketSpanSeconds: numb
}

export function hasRuntimeMappings(job: CombinedJob): boolean {
const hasDatafeed =
typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
const hasDatafeed = isPopulatedObject(job.datafeed_config);
if (hasDatafeed) {
const runtimeMappings =
typeof job.datafeed_config.runtime_mappings === 'object'
? Object.keys(job.datafeed_config.runtime_mappings)
: undefined;

if (Array.isArray(runtimeMappings) && runtimeMappings.length > 0) {
return true;
}
return isPopulatedObject(job.datafeed_config.runtime_mappings);
}
return false;
}
Expand Down Expand Up @@ -114,7 +107,11 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex
// If the datafeed uses script fields, we can only plot the time series if
// model plot is enabled. Without model plot it will be very difficult or impossible
// to invert to a reverse search of the underlying metric data.
if (isSourceDataChartable === true && typeof job.datafeed_config?.script_fields === 'object') {
if (
isSourceDataChartable === true &&
job.datafeed_config?.script_fields !== null &&
typeof job.datafeed_config?.script_fields === 'object'
) {
// Perform extra check to see if the detector is using a scripted field.
const scriptFields = Object.keys(job.datafeed_config.script_fields);
isSourceDataChartable =
Expand All @@ -123,8 +120,7 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex
scriptFields.indexOf(dtr.over_field_name!) === -1;
}

const hasDatafeed =
typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
const hasDatafeed = isPopulatedObject(job.datafeed_config);
if (hasDatafeed) {
// We cannot plot the source data for some specific aggregation configurations
const aggs = getDatafeedAggregations(job.datafeed_config);
Expand Down
10 changes: 10 additions & 0 deletions x-pack/plugins/ml/common/util/object_utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

export const isPopulatedObject = <T = Record<string, any>>(arg: any): arg is T => {
return typeof arg === 'object' && arg !== null && Object.keys(arg).length > 0;
};
2 changes: 1 addition & 1 deletion x-pack/plugins/ml/common/util/validation_utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export function findAggField(
value = returnParent === true ? aggs : aggs[k];
return true;
}
if (aggs.hasOwnProperty(k) && typeof aggs[k] === 'object') {
if (aggs.hasOwnProperty(k) && aggs[k] !== null && typeof aggs[k] === 'object') {
value = findAggField(aggs[k], fieldName, returnParent);
return value !== undefined;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ import { getNestedProperty } from '../../util/object_utils';
import { mlFieldFormatService } from '../../services/field_format_service';

import { DataGridItem, IndexPagination, RenderCellValue } from './types';
import type { RuntimeField } from '../../../../../../../src/plugins/data/common/index_patterns';
import { RuntimeMappings } from '../../../../common/types/fields';
import { isPopulatedObject } from '../../../../common/util/object_utils';

export const INIT_MAX_COLUMNS = 10;

Expand Down Expand Up @@ -86,6 +89,37 @@ export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): str
return indexPatternFields;
};

/**
* Return a map of runtime_mappings for each of the index pattern field provided
* to provide in ES search queries
* @param indexPatternFields
* @param indexPattern
* @param clonedRuntimeMappings
*/
export const getRuntimeFieldsMapping = (
indexPatternFields: string[] | undefined,
indexPattern: IndexPattern | undefined,
clonedRuntimeMappings?: RuntimeMappings
) => {
if (!Array.isArray(indexPatternFields) || indexPattern === undefined) return {};
const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields;
let combinedRuntimeMappings: RuntimeMappings = {};

if (isPopulatedObject(ipRuntimeMappings)) {
indexPatternFields.forEach((ipField) => {
if (ipRuntimeMappings.hasOwnProperty(ipField)) {
combinedRuntimeMappings[ipField] = ipRuntimeMappings[ipField];
}
});
}
if (isPopulatedObject(clonedRuntimeMappings)) {
combinedRuntimeMappings = { ...combinedRuntimeMappings, ...clonedRuntimeMappings };
}
return Object.keys(combinedRuntimeMappings).length > 0
? { runtime_mappings: combinedRuntimeMappings }
: {};
};

export interface FieldTypes {
[key: string]: ES_FIELD_TYPES;
}
Expand Down Expand Up @@ -135,6 +169,45 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results
};

export const NON_AGGREGATABLE = 'non-aggregatable';

export const getDataGridSchemaFromESFieldType = (
fieldType: ES_FIELD_TYPES | undefined | RuntimeField['type']
): string | undefined => {
// Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json']
// To fall back to the default string schema it needs to be undefined.
let schema;

switch (fieldType) {
case ES_FIELD_TYPES.GEO_POINT:
case ES_FIELD_TYPES.GEO_SHAPE:
schema = 'json';
break;
case ES_FIELD_TYPES.BOOLEAN:
schema = 'boolean';
break;
case ES_FIELD_TYPES.DATE:
case ES_FIELD_TYPES.DATE_NANOS:
schema = 'datetime';
break;
case ES_FIELD_TYPES.BYTE:
case ES_FIELD_TYPES.DOUBLE:
case ES_FIELD_TYPES.FLOAT:
case ES_FIELD_TYPES.HALF_FLOAT:
case ES_FIELD_TYPES.INTEGER:
case ES_FIELD_TYPES.LONG:
case ES_FIELD_TYPES.SCALED_FLOAT:
case ES_FIELD_TYPES.SHORT:
schema = 'numeric';
break;
// keep schema undefined for text based columns
case ES_FIELD_TYPES.KEYWORD:
case ES_FIELD_TYPES.TEXT:
break;
}

return schema;
};

export const getDataGridSchemaFromKibanaFieldType = (
field: IFieldType | undefined
): string | undefined => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@

export {
getDataGridSchemasFromFieldTypes,
getDataGridSchemaFromESFieldType,
getDataGridSchemaFromKibanaFieldType,
getFieldsFromKibanaIndexPattern,
getRuntimeFieldsMapping,
multiColumnSortFactory,
showDataGridColumnChartErrorMessageToast,
useRenderCellValue,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,16 @@ export const ConfigurationStepDetails: FC<Props> = ({ setCurrentStep, state }) =
}),
description:
includes.length > MAX_INCLUDES_LENGTH
? `${includes.slice(0, MAX_INCLUDES_LENGTH).join(', ')} ... (and ${
includes.length - MAX_INCLUDES_LENGTH
} more)`
? i18n.translate(
'xpack.ml.dataframe.analytics.create.configDetails.includedFieldsAndMoreDescription',
{
defaultMessage: '{includedFields} ... (and {extraCount} more)',
values: {
extraCount: includes.length - MAX_INCLUDES_LENGTH,
includedFields: includes.slice(0, MAX_INCLUDES_LENGTH).join(', '),
},
}
)
: includes.join(', '),
},
];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import React, { FC, Fragment, useEffect, useMemo, useRef, useState } from 'react';
import {
EuiBadge,
EuiCallOut,
EuiComboBox,
EuiComboBoxOptionOption,
EuiFormRow,
Expand All @@ -19,6 +20,7 @@ import {
import { i18n } from '@kbn/i18n';
import { debounce } from 'lodash';

import { FormattedMessage } from '@kbn/i18n/react';
import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
import { useMlContext } from '../../../../../contexts/ml';

Expand Down Expand Up @@ -62,6 +64,8 @@ const requiredFieldsErrorText = i18n.translate(
}
);

const maxRuntimeFieldsDisplayCount = 5;

export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
actions,
state,
Expand Down Expand Up @@ -314,6 +318,15 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
};
}, [jobType, dependentVariable, trainingPercent, JSON.stringify(includes), jobConfigQueryString]);

const unsupportedRuntimeFields = useMemo(
() =>
currentIndexPattern.fields
.getAll()
.filter((f) => f.runtimeField)
.map((f) => `'${f.displayName}'`),
[currentIndexPattern.fields]
);

return (
<Fragment>
<Messages messages={requestMessages} />
Expand Down Expand Up @@ -445,6 +458,36 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
>
<Fragment />
</EuiFormRow>
{Array.isArray(unsupportedRuntimeFields) && unsupportedRuntimeFields.length > 0 && (
<>
<EuiCallOut size="s" color="warning">
<FormattedMessage
id="xpack.ml.dataframe.analytics.create.unsupportedRuntimeFieldsCallout"
defaultMessage="The runtime {runtimeFieldsCount, plural, one {field} other {fields}} {unsupportedRuntimeFields} {extraCountMsg} are not supported for analysis."
values={{
runtimeFieldsCount: unsupportedRuntimeFields.length,
extraCountMsg:
unsupportedRuntimeFields.length - maxRuntimeFieldsDisplayCount > 0 ? (
<FormattedMessage
id="xpack.ml.dataframe.analytics.create.extraUnsupportedRuntimeFieldsMsg"
defaultMessage="and {count} more"
values={{
count: unsupportedRuntimeFields.length - maxRuntimeFieldsDisplayCount,
}}
/>
) : (
''
),
unsupportedRuntimeFields: unsupportedRuntimeFields
.slice(0, maxRuntimeFieldsDisplayCount)
.join(', '),
}}
/>
</EuiCallOut>
<EuiSpacer />
</>
)}

<AnalysisFieldsTable
dependentVariable={dependentVariable}
includes={includes}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import type { SearchResponse7 } from '../../../../../../common/types/es_client';
import { extractErrorMessage } from '../../../../../../common/util/errors';
import { INDEX_STATUS } from '../../../common/analytics';
import { ml } from '../../../../services/ml_api_service';
import { getRuntimeFieldsMapping } from '../../../../components/data_grid/common';

type IndexSearchResponse = SearchResponse7;

Expand All @@ -38,7 +39,9 @@ export const useIndexData = (
query: any,
toastNotifications: CoreSetup['notifications']['toasts']
): UseIndexDataReturnType => {
const indexPatternFields = getFieldsFromKibanaIndexPattern(indexPattern);
const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [
indexPattern,
]);

// EuiDataGrid State
const columns: EuiDataGridColumn[] = [
Expand Down Expand Up @@ -75,7 +78,6 @@ export const useIndexData = (
s[column.id] = { order: column.direction };
return s;
}, {} as EsSorting);

const esSearchRequest = {
index: indexPattern.title,
body: {
Expand All @@ -86,6 +88,7 @@ export const useIndexData = (
fields: ['*'],
_source: false,
...(Object.keys(sort).length > 0 ? { sort } : {}),
...getRuntimeFieldsMapping(indexPatternFields, indexPattern),
},
};

Expand All @@ -105,7 +108,7 @@ export const useIndexData = (
useEffect(() => {
getIndexData();
// custom comparison
}, [indexPattern.title, JSON.stringify([query, pagination, sortingColumns])]);
}, [indexPattern.title, indexPatternFields, JSON.stringify([query, pagination, sortingColumns])]);

const dataLoader = useMemo(() => new DataLoader(indexPattern, toastNotifications), [
indexPattern,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { stringMatch } from '../../../util/string_utils';
import { JOB_STATE, DATAFEED_STATE } from '../../../../../common/constants/states';
import { parseInterval } from '../../../../../common/util/parse_interval';
import { mlCalendarService } from '../../../services/calendar_service';
import { isPopulatedObject } from '../../../../../common/util/object_utils';

export function loadFullJob(jobId) {
return new Promise((resolve, reject) => {
Expand Down Expand Up @@ -379,7 +380,7 @@ export function checkForAutoStartDatafeed() {
mlJobService.tempJobCloningObjects.datafeed = undefined;
mlJobService.tempJobCloningObjects.createdBy = undefined;

const hasDatafeed = typeof datafeed === 'object' && Object.keys(datafeed).length > 0;
const hasDatafeed = isPopulatedObject(datafeed);
const datafeedId = hasDatafeed ? datafeed.datafeed_id : '';
return {
id: job.job_id,
Expand Down
Loading

0 comments on commit 63ac0f7

Please sign in to comment.