diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/index.ts b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/index.ts index 4f564dde8cb43..903fe5b6ed985 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/index.ts +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/index.ts @@ -4,5 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +export { useScatterplotFieldOptions } from './use_scatterplot_field_options'; export { LEGEND_TYPES } from './scatterplot_matrix_vega_lite_spec'; export { ScatterplotMatrix } from './scatterplot_matrix'; +export type { ScatterplotMatrixViewProps as ScatterplotMatrixProps } from './scatterplot_matrix_view'; diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx index b1ee9afb17788..a90fe924b91ac 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx @@ -4,316 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useMemo, useEffect, useState, FC } from 'react'; +import React, { FC, Suspense } from 'react'; -// There is still an issue with Vega Lite's typings with the strict mode Kibana is using. -// @ts-ignore -import { compile } from 'vega-lite/build-es5/vega-lite'; -import { parse, View, Warn } from 'vega'; -import { Handler } from 'vega-tooltip'; +import type { ScatterplotMatrixViewProps } from './scatterplot_matrix_view'; +import { ScatterplotMatrixLoading } from './scatterplot_matrix_loading'; -import { - htmlIdGenerator, - EuiComboBox, - EuiComboBoxOptionOption, - EuiFlexGroup, - EuiFlexItem, - EuiFormRow, - EuiLoadingSpinner, - EuiSelect, - EuiSpacer, - EuiSwitch, - EuiText, -} from '@elastic/eui'; +const ScatterplotMatrixLazy = React.lazy(() => import('./scatterplot_matrix_view')); -import { i18n } from '@kbn/i18n'; - -import type { SearchResponse7 } from '../../../../common/types/es_client'; - -import { useMlApiContext } from '../../contexts/kibana'; - -import { getProcessedFields } from '../data_grid'; -import { useCurrentEuiTheme } from '../color_range_legend'; - -import { - getScatterplotMatrixVegaLiteSpec, - LegendType, - OUTLIER_SCORE_FIELD, -} from './scatterplot_matrix_vega_lite_spec'; - -import './scatterplot_matrix.scss'; - -const SCATTERPLOT_MATRIX_DEFAULT_FIELDS = 4; -const SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE = 1000; -const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE = 1; -const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE = 10000; - -const TOGGLE_ON = i18n.translate('xpack.ml.splom.toggleOn', { - defaultMessage: 'On', -}); -const TOGGLE_OFF = i18n.translate('xpack.ml.splom.toggleOff', { - defaultMessage: 'Off', -}); - -const sampleSizeOptions = [100, 1000, 10000].map((d) => ({ value: d, text: '' + d })); - -interface ScatterplotMatrixProps { - fields: string[]; - index: string; - resultsField?: string; - color?: string; - legendType?: LegendType; -} - -export const ScatterplotMatrix: FC = ({ - fields: allFields, - index, - resultsField, - color, - legendType, -}) => { - const { esSearch } = useMlApiContext(); - - // dynamicSize is optionally used for outlier charts where the scatterplot marks - // are sized according to outlier_score - const [dynamicSize, setDynamicSize] = useState(false); - - // used to give the use the option to customize the fields used for the matrix axes - const [fields, setFields] = useState([]); - - useEffect(() => { - const defaultFields = - allFields.length > SCATTERPLOT_MATRIX_DEFAULT_FIELDS - ? allFields.slice(0, SCATTERPLOT_MATRIX_DEFAULT_FIELDS) - : allFields; - setFields(defaultFields); - }, [allFields]); - - // the amount of documents to be fetched - const [fetchSize, setFetchSize] = useState(SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE); - // flag to add a random score to the ES query to fetch documents - const [randomizeQuery, setRandomizeQuery] = useState(false); - - const [isLoading, setIsLoading] = useState(false); - - // contains the fetched documents and columns to be passed on to the Vega spec. - const [splom, setSplom] = useState<{ items: any[]; columns: string[] } | undefined>(); - - // formats the array of field names for EuiComboBox - const fieldOptions = useMemo( - () => - allFields.map((d) => ({ - label: d, - })), - [allFields] - ); - - const fieldsOnChange = (newFields: EuiComboBoxOptionOption[]) => { - setFields(newFields.map((d) => d.label)); - }; - - const fetchSizeOnChange = (e: React.ChangeEvent) => { - setFetchSize( - Math.min( - Math.max(parseInt(e.target.value, 10), SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE), - SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE - ) - ); - }; - - const randomizeQueryOnChange = () => { - setRandomizeQuery(!randomizeQuery); - }; - - const dynamicSizeOnChange = () => { - setDynamicSize(!dynamicSize); - }; - - const { euiTheme } = useCurrentEuiTheme(); - - useEffect(() => { - async function fetchSplom(options: { didCancel: boolean }) { - setIsLoading(true); - try { - const queryFields = [ - ...fields, - ...(color !== undefined ? [color] : []), - ...(legendType !== undefined ? [] : [`${resultsField}.${OUTLIER_SCORE_FIELD}`]), - ]; - - const query = randomizeQuery - ? { - function_score: { - random_score: { seed: 10, field: '_seq_no' }, - }, - } - : { match_all: {} }; - - const resp: SearchResponse7 = await esSearch({ - index, - body: { - fields: queryFields, - _source: false, - query, - from: 0, - size: fetchSize, - }, - }); - - if (!options.didCancel) { - const items = resp.hits.hits.map((d) => - getProcessedFields(d.fields, (key: string) => - key.startsWith(`${resultsField}.feature_importance`) - ) - ); - - setSplom({ columns: fields, items }); - setIsLoading(false); - } - } catch (e) { - // TODO error handling - setIsLoading(false); - } - } - - const options = { didCancel: false }; - fetchSplom(options); - return () => { - options.didCancel = true; - }; - // stringify the fields array, otherwise the comparator will trigger on new but identical instances. - }, [fetchSize, JSON.stringify(fields), index, randomizeQuery, resultsField]); - - const htmlId = useMemo(() => htmlIdGenerator()(), []); - - useEffect(() => { - if (splom === undefined) { - return; - } - - const { items, columns } = splom; - - const values = - resultsField !== undefined - ? items - : items.map((d) => { - d[`${resultsField}.${OUTLIER_SCORE_FIELD}`] = 0; - return d; - }); - - const vegaSpec = getScatterplotMatrixVegaLiteSpec( - values, - columns, - euiTheme, - resultsField, - color, - legendType, - dynamicSize - ); - - const vgSpec = compile(vegaSpec).spec; - - const view = new View(parse(vgSpec)) - .logLevel(Warn) - .renderer('canvas') - .tooltip(new Handler().call) - .initialize(`#${htmlId}`); - - view.runAsync(); // evaluate and render the view - }, [resultsField, splom, color, legendType, dynamicSize]); - - return ( - <> - {splom === undefined ? ( - - - - - - ) : ( - <> - - - - ({ - label: d, - }))} - onChange={fieldsOnChange} - isClearable={true} - data-test-subj="mlScatterplotMatrixFieldsComboBox" - /> - - - - - - - - - - - - - {resultsField !== undefined && legendType === undefined && ( - - - - - - )} - - -
- - )} - - ); -}; +export const ScatterplotMatrix: FC = (props) => ( + }> + + +); diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_loading.tsx b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_loading.tsx new file mode 100644 index 0000000000000..ccd4153769e9c --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_loading.tsx @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import { EuiLoadingSpinner, EuiSpacer, EuiText } from '@elastic/eui'; + +export const ScatterplotMatrixLoading = () => { + return ( + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.test.ts b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.test.ts index dd467161ff489..eada64b7a03ca 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.test.ts +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.test.ts @@ -163,6 +163,7 @@ describe('getScatterplotMatrixVegaLiteSpec()', () => { type: 'nominal', }); expect(vegaLiteSpec.spec.encoding.tooltip).toEqual([ + { field: 'the-color-field', type: 'nominal' }, { field: 'x', type: 'quantitative' }, { field: 'y', type: 'quantitative' }, ]); diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.ts b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.ts index 9e0834dd8b922..c943e5d1b06e3 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.ts +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.ts @@ -35,6 +35,8 @@ export const getColorSpec = ( color?: string, legendType?: LegendType ) => { + // For outlier detection result pages coloring is done based on a threshold. + // This returns a Vega spec using a conditional to return the color. if (outliers) { return { condition: { @@ -45,6 +47,8 @@ export const getColorSpec = ( }; } + // Based on the type of the color field, + // this returns either a continuous or categorical color spec. if (color !== undefined && legendType !== undefined) { return { field: color, @@ -80,6 +84,8 @@ export const getScatterplotMatrixVegaLiteSpec = ( }); } + const colorSpec = getColorSpec(euiTheme, outliers, color, legendType); + return { $schema: 'https://vega.github.io/schema/vega-lite/v4.17.0.json', background: 'transparent', @@ -115,10 +121,10 @@ export const getScatterplotMatrixVegaLiteSpec = ( : { type: 'circle', opacity: 0.75, size: 8 }), }, encoding: { - color: getColorSpec(euiTheme, outliers, color, legendType), + color: colorSpec, ...(dynamicSize ? { - stroke: getColorSpec(euiTheme, outliers, color, legendType), + stroke: colorSpec, opacity: { condition: { value: 1, @@ -163,6 +169,7 @@ export const getScatterplotMatrixVegaLiteSpec = ( scale: { zero: false }, }, tooltip: [ + ...(color !== undefined ? [{ type: colorSpec.type, field: color }] : []), ...columns.map((d) => ({ type: LEGEND_TYPES.QUANTITATIVE, field: d })), ...(outliers ? [{ type: LEGEND_TYPES.QUANTITATIVE, field: OUTLIER_SCORE_FIELD, format: '.3f' }] diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.scss b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.scss similarity index 100% rename from x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.scss rename to x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.scss diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.tsx b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.tsx new file mode 100644 index 0000000000000..0c065c1154a98 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.tsx @@ -0,0 +1,323 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useMemo, useEffect, useState, FC } from 'react'; + +// There is still an issue with Vega Lite's typings with the strict mode Kibana is using. +// @ts-ignore +import { compile } from 'vega-lite/build-es5/vega-lite'; +import { parse, View, Warn } from 'vega'; +import { Handler } from 'vega-tooltip'; + +import { + htmlIdGenerator, + EuiComboBox, + EuiComboBoxOptionOption, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiSelect, + EuiSwitch, +} from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; + +import type { SearchResponse7 } from '../../../../common/types/es_client'; +import type { ResultsSearchQuery } from '../../data_frame_analytics/common/analytics'; + +import { useMlApiContext } from '../../contexts/kibana'; + +import { getProcessedFields } from '../data_grid'; +import { useCurrentEuiTheme } from '../color_range_legend'; + +import { ScatterplotMatrixLoading } from './scatterplot_matrix_loading'; + +import { + getScatterplotMatrixVegaLiteSpec, + LegendType, + OUTLIER_SCORE_FIELD, +} from './scatterplot_matrix_vega_lite_spec'; + +import './scatterplot_matrix_view.scss'; + +const SCATTERPLOT_MATRIX_DEFAULT_FIELDS = 4; +const SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE = 1000; +const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE = 1; +const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE = 10000; + +const TOGGLE_ON = i18n.translate('xpack.ml.splom.toggleOn', { + defaultMessage: 'On', +}); +const TOGGLE_OFF = i18n.translate('xpack.ml.splom.toggleOff', { + defaultMessage: 'Off', +}); + +const sampleSizeOptions = [100, 1000, 10000].map((d) => ({ value: d, text: '' + d })); + +export interface ScatterplotMatrixViewProps { + fields: string[]; + index: string; + resultsField?: string; + color?: string; + legendType?: LegendType; + searchQuery?: ResultsSearchQuery; +} + +export const ScatterplotMatrixView: FC = ({ + fields: allFields, + index, + resultsField, + color, + legendType, + searchQuery, +}) => { + const { esSearch } = useMlApiContext(); + + // dynamicSize is optionally used for outlier charts where the scatterplot marks + // are sized according to outlier_score + const [dynamicSize, setDynamicSize] = useState(false); + + // used to give the use the option to customize the fields used for the matrix axes + const [fields, setFields] = useState([]); + + useEffect(() => { + const defaultFields = + allFields.length > SCATTERPLOT_MATRIX_DEFAULT_FIELDS + ? allFields.slice(0, SCATTERPLOT_MATRIX_DEFAULT_FIELDS) + : allFields; + setFields(defaultFields); + }, [allFields]); + + // the amount of documents to be fetched + const [fetchSize, setFetchSize] = useState(SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE); + // flag to add a random score to the ES query to fetch documents + const [randomizeQuery, setRandomizeQuery] = useState(false); + + const [isLoading, setIsLoading] = useState(false); + + // contains the fetched documents and columns to be passed on to the Vega spec. + const [splom, setSplom] = useState<{ items: any[]; columns: string[] } | undefined>(); + + // formats the array of field names for EuiComboBox + const fieldOptions = useMemo( + () => + allFields.map((d) => ({ + label: d, + })), + [allFields] + ); + + const fieldsOnChange = (newFields: EuiComboBoxOptionOption[]) => { + setFields(newFields.map((d) => d.label)); + }; + + const fetchSizeOnChange = (e: React.ChangeEvent) => { + setFetchSize( + Math.min( + Math.max(parseInt(e.target.value, 10), SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE), + SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE + ) + ); + }; + + const randomizeQueryOnChange = () => { + setRandomizeQuery(!randomizeQuery); + }; + + const dynamicSizeOnChange = () => { + setDynamicSize(!dynamicSize); + }; + + const { euiTheme } = useCurrentEuiTheme(); + + useEffect(() => { + async function fetchSplom(options: { didCancel: boolean }) { + setIsLoading(true); + try { + const queryFields = [ + ...fields, + ...(color !== undefined ? [color] : []), + ...(legendType !== undefined ? [] : [`${resultsField}.${OUTLIER_SCORE_FIELD}`]), + ]; + + const queryFallback = searchQuery !== undefined ? searchQuery : { match_all: {} }; + const query = randomizeQuery + ? { + function_score: { + query: queryFallback, + random_score: { seed: 10, field: '_seq_no' }, + }, + } + : queryFallback; + + const resp: SearchResponse7 = await esSearch({ + index, + body: { + fields: queryFields, + _source: false, + query, + from: 0, + size: fetchSize, + }, + }); + + if (!options.didCancel) { + const items = resp.hits.hits.map((d) => + getProcessedFields(d.fields, (key: string) => + key.startsWith(`${resultsField}.feature_importance`) + ) + ); + + setSplom({ columns: fields, items }); + setIsLoading(false); + } + } catch (e) { + // TODO error handling + setIsLoading(false); + } + } + + const options = { didCancel: false }; + fetchSplom(options); + return () => { + options.didCancel = true; + }; + // stringify the fields array and search, otherwise the comparator will trigger on new but identical instances. + }, [fetchSize, JSON.stringify({ fields, searchQuery }), index, randomizeQuery, resultsField]); + + const htmlId = useMemo(() => htmlIdGenerator()(), []); + + useEffect(() => { + if (splom === undefined) { + return; + } + + const { items, columns } = splom; + + const values = + resultsField !== undefined + ? items + : items.map((d) => { + d[`${resultsField}.${OUTLIER_SCORE_FIELD}`] = 0; + return d; + }); + + const vegaSpec = getScatterplotMatrixVegaLiteSpec( + values, + columns, + euiTheme, + resultsField, + color, + legendType, + dynamicSize + ); + + const vgSpec = compile(vegaSpec).spec; + + const view = new View(parse(vgSpec)) + .logLevel(Warn) + .renderer('canvas') + .tooltip(new Handler().call) + .initialize(`#${htmlId}`); + + view.runAsync(); // evaluate and render the view + }, [resultsField, splom, color, legendType, dynamicSize]); + + return ( + <> + {splom === undefined ? ( + + ) : ( + <> + + + + ({ + label: d, + }))} + onChange={fieldsOnChange} + isClearable={true} + data-test-subj="mlScatterplotMatrixFieldsComboBox" + /> + + + + + + + + + + + + + {resultsField !== undefined && legendType === undefined && ( + + + + + + )} + + +
+ + )} + + ); +}; + +// required for dynamic import using React.lazy() +// eslint-disable-next-line import/no-default-export +export default ScatterplotMatrixView; diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts new file mode 100644 index 0000000000000..f5eedbc03951f --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useMemo } from 'react'; + +import type { IndexPattern } from '../../../../../../../src/plugins/data/public'; + +import { ML__INCREMENTAL_ID } from '../../data_frame_analytics/common/fields'; + +export const useScatterplotFieldOptions = ( + indexPattern?: IndexPattern, + includes?: string[], + excludes?: string[], + resultsField = '' +): string[] => { + return useMemo(() => { + const fields: string[] = []; + + if (indexPattern === undefined || includes === undefined) { + return fields; + } + + if (includes.length > 1) { + fields.push( + ...includes.filter((d) => + indexPattern.fields.some((f) => f.name === d && f.type === 'number') + ) + ); + } else { + fields.push( + ...indexPattern.fields + .filter( + (f) => + f.type === 'number' && + !indexPattern.metaFields.includes(f.name) && + !f.name.startsWith(`${resultsField}.`) && + f.name !== ML__INCREMENTAL_ID + ) + .map((f) => f.name) + ); + } + + return Array.isArray(excludes) && excludes.length > 0 + ? fields.filter((f) => !excludes.includes(f)) + : fields; + }, [indexPattern, includes, excludes]); +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_scatterplot_matrix_legend_type.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_scatterplot_matrix_legend_type.ts new file mode 100644 index 0000000000000..8850d42577bd9 --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_scatterplot_matrix_legend_type.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ANALYSIS_CONFIG_TYPE } from './analytics'; + +import { AnalyticsJobType } from '../pages/analytics_management/hooks/use_create_analytics_form/state'; + +import { LEGEND_TYPES } from '../../components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec'; + +export const getScatterplotMatrixLegendType = (jobType: AnalyticsJobType | 'unknown') => { + switch (jobType) { + case ANALYSIS_CONFIG_TYPE.CLASSIFICATION: + return LEGEND_TYPES.NOMINAL; + case ANALYSIS_CONFIG_TYPE.REGRESSION: + return LEGEND_TYPES.QUANTITATIVE; + default: + return undefined; + } +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts index 7ba3e910ddd32..d03f73ad13575 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts @@ -41,6 +41,7 @@ export { export { getIndexData } from './get_index_data'; export { getIndexFields } from './get_index_fields'; +export { getScatterplotMatrixLegendType } from './get_scatterplot_matrix_legend_type'; export { useResultsViewConfig } from './use_results_view_config'; export { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts index 185513f75a12c..361a1262a59f8 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts @@ -102,6 +102,12 @@ export const useResultsViewConfig = (jobId: string) => { try { indexP = await mlContext.indexPatterns.get(destIndexPatternId); + + // Force refreshing the fields list here because a user directly coming + // from the job creation wizard might land on the page without the + // index pattern being fully initialized because it was created + // before the analytics job populated the destination index. + await mlContext.indexPatterns.refreshFields(indexP); } catch (e) { indexP = undefined; } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx index a5991f77e88e2..4b86f5ca12896 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx @@ -27,10 +27,10 @@ import { TRAINING_PERCENT_MAX, FieldSelectionItem, } from '../../../../common/analytics'; +import { getScatterplotMatrixLegendType } from '../../../../common/get_scatterplot_matrix_legend_type'; import { CreateAnalyticsStepProps } from '../../../analytics_management/hooks/use_create_analytics_form'; import { Messages } from '../shared'; import { - AnalyticsJobType, DEFAULT_MODEL_MEMORY_LIMIT, State, } from '../../../analytics_management/hooks/use_create_analytics_form/state'; @@ -51,18 +51,7 @@ import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/sea import { ExplorationQueryBarProps } from '../../../analytics_exploration/components/exploration_query_bar/exploration_query_bar'; import { Query } from '../../../../../../../../../../src/plugins/data/common/query'; -import { LEGEND_TYPES, ScatterplotMatrix } from '../../../../../components/scatterplot_matrix'; - -const getScatterplotMatrixLegendType = (jobType: AnalyticsJobType) => { - switch (jobType) { - case ANALYSIS_CONFIG_TYPE.CLASSIFICATION: - return LEGEND_TYPES.NOMINAL; - case ANALYSIS_CONFIG_TYPE.REGRESSION: - return LEGEND_TYPES.QUANTITATIVE; - default: - return undefined; - } -}; +import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix'; const requiredFieldsErrorText = i18n.translate( 'xpack.ml.dataframe.analytics.createWizard.requiredFieldsErrorMessage', @@ -498,6 +487,7 @@ export const ConfigurationStepForm: FC = ({ : undefined } legendType={getScatterplotMatrixLegendType(jobType)} + searchQuery={jobConfigQuery} /> diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_splom.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_splom.tsx index 5ec8963e0fc25..8c51c95d7fd63 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_splom.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_splom.tsx @@ -12,17 +12,14 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { EuiHorizontalRule, EuiSpacer } from '@elastic/eui'; -import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix'; +import { + ScatterplotMatrix, + ScatterplotMatrixProps, +} from '../../../../../components/scatterplot_matrix'; import { ExpandableSection } from './expandable_section'; -interface ExpandableSectionSplomProps { - fields: string[]; - index: string; - resultsField?: string; -} - -export const ExpandableSectionSplom: FC = (props) => { +export const ExpandableSectionSplom: FC = (props) => { const splomSectionHeaderItems = undefined; const splomSectionContent = ( <> diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx index 1329644322f33..46715af0ef0cb 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx @@ -9,16 +9,21 @@ import React, { FC, useCallback, useState } from 'react'; import { EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSpacer, EuiText } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { getAnalysisType, getDependentVar } from '../../../../../../../common/util/analytics_utils'; + +import { useScatterplotFieldOptions } from '../../../../../components/scatterplot_matrix'; + import { defaultSearchQuery, + getScatterplotMatrixLegendType, useResultsViewConfig, DataFrameAnalyticsConfig, } from '../../../../common'; -import { ResultsSearchQuery } from '../../../../common/analytics'; +import { ResultsSearchQuery, ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics'; import { DataFrameTaskStateType } from '../../../analytics_management/components/analytics_list/common'; -import { ExpandableSectionAnalytics } from '../expandable_section'; +import { ExpandableSectionAnalytics, ExpandableSectionSplom } from '../expandable_section'; import { ExplorationResultsTable } from '../exploration_results_table'; import { ExplorationQueryBar } from '../exploration_query_bar'; import { JobConfigErrorCallout } from '../job_config_error_callout'; @@ -99,6 +104,14 @@ export const ExplorationPageWrapper: FC = ({ language: pageUrlState.queryLanguage, }; + const resultsField = jobConfig?.dest.results_field ?? ''; + const scatterplotFieldOptions = useScatterplotFieldOptions( + indexPattern, + jobConfig?.analyzed_fields.includes, + jobConfig?.analyzed_fields.excludes, + resultsField + ); + if (indexPatternErrorMessage !== undefined) { return ( @@ -125,6 +138,9 @@ export const ExplorationPageWrapper: FC = ({ ); } + const jobType = + jobConfig && jobConfig.analysis ? getAnalysisType(jobConfig?.analysis) : undefined; + return ( <> {typeof jobConfig?.description !== 'undefined' && ( @@ -179,6 +195,27 @@ export const ExplorationPageWrapper: FC = ({ )} + {isLoadingJobConfig === true && jobConfig === undefined && } + {isLoadingJobConfig === false && + jobConfig !== undefined && + isInitialized === true && + typeof jobConfig?.id === 'string' && + scatterplotFieldOptions.length > 1 && + typeof jobConfig?.analysis !== 'undefined' && ( + + )} + {isLoadingJobConfig === true && jobConfig === undefined && } {isLoadingJobConfig === false && jobConfig !== undefined && diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx index 26eee9bc95d73..7e11e0bd97015 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useState, FC, useCallback } from 'react'; +import React, { useCallback, useState, FC } from 'react'; import { EuiCallOut, EuiPanel, EuiSpacer, EuiText } from '@elastic/eui'; @@ -15,6 +15,7 @@ import { COLOR_RANGE, COLOR_RANGE_SCALE, } from '../../../../../components/color_range_legend'; +import { useScatterplotFieldOptions } from '../../../../../components/scatterplot_matrix'; import { SavedSearchQuery } from '../../../../../contexts/ml'; import { defaultSearchQuery, isOutlierAnalysis, useResultsViewConfig } from '../../../../common'; @@ -90,6 +91,13 @@ export const OutlierExploration: FC = React.memo(({ jobId }) = (d) => d.id === `${resultsField}.${FEATURE_INFLUENCE}.feature_name` ) === -1; + const scatterplotFieldOptions = useScatterplotFieldOptions( + indexPattern, + jobConfig?.analyzed_fields.includes, + jobConfig?.analyzed_fields.excludes, + resultsField + ); + if (indexPatternErrorMessage !== undefined) { return ( @@ -126,11 +134,12 @@ export const OutlierExploration: FC = React.memo(({ jobId }) = )} {typeof jobConfig?.id === 'string' && } - {typeof jobConfig?.id === 'string' && jobConfig?.analyzed_fields.includes.length > 1 && ( + {typeof jobConfig?.id === 'string' && scatterplotFieldOptions.length > 1 && ( )} {showLegacyFeatureInfluenceFormatCallout && ( diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index b50a7092e108d..4369cbf35594d 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -14209,8 +14209,8 @@ "xpack.ml.splom.dynamicSizeLabel": "動的サイズ", "xpack.ml.splom.fieldSelectionLabel": "フィールド", "xpack.ml.splom.fieldSelectionPlaceholder": "フィールドを選択", - "xpack.ml.splom.RandomScoringLabel": "ランダムスコアリング", - "xpack.ml.splom.SampleSizeLabel": "サンプルサイズ", + "xpack.ml.splom.randomScoringLabel": "ランダムスコアリング", + "xpack.ml.splom.sampleSizeLabel": "サンプルサイズ", "xpack.ml.splom.toggleOff": "オフ", "xpack.ml.splom.toggleOn": "オン", "xpack.ml.splomSpec.outlierScoreThresholdName": "異常スコアしきい値：", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index a93b2c78690c1..d2504c8752c05 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -14248,8 +14248,8 @@ "xpack.ml.splom.dynamicSizeLabel": "动态大小", "xpack.ml.splom.fieldSelectionLabel": "字段", "xpack.ml.splom.fieldSelectionPlaceholder": "选择字段", - "xpack.ml.splom.RandomScoringLabel": "随机评分", - "xpack.ml.splom.SampleSizeLabel": "样例大小", + "xpack.ml.splom.randomScoringLabel": "随机评分", + "xpack.ml.splom.sampleSizeLabel": "样例大小", "xpack.ml.splom.toggleOff": "关闭", "xpack.ml.splom.toggleOn": "开启", "xpack.ml.splomSpec.outlierScoreThresholdName": "离群值分数阈值：", diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts index 6b42306c08c92..b0f1e316e626a 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts @@ -40,6 +40,17 @@ export default function ({ getService }: FtrProviderContext) { modelMemory: '60mb', createIndexPattern: true, expected: { + scatterplotMatrixColorStats: [ + // background + { key: '#000000', value: 94 }, + // tick/grid/axis + { key: '#DDDDDD', value: 1 }, + { key: '#D3DAE6', value: 1 }, + { key: '#F5F7FA', value: 1 }, + // scatterplot circles + { key: '#6A717D', value: 1 }, + { key: '#54B39A', value: 1 }, + ], row: { type: 'classification', status: 'stopped', @@ -89,6 +100,12 @@ export default function ({ getService }: FtrProviderContext) { await ml.testExecution.logTestStep('displays the include fields selection'); await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); + await ml.testExecution.logTestStep('displays the scatterplot matrix'); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow', + testData.expected.scatterplotMatrixColorStats + ); + await ml.testExecution.logTestStep('continues to the additional options step'); await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep(); @@ -207,6 +224,10 @@ export default function ({ getService }: FtrProviderContext) { await ml.dataFrameAnalyticsResults.assertResultsTableExists(); await ml.dataFrameAnalyticsResults.assertResultsTableTrainingFiltersExist(); await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty(); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlDFExpandableSection-splom', + testData.expected.scatterplotMatrixColorStats + ); }); it('displays the analytics job in the map view', async () => { diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts index 53daa0cae2522..419239d1d15ca 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts @@ -49,6 +49,27 @@ export default function ({ getService }: FtrProviderContext) { { chartAvailable: true, id: 'Exterior2nd', legend: '3 categories' }, { chartAvailable: true, id: 'Fireplaces', legend: '0 - 3' }, ], + scatterplotMatrixColorStatsWizard: [ + // background + { key: '#000000', value: 91 }, + // tick/grid/axis + { key: '#6A717D', value: 2 }, + { key: '#F5F7FA', value: 2 }, + { key: '#D3DAE6', value: 1 }, + // scatterplot circles + { key: '#54B399', value: 1 }, + { key: '#54B39A', value: 1 }, + ], + scatterplotMatrixColorStatsResults: [ + // background + { key: '#000000', value: 91 }, + // tick/grid/axis, grey markers + // the red outlier color is not above the 1% threshold. + { key: '#6A717D', value: 2 }, + { key: '#98A2B3', value: 1 }, + { key: '#F5F7FA', value: 2 }, + { key: '#D3DAE6', value: 1 }, + ], row: { type: 'outlier_detection', status: 'stopped', @@ -105,6 +126,12 @@ export default function ({ getService }: FtrProviderContext) { await ml.testExecution.logTestStep('displays the include fields selection'); await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); + await ml.testExecution.logTestStep('displays the scatterplot matrix'); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow', + testData.expected.scatterplotMatrixColorStatsWizard + ); + await ml.testExecution.logTestStep('continues to the additional options step'); await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep(); @@ -221,6 +248,10 @@ export default function ({ getService }: FtrProviderContext) { await ml.dataFrameAnalyticsResults.assertOutlierTablePanelExists(); await ml.dataFrameAnalyticsResults.assertResultsTableExists(); await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty(); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlDFExpandableSection-splom', + testData.expected.scatterplotMatrixColorStatsResults + ); }); it('displays the analytics job in the map view', async () => { diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts index fef22fcebc3ed..f1d19a82caa9b 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts @@ -39,6 +39,16 @@ export default function ({ getService }: FtrProviderContext) { modelMemory: '20mb', createIndexPattern: true, expected: { + scatterplotMatrixColorStats: [ + // background + { key: '#000000', value: 80 }, + // tick/grid/axis + { key: '#6A717D', value: 1 }, + { key: '#F5F7FA', value: 2 }, + { key: '#D3DAE6', value: 1 }, + // because a continuous color scale is used for the scatterplot circles, + // none of the generated colors is above the 1% threshold. + ], row: { type: 'regression', status: 'stopped', @@ -89,6 +99,12 @@ export default function ({ getService }: FtrProviderContext) { await ml.testExecution.logTestStep('displays the include fields selection'); await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); + await ml.testExecution.logTestStep('displays the scatterplot matrix'); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow', + testData.expected.scatterplotMatrixColorStats + ); + await ml.testExecution.logTestStep('continues to the additional options step'); await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep(); @@ -207,6 +223,10 @@ export default function ({ getService }: FtrProviderContext) { await ml.dataFrameAnalyticsResults.assertResultsTableExists(); await ml.dataFrameAnalyticsResults.assertResultsTableTrainingFiltersExist(); await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty(); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlDFExpandableSection-splom', + testData.expected.scatterplotMatrixColorStats + ); }); it('displays the analytics job in the map view', async () => { diff --git a/x-pack/test/functional/services/canvas_element.ts b/x-pack/test/functional/services/canvas_element.ts index e2a42c5dc43c3..08ac38d970225 100644 --- a/x-pack/test/functional/services/canvas_element.ts +++ b/x-pack/test/functional/services/canvas_element.ts @@ -43,9 +43,13 @@ export async function CanvasElementProvider({ getService }: FtrProviderContext) public async getImageData(selector: string): Promise { return await driver.executeScript( ` - const el = document.querySelector('${selector}'); - const ctx = el.getContext('2d'); - return ctx.getImageData(0, 0, el.width, el.height).data; + try { + const el = document.querySelector('${selector}'); + const ctx = el.getContext('2d'); + return ctx.getImageData(0, 0, el.width, el.height).data; + } catch(e) { + return []; + } ` ); } diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_scatterplot.ts b/x-pack/test/functional/services/ml/data_frame_analytics_scatterplot.ts new file mode 100644 index 0000000000000..3472e5079c79a --- /dev/null +++ b/x-pack/test/functional/services/ml/data_frame_analytics_scatterplot.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export function MachineLearningDataFrameAnalyticsScatterplotProvider({ + getService, +}: FtrProviderContext) { + const canvasElement = getService('canvasElement'); + const testSubjects = getService('testSubjects'); + + return new (class AnalyticsScatterplot { + public async assertScatterplotMatrix( + dataTestSubj: string, + expectedColorStats: Array<{ + key: string; + value: number; + }> + ) { + await testSubjects.existOrFail(dataTestSubj); + await testSubjects.existOrFail('mlScatterplotMatrix'); + + const actualColorStats = await canvasElement.getColorStats( + `[data-test-subj="mlScatterplotMatrix"] canvas`, + expectedColorStats, + 1 + ); + expect(actualColorStats.every((d) => d.withinTolerance)).to.eql( + true, + `Color stats for scatterplot matrix should be within tolerance. Expected: '${JSON.stringify( + expectedColorStats + )}' (got '${JSON.stringify(actualColorStats)}')` + ); + } + })(); +} diff --git a/x-pack/test/functional/services/ml/index.ts b/x-pack/test/functional/services/ml/index.ts index c1a9ac304dd69..aa87bc5dc4772 100644 --- a/x-pack/test/functional/services/ml/index.ts +++ b/x-pack/test/functional/services/ml/index.ts @@ -17,6 +17,7 @@ import { MachineLearningDataFrameAnalyticsProvider } from './data_frame_analytic import { MachineLearningDataFrameAnalyticsCreationProvider } from './data_frame_analytics_creation'; import { MachineLearningDataFrameAnalyticsEditProvider } from './data_frame_analytics_edit'; import { MachineLearningDataFrameAnalyticsResultsProvider } from './data_frame_analytics_results'; +import { MachineLearningDataFrameAnalyticsScatterplotProvider } from './data_frame_analytics_scatterplot'; import { MachineLearningDataFrameAnalyticsMapProvider } from './data_frame_analytics_map'; import { MachineLearningDataFrameAnalyticsTableProvider } from './data_frame_analytics_table'; import { MachineLearningDataVisualizerProvider } from './data_visualizer'; @@ -63,6 +64,9 @@ export function MachineLearningProvider(context: FtrProviderContext) { const dataFrameAnalyticsResults = MachineLearningDataFrameAnalyticsResultsProvider(context); const dataFrameAnalyticsMap = MachineLearningDataFrameAnalyticsMapProvider(context); const dataFrameAnalyticsTable = MachineLearningDataFrameAnalyticsTableProvider(context); + const dataFrameAnalyticsScatterplot = MachineLearningDataFrameAnalyticsScatterplotProvider( + context + ); const dataVisualizer = MachineLearningDataVisualizerProvider(context); const dataVisualizerTable = MachineLearningDataVisualizerTableProvider(context, commonUI); @@ -105,6 +109,7 @@ export function MachineLearningProvider(context: FtrProviderContext) { dataFrameAnalyticsResults, dataFrameAnalyticsMap, dataFrameAnalyticsTable, + dataFrameAnalyticsScatterplot, dataVisualizer, dataVisualizerFileBased, dataVisualizerIndexBased,