From 8fef5fd9e1289b7e383f8ab7e39faa5f4126386b Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Fri, 2 Apr 2021 11:06:31 -0400 Subject: [PATCH] [ML] Data Frame Analytics: adds support for runtime fields (#95734) * add runtime mapping editor in wizard * ensure depVar is updated correctly with RF changes * remove old RF from includes * ensure cloning works with RF as depVar * ensure indexPattern RF work * scatterplot supports RTF. depVar options have indexPattern RTF on first load * remove unnecessary types * ensure supported fields included by default * update types in editor * use isRuntimeMappings * fix translations. ensure runtimeMappings persist when going back to step 1 * ensure histograms support runtime fields * update types --- .../ml/common/types/data_frame_analytics.ts | 3 + x-pack/plugins/ml/common/types/fields.ts | 4 +- .../components/data_grid/common.ts | 51 +-- .../application/components/data_grid/index.ts | 2 +- .../scatterplot_matrix/scatterplot_matrix.tsx | 14 + .../configuration_step/configuration_step.tsx | 14 +- .../configuration_step_form.tsx | 332 ++++++++++++------ .../form_options_validation.ts | 67 +++- .../components/runtime_mappings/index.ts | 8 + .../runtime_mappings/runtime_mappings.tsx | 237 +++++++++++++ .../runtime_mappings_editor.tsx | 82 +++++ .../hooks/use_index_data.ts | 83 ++++- .../pages/analytics_creation/page.tsx | 1 + .../action_clone/clone_action_name.tsx | 5 + .../use_create_analytics_form/reducer.ts | 8 +- .../hooks/use_create_analytics_form/state.ts | 12 + .../index_based/data_loader/data_loader.ts | 5 +- .../routes/schemas/data_analytics_schema.ts | 3 + .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 2 - 20 files changed, 767 insertions(+), 167 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/index.ts create mode 100644 x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings.tsx create mode 100644 x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings_editor.tsx diff --git a/x-pack/plugins/ml/common/types/data_frame_analytics.ts b/x-pack/plugins/ml/common/types/data_frame_analytics.ts index 8686e3d64037e..d9632f4d4a83b 100644 --- a/x-pack/plugins/ml/common/types/data_frame_analytics.ts +++ b/x-pack/plugins/ml/common/types/data_frame_analytics.ts @@ -6,6 +6,8 @@ */ import Boom from '@hapi/boom'; +import { RuntimeMappings } from './fields'; + import { EsErrorBody } from '../util/errors'; import { ANALYSIS_CONFIG_TYPE } from '../constants/data_frame_analytics'; import { DATA_FRAME_TASK_STATE } from '../constants/data_frame_analytics'; @@ -74,6 +76,7 @@ export interface DataFrameAnalyticsConfig { source: { index: IndexName | IndexName[]; query?: any; + runtime_mappings?: RuntimeMappings; }; analysis: AnalysisConfig; analyzed_fields: { diff --git a/x-pack/plugins/ml/common/types/fields.ts b/x-pack/plugins/ml/common/types/fields.ts index f9f7f8fc7ead6..8dfe9d111ed38 100644 --- a/x-pack/plugins/ml/common/types/fields.ts +++ b/x-pack/plugins/ml/common/types/fields.ts @@ -109,8 +109,8 @@ export interface AggCardinality { export type RollupFields = Record]>; // Replace this with import once #88995 is merged -const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const; -type RuntimeType = typeof RUNTIME_FIELD_TYPES[number]; +export const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const; +export type RuntimeType = typeof RUNTIME_FIELD_TYPES[number]; export interface RuntimeField { type: RuntimeType; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/common.ts b/x-pack/plugins/ml/public/application/components/data_grid/common.ts index 312776f0d6a07..d3e58c4d7bb0d 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/common.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/common.ts @@ -49,9 +49,8 @@ import { getNestedProperty } from '../../util/object_utils'; import { mlFieldFormatService } from '../../services/field_format_service'; import { DataGridItem, IndexPagination, RenderCellValue } from './types'; -import type { RuntimeField } from '../../../../../../../src/plugins/data/common/index_patterns'; -import { RuntimeMappings } from '../../../../common/types/fields'; -import { isPopulatedObject } from '../../../../common/util/object_utils'; +import { RuntimeMappings, RuntimeField } from '../../../../common/types/fields'; +import { isRuntimeMappings } from '../../../../common/util/runtime_field_utils'; export const INIT_MAX_COLUMNS = 10; export const COLUMN_CHART_DEFAULT_VISIBILITY_ROWS_THRESHOLED = 10000; @@ -94,34 +93,36 @@ export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): str /** * Return a map of runtime_mappings for each of the index pattern field provided * to provide in ES search queries - * @param indexPatternFields * @param indexPattern - * @param clonedRuntimeMappings + * @param RuntimeMappings */ -export const getRuntimeFieldsMapping = ( - indexPatternFields: string[] | undefined, +export function getCombinedRuntimeMappings( indexPattern: IndexPattern | undefined, - clonedRuntimeMappings?: RuntimeMappings -) => { - if (!Array.isArray(indexPatternFields) || indexPattern === undefined) return {}; - const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields; - let combinedRuntimeMappings: RuntimeMappings = {}; - - if (isPopulatedObject(ipRuntimeMappings)) { - indexPatternFields.forEach((ipField) => { - if (ipRuntimeMappings.hasOwnProperty(ipField)) { - // @ts-expect-error - combinedRuntimeMappings[ipField] = ipRuntimeMappings[ipField]; + runtimeMappings?: RuntimeMappings +): RuntimeMappings | undefined { + let combinedRuntimeMappings = {}; + + // And runtime field mappings defined by index pattern + if (indexPattern) { + const computedFields = indexPattern?.getComputedFields(); + if (computedFields?.runtimeFields !== undefined) { + const indexPatternRuntimeMappings = computedFields.runtimeFields; + if (isRuntimeMappings(indexPatternRuntimeMappings)) { + combinedRuntimeMappings = { ...combinedRuntimeMappings, ...indexPatternRuntimeMappings }; } - }); + } } - if (isPopulatedObject(clonedRuntimeMappings)) { - combinedRuntimeMappings = { ...combinedRuntimeMappings, ...clonedRuntimeMappings }; + + // Use runtime field mappings defined inline from API + // and override fields with same name from index pattern + if (isRuntimeMappings(runtimeMappings)) { + combinedRuntimeMappings = { ...combinedRuntimeMappings, ...runtimeMappings }; } - return Object.keys(combinedRuntimeMappings).length > 0 - ? { runtime_mappings: combinedRuntimeMappings } - : {}; -}; + + if (isRuntimeMappings(combinedRuntimeMappings)) { + return combinedRuntimeMappings; + } +} export interface FieldTypes { [key: string]: ES_FIELD_TYPES; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/index.ts b/x-pack/plugins/ml/public/application/components/data_grid/index.ts index be37e381d1bae..481ff432e0156 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/index.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/index.ts @@ -10,7 +10,7 @@ export { getDataGridSchemaFromESFieldType, getDataGridSchemaFromKibanaFieldType, getFieldsFromKibanaIndexPattern, - getRuntimeFieldsMapping, + getCombinedRuntimeMappings, multiColumnSortFactory, showDataGridColumnChartErrorMessageToast, useRenderCellValue, diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx index 842d5fc1ae87a..bc76020d19649 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx @@ -24,9 +24,13 @@ import { import { i18n } from '@kbn/i18n'; +import { IndexPattern } from '../../../../../../../src/plugins/data/public'; import { extractErrorMessage } from '../../../../common'; +import { isRuntimeMappings } from '../../../../common/util/runtime_field_utils'; import { stringHash } from '../../../../common/util/string_utils'; +import { RuntimeMappings } from '../../../../common/types/fields'; import type { ResultsSearchQuery } from '../../data_frame_analytics/common/analytics'; +import { getCombinedRuntimeMappings } from '../../components/data_grid'; import { useMlApiContext } from '../../contexts/kibana'; @@ -84,6 +88,8 @@ export interface ScatterplotMatrixProps { color?: string; legendType?: LegendType; searchQuery?: ResultsSearchQuery; + runtimeMappings?: RuntimeMappings; + indexPattern?: IndexPattern; } export const ScatterplotMatrix: FC = ({ @@ -93,6 +99,8 @@ export const ScatterplotMatrix: FC = ({ color, legendType, searchQuery, + runtimeMappings, + indexPattern, }) => { const { esSearch } = useMlApiContext(); @@ -185,6 +193,9 @@ export const ScatterplotMatrix: FC = ({ } : searchQuery; + const combinedRuntimeMappings = + indexPattern && getCombinedRuntimeMappings(indexPattern, runtimeMappings); + const resp: estypes.SearchResponse = await esSearch({ index, body: { @@ -193,6 +204,9 @@ export const ScatterplotMatrix: FC = ({ query, from: 0, size: fetchSize, + ...(isRuntimeMappings(combinedRuntimeMappings) + ? { runtime_mappings: combinedRuntimeMappings } + : {}), }, }); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step.tsx index 3b9c84e2fa51a..710fd49f72fb6 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step.tsx @@ -13,12 +13,17 @@ import { ConfigurationStepDetails } from './configuration_step_details'; import { ConfigurationStepForm } from './configuration_step_form'; import { ANALYTICS_STEPS } from '../../page'; -export const ConfigurationStep: FC = ({ +export interface ConfigurationStepProps extends CreateAnalyticsStepProps { + isClone: boolean; +} + +export const ConfigurationStep: FC = ({ actions, state, setCurrentStep, step, stepActivated, + isClone, }) => { const showForm = step === ANALYTICS_STEPS.CONFIGURATION; const showDetails = step !== ANALYTICS_STEPS.CONFIGURATION && stepActivated === true; @@ -30,7 +35,12 @@ export const ConfigurationStep: FC = ({ return ( {showForm && ( - + )} {showDetails && } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx index 36d3de1376373..1046f1a8c3e92 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx @@ -5,10 +5,9 @@ * 2.0. */ -import React, { FC, Fragment, useEffect, useMemo, useRef, useState } from 'react'; +import React, { FC, Fragment, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { EuiBadge, - EuiCallOut, EuiComboBox, EuiComboBoxOptionOption, EuiFormRow, @@ -18,11 +17,11 @@ import { EuiText, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { debounce } from 'lodash'; +import { debounce, cloneDeep } from 'lodash'; -import { FormattedMessage } from '@kbn/i18n/react'; import { newJobCapsService } from '../../../../../services/new_job_capabilities_service'; import { useMlContext } from '../../../../../contexts/ml'; +import { getCombinedRuntimeMappings } from '../../../../../components/data_grid/common'; import { ANALYSIS_CONFIG_TYPE, @@ -31,13 +30,18 @@ import { FieldSelectionItem, } from '../../../../common/analytics'; import { getScatterplotMatrixLegendType } from '../../../../common/get_scatterplot_matrix_legend_type'; -import { CreateAnalyticsStepProps } from '../../../analytics_management/hooks/use_create_analytics_form'; +import { RuntimeMappings as RuntimeMappingsType } from '../../../../../../../common/types/fields'; +import { + isRuntimeMappings, + isRuntimeField, +} from '../../../../../../../common/util/runtime_field_utils'; +import { AnalyticsJobType } from '../../../analytics_management/hooks/use_create_analytics_form/state'; import { Messages } from '../shared'; import { DEFAULT_MODEL_MEMORY_LIMIT, State, } from '../../../analytics_management/hooks/use_create_analytics_form/state'; -import { shouldAddAsDepVarOption } from './form_options_validation'; +import { handleExplainErrorMessage, shouldAddAsDepVarOption } from './form_options_validation'; import { getToastNotifications } from '../../../../../util/dependency_cache'; import { ANALYTICS_STEPS } from '../../page'; @@ -55,6 +59,18 @@ import { ExplorationQueryBarProps } from '../../../analytics_exploration/compone import { Query } from '../../../../../../../../../../src/plugins/data/common/query'; import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix'; +import { RuntimeMappings } from '../runtime_mappings'; +import { ConfigurationStepProps } from './configuration_step'; + +const runtimeMappingKey = 'runtime_mapping'; +const notIncludedReason = 'field not in includes list'; +const requiredFieldsErrorText = i18n.translate( + 'xpack.ml.dataframe.analytics.createWizard.requiredFieldsErrorMessage', + { + defaultMessage: + 'At least one field must be included in the analysis in addition to the dependent variable.', + } +); function getIndexDataQuery(savedSearchQuery: SavedSearchQuery, jobConfigQuery: any) { // Return `undefined` if savedSearchQuery itself is `undefined`, meaning it hasn't been initialized yet. @@ -65,18 +81,23 @@ function getIndexDataQuery(savedSearchQuery: SavedSearchQuery, jobConfigQuery: a return savedSearchQuery !== null ? savedSearchQuery : jobConfigQuery; } -const requiredFieldsErrorText = i18n.translate( - 'xpack.ml.dataframe.analytics.createWizard.requiredFieldsErrorMessage', - { - defaultMessage: - 'At least one field must be included in the analysis in addition to the dependent variable.', - } -); - -const maxRuntimeFieldsDisplayCount = 5; +function getRuntimeDepVarOptions(jobType: AnalyticsJobType, runtimeMappings: RuntimeMappingsType) { + const runtimeOptions: EuiComboBoxOptionOption[] = []; + Object.keys(runtimeMappings).forEach((id) => { + const field = runtimeMappings[id]; + if (isRuntimeField(field) && shouldAddAsDepVarOption(id, field.type, jobType)) { + runtimeOptions.push({ + label: id, + key: `runtime_mapping_${id}`, + }); + } + }); + return runtimeOptions; +} -export const ConfigurationStepForm: FC = ({ +export const ConfigurationStepForm: FC = ({ actions, + isClone, state, setCurrentStep, }) => { @@ -100,7 +121,7 @@ export const ConfigurationStepForm: FC = ({ >(); const { setEstimatedModelMemoryLimit, setFormState } = actions; - const { estimatedModelMemoryLimit, form, isJobCreated, requestMessages } = state; + const { cloneJob, estimatedModelMemoryLimit, form, isJobCreated, requestMessages } = state; const firstUpdate = useRef(true); const { dependentVariable, @@ -111,10 +132,22 @@ export const ConfigurationStepForm: FC = ({ modelMemoryLimit, previousJobType, requiredFieldsError, + runtimeMappings, + previousRuntimeMapping, + runtimeMappingsUpdated, sourceIndex, trainingPercent, useEstimatedMml, } = form; + + const isJobTypeWithDepVar = + jobType === ANALYSIS_CONFIG_TYPE.REGRESSION || jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION; + const dependentVariableEmpty = isJobTypeWithDepVar && dependentVariable === ''; + const hasBasicRequiredFields = jobType !== undefined; + const hasRequiredAnalysisFields = + (isJobTypeWithDepVar && dependentVariable !== '') || + jobType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION; + const [query, setQuery] = useState({ query: jobConfigQueryString ?? '', language: SEARCH_QUERY_LANGUAGE.KUERY, @@ -132,7 +165,8 @@ export const ConfigurationStepForm: FC = ({ const indexData = useIndexData( currentIndexPattern, getIndexDataQuery(savedSearchQuery, jobConfigQuery), - toastNotifications + toastNotifications, + runtimeMappings ); const indexPreviewProps = { @@ -141,11 +175,6 @@ export const ConfigurationStepForm: FC = ({ toastNotifications, }; - const isJobTypeWithDepVar = - jobType === ANALYSIS_CONFIG_TYPE.REGRESSION || jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION; - - const dependentVariableEmpty = isJobTypeWithDepVar && dependentVariable === ''; - const isStepInvalid = dependentVariableEmpty || jobType === undefined || @@ -155,20 +184,23 @@ export const ConfigurationStepForm: FC = ({ unsupportedFieldsError !== undefined || fetchingExplainData; - const loadDepVarOptions = async (formState: State['form']) => { + const loadDepVarOptions = async ( + formState: State['form'], + runtimeOptions: EuiComboBoxOptionOption[] = [] + ) => { setLoadingDepVarOptions(true); setMaxDistinctValuesError(undefined); try { if (currentIndexPattern !== undefined) { const depVarOptions = []; - let depVarUpdate = dependentVariable; + let depVarUpdate = formState.dependentVariable; // Get fields and filter for supported types for job type const { fields } = newJobCapsService; let resetDependentVariable = true; for (const field of fields) { - if (shouldAddAsDepVarOption(field, jobType)) { + if (shouldAddAsDepVarOption(field.id, field.type, jobType)) { depVarOptions.push({ label: field.id, }); @@ -179,10 +211,21 @@ export const ConfigurationStepForm: FC = ({ } } + if ( + isRuntimeMappings(formState.runtimeMappings) && + Object.keys(formState.runtimeMappings).includes(form.dependentVariable) + ) { + resetDependentVariable = false; + depVarOptions.push({ + label: form.dependentVariable, + key: `runtime_mapping_${form.dependentVariable}`, + }); + } + if (resetDependentVariable) { depVarUpdate = ''; } - setDependentVariableOptions(depVarOptions); + setDependentVariableOptions([...runtimeOptions, ...depVarOptions]); setLoadingDepVarOptions(false); setDependentVariableFetchFail(false); setFormState({ dependentVariable: depVarUpdate }); @@ -209,8 +252,23 @@ export const ConfigurationStepForm: FC = ({ if (jobTypeChanged) { setLoadingFieldOptions(true); } + // Ensure runtime field is in 'includes' table if it is set as dependent variable + const depVarIsRuntimeField = + isJobTypeWithDepVar && + runtimeMappings && + Object.keys(runtimeMappings).includes(dependentVariable) && + includes.length > 0 && + includes.includes(dependentVariable) === false; + let formToUse = form; + + if (depVarIsRuntimeField) { + formToUse = cloneDeep(form); + formToUse.includes = [...includes, dependentVariable]; + } - const { success, expectedMemory, fieldSelection, errorMessage } = await fetchExplainData(form); + const { success, expectedMemory, fieldSelection, errorMessage } = await fetchExplainData( + formToUse + ); if (success) { if (shouldUpdateEstimatedMml) { @@ -226,53 +284,33 @@ export const ConfigurationStepForm: FC = ({ setFieldOptionsFetchFail(false); setMaxDistinctValuesError(undefined); setUnsupportedFieldsError(undefined); - setIncludesTableItems(fieldSelection ? fieldSelection : []); setFormState({ ...(shouldUpdateModelMemoryLimit ? { modelMemoryLimit: expectedMemory } : {}), requiredFieldsError: !hasRequiredFields ? requiredFieldsErrorText : undefined, + includes: formToUse.includes, }); + setIncludesTableItems(fieldSelection ? fieldSelection : []); } else { setFormState({ ...(shouldUpdateModelMemoryLimit ? { modelMemoryLimit: expectedMemory } : {}), requiredFieldsError: !hasRequiredFields ? requiredFieldsErrorText : undefined, + includes: formToUse.includes, }); } setFetchingExplainData(false); } else { - let maxDistinctValuesErrorMessage; - let unsupportedFieldsErrorMessage; - if ( - jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION && - (errorMessage.includes('must have at most') || errorMessage.includes('must have at least')) - ) { - maxDistinctValuesErrorMessage = errorMessage; - } else if ( - errorMessage.includes('status_exception') && - errorMessage.includes('unsupported type') - ) { - unsupportedFieldsErrorMessage = errorMessage; - } else if ( - errorMessage.includes('status_exception') && - errorMessage.includes('Unable to estimate memory usage as no documents') - ) { - toastNotifications.addWarning( - i18n.translate('xpack.ml.dataframe.analytics.create.allDocsMissingFieldsErrorMessage', { - defaultMessage: `Unable to estimate memory usage. There are mapped fields for source index [{index}] that do not exist in any indexed documents. You will have to switch to the JSON editor for explicit field selection and include only fields that exist in indexed documents.`, - values: { - index: sourceIndex, - }, - }) - ); - } else { - toastNotifications.addDanger({ - title: i18n.translate( - 'xpack.ml.dataframe.analytics.create.unableToFetchExplainDataMessage', - { - defaultMessage: 'An error occurred fetching analysis fields data.', - } - ), - text: errorMessage, - }); + const { + maxDistinctValuesErrorMessage, + unsupportedFieldsErrorMessage, + toastNotificationDanger, + toastNotificationWarning, + } = handleExplainErrorMessage(errorMessage, sourceIndex, jobType); + + if (toastNotificationDanger) { + toastNotifications.addDanger(toastNotificationDanger); + } + if (toastNotificationWarning) { + toastNotifications.addWarning(toastNotificationWarning); } const fallbackModelMemoryLimit = @@ -304,17 +342,126 @@ export const ConfigurationStepForm: FC = ({ useEffect(() => { if (isJobTypeWithDepVar) { - loadDepVarOptions(form); + const indexPatternRuntimeFields = getCombinedRuntimeMappings(currentIndexPattern); + let runtimeOptions; + + if (indexPatternRuntimeFields) { + runtimeOptions = getRuntimeDepVarOptions(jobType, indexPatternRuntimeFields); + } + + loadDepVarOptions(form, runtimeOptions); } }, [jobType]); - useEffect(() => { - const hasBasicRequiredFields = jobType !== undefined; + const handleRuntimeUpdate = useCallback(async () => { + if (runtimeMappingsUpdated) { + // Update dependent variable options + let resetDepVar = false; + if (isJobTypeWithDepVar) { + const filteredOptions = dependentVariableOptions.filter((option) => { + if (option.label === dependentVariable && option.key?.includes(runtimeMappingKey)) { + resetDepVar = true; + } + return !option.key?.includes(runtimeMappingKey); + }); + // Runtime mappings have been removed + if (runtimeMappings === undefined && runtimeMappingsUpdated === true) { + setDependentVariableOptions(filteredOptions); + } else if (runtimeMappings) { + // add to filteredOptions if it's the type supported + const runtimeOptions = getRuntimeDepVarOptions(jobType, runtimeMappings); + setDependentVariableOptions([...filteredOptions, ...runtimeOptions]); + } + } - const hasRequiredAnalysisFields = - (isJobTypeWithDepVar && dependentVariable !== '') || - jobType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION; + // Update includes - remove previous runtime mappings then add supported runtime fields to includes + const updatedIncludes = includes.filter((field) => { + const isRemovedRuntimeField = previousRuntimeMapping && previousRuntimeMapping[field]; + return !isRemovedRuntimeField; + }); + if (resetDepVar) { + setFormState({ + dependentVariable: '', + includes: updatedIncludes, + }); + setIncludesTableItems( + includesTableItems.filter(({ name }) => { + const isRemovedRuntimeField = previousRuntimeMapping && previousRuntimeMapping[name]; + return !isRemovedRuntimeField; + }) + ); + } + + if (!resetDepVar && hasBasicRequiredFields && hasRequiredAnalysisFields) { + const formCopy = cloneDeep(form); + // When switching back to step ensure runtime field is in 'includes' table if it is set as dependent variable + const depVarIsRuntimeField = + isJobTypeWithDepVar && + runtimeMappings && + Object.keys(runtimeMappings).includes(dependentVariable) && + formCopy.includes.length > 0 && + formCopy.includes.includes(dependentVariable) === false; + + formCopy.includes = depVarIsRuntimeField + ? [...updatedIncludes, dependentVariable] + : updatedIncludes; + + const { success, fieldSelection, errorMessage } = await fetchExplainData(formCopy); + if (success) { + // update the field selection table + const hasRequiredFields = fieldSelection.some( + (field) => field.is_included === true && field.is_required === false + ); + let updatedFieldSelection; + // Update field selection to select supported runtime fields by default. Add those fields to 'includes'. + if (isRuntimeMappings(runtimeMappings)) { + updatedFieldSelection = fieldSelection.map((field) => { + if ( + runtimeMappings[field.name] !== undefined && + field.is_included === false && + field.reason?.includes(notIncludedReason) + ) { + updatedIncludes.push(field.name); + field.is_included = true; + } + return field; + }); + } + setIncludesTableItems(updatedFieldSelection ? updatedFieldSelection : fieldSelection); + setMaxDistinctValuesError(undefined); + setUnsupportedFieldsError(undefined); + setFormState({ + includes: updatedIncludes, + requiredFieldsError: !hasRequiredFields ? requiredFieldsErrorText : undefined, + }); + } else { + const { + maxDistinctValuesErrorMessage, + unsupportedFieldsErrorMessage, + toastNotificationDanger, + toastNotificationWarning, + } = handleExplainErrorMessage(errorMessage, sourceIndex, jobType); + + if (toastNotificationDanger) { + toastNotifications.addDanger(toastNotificationDanger); + } + if (toastNotificationWarning) { + toastNotifications.addWarning(toastNotificationWarning); + } + + setMaxDistinctValuesError(maxDistinctValuesErrorMessage); + setUnsupportedFieldsError(unsupportedFieldsErrorMessage); + } + } + } + }, [JSON.stringify(runtimeMappings)]); + + useEffect(() => { + handleRuntimeUpdate(); + }, [JSON.stringify(runtimeMappings)]); + + useEffect(() => { if (hasBasicRequiredFields && hasRequiredAnalysisFields) { debouncedGetExplainData(); } @@ -324,15 +471,6 @@ export const ConfigurationStepForm: FC = ({ }; }, [jobType, dependentVariable, trainingPercent, JSON.stringify(includes), jobConfigQueryString]); - const unsupportedRuntimeFields = useMemo( - () => - currentIndexPattern.fields - .getAll() - .filter((f) => f.runtimeField) - .map((f) => `'${f.displayName}'`), - [currentIndexPattern.fields] - ); - const scatterplotMatrixProps = useMemo( () => ({ color: isJobTypeWithDepVar ? dependentVariable : undefined, @@ -342,6 +480,8 @@ export const ConfigurationStepForm: FC = ({ index: currentIndexPattern.title, legendType: getScatterplotMatrixLegendType(jobType), searchQuery: jobConfigQuery, + runtimeMappings, + indexPattern: currentIndexPattern, }), [ currentIndexPattern.title, @@ -388,6 +528,7 @@ export const ConfigurationStepForm: FC = ({ /> )} + {((isClone && cloneJob) || !isClone) && } @@ -476,11 +617,11 @@ export const ConfigurationStepForm: FC = ({ singleSelection={true} options={dependentVariableOptions} selectedOptions={dependentVariable ? [{ label: dependentVariable }] : []} - onChange={(selectedOptions) => + onChange={(selectedOptions) => { setFormState({ dependentVariable: selectedOptions[0].label || '', - }) - } + }); + }} isClearable={false} isInvalid={dependentVariable === ''} data-test-subj={`mlAnalyticsCreateJobWizardDependentVariableSelect${ @@ -500,35 +641,6 @@ export const ConfigurationStepForm: FC = ({ > - {Array.isArray(unsupportedRuntimeFields) && unsupportedRuntimeFields.length > 0 && ( - <> - - 0 ? ( - - ) : ( - '' - ), - unsupportedRuntimeFields: unsupportedRuntimeFields - .slice(0, maxRuntimeFieldsDisplayCount) - .join(', '), - }} - /> - - - - )} { - if (field.id === EVENT_RATE_FIELD_ID) return false; +export const shouldAddAsDepVarOption = ( + fieldId: string, + fieldType: ES_FIELD_TYPES | RuntimeType, + jobType: AnalyticsJobType +) => { + if (fieldId === EVENT_RATE_FIELD_ID) return false; - const isBasicNumerical = BASIC_NUMERICAL_TYPES.has(field.type); + const isBasicNumerical = BASIC_NUMERICAL_TYPES.has(fieldType as ES_FIELD_TYPES); const isSupportedByClassification = - isBasicNumerical || CATEGORICAL_TYPES.has(field.type) || field.type === ES_FIELD_TYPES.BOOLEAN; + isBasicNumerical || CATEGORICAL_TYPES.has(fieldType) || fieldType === ES_FIELD_TYPES.BOOLEAN; if (jobType === ANALYSIS_CONFIG_TYPE.REGRESSION) { - return isBasicNumerical || EXTENDED_NUMERICAL_TYPES.has(field.type); + return isBasicNumerical || EXTENDED_NUMERICAL_TYPES.has(fieldType as ES_FIELD_TYPES); } if (jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION) return isSupportedByClassification; }; + +export const handleExplainErrorMessage = ( + errorMessage: string, + sourceIndex: string, + jobType: AnalyticsJobType +) => { + let maxDistinctValuesErrorMessage; + let unsupportedFieldsErrorMessage; + let toastNotificationWarning; + let toastNotificationDanger; + if ( + jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION && + (errorMessage.includes('must have at most') || errorMessage.includes('must have at least')) + ) { + maxDistinctValuesErrorMessage = errorMessage; + } else if ( + errorMessage.includes('status_exception') && + errorMessage.includes('unsupported type') + ) { + unsupportedFieldsErrorMessage = errorMessage; + } else if ( + errorMessage.includes('status_exception') && + errorMessage.includes('Unable to estimate memory usage as no documents') + ) { + toastNotificationWarning = i18n.translate( + 'xpack.ml.dataframe.analytics.create.allDocsMissingFieldsErrorMessage', + { + defaultMessage: `Unable to estimate memory usage. There are mapped fields for source index [{index}] that do not exist in any indexed documents. You will have to switch to the JSON editor for explicit field selection and include only fields that exist in indexed documents.`, + values: { + index: sourceIndex, + }, + } + ); + } else { + toastNotificationDanger = { + title: i18n.translate('xpack.ml.dataframe.analytics.create.unableToFetchExplainDataMessage', { + defaultMessage: 'An error occurred fetching analysis fields data.', + }), + text: errorMessage, + }; + } + + return { + maxDistinctValuesErrorMessage, + unsupportedFieldsErrorMessage, + toastNotificationDanger, + toastNotificationWarning, + }; +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/index.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/index.ts new file mode 100644 index 0000000000000..8b93ddaa4a26a --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { RuntimeMappings } from './runtime_mappings'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings.tsx new file mode 100644 index 0000000000000..d9f1d78c302fd --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings.tsx @@ -0,0 +1,237 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useState, useEffect } from 'react'; +import { + EuiButton, + EuiButtonIcon, + EuiCopy, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiSpacer, + EuiSwitch, + EuiText, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { XJsonMode } from '@kbn/ace'; +import { RuntimeField } from '../../../../../../../../../../src/plugins/data/common/index_patterns'; +import { useMlContext } from '../../../../../contexts/ml'; +import { CreateAnalyticsFormProps } from '../../../analytics_management/hooks/use_create_analytics_form'; +import { XJson } from '../../../../../../../../../../src/plugins/es_ui_shared/public'; +import { getCombinedRuntimeMappings } from '../../../../../components/data_grid/common'; +import { isPopulatedObject } from '../../../../../../../common/util/object_utils'; +import { RuntimeMappingsEditor } from './runtime_mappings_editor'; + +const advancedEditorsSidebarWidth = '220px'; +const COPY_TO_CLIPBOARD_RUNTIME_MAPPINGS = i18n.translate( + 'xpack.ml.dataframe.analytics.createWizard.indexPreview.copyRuntimeMappingsClipboardTooltip', + { + defaultMessage: 'Copy Dev Console statement of the runtime mappings to the clipboard.', + } +); + +const { useXJsonMode } = XJson; +const xJsonMode = new XJsonMode(); + +interface Props { + actions: CreateAnalyticsFormProps['actions']; + state: CreateAnalyticsFormProps['state']; +} + +type RuntimeMappings = Record; + +export const RuntimeMappings: FC = ({ actions, state }) => { + const [isRuntimeMappingsEditorEnabled, setIsRuntimeMappingsEditorEnabled] = useState( + false + ); + const [ + isRuntimeMappingsEditorApplyButtonEnabled, + setIsRuntimeMappingsEditorApplyButtonEnabled, + ] = useState(false); + const [ + advancedEditorRuntimeMappingsLastApplied, + setAdvancedEditorRuntimeMappingsLastApplied, + ] = useState(); + const [advancedEditorRuntimeMappings, setAdvancedEditorRuntimeMappings] = useState(); + + const { setFormState } = actions; + const { jobType, previousRuntimeMapping, runtimeMappings } = state.form; + + const { + convertToJson, + setXJson: setAdvancedRuntimeMappingsConfig, + xJson: advancedRuntimeMappingsConfig, + } = useXJsonMode(runtimeMappings || ''); + + const mlContext = useMlContext(); + const { currentIndexPattern } = mlContext; + + const applyChanges = () => { + const removeRuntimeMappings = advancedRuntimeMappingsConfig === ''; + const parsedRuntimeMappings = removeRuntimeMappings + ? undefined + : JSON.parse(advancedRuntimeMappingsConfig); + const prettySourceConfig = removeRuntimeMappings + ? '' + : JSON.stringify(parsedRuntimeMappings, null, 2); + const previous = + previousRuntimeMapping === undefined && runtimeMappings === undefined + ? parsedRuntimeMappings + : runtimeMappings; + setFormState({ + runtimeMappings: parsedRuntimeMappings, + runtimeMappingsUpdated: true, + previousRuntimeMapping: previous, + }); + setAdvancedEditorRuntimeMappings(prettySourceConfig); + setAdvancedEditorRuntimeMappingsLastApplied(prettySourceConfig); + setIsRuntimeMappingsEditorApplyButtonEnabled(false); + }; + + // If switching to KQL after updating via editor - reset search + const toggleEditorHandler = (reset = false) => { + if (reset === true) { + setFormState({ runtimeMappingsUpdated: false }); + } + if (isRuntimeMappingsEditorEnabled === false) { + setAdvancedEditorRuntimeMappingsLastApplied(advancedEditorRuntimeMappings); + } + + setIsRuntimeMappingsEditorEnabled(!isRuntimeMappingsEditorEnabled); + setIsRuntimeMappingsEditorApplyButtonEnabled(false); + }; + + useEffect(function getInitialRuntimeMappings() { + const combinedRuntimeMappings = getCombinedRuntimeMappings( + currentIndexPattern, + runtimeMappings + ); + + if (combinedRuntimeMappings) { + setAdvancedRuntimeMappingsConfig(JSON.stringify(combinedRuntimeMappings, null, 2)); + setFormState({ + runtimeMappings: combinedRuntimeMappings, + }); + } + }, []); + + return ( + <> + + + + + {isPopulatedObject(runtimeMappings) ? ( + + {Object.keys(runtimeMappings).join(',')} + + ) : ( + + )} + + {isRuntimeMappingsEditorEnabled && ( + <> + + + + )} + + + + + + + + toggleEditorHandler()} + data-test-subj="mlDataFrameAnalyticsRuntimeMappingsEditorSwitch" + /> + + + + {(copy: () => void) => ( + + )} + + + + + + {isRuntimeMappingsEditorEnabled && ( + + + + {i18n.translate( + 'xpack.ml.dataframe.analytics.createWizard.advancedRuntimeMappingsEditorHelpText', + { + defaultMessage: + 'The advanced editor allows you to edit the runtime mappings of the source.', + } + )} + + + + {i18n.translate( + 'xpack.ml.dataframe.analytics.createWizard.advancedSourceEditorApplyButtonText', + { + defaultMessage: 'Apply changes', + } + )} + + + )} + + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings_editor.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings_editor.tsx new file mode 100644 index 0000000000000..70544cc14ba08 --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings_editor.tsx @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEqual } from 'lodash'; +import React, { memo, FC } from 'react'; +import { EuiCodeEditor } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { isRuntimeMappings } from '../../../../../../../common/util/runtime_field_utils'; + +interface Props { + convertToJson: (data: string) => string; + setAdvancedRuntimeMappingsConfig: React.Dispatch; + setIsRuntimeMappingsEditorApplyButtonEnabled: React.Dispatch>; + advancedEditorRuntimeMappingsLastApplied: string | undefined; + advancedRuntimeMappingsConfig: string; + xJsonMode: any; +} + +export const RuntimeMappingsEditor: FC = memo( + ({ + convertToJson, + xJsonMode, + setAdvancedRuntimeMappingsConfig, + setIsRuntimeMappingsEditorApplyButtonEnabled, + advancedEditorRuntimeMappingsLastApplied, + advancedRuntimeMappingsConfig, + }) => { + return ( + { + setAdvancedRuntimeMappingsConfig(d); + + // Disable the "Apply"-Button if the config hasn't changed. + if (advancedEditorRuntimeMappingsLastApplied === d) { + setIsRuntimeMappingsEditorApplyButtonEnabled(false); + return; + } + + // Enable Apply button so user can remove previously created runtime field + if (d === '') { + setIsRuntimeMappingsEditorApplyButtonEnabled(true); + return; + } + + // Try to parse the string passed on from the editor. + // If parsing fails, the "Apply"-Button will be disabled + try { + const parsedJson = JSON.parse(convertToJson(d)); + setIsRuntimeMappingsEditorApplyButtonEnabled(isRuntimeMappings(parsedJson)); + } catch (e) { + setIsRuntimeMappingsEditorApplyButtonEnabled(false); + } + }} + setOptions={{ + fontSize: '12px', + }} + theme="textmate" + aria-label={i18n.translate( + 'xpack.ml.dataframe.analytics.createWizard.runtimeMappings.advancedEditorAriaLabel', + { + defaultMessage: 'Advanced runtime editor', + } + )} + /> + ); + }, + (prevProps, nextProps) => isEqual(pickProps(prevProps), pickProps(nextProps)) +); + +function pickProps(props: Props) { + return [props.advancedEditorRuntimeMappingsLastApplied, props.advancedRuntimeMappingsConfig]; +} diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts index 4552ca34ebbae..f48f4a62f5a7d 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts @@ -5,20 +5,23 @@ * 2.0. */ -import { useEffect, useMemo } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { estypes } from '@elastic/elasticsearch'; import { EuiDataGridColumn } from '@elastic/eui'; - import { CoreSetup } from 'src/core/public'; import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import { isRuntimeMappings } from '../../../../../../common/util/runtime_field_utils'; +import { RuntimeMappings, RuntimeField } from '../../../../../../common/types/fields'; +import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../../../common/constants/field_histograms'; import { DataLoader } from '../../../../datavisualizer/index_based/data_loader'; import { getFieldType, getDataGridSchemaFromKibanaFieldType, + getDataGridSchemaFromESFieldType, getFieldsFromKibanaIndexPattern, showDataGridColumnChartErrorMessageToast, useDataGrid, @@ -26,31 +29,51 @@ import { EsSorting, UseIndexDataReturnType, getProcessedFields, + getCombinedRuntimeMappings, } from '../../../../components/data_grid'; import { extractErrorMessage } from '../../../../../../common/util/errors'; import { INDEX_STATUS } from '../../../common/analytics'; import { ml } from '../../../../services/ml_api_service'; -import { getRuntimeFieldsMapping } from '../../../../components/data_grid/common'; type IndexSearchResponse = estypes.SearchResponse; +interface MLEuiDataGridColumn extends EuiDataGridColumn { + isRuntimeFieldColumn?: boolean; +} + +function getRuntimeFieldColumns(runtimeMappings: RuntimeMappings) { + return Object.keys(runtimeMappings).map((id) => { + const field = runtimeMappings[id]; + const schema = getDataGridSchemaFromESFieldType(field.type as RuntimeField['type']); + return { id, schema, isExpandable: schema !== 'boolean', isRuntimeFieldColumn: true }; + }); +} + export const useIndexData = ( indexPattern: IndexPattern, query: Record | undefined, - toastNotifications: CoreSetup['notifications']['toasts'] + toastNotifications: CoreSetup['notifications']['toasts'], + runtimeMappings?: RuntimeMappings ): UseIndexDataReturnType => { const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [ indexPattern, ]); - // EuiDataGrid State - const columns: EuiDataGridColumn[] = [ + const [columns, setColumns] = useState([ ...indexPatternFields.map((id) => { const field = indexPattern.fields.getByName(id); - const schema = getDataGridSchemaFromKibanaFieldType(field); - return { id, schema, isExpandable: schema !== 'boolean' }; + const isRuntimeFieldColumn = field?.runtimeField !== undefined; + const schema = isRuntimeFieldColumn + ? getDataGridSchemaFromESFieldType(field?.type as RuntimeField['type']) + : getDataGridSchemaFromKibanaFieldType(field); + return { + id, + schema, + isExpandable: schema !== 'boolean', + isRuntimeFieldColumn, + }; }), - ]; + ]); const dataGrid = useDataGrid(columns); @@ -75,6 +98,8 @@ export const useIndexData = ( setErrorMessage(''); setStatus(INDEX_STATUS.LOADING); + const combinedRuntimeMappings = getCombinedRuntimeMappings(indexPattern, runtimeMappings); + const sort: EsSorting = sortingColumns.reduce((s, column) => { s[column.id] = { order: column.direction }; return s; @@ -88,14 +113,37 @@ export const useIndexData = ( fields: ['*'], _source: false, ...(Object.keys(sort).length > 0 ? { sort } : {}), - ...getRuntimeFieldsMapping(indexPatternFields, indexPattern), + ...(isRuntimeMappings(combinedRuntimeMappings) + ? { runtime_mappings: combinedRuntimeMappings } + : {}), }, }; try { const resp: IndexSearchResponse = await ml.esSearch(esSearchRequest); - const docs = resp.hits.hits.map((d) => getProcessedFields(d.fields ?? {})); + + if (isRuntimeMappings(runtimeMappings)) { + // remove old runtime field from columns + const updatedColumns = columns.filter((col) => col.isRuntimeFieldColumn === false); + setColumns([ + ...updatedColumns, + ...(combinedRuntimeMappings ? getRuntimeFieldColumns(combinedRuntimeMappings) : []), + ]); + } else { + setColumns([ + ...indexPatternFields.map((id) => { + const field = indexPattern.fields.getByName(id); + const schema = getDataGridSchemaFromKibanaFieldType(field); + return { + id, + schema, + isExpandable: schema !== 'boolean', + isRuntimeFieldColumn: field?.runtimeField !== undefined, + }; + }), + ]); + } setRowCount(typeof resp.hits.total === 'number' ? resp.hits.total : resp.hits.total.value); setRowCountRelation( typeof resp.hits.total === 'number' @@ -115,13 +163,18 @@ export const useIndexData = ( getIndexData(); } // custom comparison - }, [indexPattern.title, indexPatternFields, JSON.stringify([query, pagination, sortingColumns])]); + }, [ + indexPattern.title, + indexPatternFields, + JSON.stringify([query, pagination, sortingColumns, runtimeMappings]), + ]); const dataLoader = useMemo(() => new DataLoader(indexPattern, toastNotifications), [ indexPattern, ]); const fetchColumnChartsData = async function (fieldHistogramsQuery: Record) { + const combinedRuntimeMappings = getCombinedRuntimeMappings(indexPattern, runtimeMappings); try { const columnChartsData = await dataLoader.loadFieldHistograms( columns @@ -130,7 +183,9 @@ export const useIndexData = ( fieldName: cT.id, type: getFieldType(cT.schema), })), - fieldHistogramsQuery + fieldHistogramsQuery, + DEFAULT_SAMPLER_SHARD_SIZE, + combinedRuntimeMappings ); dataGrid.setColumnCharts(columnChartsData); } catch (e) { @@ -146,7 +201,7 @@ export const useIndexData = ( }, [ dataGrid.chartsVisible, indexPattern.title, - JSON.stringify([query, dataGrid.visibleColumns]), + JSON.stringify([query, dataGrid.visibleColumns, runtimeMappings]), ]); const renderCellValue = useRenderCellValue(indexPattern, pagination, tableItems); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx index 8fd0ae86d240c..830870cf1ca74 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx @@ -104,6 +104,7 @@ export const Page: FC = ({ jobId }) => { children: ( ({ requiredFieldsError: undefined, randomizeSeed: undefined, resultsField: undefined, + runtimeMappings: undefined, + runtimeMappingsUpdated: false, + previousRuntimeMapping: undefined, softTreeDepthLimit: undefined, softTreeDepthTolerance: undefined, sourceIndex: '', @@ -212,6 +220,9 @@ export const getJobConfigFromFormState = ( ? formState.sourceIndex.split(',').map((d) => d.trim()) : formState.sourceIndex, query: formState.jobConfigQuery, + ...(isRuntimeMappings(formState.runtimeMappings) + ? { runtime_mappings: formState.runtimeMappings } + : {}), }, dest: { index: formState.destinationIndex, @@ -340,6 +351,7 @@ export function getFormStateFromJobConfig( sourceIndex: Array.isArray(analyticsJobConfig.source.index) ? analyticsJobConfig.source.index.join(',') : analyticsJobConfig.source.index, + runtimeMappings: analyticsJobConfig.source.runtime_mappings, modelMemoryLimit: analyticsJobConfig.model_memory_limit, maxNumThreads: analyticsJobConfig.max_num_threads, includes: analyticsJobConfig.analyzed_fields?.includes ?? [], diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts index 38b9aa2ce29f2..0da7d3d6b63d8 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts @@ -110,14 +110,15 @@ export class DataLoader { async loadFieldHistograms( fields: FieldHistogramRequestConfig[], query: string | SavedSearchQuery, - samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE + samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE, + editorRuntimeMappings?: RuntimeMappings ): Promise { const stats = await ml.getVisualizerFieldHistograms({ indexPatternTitle: this._indexPatternTitle, query, fields, samplerShardSize, - runtimeMappings: this._runtimeMappings, + runtimeMappings: editorRuntimeMappings || this._runtimeMappings, }); return stats; diff --git a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts index 70ffecd11c96c..1f5bcbc23423a 100644 --- a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts @@ -6,6 +6,7 @@ */ import { schema } from '@kbn/config-schema'; +import { runtimeMappingsSchema } from './runtime_mappings_schema'; export const dataAnalyticsJobConfigSchema = schema.object({ description: schema.maybe(schema.string()), @@ -16,6 +17,7 @@ export const dataAnalyticsJobConfigSchema = schema.object({ source: schema.object({ index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]), query: schema.maybe(schema.any()), + runtime_mappings: runtimeMappingsSchema, _source: schema.maybe( schema.object({ /** Fields to include in results */ @@ -51,6 +53,7 @@ export const dataAnalyticsExplainSchema = schema.object({ source: schema.object({ index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]), query: schema.maybe(schema.any()), + runtime_mappings: runtimeMappingsSchema, }), analysis: schema.any(), analyzed_fields: schema.maybe(schema.any()), diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 9602a324e5d51..133b4d0b6aaa8 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -13329,7 +13329,6 @@ "xpack.ml.dataframe.analytics.create.etaInputAriaLabel": "縮小が重みに適用されました。", "xpack.ml.dataframe.analytics.create.etaLabel": "Eta", "xpack.ml.dataframe.analytics.create.etaText": "縮小が重みに適用されました。0.001から1の範囲でなければなりません。", - "xpack.ml.dataframe.analytics.create.extraUnsupportedRuntimeFieldsMsg": "{count}以上", "xpack.ml.dataframe.analytics.create.featureBagFractionInputAriaLabel": "各候補分割のランダムなbagを選択したときに使用される特徴量の割合", "xpack.ml.dataframe.analytics.create.featureBagFractionLabel": "特徴量bag割合", "xpack.ml.dataframe.analytics.create.featureBagFractionText": "各候補分割のランダムなbagを選択したときに使用される特徴量の割合。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 12a3c8925cfc6..0f9d8b90a2578 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -13500,7 +13500,6 @@ "xpack.ml.dataframe.analytics.create.etaInputAriaLabel": "缩小量已应用于权重。", "xpack.ml.dataframe.analytics.create.etaLabel": "Eta", "xpack.ml.dataframe.analytics.create.etaText": "缩小量已应用于权重。必须介于 0.001 和 1 之间。", - "xpack.ml.dataframe.analytics.create.extraUnsupportedRuntimeFieldsMsg": "及另外 {count} 个", "xpack.ml.dataframe.analytics.create.featureBagFractionInputAriaLabel": "选择为每个候选拆分选择随机袋时使用的特征比例", "xpack.ml.dataframe.analytics.create.featureBagFractionLabel": "特征袋比例", "xpack.ml.dataframe.analytics.create.featureBagFractionText": "选择为每个候选拆分选择随机袋时使用的特征比例。", @@ -13604,7 +13603,6 @@ "xpack.ml.dataframe.analytics.create.trainingPercentLabel": "训练百分比", "xpack.ml.dataframe.analytics.create.unableToFetchExplainDataMessage": "提取分析字段数据时发生错误。", "xpack.ml.dataframe.analytics.create.unsupportedFieldsError": "无效。{message}", - "xpack.ml.dataframe.analytics.create.unsupportedRuntimeFieldsCallout": "不支持分析运行时{runtimeFieldsCount, plural, other {字段}} {unsupportedRuntimeFields} {extraCountMsg}。", "xpack.ml.dataframe.analytics.create.useEstimatedMmlLabel": "使用估计的模型内存限制", "xpack.ml.dataframe.analytics.create.UseResultsFieldDefaultLabel": "使用结果字段默认值“{defaultValue}”", "xpack.ml.dataframe.analytics.create.viewResultsCardDescription": "查看分析作业的结果。",