[ML] Data Frame Analytics: adds support for runtime fields (#95734)

* add runtime mapping editor in wizard

* ensure depVar is updated correctly with RF changes

* remove old RF from includes

* ensure cloning works with RF as depVar

* ensure indexPattern RF work

* scatterplot supports RTF. depVar options have indexPattern RTF on first load

* remove unnecessary types

* ensure supported fields included by default

* update types in editor

* use isRuntimeMappings

* fix translations. ensure runtimeMappings persist when going back to step 1

* ensure histograms support runtime fields

* update types

x-pack/plugins/ml/common/types/data_frame_analytics.ts

@@ -6,6 +6,8 @@
  */
 
 import Boom from '@hapi/boom';
+import { RuntimeMappings } from './fields';
+
 import { EsErrorBody } from '../util/errors';
 import { ANALYSIS_CONFIG_TYPE } from '../constants/data_frame_analytics';
 import { DATA_FRAME_TASK_STATE } from '../constants/data_frame_analytics';
@@ -74,6 +76,7 @@ export interface DataFrameAnalyticsConfig {
   source: {
     index: IndexName | IndexName[];
     query?: any;
+    runtime_mappings?: RuntimeMappings;
   };
   analysis: AnalysisConfig;
   analyzed_fields: {

x-pack/plugins/ml/common/types/fields.ts

@@ -109,8 +109,8 @@ export interface AggCardinality {
 export type RollupFields = Record<FieldId, [Record<'agg', ES_AGGREGATION>]>;
 
 // Replace this with import once #88995 is merged
-const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const;
-type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
+export const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const;
+export type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
 
 export interface RuntimeField {
   type: RuntimeType;

x-pack/plugins/ml/public/application/components/data_grid/common.ts

@@ -49,9 +49,8 @@ import { getNestedProperty } from '../../util/object_utils';
 import { mlFieldFormatService } from '../../services/field_format_service';
 
 import { DataGridItem, IndexPagination, RenderCellValue } from './types';
-import type { RuntimeField } from '../../../../../../../src/plugins/data/common/index_patterns';
-import { RuntimeMappings } from '../../../../common/types/fields';
-import { isPopulatedObject } from '../../../../common/util/object_utils';
+import { RuntimeMappings, RuntimeField } from '../../../../common/types/fields';
+import { isRuntimeMappings } from '../../../../common/util/runtime_field_utils';
 
 export const INIT_MAX_COLUMNS = 10;
 export const COLUMN_CHART_DEFAULT_VISIBILITY_ROWS_THRESHOLED = 10000;
@@ -94,34 +93,36 @@ export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): str
 /**
  * Return a map of runtime_mappings for each of the index pattern field provided
  * to provide in ES search queries
- * @param indexPatternFields
  * @param indexPattern
- * @param clonedRuntimeMappings
+ * @param RuntimeMappings
  */
-export const getRuntimeFieldsMapping = (
-  indexPatternFields: string[] | undefined,
+export function getCombinedRuntimeMappings(
   indexPattern: IndexPattern | undefined,
-  clonedRuntimeMappings?: RuntimeMappings
-) => {
-  if (!Array.isArray(indexPatternFields) || indexPattern === undefined) return {};
-  const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields;
-  let combinedRuntimeMappings: RuntimeMappings = {};
-
-  if (isPopulatedObject(ipRuntimeMappings)) {
-    indexPatternFields.forEach((ipField) => {
-      if (ipRuntimeMappings.hasOwnProperty(ipField)) {
-        // @ts-expect-error
-        combinedRuntimeMappings[ipField] = ipRuntimeMappings[ipField];
+  runtimeMappings?: RuntimeMappings
+): RuntimeMappings | undefined {
+  let combinedRuntimeMappings = {};
+
+  // And runtime field mappings defined by index pattern
+  if (indexPattern) {
+    const computedFields = indexPattern?.getComputedFields();
+    if (computedFields?.runtimeFields !== undefined) {
+      const indexPatternRuntimeMappings = computedFields.runtimeFields;
+      if (isRuntimeMappings(indexPatternRuntimeMappings)) {
+        combinedRuntimeMappings = { ...combinedRuntimeMappings, ...indexPatternRuntimeMappings };
       }
-    });
+    }
   }
-  if (isPopulatedObject(clonedRuntimeMappings)) {
-    combinedRuntimeMappings = { ...combinedRuntimeMappings, ...clonedRuntimeMappings };
+
+  // Use runtime field mappings defined inline from API
+  // and override fields with same name from index pattern
+  if (isRuntimeMappings(runtimeMappings)) {
+    combinedRuntimeMappings = { ...combinedRuntimeMappings, ...runtimeMappings };
   }
-  return Object.keys(combinedRuntimeMappings).length > 0
-    ? { runtime_mappings: combinedRuntimeMappings }
-    : {};
-};
+
+  if (isRuntimeMappings(combinedRuntimeMappings)) {
+    return combinedRuntimeMappings;
+  }
+}
 
 export interface FieldTypes {
   [key: string]: ES_FIELD_TYPES;

x-pack/plugins/ml/public/application/components/data_grid/index.ts

@@ -10,7 +10,7 @@ export {
   getDataGridSchemaFromESFieldType,
   getDataGridSchemaFromKibanaFieldType,
   getFieldsFromKibanaIndexPattern,
-  getRuntimeFieldsMapping,
+  getCombinedRuntimeMappings,
   multiColumnSortFactory,
   showDataGridColumnChartErrorMessageToast,
   useRenderCellValue,

x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx

@@ -24,9 +24,13 @@ import {
 
 import { i18n } from '@kbn/i18n';
 
+import { IndexPattern } from '../../../../../../../src/plugins/data/public';
 import { extractErrorMessage } from '../../../../common';
+import { isRuntimeMappings } from '../../../../common/util/runtime_field_utils';
 import { stringHash } from '../../../../common/util/string_utils';
+import { RuntimeMappings } from '../../../../common/types/fields';
 import type { ResultsSearchQuery } from '../../data_frame_analytics/common/analytics';
+import { getCombinedRuntimeMappings } from '../../components/data_grid';
 
 import { useMlApiContext } from '../../contexts/kibana';
 
@@ -84,6 +88,8 @@ export interface ScatterplotMatrixProps {
   color?: string;
   legendType?: LegendType;
   searchQuery?: ResultsSearchQuery;
+  runtimeMappings?: RuntimeMappings;
+  indexPattern?: IndexPattern;
 }
 
 export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
@@ -93,6 +99,8 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
   color,
   legendType,
   searchQuery,
+  runtimeMappings,
+  indexPattern,
 }) => {
   const { esSearch } = useMlApiContext();
 
@@ -185,6 +193,9 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
             }
           : searchQuery;
 
+        const combinedRuntimeMappings =
+          indexPattern && getCombinedRuntimeMappings(indexPattern, runtimeMappings);
+
         const resp: estypes.SearchResponse = await esSearch({
           index,
           body: {
@@ -193,6 +204,9 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
             query,
             from: 0,
             size: fetchSize,
+            ...(isRuntimeMappings(combinedRuntimeMappings)
+              ? { runtime_mappings: combinedRuntimeMappings }
+              : {}),
           },
         });
 

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step.tsx

@@ -13,12 +13,17 @@ import { ConfigurationStepDetails } from './configuration_step_details';
 import { ConfigurationStepForm } from './configuration_step_form';
 import { ANALYTICS_STEPS } from '../../page';
 
-export const ConfigurationStep: FC<CreateAnalyticsStepProps> = ({
+export interface ConfigurationStepProps extends CreateAnalyticsStepProps {
+  isClone: boolean;
+}
+
+export const ConfigurationStep: FC<ConfigurationStepProps> = ({
   actions,
   state,
   setCurrentStep,
   step,
   stepActivated,
+  isClone,
 }) => {
   const showForm = step === ANALYTICS_STEPS.CONFIGURATION;
   const showDetails = step !== ANALYTICS_STEPS.CONFIGURATION && stepActivated === true;
@@ -30,7 +35,12 @@ export const ConfigurationStep: FC<CreateAnalyticsStepProps> = ({
   return (
     <EuiForm className="mlDataFrameAnalyticsCreateForm" data-test-subj={dataTestSubj}>
       {showForm && (
-        <ConfigurationStepForm actions={actions} state={state} setCurrentStep={setCurrentStep} />
+        <ConfigurationStepForm
+          actions={actions}
+          isClone={isClone}
+          state={state}
+          setCurrentStep={setCurrentStep}
+        />
       )}
       {showDetails && <ConfigurationStepDetails setCurrentStep={setCurrentStep} state={state} />}
     </EuiForm>