[ML] Improve support for script and aggregation fields in anomaly detection jobs

x-pack/plugins/ml/common/constants/messages.ts

@@ -442,6 +442,16 @@ export const getMessages = once(() => {
       url:
         'https://www.elastic.co/guide/en/elasticsearch/reference/{{version}}/ml-job-resource.html#ml-job-resource',
     },
+    missing_summary_count_field_name: {
+      status: VALIDATION_STATUS.ERROR,
+      text: i18n.translate(
+        'xpack.ml.models.jobValidation.messages.missingSummaryCountFieldNameMessage',
+        {
+          defaultMessage:
+            'A job configured with a datafeed with aggregations must set summary_count_field_name; use doc_count or suitable alternative.',
+        }
+      ),
+    },
     skipped_extended_tests: {
       status: VALIDATION_STATUS.WARNING,
       text: i18n.translate('xpack.ml.models.jobValidation.messages.skippedExtendedTestsMessage', {

x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts

@@ -19,7 +19,9 @@ export interface Datafeed {
   job_id: JobId;
   query: object;
   query_delay?: string;
-  script_fields?: object;
+  script_fields?: {
+    [key: string]: any;
+  };
   scroll_size?: number;
   delayed_data_check_config?: object;
   indices_options?: IndicesOptions;
@@ -30,8 +32,8 @@ export interface ChunkingConfig {
   time_span?: string;
 }
 
-interface Aggregation {
-  buckets: {
+export interface Aggregation {
+  [key: string]: {
     date_histogram: {
       field: string;
       fixed_interval: string;

x-pack/plugins/ml/common/types/fields.ts

@@ -89,3 +89,16 @@ export const mlCategory: Field = {
   type: ES_FIELD_TYPES.KEYWORD,
   aggregatable: false,
 };
+
+export interface FieldAggCardinality {
+  field: string;
+  percent?: any;
+}
+
+export interface ScriptAggCardinality {
+  script: any;
+}
+
+export interface AggCardinality {
+  cardinality: FieldAggCardinality | ScriptAggCardinality;
+}

x-pack/plugins/ml/common/util/job_utils.test.ts

@@ -188,8 +188,8 @@ describe('ML - job utils', () => {
       expect(isTimeSeriesViewDetector(job, 3)).toBe(false);
     });
 
-    test('returns false for a detector using a script field as a metric field_name', () => {
-      expect(isTimeSeriesViewDetector(job, 4)).toBe(false);
+    test('returns true for a detector using a script field as a metric field_name', () => {
+      expect(isTimeSeriesViewDetector(job, 4)).toBe(true);
     });
   });
 
@@ -281,6 +281,7 @@ describe('ML - job utils', () => {
       expect(isSourceDataChartableForDetector(job, 22)).toBe(true);
       expect(isSourceDataChartableForDetector(job, 23)).toBe(true);
       expect(isSourceDataChartableForDetector(job, 24)).toBe(true);
+      expect(isSourceDataChartableForDetector(job, 37)).toBe(true);
     });
 
     test('returns false for expected detectors', () => {
@@ -296,7 +297,6 @@ describe('ML - job utils', () => {
       expect(isSourceDataChartableForDetector(job, 34)).toBe(false);
       expect(isSourceDataChartableForDetector(job, 35)).toBe(false);
       expect(isSourceDataChartableForDetector(job, 36)).toBe(false);
-      expect(isSourceDataChartableForDetector(job, 37)).toBe(false);
     });
   });
 

x-pack/plugins/ml/common/util/job_utils.ts

@@ -95,7 +95,6 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex
       // Perform extra check to see if the detector is using a scripted field.
       const scriptFields = Object.keys(job.datafeed_config.script_fields);
       isSourceDataChartable =
-        scriptFields.indexOf(dtr.field_name!) === -1 &&
         scriptFields.indexOf(dtr.partition_field_name!) === -1 &&
         scriptFields.indexOf(dtr.by_field_name!) === -1 &&
         scriptFields.indexOf(dtr.over_field_name!) === -1;

x-pack/plugins/ml/common/util/validation_utils.ts

@@ -31,3 +31,22 @@ export function isValidJson(json: string) {
     return false;
   }
 }
+
+export function findAggField(aggs: Record<string, any>, fieldName: string): any {
+  let value;
+  Object.keys(aggs).some(function (k) {
+    if (k === fieldName) {
+      value = aggs[k];
+      return true;
+    }
+    if (aggs.hasOwnProperty(k) && typeof aggs[k] === 'object') {
+      value = findAggField(aggs[k], fieldName);
+      return value !== undefined;
+    }
+  });
+  return value;
+}
+
+export function isValidAggregationField(aggs: Record<string, any>, fieldName: string): boolean {
+  return findAggField(aggs, fieldName) !== undefined;
+}

x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js

@@ -125,7 +125,8 @@ export const anomalyDataChange = function (
           config.timeField,
           range.min,
           range.max,
-          bucketSpanSeconds * 1000
+          bucketSpanSeconds * 1000,
+          config.datafeedConfig
         )
         .toPromise();
     } else {

x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/model_memory_estimator.ts

@@ -134,6 +134,7 @@ export const useModelMemoryEstimator = (
   // Update model memory estimation payload on the job creator updates
   useEffect(() => {
     modelMemoryEstimator.update({
+      datafeedConfig: jobCreator.datafeedConfig,
       analysisConfig: jobCreator.jobConfig.analysis_config,
       indexPattern: jobCreator.indexPatternTitle,
       query: jobCreator.datafeedConfig.query,

x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx

@@ -61,9 +61,12 @@ export const DatafeedPreview: FC<{
     if (combinedJob.datafeed_config && combinedJob.datafeed_config.indices.length) {
       try {
         const resp = await mlJobService.searchPreview(combinedJob);
-        const data = resp.aggregations
-          ? resp.aggregations.buckets.buckets.slice(0, ML_DATA_PREVIEW_COUNT)
-          : resp.hits.hits;
+        let data = resp.hits.hits;
+        // the first item under aggregations can be any name
+        if (typeof resp.aggregations === 'object' && Object.keys(resp.aggregations).length > 0) {
+          const accessor = Object.keys(resp.aggregations)[0];
+          data = resp.aggregations[accessor].buckets.slice(0, ML_DATA_PREVIEW_COUNT);
+        }
 
         setPreviewJsonString(JSON.stringify(data, null, 2));
       } catch (error) {

x-pack/plugins/ml/public/application/services/ml_api_service/index.ts

@@ -32,7 +32,11 @@ import {
   FieldHistogramRequestConfig,
   FieldRequestConfig,
 } from '../../datavisualizer/index_based/common';
-import { DataRecognizerConfigResponse, Module } from '../../../../common/types/modules';
+import {
+  DatafeedOverride,
+  DataRecognizerConfigResponse,
+  Module,
+} from '../../../../common/types/modules';
 import { getHttp } from '../../util/dependency_cache';
 
 export interface MlInfoResponse {
@@ -628,13 +632,15 @@ export function mlApiServicesProvider(httpService: HttpService) {
     },
 
     calculateModelMemoryLimit$({
+      datafeedConfig,
       analysisConfig,
       indexPattern,
       query,
       timeFieldName,
       earliestMs,
       latestMs,
     }: {
+      datafeedConfig: DatafeedOverride;
       analysisConfig: AnalysisConfig;
       indexPattern: string;
       query: any;
@@ -643,6 +649,7 @@ export function mlApiServicesProvider(httpService: HttpService) {
       latestMs: number;
     }) {
       const body = JSON.stringify({
+        datafeedConfig,
         analysisConfig,
         indexPattern,
         query,

x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts

@@ -19,6 +19,9 @@ import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils';
 import { JobId } from '../../../../common/types/anomaly_detection_jobs';
 import { MlApiServices } from '../ml_api_service';
 import { CriteriaField } from './index';
+import type { DatafeedOverride } from '../../../../common/types/modules';
+import type { Aggregation } from '../../../../common/types/anomaly_detection_jobs/datafeed';
+import { findAggField } from '../../../../common/util/validation_utils';
 
 interface ResultResponse {
   success: boolean;
@@ -68,8 +71,12 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) {
       timeFieldName: string,
       earliestMs: number,
       latestMs: number,
-      intervalMs: number
+      intervalMs: number,
+      dataFeedConfig?: DatafeedOverride
     ): Observable<MetricData> {
+      const scriptFields: any | undefined = dataFeedConfig?.script_fields;
+      const aggFields: Aggregation | undefined = dataFeedConfig?.aggregations;
+
       // Build the criteria to use in the bool filter part of the request.
       // Add criteria for the time range, entity fields,
       // plus any additional supplied query.
@@ -150,15 +157,35 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) {
         body.aggs.byTime.aggs = {};
 
         const metricAgg: any = {
-          [metricFunction]: {
-            field: metricFieldName,
-          },
+          [metricFunction]: {},
         };
+        if (scriptFields !== undefined && scriptFields[metricFieldName] !== undefined) {
+          metricAgg[metricFunction].script = scriptFields[metricFieldName].script;
+        } else {
+          metricAgg[metricFunction].field = metricFieldName;
+        }
 
         if (metricFunction === 'percentiles') {
           metricAgg[metricFunction].percents = [ML_MEDIAN_PERCENTS];
         }
-        body.aggs.byTime.aggs.metric = metricAgg;
+
+        // when the field is an aggregation field, because the field doesn't actually exist in the indices
+        // we need to pass all the sub aggs from the original datafeed config
+        // so that we can access the aggregated field
+        if (typeof aggFields === 'object' && Object.keys(aggFields).length > 0) {
+          // first item under aggregations can be any name, not necessarily 'buckets'
+          const accessor = Object.keys(aggFields)[0];
+          const tempAggs = { ...(aggFields[accessor].aggs ?? aggFields[accessor].aggregations) };
+          const foundValue = findAggField(tempAggs, metricFieldName);
+
+          if (foundValue !== undefined) {
+            tempAggs.metric = foundValue;
+            delete tempAggs[metricFieldName];
+          }
+          body.aggs.byTime.aggs = tempAggs;
+        } else {
+          body.aggs.byTime.aggs.metric = metricAgg;
+        }
       }
 
       return mlApiServices.esSearch$({ index, body }).pipe(

x-pack/plugins/ml/public/application/services/results_service/results_service.js

@@ -285,7 +285,7 @@ export function resultsServiceProvider(mlApiServices) {
                       influencerFieldValues: {
                         terms: {
                           field: 'influencer_field_value',
-                          size: maxFieldValues,
+                          size: !!maxFieldValues ? maxFieldValues : ANOMALY_SWIM_LANE_HARD_LIMIT,
                           order: {
                             maxAnomalyScore: 'desc',
                           },
@@ -415,7 +415,7 @@ export function resultsServiceProvider(mlApiServices) {
                   influencerFieldValues: {
                     terms: {
                       field: 'influencer_field_value',
-                      size: maxResults !== undefined ? maxResults : 2,
+                      size: !!maxResults ? maxResults : 2,
                       order: {
                         maxAnomalyScore: 'desc',
                       },

x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts

@@ -93,7 +93,8 @@ function getMetricData(
         chartConfig.timeField,
         earliestMs,
         latestMs,
-        intervalMs
+        intervalMs,
+        chartConfig?.datafeedConfig
       )
       .pipe(
         map((resp) => {

x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts

@@ -10,6 +10,7 @@ import { MLCATEGORY } from '../../../common/constants/field_types';
 import { AnalysisConfig } from '../../../common/types/anomaly_detection_jobs';
 import { fieldsServiceProvider } from '../fields_service';
 import { MlInfoResponse } from '../../../common/types/ml_server_info';
+import { DatafeedOverride } from '../../../common/types/modules';
 import type { MlClient } from '../../lib/ml_client';
 
 export interface ModelMemoryEstimationResult {
@@ -46,7 +47,8 @@ const cardinalityCheckProvider = (client: IScopedClusterClient) => {
     query: any,
     timeFieldName: string,
     earliestMs: number,
-    latestMs: number
+    latestMs: number,
+    datafeedConfig?: DatafeedOverride
   ): Promise<{
     overallCardinality: { [key: string]: number };
     maxBucketCardinality: { [key: string]: number };
@@ -101,7 +103,8 @@ const cardinalityCheckProvider = (client: IScopedClusterClient) => {
         query,
         timeFieldName,
         earliestMs,
-        latestMs
+        latestMs,
+        datafeedConfig
       );
     }
 
@@ -142,7 +145,8 @@ export function calculateModelMemoryLimitProvider(
     timeFieldName: string,
     earliestMs: number,
     latestMs: number,
-    allowMMLGreaterThanMax = false
+    allowMMLGreaterThanMax = false,
+    datafeedConfig?: DatafeedOverride
   ): Promise<ModelMemoryEstimationResult> {
     const { body: info } = await mlClient.info<MlInfoResponse>();
     const maxModelMemoryLimit = info.limits.max_model_memory_limit?.toUpperCase();
@@ -154,7 +158,8 @@ export function calculateModelMemoryLimitProvider(
       query,
       timeFieldName,
       earliestMs,
-      latestMs
+      latestMs,
+      datafeedConfig
     );
 
     const { body } = await mlClient.estimateModelMemory<ModelMemoryEstimateResponse>({