[ML] Adding job overrides to the module setup endpoint (elastic#42946)

* [ML] Adding job overrides to the module setup endpoint * removing text code * throw error for incompatible data * changes based on review
jgowdyelastic · Aug 9, 2019 · dfdb1ab · dfdb1ab
1 parent da44c70
commit dfdb1ab
Show file tree

Hide file tree

Showing 4 changed files with 109 additions and 5 deletions.
diff --git a/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js b/x-pack/legacy/plugins/ml/common/util/__tests__/job_utils.js
@@ -521,12 +521,12 @@ describe('ML - job utils', () => {
 
   describe('prefixDatafeedId', () => {
 
-    it('returns datafeed-prefix-job"', () => {
+    it('returns datafeed-prefix-job from datafeed-job"', () => {
       expect(prefixDatafeedId('datafeed-job', 'prefix-')).to.be('datafeed-prefix-job');
     });
 
-    it('returns prefix-job"', () => {
-      expect(prefixDatafeedId('job', 'prefix-')).to.be('prefix-job');
+    it('returns datafeed-prefix-job from job"', () => {
+      expect(prefixDatafeedId('job', 'prefix-')).to.be('datafeed-prefix-job');
     });
   });
 

diff --git a/x-pack/legacy/plugins/ml/common/util/job_utils.js b/x-pack/legacy/plugins/ml/common/util/job_utils.js
@@ -239,7 +239,7 @@ export const ML_DATA_PREVIEW_COUNT = 10;
 export function prefixDatafeedId(datafeedId, prefix) {
   return (datafeedId.match(/^datafeed-/)) ?
     datafeedId.replace(/^datafeed-/, `datafeed-${prefix}`) :
-    `${prefix}${datafeedId}`;
+    `datafeed-${prefix}${datafeedId}`;
 }
 
 // Returns a name which is safe to use in elasticsearch aggregations for the supplied

diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js b/x-pack/legacy/plugins/ml/server/models/data_recognizer/data_recognizer.js
@@ -8,6 +8,7 @@
 
 import fs from 'fs';
 import Boom from 'boom';
+import { merge } from 'lodash';
 import { getLatestDataOrBucketTimestamp, prefixDatafeedId } from '../../../common/util/job_utils';
 import { mlLog } from '../../client/log';
 import { jobServiceProvider } from '../job_service';
@@ -261,6 +262,8 @@ export class DataRecognizer {
     startDatafeed,
     start,
     end,
+    jobOverrides,
+    datafeedOverrides,
     request
   ) {
 
@@ -300,6 +303,9 @@ export class DataRecognizer {
       datafeeds: [],
       savedObjects: []
     };
+
+    this.applyJobConfigOverrides(moduleConfig, jobOverrides, jobPrefix);
+    this.applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix);
     this.updateDatafeedIndices(moduleConfig);
     this.updateJobUrlIndexPatterns(moduleConfig);
 
@@ -754,4 +760,94 @@ export class DataRecognizer {
     return false;
   }
 
+  applyJobConfigOverrides(moduleConfig, jobOverrides, jobPrefix = '') {
+    if(jobOverrides !== undefined && jobOverrides !== null) {
+      if (typeof jobOverrides !== 'object') {
+        throw Boom.badRequest(
+          `Incompatible jobOverrides type (${typeof jobOverrides}). It needs to be an object or array of objects.`
+        );
+      }
+
+      // jobOverrides could be a single object or an array of objects.
+      // if single, convert to an array
+      const overrides = Array.isArray(jobOverrides) ? jobOverrides : [jobOverrides];
+      const { jobs } = moduleConfig;
+
+      // separate all the overrides.
+      // the overrides which don't contain a job id will be applied to all jobs in the module
+      const generalOverrides = [];
+      const jobSpecificOverrides = [];
+      overrides.forEach(o => {
+        if (o.job_id === undefined) {
+          generalOverrides.push(o);
+        } else {
+          jobSpecificOverrides.push(o);
+        }
+      });
+
+      generalOverrides.forEach(o => {
+        jobs.forEach(({ config }) => merge(config, o));
+      });
+
+      jobSpecificOverrides.forEach(o => {
+        // for each override, find the relevant job.
+        // note, the job id already has the prefix prepended to it
+        const job = jobs.find(j => j.id === `${jobPrefix}${o.job_id}`);
+        if (job !== undefined) {
+          // delete the job_id in the override as this shouldn't be overridden
+          delete o.job_id;
+          merge(job.config, o);
+        }
+      });
+    }
+  }
+
+  applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix = '') {
+    if(datafeedOverrides !== undefined && datafeedOverrides !== null) {
+      if (typeof datafeedOverrides !== 'object') {
+        throw Boom.badRequest(
+          `Incompatible datafeedOverrides type (${typeof datafeedOverrides}). It needs to be an object or array of objects.`
+        );
+      }
+
+      // jobOverrides could be a single object or an array of objects.
+      // if single, convert to an array
+      const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides];
+      const { datafeeds } = moduleConfig;
+
+      // separate all the overrides.
+      // the overrides which don't contain a datafeed id or a job id will be applied to all jobs in the module
+      const generalOverrides = [];
+      const datafeedSpecificOverrides = [];
+      overrides.forEach(o => {
+        if (o.datafeed_id === undefined && o.job_id === undefined) {
+          generalOverrides.push(o);
+        } else {
+          datafeedSpecificOverrides.push(o);
+        }
+      });
+
+      generalOverrides.forEach(o => {
+        datafeeds.forEach(({ config }) => {
+          merge(config, o);
+        });
+      });
+
+      // collect all the overrides which contain either a job id or a datafeed id
+      datafeedSpecificOverrides.forEach(o => {
+        // either a job id or datafeed id has been specified, so create a new id
+        // containing either one plus the prefix
+        const tempId = o.datafeed_id !== undefined ? o.datafeed_id : o.job_id;
+        const dId = prefixDatafeedId(tempId, jobPrefix);
+
+        const datafeed = datafeeds.find(d => d.id === dId);
+        if (datafeed !== undefined) {
+          delete o.job_id;
+          delete o.datafeed_id;
+          merge(datafeed.config, o);
+        }
+      });
+    }
+  }
+
 }
diff --git a/x-pack/legacy/plugins/ml/server/routes/modules.js b/x-pack/legacy/plugins/ml/server/routes/modules.js
@@ -36,6 +36,8 @@ function saveModuleItems(
   startDatafeed,
   start,
   end,
+  jobOverrides,
+  datafeedOverrides,
   request
 ) {
   const dr = new DataRecognizer(callWithRequest);
@@ -49,6 +51,8 @@ function saveModuleItems(
     startDatafeed,
     start,
     end,
+    jobOverrides,
+    datafeedOverrides,
     request);
 }
 
@@ -107,7 +111,9 @@ export function dataRecognizer({ commonRouteConfig, elasticsearchPlugin, route }
         useDedicatedIndex,
         startDatafeed,
         start,
-        end
+        end,
+        jobOverrides,
+        datafeedOverrides,
       } = request.payload;
 
       return saveModuleItems(
@@ -121,6 +127,8 @@ export function dataRecognizer({ commonRouteConfig, elasticsearchPlugin, route }
         startDatafeed,
         start,
         end,
+        jobOverrides,
+        datafeedOverrides,
         request
       )
         .catch(resp => wrapError(resp));