[ML] Additional job spaces initialization (#83127) (#83516)

* [ML] Additional job spaces initialization * adding logs test * updating integrations * updating test text * fixing logs jobs error * fix bug with duplicate ids * updating initialization log text * fixing initialization text * adding metrics overrides Co-authored-by: Kibana Machine <[email protected]> Co-authored-by: Kibana Machine <[email protected]>
elastic · Nov 17, 2020 · 72933f6 · 72933f6
1 parent 076b2cd
commit 72933f6
Show file tree

Hide file tree

Showing 22 changed files with 583 additions and 154 deletions.
diff --git a/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts b/x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts
@@ -77,6 +77,7 @@ async function createAnomalyDetectionJob({
     prefix: `${APM_ML_JOB_GROUP}-${snakeCase(environment)}-${randomToken}-`,
     groups: [APM_ML_JOB_GROUP],
     indexPatternName,
+    applyToAllSpaces: true,
     query: {
       bool: {
         filter: [

diff --git a/x-pack/plugins/ml/common/types/saved_objects.ts b/x-pack/plugins/ml/common/types/saved_objects.ts
@@ -6,3 +6,12 @@
 
 export type JobType = 'anomaly-detector' | 'data-frame-analytics';
 export const ML_SAVED_OBJECT_TYPE = 'ml-job';
+
+type Result = Record<string, { success: boolean; error?: any }>;
+
+export interface RepairSavedObjectResponse {
+  savedObjectsCreated: Result;
+  savedObjectsDeleted: Result;
+  datafeedsAdded: Result;
+  datafeedsRemoved: Result;
+}
diff --git a/x-pack/plugins/ml/server/lib/route_guard.ts b/x-pack/plugins/ml/server/lib/route_guard.ts
@@ -13,6 +13,7 @@ import {
   SavedObjectsClientContract,
 } from 'kibana/server';
 import { SpacesPluginSetup } from '../../../spaces/server';
+import type { SecurityPluginSetup } from '../../../security/server';
 
 import { jobSavedObjectServiceFactory, JobSavedObjectService } from '../saved_objects';
 import { MlLicense } from '../../common/license';
@@ -36,19 +37,22 @@ export class RouteGuard {
   private _getMlSavedObjectClient: GetMlSavedObjectClient;
   private _getInternalSavedObjectClient: GetInternalSavedObjectClient;
   private _spacesPlugin: SpacesPluginSetup | undefined;
+  private _authorization: SecurityPluginSetup['authz'] | undefined;
   private _isMlReady: () => Promise<void>;
 
   constructor(
     mlLicense: MlLicense,
     getSavedObject: GetMlSavedObjectClient,
     getInternalSavedObject: GetInternalSavedObjectClient,
     spacesPlugin: SpacesPluginSetup | undefined,
+    authorization: SecurityPluginSetup['authz'] | undefined,
     isMlReady: () => Promise<void>
   ) {
     this._mlLicense = mlLicense;
     this._getMlSavedObjectClient = getSavedObject;
     this._getInternalSavedObjectClient = getInternalSavedObject;
     this._spacesPlugin = spacesPlugin;
+    this._authorization = authorization;
     this._isMlReady = isMlReady;
   }
 
@@ -81,6 +85,7 @@ export class RouteGuard {
         mlSavedObjectClient,
         internalSavedObjectsClient,
         this._spacesPlugin !== undefined,
+        this._authorization,
         this._isMlReady
       );
       const client = context.core.elasticsearch.client;

diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.test.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.test.ts
@@ -8,6 +8,7 @@ import { SavedObjectsClientContract, KibanaRequest, IScopedClusterClient } from
 import { Module } from '../../../common/types/modules';
 import { DataRecognizer } from '../data_recognizer';
 import type { MlClient } from '../../lib/ml_client';
+import { JobSavedObjectService } from '../../saved_objects';
 
 const callAs = () => Promise.resolve({ body: {} });
 
@@ -26,6 +27,7 @@ describe('ML - data recognizer', () => {
       find: jest.fn(),
       bulkCreate: jest.fn(),
     } as unknown) as SavedObjectsClientContract,
+    {} as JobSavedObjectService,
     { headers: { authorization: '' } } as KibanaRequest
   );
 

diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts
@@ -44,6 +44,7 @@ import { jobServiceProvider } from '../job_service';
 import { resultsServiceProvider } from '../results_service';
 import { JobExistResult, JobStat } from '../../../common/types/data_recognizer';
 import { MlJobsStatsResponse } from '../job_service/jobs';
+import { JobSavedObjectService } from '../../saved_objects';
 
 const ML_DIR = 'ml';
 const KIBANA_DIR = 'kibana';
@@ -108,6 +109,8 @@ export class DataRecognizer {
   private _client: IScopedClusterClient;
   private _mlClient: MlClient;
   private _savedObjectsClient: SavedObjectsClientContract;
+  private _jobSavedObjectService: JobSavedObjectService;
+  private _request: KibanaRequest;
 
   private _authorizationHeader: object;
   private _modulesDir = `${__dirname}/modules`;
@@ -127,11 +130,14 @@ export class DataRecognizer {
     mlClusterClient: IScopedClusterClient,
     mlClient: MlClient,
     savedObjectsClient: SavedObjectsClientContract,
+    jobSavedObjectService: JobSavedObjectService,
     request: KibanaRequest
   ) {
     this._client = mlClusterClient;
     this._mlClient = mlClient;
     this._savedObjectsClient = savedObjectsClient;
+    this._jobSavedObjectService = jobSavedObjectService;
+    this._request = request;
     this._authorizationHeader = getAuthorizationHeader(request);
     this._jobsService = jobServiceProvider(mlClusterClient, mlClient);
     this._resultsService = resultsServiceProvider(mlClient);
@@ -394,7 +400,8 @@ export class DataRecognizer {
     end?: number,
     jobOverrides?: JobOverride | JobOverride[],
     datafeedOverrides?: DatafeedOverride | DatafeedOverride[],
-    estimateModelMemory: boolean = true
+    estimateModelMemory: boolean = true,
+    applyToAllSpaces: boolean = false
   ) {
     // load the config from disk
     const moduleConfig = await this.getModule(moduleId, jobPrefix);
@@ -458,7 +465,7 @@ export class DataRecognizer {
       if (useDedicatedIndex === true) {
         moduleConfig.jobs.forEach((job) => (job.config.results_index_name = job.id));
       }
-      saveResults.jobs = await this.saveJobs(moduleConfig.jobs);
+      saveResults.jobs = await this.saveJobs(moduleConfig.jobs, applyToAllSpaces);
     }
 
     // create the datafeeds
@@ -699,8 +706,8 @@ export class DataRecognizer {
   // save the jobs.
   // if any fail (e.g. it already exists), catch the error and mark the result
   // as success: false
-  async saveJobs(jobs: ModuleJob[]): Promise<JobResponse[]> {
-    return await Promise.all(
+  async saveJobs(jobs: ModuleJob[], applyToAllSpaces: boolean = false): Promise<JobResponse[]> {
+    const resp = await Promise.all(
       jobs.map(async (job) => {
         const jobId = job.id;
         try {
@@ -712,6 +719,19 @@ export class DataRecognizer {
         }
       })
     );
+    if (applyToAllSpaces === true) {
+      const canCreateGlobalJobs = await this._jobSavedObjectService.canCreateGlobalJobs(
+        this._request
+      );
+      if (canCreateGlobalJobs === true) {
+        await this._jobSavedObjectService.assignJobsToSpaces(
+          'anomaly-detector',
+          jobs.map((j) => j.id),
+          ['*']
+        );
+      }
+    }
+    return resp;
   }
 
   async saveJob(job: ModuleJob) {

diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts
@@ -16,6 +16,7 @@ import {
   IClusterClient,
   SavedObjectsServiceStart,
 } from 'kibana/server';
+import type { SecurityPluginSetup } from '../../security/server';
 import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
 import { SpacesPluginSetup } from '../../spaces/server';
 import { PluginsSetup, RouteInitialization } from './types';
@@ -68,6 +69,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
   private clusterClient: IClusterClient | null = null;
   private savedObjectsStart: SavedObjectsServiceStart | null = null;
   private spacesPlugin: SpacesPluginSetup | undefined;
+  private security: SecurityPluginSetup | undefined;
   private isMlReady: Promise<void>;
   private setMlReady: () => void = () => {};
 
@@ -80,6 +82,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
 
   public setup(coreSetup: CoreSetup, plugins: PluginsSetup): MlPluginSetup {
     this.spacesPlugin = plugins.spaces;
+    this.security = plugins.security;
     const { admin, user, apmUser } = getPluginPrivileges();
 
     plugins.features.registerKibanaFeature({
@@ -140,6 +143,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
         getMlSavedObjectsClient,
         getInternalSavedObjectsClient,
         plugins.spaces,
+        plugins.security?.authz,
         () => this.isMlReady
       ),
       mlLicense: this.mlLicense,
@@ -185,6 +189,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
         this.mlLicense,
         plugins.spaces,
         plugins.cloud,
+        plugins.security?.authz,
         resolveMlCapabilities,
         () => this.clusterClient,
         () => getInternalSavedObjectsClient(),
@@ -202,6 +207,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
     // and create them if needed.
     const { initializeJobs } = jobSavedObjectsInitializationFactory(
       coreStart,
+      this.security,
       this.spacesPlugin !== undefined
     );
     initializeJobs().finally(() => {