[Logs UI] Create screen to set up analysis ML jobs

x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts

@@ -6,8 +6,13 @@
 
 import { JobType } from './log_analysis';
 
+export const bucketSpan = 900000;
+
 export const getJobIdPrefix = (spaceId: string, sourceId: string) =>
   `kibana-logs-ui-${spaceId}-${sourceId}-`;
 
 export const getJobId = (spaceId: string, sourceId: string, jobType: JobType) =>
   `${getJobIdPrefix(spaceId, sourceId)}${jobType}`;
+
+export const getDatafeedId = (spaceId: string, sourceId: string, jobType: JobType) =>
+  `datafeed-${getJobId(spaceId, sourceId, jobType)}`;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts

@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+import { kfetch } from 'ui/kfetch';
+
+import { getJobIdPrefix } from '../../../../../common/log_analysis';
+import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
+
+const MODULE_ID = 'logs_ui_analysis';
+
+// This is needed due to: https://github.com/elastic/kibana/issues/43671
+const removeSampleDataIndex = (indexPattern: string) => {
+  const SAMPLE_DATA_INDEX = 'kibana_sample_data_logs*';
+  const indices = indexPattern.split(',');
+  const sampleDataIndex = indices.findIndex((index: string) => {
+    return index === SAMPLE_DATA_INDEX;
+  });
+  if (sampleDataIndex > -1) {
+    indices.splice(sampleDataIndex, 1);
+    return indices.join(',');
+  } else {
+    return indexPattern;
+  }
+};
+
+export const callSetupMlModuleAPI = async (
+  start: number | undefined,
+  end: number | undefined,
+  spaceId: string,
+  sourceId: string,
+  indexPattern: string,
+  timeField: string,
+  bucketSpan: number
+) => {
+  const response = await kfetch({
+    method: 'POST',
+    pathname: `/api/ml/modules/setup/${MODULE_ID}`,
+    body: JSON.stringify(
+      setupMlModuleRequestPayloadRT.encode({
+        start,
+        end,
+        indexPatternName: removeSampleDataIndex(indexPattern),
+        prefix: getJobIdPrefix(spaceId, sourceId),
+        startDatafeed: true,
+        jobOverrides: [
+          {
+            job_id: 'log-entry-rate',
+            analysis_config: {
+              bucket_span: `${bucketSpan}ms`,
+            },
+            data_description: {
+              time_field: timeField,
+            },
+          },
+        ],
+        datafeedOverrides: [
+          {
+            job_id: 'log-entry-rate',
+            aggregations: {
+              buckets: {
+                date_histogram: {
+                  field: timeField,
+                  fixed_interval: `${bucketSpan}ms`,
+                },
+                aggregations: {
+                  [timeField]: {
+                    max: {
+                      field: `${timeField}`,
+                    },
+                  },
+                  doc_count_per_minute: {
+                    bucket_script: {
+                      script: {
+                        params: {
+                          bucket_span_in_ms: bucketSpan,
+                        },
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        ],
+      })
+    ),
+  });
+
+  return setupMlModuleResponsePayloadRT.decode(response).getOrElseL(throwErrors(createPlainError));
+};
+
+const setupMlModuleTimeParamsRT = rt.partial({
+  start: rt.number,
+  end: rt.number,
+});
+
+const setupMlModuleRequestParamsRT = rt.type({
+  indexPatternName: rt.string,
+  prefix: rt.string,
+  startDatafeed: rt.boolean,
+  jobOverrides: rt.array(rt.object),
+  datafeedOverrides: rt.array(rt.object),
+});
+
+const setupMlModuleRequestPayloadRT = rt.intersection([
+  setupMlModuleTimeParamsRT,
+  setupMlModuleRequestParamsRT,
+]);
+
+const setupMlModuleResponsePayloadRT = rt.type({
+  datafeeds: rt.array(
+    rt.type({
+      id: rt.string,
+      started: rt.boolean,
+      success: rt.boolean,
+    })
+  ),
+  jobs: rt.array(
+    rt.type({
+      id: rt.string,
+      success: rt.boolean,
+    })
+  ),
+});
+
+export type SetupMlModuleResponsePayload = rt.TypeOf<typeof setupMlModuleResponsePayloadRT>;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx

@@ -6,48 +6,70 @@
 
 import createContainer from 'constate-latest';
 import { useMemo, useEffect, useState } from 'react';
-import { values } from 'lodash';
-import { getJobId } from '../../../../common/log_analysis';
+import { bucketSpan, getJobId } from '../../../../common/log_analysis';
 import { useTrackedPromise } from '../../../utils/use_tracked_promise';
+import { callSetupMlModuleAPI, SetupMlModuleResponsePayload } from './api/ml_setup_module_api';
 import { callJobsSummaryAPI } from './api/ml_get_jobs_summary_api';
 
-type JobStatus = 'unknown' | 'closed' | 'closing' | 'failed' | 'opened' | 'opening' | 'deleted';
-// type DatafeedStatus = 'unknown' | 'started' | 'starting' | 'stopped' | 'stopping' | 'deleted';
+// combines and abstracts job and datafeed status
+type JobStatus =
+  | 'unknown'
+  | 'missing'
+  | 'inconsistent'
+  | 'created'
+  | 'started'
+  | 'opening'
+  | 'opened';
 
 export const useLogAnalysisJobs = ({
   indexPattern,
   sourceId,
   spaceId,
+  timeField,
 }: {
   indexPattern: string;
   sourceId: string;
   spaceId: string;
+  timeField: string;
 }) => {
   const [jobStatus, setJobStatus] = useState<{
     logEntryRate: JobStatus;
   }>({
     logEntryRate: 'unknown',
   });
 
-  // const [setupMlModuleRequest, setupMlModule] = useTrackedPromise(
-  //   {
-  //     cancelPreviousOn: 'resolution',
-  //     createPromise: async () => {
-  //       kfetch({
-  //         method: 'POST',
-  //         pathname: '/api/ml/modules/setup',
-  //         body: JSON.stringify(
-  //           setupMlModuleRequestPayloadRT.encode({
-  //             indexPatternName: indexPattern,
-  //             prefix: getJobIdPrefix(spaceId, sourceId),
-  //             startDatafeed: true,
-  //           })
-  //         ),
-  //       });
-  //     },
-  //   },
-  //   [indexPattern, spaceId, sourceId]
-  // );
+  const [setupMlModuleRequest, setupMlModule] = useTrackedPromise(
+    {
+      cancelPreviousOn: 'resolution',
+      createPromise: async (start, end) => {
+        return await callSetupMlModuleAPI(
+          start,
+          end,
+          spaceId,
+          sourceId,
+          indexPattern,
+          timeField,
+          bucketSpan
+        );
+      },
+      onResolve: ({ datafeeds, jobs }: SetupMlModuleResponsePayload) => {
+        const hasSuccessfullyCreatedJobs = jobs.every(job => job.success);
+        const hasSuccessfullyStartedDatafeeds = datafeeds.every(
+          datafeed => datafeed.success && datafeed.started
+        );
+
+        setJobStatus(currentJobStatus => ({
+          ...currentJobStatus,
+          logEntryRate: hasSuccessfullyCreatedJobs
+            ? hasSuccessfullyStartedDatafeeds
+              ? 'started'
+              : 'created'
+            : 'inconsistent',
+        }));
+      },
+    },
+    [indexPattern, spaceId, sourceId]
+  );
 
   const [fetchJobStatusRequest, fetchJobStatus] = useTrackedPromise(
     {
@@ -77,20 +99,34 @@ export const useLogAnalysisJobs = ({
   }, []);
 
   const isSetupRequired = useMemo(() => {
-    const jobStates = values(jobStatus);
+    const jobStates = Object.values(jobStatus);
     return (
-      jobStates.filter(state => state === 'opened' || state === 'opening').length < jobStates.length
+      jobStates.filter(state => ['opened', 'opening', 'created', 'started'].includes(state))
+        .length < jobStates.length
     );
   }, [jobStatus]);
 
   const isLoadingSetupStatus = useMemo(() => fetchJobStatusRequest.state === 'pending', [
     fetchJobStatusRequest.state,
   ]);
 
+  const isSettingUpMlModule = useMemo(() => setupMlModuleRequest.state === 'pending', [
+    setupMlModuleRequest.state,
+  ]);
+
+  const didSetupFail = useMemo(
+    () => !isSettingUpMlModule && setupMlModuleRequest.state !== 'uninitialized' && isSetupRequired,
+    [setupMlModuleRequest.state, jobStatus]
+  );
+
   return {
     jobStatus,
     isSetupRequired,
     isLoadingSetupStatus,
+    setupMlModule,
+    setupMlModuleRequest,
+    isSettingUpMlModule,
+    didSetupFail,
   };
 };