[Logs UI] Create screen to set up analysis ML jobs

x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts

@@ -6,8 +6,13 @@
 
 import { JobType } from './log_analysis';
 
+export const bucketSpan = 900000;
+
 export const getJobIdPrefix = (spaceId: string, sourceId: string) =>
   `kibana-logs-ui-${spaceId}-${sourceId}-`;
 
 export const getJobId = (spaceId: string, sourceId: string, jobType: JobType) =>
   `${getJobIdPrefix(spaceId, sourceId)}${jobType}`;
+
+export const getDatafeedId = (spaceId: string, sourceId: string, jobType: JobType) =>
+  `datafeed-${getJobId(spaceId, sourceId, jobType)}`;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+import { kfetch } from 'ui/kfetch';
+
+import { getJobIdPrefix } from '../../../../../common/log_analysis';
+import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
+
+const MODULE_ID = 'logs_ui_analysis';
+
+export const callSetupMlModuleAPI = async (
+  start: number | undefined,
+  end: number | undefined,
+  spaceId: string,
+  sourceId: string,
+  indexPattern: string,
+  timeField: string,
+  bucketSpan: number
+) => {
+  const response = await kfetch({
+    method: 'POST',
+    pathname: `/api/ml/modules/setup/${MODULE_ID}`,
+    body: JSON.stringify(
+      setupMlModuleRequestPayloadRT.encode({
+        start,
+        end,
+        indexPatternName: indexPattern,
+        prefix: getJobIdPrefix(spaceId, sourceId),
+        startDatafeed: true,
+        jobOverrides: [
+          {
+            job_id: 'log-entry-rate',
+            analysis_config: {
+              bucket_span: `${bucketSpan}ms`,
+            },
+            data_description: {
+              time_field: timeField,
+            },
+          },
+        ],
+        datafeedOverrides: [
+          {
+            job_id: 'log-entry-rate',
+            aggregations: {
+              buckets: {
+                date_histogram: {
+                  field: timeField,
+                  fixed_interval: `${bucketSpan}ms`,
+                },
+                aggregations: {
+                  [timeField]: {
+                    max: {
+                      field: `${timeField}`,
+                    },
+                  },
+                  doc_count_per_minute: {
+                    bucket_script: {
+                      script: {
+                        params: {
+                          bucket_span_in_ms: bucketSpan,
+                        },
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        ],
+      })
+    ),
+  });
+
+  return setupMlModuleResponsePayloadRT.decode(response).getOrElseL(throwErrors(createPlainError));
+};
+
+const setupMlModuleTimeParamsRT = rt.partial({
+  start: rt.number,
+  end: rt.number,
+});
+
+const setupMlModuleRequestParamsRT = rt.type({
+  indexPatternName: rt.string,
+  prefix: rt.string,
+  startDatafeed: rt.boolean,
+  jobOverrides: rt.array(rt.object),
+  datafeedOverrides: rt.array(rt.object),
+});
+
+const setupMlModuleRequestPayloadRT = rt.intersection([
+  setupMlModuleTimeParamsRT,
+  setupMlModuleRequestParamsRT,
+]);
+
+const setupMlModuleResponsePayloadRT = rt.type({
+  datafeeds: rt.array(
+    rt.type({
+      id: rt.string,
+      started: rt.boolean,
+      success: rt.boolean,
+    })
+  ),
+  jobs: rt.array(
+    rt.type({
+      id: rt.string,
+      success: rt.boolean,
+    })
+  ),
+});
+
+export type SetupMlModuleResponsePayload = rt.TypeOf<typeof setupMlModuleResponsePayloadRT>;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx

@@ -6,48 +6,70 @@
 
 import createContainer from 'constate-latest';
 import { useMemo, useEffect, useState } from 'react';
-import { values } from 'lodash';
-import { getJobId } from '../../../../common/log_analysis';
+import { bucketSpan, getJobId } from '../../../../common/log_analysis';
 import { useTrackedPromise } from '../../../utils/use_tracked_promise';
+import { callSetupMlModuleAPI, SetupMlModuleResponsePayload } from './api/ml_setup_module_api';
 import { callJobsSummaryAPI } from './api/ml_get_jobs_summary_api';
 
-type JobStatus = 'unknown' | 'closed' | 'closing' | 'failed' | 'opened' | 'opening' | 'deleted';
-// type DatafeedStatus = 'unknown' | 'started' | 'starting' | 'stopped' | 'stopping' | 'deleted';
+// combines and abstracts job and datafeed status
+type JobStatus =
+  | 'unknown'
+  | 'missing'
+  | 'inconsistent'
+  | 'created'
+  | 'started'
+  | 'opening'
+  | 'opened';
 
 export const useLogAnalysisJobs = ({
   indexPattern,
   sourceId,
   spaceId,
+  timeField,
 }: {
   indexPattern: string;
   sourceId: string;
   spaceId: string;
+  timeField: string;
 }) => {
   const [jobStatus, setJobStatus] = useState<{
     logEntryRate: JobStatus;
   }>({
     logEntryRate: 'unknown',
   });
 
-  // const [setupMlModuleRequest, setupMlModule] = useTrackedPromise(
-  //   {
-  //     cancelPreviousOn: 'resolution',
-  //     createPromise: async () => {
-  //       kfetch({
-  //         method: 'POST',
-  //         pathname: '/api/ml/modules/setup',
-  //         body: JSON.stringify(
-  //           setupMlModuleRequestPayloadRT.encode({
-  //             indexPatternName: indexPattern,
-  //             prefix: getJobIdPrefix(spaceId, sourceId),
-  //             startDatafeed: true,
-  //           })
-  //         ),
-  //       });
-  //     },
-  //   },
-  //   [indexPattern, spaceId, sourceId]
-  // );
+  const [setupMlModuleRequest, setupMlModule] = useTrackedPromise(
+    {
+      cancelPreviousOn: 'resolution',
+      createPromise: async (start, end) => {
+        return await callSetupMlModuleAPI(
+          start,
+          end,
+          spaceId,
+          sourceId,
+          indexPattern,
+          timeField,
+          bucketSpan
+        );
+      },
+      onResolve: ({ datafeeds, jobs }: SetupMlModuleResponsePayload) => {
+        const hasSuccessfullyCreatedJobs = jobs.every(job => job.success);
+        const hasSuccessfullyStartedDatafeeds = datafeeds.every(
+          datafeed => datafeed.success && datafeed.started
+        );
+
+        setJobStatus(currentJobStatus => ({
+          ...currentJobStatus,
+          logEntryRate: hasSuccessfullyCreatedJobs
+            ? hasSuccessfullyStartedDatafeeds
+              ? 'started'
+              : 'created'
+            : 'inconsistent',
+        }));
+      },
+    },
+    [indexPattern, spaceId, sourceId]
+  );
 
   const [fetchJobStatusRequest, fetchJobStatus] = useTrackedPromise(
     {
@@ -77,20 +99,34 @@ export const useLogAnalysisJobs = ({
   }, []);
 
   const isSetupRequired = useMemo(() => {
-    const jobStates = values(jobStatus);
+    const jobStates = Object.values(jobStatus);
     return (
-      jobStates.filter(state => state === 'opened' || state === 'opening').length < jobStates.length
+      jobStates.filter(state => ['opened', 'opening', 'created', 'started'].includes(state))
+        .length < jobStates.length
     );
   }, [jobStatus]);
 
   const isLoadingSetupStatus = useMemo(() => fetchJobStatusRequest.state === 'pending', [
     fetchJobStatusRequest.state,
   ]);
 
+  const isSettingUpMlModule = useMemo(() => setupMlModuleRequest.state === 'pending', [
+    setupMlModuleRequest.state,
+  ]);
+
+  const didSetupFail = useMemo(
+    () => !isSettingUpMlModule && setupMlModuleRequest.state !== 'uninitialized' && isSetupRequired,
+    [setupMlModuleRequest.state, jobStatus]
+  );
+
   return {
     jobStatus,
     isSetupRequired,
     isLoadingSetupStatus,
+    setupMlModule,
+    setupMlModuleRequest,
+    isSettingUpMlModule,
+    didSetupFail,
   };
 };
 

x-pack/legacy/plugins/infra/public/pages/logs/analysis/analysis_setup_timerange_form.tsx

@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { useMemo, useState } from 'react';
+import moment, { Moment } from 'moment';
+
+import { i18n } from '@kbn/i18n';
+import {
+  EuiForm,
+  EuiDescribedFormGroup,
+  EuiFormRow,
+  EuiDatePicker,
+  EuiFlexGroup,
+  EuiFormControlLayout,
+} from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+import { CreateMLJobsButton } from './create_ml_jobs_button';
+
+const startTimeLabel = i18n.translate('xpack.infra.analysisSetup.startTimeLabel', {
+  defaultMessage: 'Start time',
+});
+const endTimeLabel = i18n.translate('xpack.infra.analysisSetup.endTimeLabel', {
+  defaultMessage: 'End time',
+});
+const startTimeDefaultDescription = i18n.translate(
+  'xpack.infra.analysisSetup.startTimeDefaultDescription',
+  {
+    defaultMessage: 'Start of log indices',
+  }
+);
+const endTimeDefaultDescription = i18n.translate(
+  'xpack.infra.analysisSetup.endTimeDefaultDescription',
+  {
+    defaultMessage: 'Indefinitely',
+  }
+);
+
+function selectedDateToParam(selectedDate: Moment | null) {
+  if (selectedDate) {
+    return selectedDate.valueOf(); // To ms unix timestamp
+  }
+  return undefined;
+}
+
+export const AnalysisSetupTimerangeForm: React.FunctionComponent<{
+  isSettingUp: boolean;
+  setupMlModule: (startTime: number | undefined, endTime: number | undefined) => Promise<any>;
+}> = ({ isSettingUp, setupMlModule }) => {
+  const [startTime, setStartTime] = useState<Moment | null>(null);
+  const [endTime, setEndTime] = useState<Moment | null>(null);
+
+  const now = useMemo(() => moment(), []);
+  const selectedEndTimeIsToday = !endTime || endTime.isSame(now, 'day');
+
+  const onClickCreateJob = () =>
+    setupMlModule(selectedDateToParam(startTime), selectedDateToParam(endTime));
+
+  return (
+    <EuiForm>
+      <EuiDescribedFormGroup
+        idAria="timeRange"
+        title={
+          <FormattedMessage
+            id="xpack.infra.analysisSetup.timeRangeTitle"
+            defaultMessage="Choose a time range"
+          />
+        }
+        description={
+          <FormattedMessage
+            id="xpack.infra.analysisSetup.timeRangeDescription"
+            defaultMessage="By default, Machine Learning analyzes log messages from the start of your log indices and continues indefinitely. You can specify a different date to begin, to end, or both."
+          />
+        }
+      >
+        <EuiFormRow
+          describedByIds={['timeRange']}
+          error={false}
+          fullWidth
+          isInvalid={false}
+          label={startTimeLabel}
+        >
+          <EuiFlexGroup gutterSize="s">
+            <EuiFormControlLayout
+              clear={startTime ? { onClick: () => setStartTime(null) } : undefined}
+            >
+              <EuiDatePicker
+                showTimeSelect
+                selected={startTime}
+                onChange={setStartTime}
+                placeholder={startTimeDefaultDescription}
+                maxDate={now}
+              />
+            </EuiFormControlLayout>
+          </EuiFlexGroup>
+        </EuiFormRow>
+        <EuiFormRow
+          describedByIds={['timeRange']}
+          error={false}
+          fullWidth
+          isInvalid={false}
+          label={endTimeLabel}
+        >
+          <EuiFlexGroup gutterSize="s">
+            <EuiFormControlLayout clear={endTime ? { onClick: () => setEndTime(null) } : undefined}>
+              <EuiDatePicker
+                showTimeSelect
+                selected={endTime}
+                onChange={setEndTime}
+                placeholder={endTimeDefaultDescription}
+                openToDate={now}
+                minDate={now}
+                minTime={
+                  selectedEndTimeIsToday
+                    ? now
+                    : moment()
+                        .hour(0)
+                        .minutes(0)
+                }
+                maxTime={moment()
+                  .hour(23)
+                  .minutes(59)}
+              />
+            </EuiFormControlLayout>
+          </EuiFlexGroup>
+        </EuiFormRow>
+        <CreateMLJobsButton isLoading={isSettingUp} onClick={onClickCreateJob} />
+      </EuiDescribedFormGroup>
+    </EuiForm>
+  );
+};