[Logs UI] log rate setup index validation (#50008)

* Scaffold API endpoint

* Implement the API endpoint

* Implement API client

* Set error messages in `useAnalysisSetupState`

* Show validation errors next to the submit button

* Check for setup errors regarding the selected indexes

* Call validation only once

Enrich the `availableIndices` array with validation information to show
it later in the form.

* Ensure validation runs before showing the indices

* Adjust naming conventions

- Replace `index_pattern` with `indices`, since it means something
  different in kibana.
- Group validation actions under the `validation` namespace.

* Move index error messages to the `InitialConfigurationStep`

* Move error messages to the UI layer

* Move validation call to `useAnalysisSetupState`

* Pass timestamp as a parameter of `useAnalysisSetupState`

* Fix regression with the index names in the API response

* Use `_field_caps` api

* s/timestamp/timestampField/g

* Tweak error messages

* Move `ValidationIndicesUIError` to `log_analysis_setup_state`

* Track validation status

It's safer to rely on the state of the promise instead of treating an
empty array as "loading"

* Handle network errors

* Use individual `<EuiCheckbox />` elements for the indices

This allows to disable individual checkboxes

* Pass the whole `validatedIndices` array to the inner objects

This will make easier to determine which indeces have errors in the
checkbox list itself and simplify the state we keep track of.

* Disable indices with errors

Show a tooltip above the disabled index to explain why it cannot be
selected.

* Pass indices to the API as an array

* Show overlay while the validation loads

* Wrap tooltips on a `block` element

Prevents the checkboxes from collapsing on the same line

* Use the right dependencies for `useEffect => validateIndices()`

* Restore formatter function name

* Simplify mapping of selected indices to errors

* s/checked/isSelected/g

* Make errors field-generic

* Allow multiple errors per index

* Simplify code a bit

x-pack/legacy/plugins/infra/common/http_api/log_analysis/index.ts

@@ -5,3 +5,4 @@
  */
 
 export * from './results';
+export * from './validation';

x-pack/legacy/plugins/infra/common/http_api/log_analysis/validation/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './indices';

x-pack/legacy/plugins/infra/common/http_api/log_analysis/validation/indices.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+export const LOG_ANALYSIS_VALIDATION_INDICES_PATH = '/api/infra/log_analysis/validation/indices';
+
+/**
+ * Request types
+ */
+export const validationIndicesRequestPayloadRT = rt.type({
+  data: rt.type({
+    timestampField: rt.string,
+    indices: rt.array(rt.string),
+  }),
+});
+
+export type ValidationIndicesRequestPayload = rt.TypeOf<typeof validationIndicesRequestPayloadRT>;
+
+/**
+ * Response types
+ * */
+export const validationIndicesErrorRT = rt.union([
+  rt.type({
+    error: rt.literal('INDEX_NOT_FOUND'),
+    index: rt.string,
+  }),
+  rt.type({
+    error: rt.literal('FIELD_NOT_FOUND'),
+    index: rt.string,
+    field: rt.string,
+  }),
+  rt.type({
+    error: rt.literal('FIELD_NOT_VALID'),
+    index: rt.string,
+    field: rt.string,
+  }),
+]);
+
+export type ValidationIndicesError = rt.TypeOf<typeof validationIndicesErrorRT>;
+
+export const validationIndicesResponsePayloadRT = rt.type({
+  data: rt.type({
+    errors: rt.array(validationIndicesErrorRT),
+  }),
+});
+
+export type ValidationIndicesResponsePayload = rt.TypeOf<typeof validationIndicesResponsePayloadRT>;

x-pack/legacy/plugins/infra/package.json

@@ -16,4 +16,4 @@
     "boom": "7.3.0",
     "lodash": "^4.17.15"
   }
-}
+}

x-pack/legacy/plugins/infra/public/components/loading_overlay_wrapper.tsx

@@ -40,4 +40,5 @@ const OverlayDiv = euiStyled.div`
   position: absolute;
   top: 0;
   width: 100%;
+  z-index: ${props => props.theme.eui.euiZLevel1};
 `;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/index_patterns_validate.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { fold } from 'fp-ts/lib/Either';
+import { pipe } from 'fp-ts/lib/pipeable';
+import { identity } from 'fp-ts/lib/function';
+import { kfetch } from 'ui/kfetch';
+
+import {
+  LOG_ANALYSIS_VALIDATION_INDICES_PATH,
+  validationIndicesRequestPayloadRT,
+  validationIndicesResponsePayloadRT,
+} from '../../../../../common/http_api';
+
+import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
+
+export const callIndexPatternsValidate = async (timestampField: string, indices: string[]) => {
+  const response = await kfetch({
+    method: 'POST',
+    pathname: LOG_ANALYSIS_VALIDATION_INDICES_PATH,
+    body: JSON.stringify(
+      validationIndicesRequestPayloadRT.encode({ data: { timestampField, indices } })
+    ),
+  });
+
+  return pipe(
+    validationIndicesResponsePayloadRT.decode(response),
+    fold(throwErrors(createPlainError), identity)
+  );
+};

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx

@@ -94,8 +94,8 @@ export const useLogAnalysisJobs = ({
         dispatch({ type: 'fetchingJobStatuses' });
         return await callJobsSummaryAPI(spaceId, sourceId);
       },
-      onResolve: response => {
-        dispatch({ type: 'fetchedJobStatuses', payload: response, spaceId, sourceId });
+      onResolve: jobResponse => {
+        dispatch({ type: 'fetchedJobStatuses', payload: jobResponse, spaceId, sourceId });
       },
       onReject: err => {
         dispatch({ type: 'failedFetchingJobStatuses' });
@@ -158,6 +158,7 @@ export const useLogAnalysisJobs = ({
     setup: setupMlModule,
     setupMlModuleRequest,
     setupStatus: statusState.setupStatus,
+    timestampField: timeField,
     viewSetupForReconfiguration,
     viewSetupForUpdate,
     viewResults,

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.tsx

@@ -4,52 +4,91 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { useState, useCallback, useMemo } from 'react';
+import { useState, useCallback, useMemo, useEffect } from 'react';
 
 import { isExampleDataIndex } from '../../../../common/log_analysis';
+import {
+  ValidationIndicesError,
+  ValidationIndicesResponsePayload,
+} from '../../../../common/http_api';
+import { useTrackedPromise } from '../../../utils/use_tracked_promise';
+import { callIndexPatternsValidate } from './api/index_patterns_validate';
 
 type SetupHandler = (
   indices: string[],
   startTime: number | undefined,
   endTime: number | undefined
 ) => void;
 
+export type ValidationIndicesUIError =
+  | ValidationIndicesError
+  | { error: 'NETWORK_ERROR' }
+  | { error: 'TOO_FEW_SELECTED_INDICES' };
+
+export interface ValidatedIndex {
+  index: string;
+  errors: ValidationIndicesError[];
+  isSelected: boolean;
+}
+
 interface AnalysisSetupStateArguments {
   availableIndices: string[];
   cleanupAndSetupModule: SetupHandler;
   setupModule: SetupHandler;
+  timestampField: string;
 }
 
-type IndicesSelection = Record<string, boolean>;
-
-type ValidationErrors = 'TOO_FEW_SELECTED_INDICES';
-
 const fourWeeksInMs = 86400000 * 7 * 4;
 
 export const useAnalysisSetupState = ({
   availableIndices,
   cleanupAndSetupModule,
   setupModule,
+  timestampField,
 }: AnalysisSetupStateArguments) => {
   const [startTime, setStartTime] = useState<number | undefined>(Date.now() - fourWeeksInMs);
   const [endTime, setEndTime] = useState<number | undefined>(undefined);
 
-  const [selectedIndices, setSelectedIndices] = useState<IndicesSelection>(
-    availableIndices.reduce(
-      (indexMap, indexName) => ({
-        ...indexMap,
-        [indexName]: !(availableIndices.length > 1 && isExampleDataIndex(indexName)),
-      }),
-      {}
-    )
+  // Prepare the validation
+  const [validatedIndices, setValidatedIndices] = useState<ValidatedIndex[]>(
+    availableIndices.map(index => ({
+      index,
+      errors: [],
+      isSelected: false,
+    }))
+  );
+  const [validateIndicesRequest, validateIndices] = useTrackedPromise(
+    {
+      cancelPreviousOn: 'resolution',
+      createPromise: async () => {
+        return await callIndexPatternsValidate(timestampField, availableIndices);
+      },
+      onResolve: ({ data }: ValidationIndicesResponsePayload) => {
+        setValidatedIndices(
+          availableIndices.map(index => {
+            const errors = data.errors.filter(error => error.index === index);
+            return {
+              index,
+              errors,
+              isSelected: errors.length === 0 && !isExampleDataIndex(index),
+            };
+          })
+        );
+      },
+      onReject: () => {
+        setValidatedIndices([]);
+      },
+    },
+    [availableIndices, timestampField]
   );
 
+  useEffect(() => {
+    validateIndices();
+  }, [validateIndices]);
+
   const selectedIndexNames = useMemo(
-    () =>
-      Object.entries(selectedIndices)
-        .filter(([_indexName, isSelected]) => isSelected)
-        .map(([indexName]) => indexName),
-    [selectedIndices]
+    () => validatedIndices.filter(i => i.isSelected).map(i => i.index),
+    [validatedIndices]
   );
 
   const setup = useCallback(() => {
@@ -60,24 +99,42 @@ export const useAnalysisSetupState = ({
     return cleanupAndSetupModule(selectedIndexNames, startTime, endTime);
   }, [cleanupAndSetupModule, selectedIndexNames, startTime, endTime]);
 
-  const validationErrors: ValidationErrors[] = useMemo(
+  const isValidating = useMemo(
     () =>
-      Object.values(selectedIndices).some(isSelected => isSelected)
-        ? []
-        : ['TOO_FEW_SELECTED_INDICES' as const],
-    [selectedIndices]
+      validateIndicesRequest.state === 'pending' ||
+      validateIndicesRequest.state === 'uninitialized',
+    [validateIndicesRequest.state]
   );
 
+  const validationErrors = useMemo<ValidationIndicesUIError[]>(() => {
+    if (isValidating) {
+      return [];
+    }
+
+    if (validateIndicesRequest.state === 'rejected') {
+      return [{ error: 'NETWORK_ERROR' }];
+    }
+
+    if (selectedIndexNames.length === 0) {
+      return [{ error: 'TOO_FEW_SELECTED_INDICES' }];
+    }
+
+    return validatedIndices.reduce<ValidationIndicesUIError[]>((errors, index) => {
+      return selectedIndexNames.includes(index.index) ? errors.concat(index.errors) : errors;
+    }, []);
+  }, [selectedIndexNames, validatedIndices, validateIndicesRequest.state]);
+
   return {
     cleanupAndSetup,
     endTime,
+    isValidating,
     selectedIndexNames,
-    selectedIndices,
     setEndTime,
-    setSelectedIndices,
     setStartTime,
     setup,
     startTime,
+    validatedIndices,
+    setValidatedIndices,
     validationErrors,
   };
 };

x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_content.tsx

@@ -27,6 +27,7 @@ export const AnalysisPageContent = () => {
     lastSetupErrorMessages,
     setup,
     setupStatus,
+    timestampField,
     viewResults,
   } = useContext(LogAnalysisJobs.Context);
 
@@ -61,6 +62,7 @@ export const AnalysisPageContent = () => {
         errorMessages={lastSetupErrorMessages}
         setup={setup}
         setupStatus={setupStatus}
+        timestampField={timestampField}
         viewResults={viewResults}
       />
     );

x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_setup_content.tsx

@@ -34,6 +34,7 @@ interface AnalysisSetupContentProps {
   errorMessages: string[];
   setup: SetupHandler;
   setupStatus: SetupStatus;
+  timestampField: string;
   viewResults: () => void;
 }
 
@@ -43,6 +44,7 @@ export const AnalysisSetupContent: React.FunctionComponent<AnalysisSetupContentP
   errorMessages,
   setup,
   setupStatus,
+  timestampField,
   viewResults,
 }) => {
   useTrackPageview({ app: 'infra_logs', path: 'analysis_setup' });
@@ -82,6 +84,7 @@ export const AnalysisSetupContent: React.FunctionComponent<AnalysisSetupContentP
               errorMessages={errorMessages}
               setup={setup}
               setupStatus={setupStatus}
+              timestampField={timestampField}
               viewResults={viewResults}
             />
           </EuiPageContentBody>