[ML] Alerting rule for Anomaly Detection jobs monitoring

src/core/public/doc_links/doc_links_service.ts

@@ -242,6 +242,7 @@ export class DocLinksService {
           anomalyDetectionJobResource: `${ELASTICSEARCH_DOCS}ml-put-job.html#ml-put-job-path-parms`,
           anomalyDetectionJobResourceAnalysisConfig: `${ELASTICSEARCH_DOCS}ml-put-job.html#put-analysisconfig`,
           anomalyDetectionJobTips: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-job-tips`,
+          alertingRules: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-configuring-alerts.html`,
           anomalyDetectionModelMemoryLimits: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-model-memory-limits`,
           calendars: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-calendars`,
           classificationEvaluation: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-dfa-classification.html#ml-dfanalytics-classification-evaluation`,

x-pack/plugins/ml/common/constants/alerts.ts

@@ -6,46 +6,22 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { ActionGroup } from '../../../alerting/common';
-import { MINIMUM_FULL_LICENSE } from '../license';
-import { PLUGIN_ID } from './app';
 
 export const ML_ALERT_TYPES = {
   ANOMALY_DETECTION: 'xpack.ml.anomaly_detection_alert',
+  AD_JOBS_HEALTH: 'xpack.ml.anomaly_detection_jobs_health',
 } as const;
 
 export type MlAlertType = typeof ML_ALERT_TYPES[keyof typeof ML_ALERT_TYPES];
 
-export const ANOMALY_SCORE_MATCH_GROUP_ID = 'anomaly_score_match';
-export type AnomalyScoreMatchGroupId = typeof ANOMALY_SCORE_MATCH_GROUP_ID;
-export const THRESHOLD_MET_GROUP: ActionGroup<AnomalyScoreMatchGroupId> = {
-  id: ANOMALY_SCORE_MATCH_GROUP_ID,
-  name: i18n.translate('xpack.ml.anomalyDetectionAlert.actionGroupName', {
-    defaultMessage: 'Anomaly score matched the condition',
-  }),
-};
-
-export const ML_ALERT_TYPES_CONFIG: Record<
-  MlAlertType,
-  {
-    name: string;
-    actionGroups: Array<ActionGroup<AnomalyScoreMatchGroupId>>;
-    defaultActionGroupId: AnomalyScoreMatchGroupId;
-    minimumLicenseRequired: string;
-    producer: string;
-  }
-> = {
-  [ML_ALERT_TYPES.ANOMALY_DETECTION]: {
-    name: i18n.translate('xpack.ml.anomalyDetectionAlert.name', {
-      defaultMessage: 'Anomaly detection alert',
-    }),
-    actionGroups: [THRESHOLD_MET_GROUP],
-    defaultActionGroupId: ANOMALY_SCORE_MATCH_GROUP_ID,
-    minimumLicenseRequired: MINIMUM_FULL_LICENSE,
-    producer: PLUGIN_ID,
-  },
-};
-
 export const ALERT_PREVIEW_SAMPLE_SIZE = 5;
 
 export const TOP_N_BUCKETS_COUNT = 1;
+
+export const ALL_JOBS_SELECTION = '*';
+
+export const HEALTH_CHECK_NAMES = {
+  datafeed: i18n.translate('xpack.ml.alertTypes.jobsHealthAlertingRule.datafeedCheckName', {
+    defaultMessage: 'Datafeed is not started',
+  }),
+};

x-pack/plugins/ml/common/types/alerts.ts

@@ -108,3 +108,38 @@ export type MlAnomalyDetectionAlertRule = Omit<Alert<MlAnomalyDetectionAlertPara
 export interface JobAlertingRuleStats {
   alerting_rules?: MlAnomalyDetectionAlertRule[];
 }
+
+interface CommonHealthCheckConfig {
+  enabled: boolean;
+}
+
+export type MlAnomalyDetectionJobsHealthRuleParams = {
+  includeJobs: {
+    jobIds?: string[];
+    groupIds?: string[];
+  };
+  excludeJobs?: {
+    jobIds?: string[];
+    groupIds?: string[];
+  } | null;
+  testsConfig?: {
+    datafeed?: CommonHealthCheckConfig | null;
+    mml?: CommonHealthCheckConfig | null;
+    delayedData?:
+      | (CommonHealthCheckConfig & {
+          docsCount?: number | null;
+          timeInterval?: string | null;
+        })
+      | null;
+    behindRealtime?:
+      | (CommonHealthCheckConfig & {
+          timeInterval?: string | null;
+        })
+      | null;
+    errorMessages?: CommonHealthCheckConfig | null;
+  } | null;
+} & AlertTypeParams;
+
+export type JobsHealthRuleTestsConfig = MlAnomalyDetectionJobsHealthRuleParams['testsConfig'];
+
+export type JobsHealthTests = keyof Exclude<JobsHealthRuleTestsConfig, null | undefined>;

x-pack/plugins/ml/common/util/alerts.test.ts

@@ -5,7 +5,11 @@
  * 2.0.
  */
 
-import { getLookbackInterval, resolveLookbackInterval } from './alerts';
+import {
+  getLookbackInterval,
+  getResultJobsHealthRuleConfig,
+  resolveLookbackInterval,
+} from './alerts';
 import type { CombinedJobWithStats, Datafeed, Job } from '../types/anomaly_detection_jobs';
 
 describe('resolveLookbackInterval', () => {
@@ -76,3 +80,49 @@ describe('getLookbackInterval', () => {
     expect(getLookbackInterval(testJobs)).toBe('32m');
   });
 });
+
+describe('getResultJobsHealthRuleConfig', () => {
+  test('returns default config for empty configuration', () => {
+    expect(getResultJobsHealthRuleConfig(null)).toEqual({
+      datafeed: {
+        enabled: true,
+      },
+      mml: {
+        enabled: true,
+      },
+      delayedData: {
+        enabled: true,
+      },
+      behindRealtime: {
+        enabled: true,
+      },
+      errorMessages: {
+        enabled: true,
+      },
+    });
+  });
+  test('returns config with overridden values based on provided configuration', () => {
+    expect(
+      getResultJobsHealthRuleConfig({
+        mml: { enabled: false },
+        errorMessages: { enabled: true },
+      })
+    ).toEqual({
+      datafeed: {
+        enabled: true,
+      },
+      mml: {
+        enabled: false,
+      },
+      delayedData: {
+        enabled: true,
+      },
+      behindRealtime: {
+        enabled: true,
+      },
+      errorMessages: {
+        enabled: true,
+      },
+    });
+  });
+});

x-pack/plugins/ml/common/util/alerts.ts

@@ -9,6 +9,7 @@ import { CombinedJobWithStats, Datafeed, Job } from '../types/anomaly_detection_
 import { resolveMaxTimeInterval } from './job_utils';
 import { isDefined } from '../types/guards';
 import { parseInterval } from './parse_interval';
+import { JobsHealthRuleTestsConfig } from '../types/alerts';
 
 const narrowBucketLength = 60;
 
@@ -51,3 +52,27 @@ export function getTopNBuckets(job: Job): number {
 
   return Math.ceil(narrowBucketLength / bucketSpan.asSeconds());
 }
+
+/**
+ * Returns tests configuration combined with default values.
+ * @param config
+ */
+export function getResultJobsHealthRuleConfig(config: JobsHealthRuleTestsConfig) {
+  return {
+    datafeed: {
+      enabled: config?.datafeed?.enabled ?? true,
+    },
+    mml: {
+      enabled: config?.mml?.enabled ?? true,
+    },
+    delayedData: {
+      enabled: config?.delayedData?.enabled ?? true,
+    },
+    behindRealtime: {
+      enabled: config?.behindRealtime?.enabled ?? true,
+    },
+    errorMessages: {
+      enabled: config?.errorMessages?.enabled ?? true,
+    },
+  };
+}

x-pack/plugins/ml/public/alerting/job_selector.tsx

@@ -5,12 +5,13 @@
  * 2.0.
  */
 
-import React, { FC, useCallback, useEffect, useMemo, useState } from 'react';
+import React, { FC, ReactNode, useCallback, useEffect, useMemo, useState } from 'react';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { EuiComboBox, EuiComboBoxOptionOption, EuiComboBoxProps, EuiFormRow } from '@elastic/eui';
 import { JobId } from '../../common/types/anomaly_detection_jobs';
 import { MlApiServices } from '../application/services/ml_api_service';
+import { ALL_JOBS_SELECTION } from '../../common/constants/alerts';
 
 interface JobSelection {
   jobIds?: JobId[];
@@ -25,13 +26,28 @@ export interface JobSelectorControlProps {
    * Validation is handled by alerting framework
    */
   errors: string[];
+  /** Enables multiple selection of jobs and groups */
+  multiSelect?: boolean;
+  label?: ReactNode;
+  /**
+   * Allows selecting all jobs, even those created afterward.
+   */
+  allowSelectAll?: boolean;
+  /**
+   * Available options to select. By default suggest all existing jobs.
+   */
+  options?: Array<EuiComboBoxOptionOption<string>>;
 }
 
 export const JobSelectorControl: FC<JobSelectorControlProps> = ({
   jobsAndGroupIds,
   onChange,
   adJobsApiService,
   errors,
+  multiSelect = false,
+  label,
+  allowSelectAll = false,
+  options: defaultOptions,
 }) => {
   const [options, setOptions] = useState<Array<EuiComboBoxOptionOption<string>>>([]);
   const jobIds = useMemo(() => new Set(), []);
@@ -60,54 +76,94 @@ export const JobSelectorControl: FC<JobSelectorControlProps> = ({
       });
 
       setOptions([
+        ...(allowSelectAll
+          ? [
+              {
+                label: i18n.translate('xpack.ml.jobSelector.selectAllGroupLabel', {
+                  defaultMessage: 'Select all',
+                }),
+                options: [
+                  {
+                    label: i18n.translate('xpack.ml.jobSelector.selectAllOptionLabel', {
+                      defaultMessage: '*',
+                    }),
+                    value: ALL_JOBS_SELECTION,
+                  },
+                ],
+              },
+            ]
+          : []),
         {
           label: i18n.translate('xpack.ml.jobSelector.jobOptionsLabel', {
             defaultMessage: 'Jobs',
           }),
           options: jobIdOptions.map((v) => ({ label: v })),
         },
+        ...(multiSelect
+          ? [
+              {
+                label: i18n.translate('xpack.ml.jobSelector.groupOptionsLabel', {
+                  defaultMessage: 'Groups',
+                }),
+                options: groupIdOptions.map((v) => ({ label: v })),
+              },
+            ]
+          : []),
       ]);
     } catch (e) {
       // TODO add error handling
     }
   }, [adJobsApiService]);
 
   const onSelectionChange: EuiComboBoxProps<string>['onChange'] = useCallback(
-    (selectionUpdate) => {
+    ((selectionUpdate) => {
+      if (selectionUpdate.some((selectedOption) => selectedOption.value === ALL_JOBS_SELECTION)) {
+        onChange({ jobIds: [ALL_JOBS_SELECTION] });
+        return;
+      }
+
       const selectedJobIds: JobId[] = [];
       const selectedGroupIds: string[] = [];
-      selectionUpdate.forEach(({ label }: { label: string }) => {
-        if (jobIds.has(label)) {
-          selectedJobIds.push(label);
-        } else if (groupIds.has(label)) {
-          selectedGroupIds.push(label);
+      selectionUpdate.forEach(({ label: selectedLabel }: { label: string }) => {
+        if (jobIds.has(selectedLabel)) {
+          selectedJobIds.push(selectedLabel);
+        } else if (groupIds.has(selectedLabel)) {
+          selectedGroupIds.push(selectedLabel);
+        } else if (defaultOptions?.some((v) => v.options?.some((o) => o.label === selectedLabel))) {
+          selectedJobIds.push(selectedLabel);
         }
       });
       onChange({
         ...(selectedJobIds.length > 0 ? { jobIds: selectedJobIds } : {}),
         ...(selectedGroupIds.length > 0 ? { groupIds: selectedGroupIds } : {}),
       });
-    },
-    [jobIds, groupIds]
+    }) as Exclude<EuiComboBoxProps<string>['onChange'], undefined>,
+    [jobIds, groupIds, defaultOptions]
   );
 
   useEffect(() => {
+    if (defaultOptions) return;
     fetchOptions();
   }, []);
 
   return (
     <EuiFormRow
       fullWidth
       label={
-        <FormattedMessage id="xpack.ml.jobSelector.formControlLabel" defaultMessage="Select job" />
+        label ?? (
+          <FormattedMessage
+            id="xpack.ml.jobSelector.formControlLabel"
+            defaultMessage="Select job"
+          />
+        )
       }
       isInvalid={!!errors?.length}
       error={errors}
     >
       <EuiComboBox<string>
-        singleSelection
+        singleSelection={!multiSelect}
         selectedOptions={selectedOptions}
-        options={options}
+        options={defaultOptions ?? options}
         onChange={onSelectionChange}
         fullWidth
         data-test-subj={'mlAnomalyAlertJobSelection'}