[Alerting] formalize alert status and add status fields to alert save…

…d object

resolves elastic#51099

This formalizes the concept of "alert status", in terms of it's execution, with
some new fields in the alert saved object and types used with the alert client
and http APIs.

These fields are read-only from the client point-of-view; they are provided in
the alert structures, but are only updated by the alerting framework itself.
The values will be updated after each run of the alert type executor.

The data is added to the alert as the `executionStatus` field, with the
following shape:

```ts
interface AlertExecutionStatus {
  status: 'ok' | 'active' | 'error' | 'unknown';
  date: Date;
  error?: {
    reason: 'read' | 'decrypt' | 'execute' | 'unknown';
    message: string;
  };
}
```

interim commits:

calculate the execution status, some refactoring
write the execution status to the alert after execution
use real date in execution status on create
add an await to an async fn
comment out status update to see if SIEM FT succeeds
fix SIEM FT alert deletion issue
use partial updates and retries in alerts clients to avoid conflicts
fix jest tests
clean up conflict-fixin code
moar conflict-prevention fixing
fix type error with find result
add reasons to alert execution errors
add some jest tests
add some function tests
fix status update to use alert namespace
fix function test
finish function tests
more fixes after rebase
fix type checks and jest tests after rebase
add migration and find functional tests
fix relative import

x-pack/plugins/alerts/common/alert.ts

@@ -15,6 +15,18 @@ export interface IntervalSchedule extends SavedObjectAttributes {
   interval: string;
 }
 
+export type AlertExecutionStatuses = 'ok' | 'active' | 'error' | 'unknown';
+export type AlertExecutionStatusErrorReasons = 'read' | 'decrypt' | 'execute' | 'unknown';
+
+export interface AlertExecutionStatus {
+  status: AlertExecutionStatuses;
+  date: Date;
+  error?: {
+    reason: AlertExecutionStatusErrorReasons;
+    message: string;
+  };
+}
+
 export type AlertActionParams = SavedObjectAttributes;
 
 export interface AlertAction {
@@ -44,6 +56,7 @@ export interface Alert {
   throttle: string | null;
   muteAll: boolean;
   mutedInstanceIds: string[];
+  executionStatus: AlertExecutionStatus;
 }
 
 export type SanitizedAlert = Omit<Alert, 'apiKey'>;

x-pack/plugins/alerts/server/alerts_client.test.ts

@@ -393,6 +393,11 @@ describe('create()', () => {
         "createdAt": "2019-02-12T21:01:22.479Z",
         "createdBy": "elastic",
         "enabled": true,
+        "executionStatus": Object {
+          "date": "2019-02-12T21:01:22.479Z",
+          "error": null,
+          "status": "unknown",
+        },
         "meta": Object {
           "versionApiKeyLastmodified": "v7.10.0",
         },
@@ -1029,6 +1034,11 @@ describe('create()', () => {
         muteAll: false,
         mutedInstanceIds: [],
         tags: ['foo'],
+        executionStatus: {
+          date: '2019-02-12T21:01:22.479Z',
+          status: 'unknown',
+          error: null,
+        },
       },
       {
         references: [
@@ -1145,6 +1155,11 @@ describe('create()', () => {
         muteAll: false,
         mutedInstanceIds: [],
         tags: ['foo'],
+        executionStatus: {
+          date: '2019-02-12T21:01:22.479Z',
+          status: 'unknown',
+          error: null,
+        },
       },
       {
         references: [
@@ -2496,6 +2511,11 @@ const BaseAlertInstanceSummarySavedObject: SavedObject<RawAlert> = {
     throttle: null,
     muteAll: false,
     mutedInstanceIds: [],
+    executionStatus: {
+      status: 'unknown',
+      date: '2020-08-20T19:23:38Z',
+      error: null,
+    },
   },
   references: [],
 };

x-pack/plugins/alerts/server/alerts_client.ts

@@ -28,7 +28,7 @@ import {
   AlertTaskState,
   AlertInstanceSummary,
 } from './types';
-import { validateAlertTypeParams } from './lib';
+import { validateAlertTypeParams, alertExecutionStatusFromRaw } from './lib';
 import {
   InvalidateAPIKeyParams,
   GrantAPIKeyResult as SecurityPluginGrantAPIKeyResult,
@@ -122,6 +122,7 @@ export interface CreateOptions {
     | 'muteAll'
     | 'mutedInstanceIds'
     | 'actions'
+    | 'executionStatus'
   > & { actions: NormalizedAlertAction[] };
   options?: {
     migrationVersion?: Record<string, string>;
@@ -228,6 +229,11 @@ export class AlertsClient {
       params: validatedAlertTypeParams as RawAlert['params'],
       muteAll: false,
       mutedInstanceIds: [],
+      executionStatus: {
+        status: 'unknown',
+        date: new Date().toISOString(),
+        error: null,
+      },
     };
     const createdAlert = await this.unsecuredSavedObjectsClient.create(
       'alert',
@@ -961,9 +967,14 @@ export class AlertsClient {
     updatedAt: SavedObject['updated_at'] = createdAt,
     references: SavedObjectReference[] | undefined
   ): PartialAlert {
+    const rawAlertWithoutExecutionStatus: Partial<Omit<RawAlert, 'executionStatus'>> = {
+      ...rawAlert,
+    };
+    delete rawAlertWithoutExecutionStatus.executionStatus;
+    const executionStatus = alertExecutionStatusFromRaw(rawAlert.executionStatus);
     return {
       id,
-      ...rawAlert,
+      ...rawAlertWithoutExecutionStatus,
       // we currently only support the Interval Schedule type
       // Once we support additional types, this type signature will likely change
       schedule: rawAlert.schedule as IntervalSchedule,
@@ -973,6 +984,7 @@ export class AlertsClient {
       ...(updatedAt ? { updatedAt: new Date(updatedAt) } : {}),
       ...(createdAt ? { createdAt: new Date(createdAt) } : {}),
       ...(scheduledTaskId ? { scheduledTaskId } : {}),
+      ...(executionStatus ? { executionStatus } : {}),
     };
   }
 

x-pack/plugins/alerts/server/lib/alert_execution_status.test.ts

@@ -0,0 +1,163 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { AlertExecutionStatusErrorReasons } from '../types';
+import {
+  executionStatusFromState,
+  executionStatusFromError,
+  alertExecutionStatusToRaw,
+  alertExecutionStatusFromRaw,
+} from './alert_execution_status';
+import { ErrorWithReason } from './error_with_reason';
+
+describe('AlertExecutionStatus', () => {
+  describe('executionStatusFromState()', () => {
+    test('empty task state', () => {
+      const status = executionStatusFromState({});
+      checkDateIsNearNow(status.date);
+      expect(status.status).toBe('ok');
+      expect(status.error).toBe(undefined);
+    });
+
+    test('task state with no instances', () => {
+      const status = executionStatusFromState({ alertInstances: {} });
+      checkDateIsNearNow(status.date);
+      expect(status.status).toBe('ok');
+      expect(status.error).toBe(undefined);
+    });
+
+    test('task state with one instance', () => {
+      const status = executionStatusFromState({ alertInstances: { a: {} } });
+      checkDateIsNearNow(status.date);
+      expect(status.status).toBe('active');
+      expect(status.error).toBe(undefined);
+    });
+  });
+
+  describe('executionStatusFromError()', () => {
+    test('error with no reason', () => {
+      const status = executionStatusFromError(new Error('boo!'));
+      expect(status.status).toBe('error');
+      expect(status.error).toMatchInlineSnapshot(`
+        Object {
+          "message": "boo!",
+          "reason": "unknown",
+        }
+      `);
+    });
+
+    test('error with a reason', () => {
+      const status = executionStatusFromError(new ErrorWithReason('execute', new Error('hoo!')));
+      expect(status.status).toBe('error');
+      expect(status.error).toMatchInlineSnapshot(`
+        Object {
+          "message": "hoo!",
+          "reason": "execute",
+        }
+      `);
+    });
+  });
+
+  describe('alertExecutionStatusToRaw()', () => {
+    const date = new Date('2020-09-03T16:26:58Z');
+    const status = 'ok';
+    const reason: AlertExecutionStatusErrorReasons = 'decrypt';
+    const error = { reason, message: 'wops' };
+
+    test('status without an error', () => {
+      expect(alertExecutionStatusToRaw({ date, status })).toMatchInlineSnapshot(`
+        Object {
+          "date": "2020-09-03T16:26:58.000Z",
+          "error": null,
+          "status": "ok",
+        }
+      `);
+    });
+
+    test('status with an error', () => {
+      expect(alertExecutionStatusToRaw({ date, status, error })).toMatchInlineSnapshot(`
+        Object {
+          "date": "2020-09-03T16:26:58.000Z",
+          "error": Object {
+            "message": "wops",
+            "reason": "decrypt",
+          },
+          "status": "ok",
+        }
+      `);
+    });
+  });
+
+  describe('alertExecutionStatusFromRaw()', () => {
+    const date = new Date('2020-09-03T16:26:58Z').toISOString();
+    const status = 'active';
+    const reason: AlertExecutionStatusErrorReasons = 'execute';
+    const error = { reason, message: 'wops' };
+
+    test('no input', () => {
+      const result = alertExecutionStatusFromRaw();
+      expect(result).toBe(undefined);
+    });
+
+    test('undefined input', () => {
+      const result = alertExecutionStatusFromRaw(undefined);
+      expect(result).toBe(undefined);
+    });
+
+    test('null input', () => {
+      const result = alertExecutionStatusFromRaw(null);
+      expect(result).toBe(undefined);
+    });
+
+    test('invalid date', () => {
+      const result = alertExecutionStatusFromRaw({ date: 'an invalid date' })!;
+      checkDateIsNearNow(result.date);
+      expect(result.status).toBe('unknown');
+      expect(result.error).toBe(undefined);
+    });
+
+    test('valid date', () => {
+      const result = alertExecutionStatusFromRaw({ date });
+      expect(result).toMatchInlineSnapshot(`
+        Object {
+          "date": 2020-09-03T16:26:58.000Z,
+          "status": "unknown",
+        }
+      `);
+    });
+
+    test('valid status and date', () => {
+      const result = alertExecutionStatusFromRaw({ status, date });
+      expect(result).toMatchInlineSnapshot(`
+        Object {
+          "date": 2020-09-03T16:26:58.000Z,
+          "status": "active",
+        }
+      `);
+    });
+
+    test('valid status, date and error', () => {
+      const result = alertExecutionStatusFromRaw({ status, date, error });
+      expect(result).toMatchInlineSnapshot(`
+        Object {
+          "date": 2020-09-03T16:26:58.000Z,
+          "error": Object {
+            "message": "wops",
+            "reason": "execute",
+          },
+          "status": "active",
+        }
+      `);
+    });
+  });
+});
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function checkDateIsNearNow(date: any) {
+  expect(date instanceof Date).toBe(true);
+  // allow for lots of slop in the time difference
+  expect(Date.now() - date.valueOf()).toBeLessThanOrEqual(10000);
+}

x-pack/plugins/alerts/server/lib/alert_execution_status.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { AlertTaskState, AlertExecutionStatus, RawAlertExecutionStatus } from '../types';
+import { getReasonFromError } from './error_with_reason';
+
+export function executionStatusFromState(state: AlertTaskState): AlertExecutionStatus {
+  const instanceIds = Object.keys(state.alertInstances ?? {});
+  return {
+    date: new Date(),
+    status: instanceIds.length === 0 ? 'ok' : 'active',
+  };
+}
+
+export function executionStatusFromError(error: Error): AlertExecutionStatus {
+  return {
+    date: new Date(),
+    status: 'error',
+    error: {
+      reason: getReasonFromError(error),
+      message: error.message,
+    },
+  };
+}
+
+export function alertExecutionStatusToRaw({
+  date,
+  status,
+  error,
+}: AlertExecutionStatus): RawAlertExecutionStatus {
+  return {
+    date: date.toISOString(),
+    status,
+    error: error ?? null,
+  };
+}
+
+export function alertExecutionStatusFromRaw(
+  rawAlertExecutionStatus?: Partial<RawAlertExecutionStatus> | null | undefined
+): AlertExecutionStatus | undefined {
+  if (!rawAlertExecutionStatus) return undefined;
+
+  const { date, status = 'unknown', error } = rawAlertExecutionStatus;
+
+  let parsedDateMillis = date ? Date.parse(date) : Date.now();
+  if (isNaN(parsedDateMillis)) {
+    // TODO: log a message?
+    parsedDateMillis = Date.now();
+  }
+
+  const parsedDate = new Date(parsedDateMillis);
+  if (error) {
+    return { date: parsedDate, status, error };
+  } else {
+    return { date: parsedDate, status };
+  }
+}

x-pack/plugins/alerts/server/lib/alert_instance_summary_from_event_log.test.ts

@@ -511,4 +511,8 @@ const BaseAlert: SanitizedAlert = {
   createdAt: new Date(),
   updatedAt: new Date(),
   apiKeyOwner: null,
+  executionStatus: {
+    status: 'unknown',
+    date: new Date('2020-08-20T19:23:38Z'),
+  },
 };

x-pack/plugins/alerts/server/lib/error_with_reason.test.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ErrorWithReason, getReasonFromError, isErrorWithReason } from './error_with_reason';
+
+describe('ErrorWithReason', () => {
+  const plainError = new Error('well, actually');
+  const errorWithReason = new ErrorWithReason('decrypt', plainError);
+
+  test('ErrorWithReason class', () => {
+    expect(errorWithReason.message).toBe(plainError.message);
+    expect(errorWithReason.error).toBe(plainError);
+    expect(errorWithReason.reason).toBe('decrypt');
+  });
+
+  test('getReasonFromError()', () => {
+    expect(getReasonFromError(plainError)).toBe('unknown');
+    expect(getReasonFromError(errorWithReason)).toBe('decrypt');
+  });
+
+  test('isErrorWithReason()', () => {
+    expect(isErrorWithReason(plainError)).toBe(false);
+    expect(isErrorWithReason(errorWithReason)).toBe(true);
+  });
+});