[Alerting] Initial implementation of alerting task cancel()

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -199,6 +199,7 @@ kibana_vars=(
     xpack.alerting.invalidateApiKeysTask.interval
     xpack.alerting.invalidateApiKeysTask.removalDelay
     xpack.alerting.defaultRuleTaskTimeout
+    xpack.alerting.cancelAlertsOnRuleTimeout
     xpack.alerts.healthCheck.interval
     xpack.alerts.invalidateApiKeysTask.interval
     xpack.alerts.invalidateApiKeysTask.removalDelay

x-pack/plugins/alerting/README.md

@@ -119,6 +119,7 @@ The following table describes the properties of the `options` object.
 |producer|The id of the application producing this rule type.|string|
 |minimumLicenseRequired|The value of a minimum license. Most of the rules are licensed as "basic".|string|
 |ruleTaskTimeout|The length of time a rule can run before being cancelled due to timeout. By default, this value is "5m".|string|
+|cancelAlertsOnRuleTimeout|Whether to skip writing alerts and scheduling actions if a rule execution is cancelled due to timeout. By default, this value is set to "true".|boolean|
 |useSavedObjectReferences.extractReferences|(Optional) When developing a rule type, you can choose to implement hooks for extracting saved object references from rule parameters. This hook will be invoked when a rule is created or updated. Implementing this hook is optional, but if an extract hook is implemented, an inject hook must also be implemented.|Function
 |useSavedObjectReferences.injectReferences|(Optional) When developing a rule type, you can choose to implement hooks for injecting saved object references into rule parameters. This hook will be invoked when a rule is retrieved (get or find). Implementing this hook is optional, but if an inject hook is implemented, an extract hook must also be implemented.|Function
 |isExportable|Whether the rule type is exportable from the Saved Objects Management UI.|boolean|

x-pack/plugins/alerting/common/alert.ts

@@ -30,6 +30,7 @@ export enum AlertExecutionStatusErrorReasons {
   Execute = 'execute',
   Unknown = 'unknown',
   License = 'license',
+  Timeout = 'timeout',
 }
 
 export interface AlertExecutionStatus {

x-pack/plugins/alerting/server/config.test.ts

@@ -12,6 +12,7 @@ describe('config validation', () => {
     const config: Record<string, unknown> = {};
     expect(configSchema.validate(config)).toMatchInlineSnapshot(`
       Object {
+        "cancelAlertsOnRuleTimeout": true,
         "defaultRuleTaskTimeout": "5m",
         "healthCheck": Object {
           "interval": "60m",

x-pack/plugins/alerting/server/config.ts

@@ -21,6 +21,7 @@ export const configSchema = schema.object({
     defaultValue: DEFAULT_MAX_EPHEMERAL_ACTIONS_PER_ALERT,
   }),
   defaultRuleTaskTimeout: schema.string({ validate: validateDurationSchema, defaultValue: '5m' }),
+  cancelAlertsOnRuleTimeout: schema.boolean({ defaultValue: true }),
 });
 
 export type AlertsConfig = TypeOf<typeof configSchema>;

x-pack/plugins/alerting/server/health/get_state.test.ts

@@ -73,6 +73,7 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
         },
         maxEphemeralActionsPerAlert: 100,
         defaultRuleTaskTimeout: '20m',
+        cancelAlertsOnRuleTimeout: true,
       }),
       pollInterval
     ).subscribe();
@@ -108,6 +109,7 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
         },
         maxEphemeralActionsPerAlert: 100,
         defaultRuleTaskTimeout: '20m',
+        cancelAlertsOnRuleTimeout: true,
       }),
       pollInterval,
       retryDelay
@@ -154,6 +156,7 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
         },
         maxEphemeralActionsPerAlert: 100,
         defaultRuleTaskTimeout: '20m',
+        cancelAlertsOnRuleTimeout: true,
       })
     ).toPromise();
 
@@ -186,6 +189,7 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
         },
         maxEphemeralActionsPerAlert: 100,
         defaultRuleTaskTimeout: '20m',
+        cancelAlertsOnRuleTimeout: true,
       })
     ).toPromise();
 
@@ -218,6 +222,7 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
         },
         maxEphemeralActionsPerAlert: 100,
         defaultRuleTaskTimeout: '20m',
+        cancelAlertsOnRuleTimeout: true,
       })
     ).toPromise();
 
@@ -247,6 +252,7 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
         },
         maxEphemeralActionsPerAlert: 100,
         defaultRuleTaskTimeout: '20m',
+        cancelAlertsOnRuleTimeout: true,
       }),
       retryDelay
     ).subscribe((status) => {
@@ -279,6 +285,7 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
         },
         maxEphemeralActionsPerAlert: 100,
         defaultRuleTaskTimeout: '20m',
+        cancelAlertsOnRuleTimeout: true,
       }),
       retryDelay
     ).subscribe((status) => {
@@ -317,6 +324,7 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
         },
         maxEphemeralActionsPerAlert: 100,
         defaultRuleTaskTimeout: '20m',
+        cancelAlertsOnRuleTimeout: true,
       })
     ).toPromise();
 

x-pack/plugins/alerting/server/plugin.test.ts

@@ -39,6 +39,7 @@ describe('Alerting Plugin', () => {
         },
         maxEphemeralActionsPerAlert: 10,
         defaultRuleTaskTimeout: '5m',
+        cancelAlertsOnRuleTimeout: true,
       });
       plugin = new AlertingPlugin(context);
 
@@ -73,6 +74,7 @@ describe('Alerting Plugin', () => {
         },
         maxEphemeralActionsPerAlert: 10,
         defaultRuleTaskTimeout: '5m',
+        cancelAlertsOnRuleTimeout: true,
       });
       plugin = new AlertingPlugin(context);
 
@@ -153,6 +155,15 @@ describe('Alerting Plugin', () => {
         await setup.registerType(ruleType);
         expect(ruleType.ruleTaskTimeout).toBe('5m');
       });
+
+      it('should apply default config value for cancelAlertsOnRuleTimeout if undefined on rule type', async () => {
+        const ruleType = {
+          ...sampleAlertType,
+          minimumLicenseRequired: 'basic',
+        } as AlertType<never, never, never, never, never, 'default', never>;
+        await setup.registerType(ruleType);
+        expect(ruleType.cancelAlertsOnRuleTimeout).toBe(true);
+      });
     });
   });
 
@@ -169,6 +180,7 @@ describe('Alerting Plugin', () => {
           },
           maxEphemeralActionsPerAlert: 10,
           defaultRuleTaskTimeout: '5m',
+          cancelAlertsOnRuleTimeout: true,
         });
         const plugin = new AlertingPlugin(context);
 
@@ -210,6 +222,7 @@ describe('Alerting Plugin', () => {
           },
           maxEphemeralActionsPerAlert: 10,
           defaultRuleTaskTimeout: '5m',
+          cancelAlertsOnRuleTimeout: true,
         });
         const plugin = new AlertingPlugin(context);
 
@@ -265,6 +278,7 @@ describe('Alerting Plugin', () => {
         },
         maxEphemeralActionsPerAlert: 100,
         defaultRuleTaskTimeout: '5m',
+        cancelAlertsOnRuleTimeout: true,
       });
       const plugin = new AlertingPlugin(context);
 

x-pack/plugins/alerting/server/plugin.ts

@@ -74,6 +74,7 @@ export const EVENT_LOG_ACTIONS = {
   newInstance: 'new-instance',
   recoveredInstance: 'recovered-instance',
   activeInstance: 'active-instance',
+  executeTimeout: 'execute-timeout',
 };
 export const LEGACY_EVENT_LOG_ACTIONS = {
   resolvedInstance: 'resolved-instance',
@@ -306,9 +307,11 @@ export class AlertingPlugin {
         if (!(alertType.minimumLicenseRequired in LICENSE_TYPE)) {
           throw new Error(`"${alertType.minimumLicenseRequired}" is not a valid license type`);
         }
-        if (!alertType.ruleTaskTimeout) {
+        if (!alertType.ruleTaskTimeout || alertType.cancelAlertsOnRuleTimeout === undefined) {
           alertingConfig.then((config) => {
-            alertType.ruleTaskTimeout = config.defaultRuleTaskTimeout;
+            alertType.ruleTaskTimeout = alertType.ruleTaskTimeout ?? config.defaultRuleTaskTimeout;
+            alertType.cancelAlertsOnRuleTimeout =
+              alertType.cancelAlertsOnRuleTimeout ?? config.cancelAlertsOnRuleTimeout;
             ruleTypeRegistry.register(alertType);
           });
         } else {
@@ -386,21 +389,24 @@ export class AlertingPlugin {
       return alertingAuthorizationClientFactory!.create(request);
     };
 
-    taskRunnerFactory.initialize({
-      logger,
-      getServices: this.getServicesFactory(core.savedObjects, core.elasticsearch),
-      getRulesClientWithRequest,
-      spaceIdToNamespace,
-      actionsPlugin: plugins.actions,
-      encryptedSavedObjectsClient,
-      basePathService: core.http.basePath,
-      eventLogger: this.eventLogger!,
-      internalSavedObjectsRepository: core.savedObjects.createInternalRepository(['alert']),
-      executionContext: core.executionContext,
-      ruleTypeRegistry: this.ruleTypeRegistry!,
-      kibanaBaseUrl: this.kibanaBaseUrl,
-      supportsEphemeralTasks: plugins.taskManager.supportsEphemeralTasks(),
-      maxEphemeralActionsPerAlert: this.config.then((config) => config.maxEphemeralActionsPerAlert),
+    this.config.then((config) => {
+      taskRunnerFactory.initialize({
+        logger,
+        getServices: this.getServicesFactory(core.savedObjects, core.elasticsearch),
+        getRulesClientWithRequest,
+        spaceIdToNamespace,
+        actionsPlugin: plugins.actions,
+        encryptedSavedObjectsClient,
+        basePathService: core.http.basePath,
+        eventLogger: this.eventLogger!,
+        internalSavedObjectsRepository: core.savedObjects.createInternalRepository(['alert']),
+        executionContext: core.executionContext,
+        ruleTypeRegistry: this.ruleTypeRegistry!,
+        kibanaBaseUrl: this.kibanaBaseUrl,
+        supportsEphemeralTasks: plugins.taskManager.supportsEphemeralTasks(),
+        maxEphemeralActionsPerAlert: config.maxEphemeralActionsPerAlert,
+        cancelAlertsOnRuleTimeout: config.cancelAlertsOnRuleTimeout,
+      });
     });
 
     this.eventLogService!.registerSavedObjectProvider('alert', (request) => {

x-pack/plugins/alerting/server/task_runner/create_execution_handler.test.ts

@@ -99,7 +99,7 @@ const createExecutionHandlerParams: jest.Mocked<
     stateVal: 'My other {{state.value}} goes here',
   },
   supportsEphemeralTasks: false,
-  maxEphemeralActionsPerAlert: Promise.resolve(10),
+  maxEphemeralActionsPerAlert: 10,
 };
 
 beforeEach(() => {

x-pack/plugins/alerting/server/task_runner/create_execution_handler.ts

@@ -56,7 +56,7 @@ export interface CreateExecutionHandlerOptions<
   request: KibanaRequest;
   alertParams: AlertTypeParams;
   supportsEphemeralTasks: boolean;
-  maxEphemeralActionsPerAlert: Promise<number>;
+  maxEphemeralActionsPerAlert: number;
 }
 
 interface ExecutionHandlerOptions<ActionGroupIds extends string> {
@@ -157,7 +157,7 @@ export function createExecutionHandler<
     const alertLabel = `${alertType.id}:${alertId}: '${alertName}'`;
 
     const actionsClient = await actionsPlugin.getActionsClientWithRequest(request);
-    let ephemeralActionsToSchedule = await maxEphemeralActionsPerAlert;
+    let ephemeralActionsToSchedule = maxEphemeralActionsPerAlert;
     for (const action of actions) {
       if (
         !actionsPlugin.isActionExecutable(action.id, action.actionTypeId, { notifyUsage: true })

x-pack/plugins/alerting/server/task_runner/task_runner.test.ts

@@ -107,7 +107,8 @@ describe('Task Runner', () => {
     ruleTypeRegistry,
     kibanaBaseUrl: 'https://localhost:5601',
     supportsEphemeralTasks: false,
-    maxEphemeralActionsPerAlert: new Promise((resolve) => resolve(10)),
+    maxEphemeralActionsPerAlert: 10,
+    cancelAlertsOnRuleTimeout: true,
   };
 
   function testAgainstEphemeralSupport(
@@ -285,7 +286,7 @@ describe('Task Runner', () => {
     expect(call.services).toBeTruthy();
 
     const logger = taskRunnerFactoryInitializerParams.logger;
-    expect(logger.debug).toHaveBeenCalledTimes(2);
+    expect(logger.debug).toHaveBeenCalledTimes(3);
     expect(logger.debug).nthCalledWith(1, 'executing alert test:1 at 1970-01-01T00:00:00.000Z');
     expect(logger.debug).nthCalledWith(
       2,
@@ -432,7 +433,7 @@ describe('Task Runner', () => {
     `);
 
         const logger = customTaskRunnerFactoryInitializerParams.logger;
-        expect(logger.debug).toHaveBeenCalledTimes(3);
+        expect(logger.debug).toHaveBeenCalledTimes(4);
         expect(logger.debug).nthCalledWith(1, 'executing alert test:1 at 1970-01-01T00:00:00.000Z');
         expect(logger.debug).nthCalledWith(
           2,
@@ -648,7 +649,7 @@ describe('Task Runner', () => {
     expect(actionsClient.ephemeralEnqueuedExecution).toHaveBeenCalledTimes(0);
 
     const logger = taskRunnerFactoryInitializerParams.logger;
-    expect(logger.debug).toHaveBeenCalledTimes(4);
+    expect(logger.debug).toHaveBeenCalledTimes(5);
     expect(logger.debug).nthCalledWith(1, 'executing alert test:1 at 1970-01-01T00:00:00.000Z');
     expect(logger.debug).nthCalledWith(
       2,
@@ -848,7 +849,7 @@ describe('Task Runner', () => {
         expect(enqueueFunction).toHaveBeenCalledTimes(1);
 
         const logger = customTaskRunnerFactoryInitializerParams.logger;
-        expect(logger.debug).toHaveBeenCalledTimes(4);
+        expect(logger.debug).toHaveBeenCalledTimes(5);
         expect(logger.debug).nthCalledWith(1, 'executing alert test:1 at 1970-01-01T00:00:00.000Z');
         expect(logger.debug).nthCalledWith(
           2,
@@ -1537,7 +1538,7 @@ describe('Task Runner', () => {
         `);
 
         const logger = customTaskRunnerFactoryInitializerParams.logger;
-        expect(logger.debug).toHaveBeenCalledTimes(4);
+        expect(logger.debug).toHaveBeenCalledTimes(5);
         expect(logger.debug).nthCalledWith(1, 'executing alert test:1 at 1970-01-01T00:00:00.000Z');
         expect(logger.debug).nthCalledWith(
           2,
@@ -4339,7 +4340,7 @@ describe('Task Runner', () => {
     expect(call.services).toBeTruthy();
 
     const logger = taskRunnerFactoryInitializerParams.logger;
-    expect(logger.debug).toHaveBeenCalledTimes(2);
+    expect(logger.debug).toHaveBeenCalledTimes(3);
     expect(logger.debug).nthCalledWith(1, 'executing alert test:1 at 1970-01-01T00:00:00.000Z');
     expect(logger.debug).nthCalledWith(
       2,