[Alerting] Skip writing alerts when rule execution times out

x-pack/plugins/alerting/server/mocks.ts

@@ -74,6 +74,7 @@ const createAlertServicesMock = <
       .mockReturnValue(alertInstanceFactoryMock),
     savedObjectsClient: savedObjectsClientMock.create(),
     scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(),
+    shouldWriteAlerts: () => true,
   };
 };
 export type AlertServicesMock = ReturnType<typeof createAlertServicesMock>;

x-pack/plugins/alerting/server/task_runner/task_runner.ts

@@ -323,6 +323,7 @@ export class TaskRunner<
               InstanceContext,
               WithoutReservedActionGroups<ActionGroupIds, RecoveryActionGroupId>
             >(alertInstances),
+            shouldWriteAlerts: () => this.shouldLogAndScheduleActionsForAlerts(),
           },
           params,
           state: alertTypeState as State,

x-pack/plugins/alerting/server/types.ts

@@ -75,6 +75,7 @@ export interface AlertServices<
   alertInstanceFactory: (
     id: string
   ) => PublicAlertInstance<InstanceState, InstanceContext, ActionGroupIds>;
+  shouldWriteAlerts: () => boolean;
 }
 
 export interface AlertExecutorOptions<

x-pack/plugins/apm/server/routes/alerts/test_utils/index.ts

@@ -45,6 +45,7 @@ export const createRuleTypeMocks = () => {
     alertInstanceFactory: jest.fn(() => ({ scheduleActions })),
     alertWithLifecycle: jest.fn(),
     logger: loggerMock,
+    shouldWriteAlerts: () => true,
   };
 
   return {

x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts

@@ -350,6 +350,33 @@ describe('createLifecycleExecutor', () => {
       })
     );
   });
+
+  it('does not write alert documents when rule execution is cancelled and feature flags indicate to skip', async () => {
+    const logger = loggerMock.create();
+    const ruleDataClientMock = createRuleDataClientMock();
+    const executor = createLifecycleExecutor(
+      logger,
+      ruleDataClientMock
+    )<{}, TestRuleState, never, never, never>(async (options) => {
+      expect(options.state).toEqual(initialRuleState);
+
+      const nextRuleState: TestRuleState = {
+        aRuleStateKey: 'NEXT_RULE_STATE_VALUE',
+      };
+
+      return nextRuleState;
+    });
+
+    await executor(
+      createDefaultAlertExecutorOptions({
+        params: {},
+        state: { wrapped: initialRuleState, trackedAlerts: {} },
+        shouldWriteAlerts: false,
+      })
+    );
+
+    expect(ruleDataClientMock.getWriter).not.toHaveBeenCalled();
+  });
 });
 
 type TestRuleState = Record<string, unknown> & {

x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts

@@ -140,7 +140,7 @@ export const createLifecycleExecutor =
     >
   ): Promise<WrappedLifecycleRuleState<State>> => {
     const {
-      services: { alertInstanceFactory },
+      services: { alertInstanceFactory, shouldWriteAlerts },
       state: previousState,
     } = options;
 
@@ -281,7 +281,15 @@ export const createLifecycleExecutor =
     const newEventsToIndex = makeEventsDataMapFor(newAlertIds);
     const allEventsToIndex = [...trackedEventsToIndex, ...newEventsToIndex];
 
-    if (allEventsToIndex.length > 0 && ruleDataClient.isWriteEnabled()) {
+    // Only write alerts if:
+    // - writing is enabled
+    //   AND
+    //   - rule execution has not been cancelled due to timeout
+    //     OR
+    //   - if execution has been cancelled due to timeout, if feature flags are configured to write alerts anyway
+    const writeAlerts = ruleDataClient.isWriteEnabled() && shouldWriteAlerts();
+
+    if (allEventsToIndex.length > 0 && writeAlerts) {
       logger.debug(`Preparing to index ${allEventsToIndex.length} alerts.`);
 
       await ruleDataClient.getWriter().bulk({
@@ -307,6 +315,6 @@ export const createLifecycleExecutor =
 
     return {
       wrapped: nextWrappedState ?? ({} as State),
-      trackedAlerts: ruleDataClient.isWriteEnabled() ? nextTrackedAlerts : {},
+      trackedAlerts: writeAlerts ? nextTrackedAlerts : {},
     };
   };

x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts

@@ -21,7 +21,7 @@ import { createLifecycleRuleTypeFactory } from './create_lifecycle_rule_type_fac
 
 type RuleTestHelpers = ReturnType<typeof createRule>;
 
-function createRule() {
+function createRule(shouldWriteAlerts: boolean = true) {
   const ruleDataClientMock = createRuleDataClientMock();
 
   const factory = createLifecycleRuleTypeFactory({
@@ -110,6 +110,7 @@ function createRule() {
           alertInstanceFactory,
           savedObjectsClient: {} as any,
           scopedClusterClient: {} as any,
+          shouldWriteAlerts: () => shouldWriteAlerts,
         },
         spaceId: 'spaceId',
         state,
@@ -152,6 +153,26 @@ describe('createLifecycleRuleTypeFactory', () => {
       });
     });
 
+    describe('when rule is cancelled due to timeout and config flags indicate to skip actions', () => {
+      beforeEach(() => {
+        helpers = createRule(false);
+        helpers.ruleDataClientMock.isWriteEnabled.mockReturnValue(true);
+      });
+
+      it("doesn't persist anything", async () => {
+        await helpers.alertWithLifecycle([
+          {
+            id: 'opbeans-java',
+            fields: {
+              'service.name': 'opbeans-java',
+            },
+          },
+        ]);
+
+        expect(helpers.ruleDataClientMock.getWriter().bulk).toHaveBeenCalledTimes(0);
+      });
+    });
+
     describe('when alerts are new', () => {
       beforeEach(async () => {
         await helpers.alertWithLifecycle([

x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts

@@ -25,7 +25,16 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
               const numAlerts = alerts.length;
               logger.debug(`Found ${numAlerts} alerts.`);
 
-              if (ruleDataClient.isWriteEnabled() && numAlerts) {
+              // Only write alerts if:
+              // - writing is enabled
+              //   AND
+              //   - rule execution has not been cancelled due to timeout
+              //     OR
+              //   - if execution has been cancelled due to timeout, if feature flags are configured to write alerts anyway
+              const writeAlerts =
+                ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts();
+
+              if (writeAlerts && numAlerts) {
                 const commonRuleFields = getCommonAlertFields(options);
 
                 const CHUNK_SIZE = 10000;

x-pack/plugins/rule_registry/server/utils/rule_executor_test_utils.ts

@@ -31,6 +31,7 @@ export const createDefaultAlertExecutorOptions = <
   createdAt = new Date(),
   startedAt = new Date(),
   updatedAt = new Date(),
+  shouldWriteAlerts = true,
 }: {
   alertId?: string;
   ruleName?: string;
@@ -39,6 +40,7 @@ export const createDefaultAlertExecutorOptions = <
   createdAt?: Date;
   startedAt?: Date;
   updatedAt?: Date;
+  shouldWriteAlerts?: boolean;
 }): AlertExecutorOptions<Params, State, InstanceState, InstanceContext, ActionGroupIds> => ({
   alertId,
   createdBy: 'CREATED_BY',
@@ -69,6 +71,7 @@ export const createDefaultAlertExecutorOptions = <
       .alertInstanceFactory,
     savedObjectsClient: savedObjectsClientMock.create(),
     scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(),
+    shouldWriteAlerts: () => shouldWriteAlerts,
   },
   state,
   updatedBy: null,

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/preview_rules_route.ts

@@ -122,6 +122,7 @@ export const previewRulesRoute = async (
           ruleTypeId: string,
           ruleTypeName: string,
           params: TParams,
+          shouldWriteAlerts: () => boolean,
           alertInstanceFactory: (
             id: string
           ) => Pick<
@@ -157,6 +158,7 @@ export const previewRulesRoute = async (
               previousStartedAt,
               rule,
               services: {
+                shouldWriteAlerts,
                 alertInstanceFactory,
                 savedObjectsClient: context.core.savedObjects.client,
                 scopedClusterClient: context.core.elasticsearch.client,
@@ -191,6 +193,7 @@ export const previewRulesRoute = async (
           signalRuleAlertType.id,
           signalRuleAlertType.name,
           previewRuleParams,
+          () => true,
           alertInstanceFactoryStub
         );
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts

@@ -78,6 +78,7 @@ export const createRuleTypeMocks = (
     findAlerts: jest.fn(), // TODO: does this stay?
     alertWithPersistence: jest.fn(),
     logger: loggerMock,
+    shouldWriteAlerts: () => true,
   };
 
   return {