[Security Solution] Improve find rule and find rule status route performance

src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts

@@ -68,7 +68,7 @@ export const metricsAggsSchemas: Record<string, ObjectType> = {
     stored_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])),
     from: s.maybe(s.number()),
     size: s.maybe(s.number()),
-    sort: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])),
+    sort: s.maybe(s.any()),
     seq_no_primary_term: s.maybe(s.boolean()),
     version: s.maybe(s.boolean()),
     track_scores: s.maybe(s.boolean()),

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts

@@ -14,10 +14,10 @@ import type { SecuritySolutionPluginRouter } from '../../../../types';
 import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
 import { findRules } from '../../rules/find_rules';
 import { transformError, buildSiemResponse } from '../utils';
-import { getRuleActionsSavedObject } from '../../rule_actions/get_rule_actions_saved_object';
 import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_saved_objects_client';
 import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
 import { transformFindAlerts } from './utils';
+import { getBulkRuleActionsSavedObject } from '../../rule_actions/get_bulk_rule_actions_saved_object';
 
 export const findRulesRoute = (router: SecuritySolutionPluginRouter) => {
   router.get(
@@ -58,44 +58,11 @@ export const findRulesRoute = (router: SecuritySolutionPluginRouter) => {
           filter: query.filter,
           fields: query.fields,
         });
-
-        // if any rules attempted to execute but failed before the rule executor is called,
-        // an execution status will be written directly onto the rule via the kibana alerting framework,
-        // which we are filtering on and will write a failure status
-        // for any rules found to be in a failing state into our rule status saved objects
-        const failingRules = rules.data.filter(
-          (rule) => rule.executionStatus != null && rule.executionStatus.status === 'error'
-        );
-
-        const ruleStatuses = await Promise.all(
-          rules.data.map(async (rule) => {
-            const results = await ruleStatusClient.find({
-              perPage: 1,
-              sortField: 'statusDate',
-              sortOrder: 'desc',
-              search: rule.id,
-              searchFields: ['alertId'],
-            });
-            const failingRule = failingRules.find((badRule) => badRule.id === rule.id);
-            if (failingRule != null) {
-              if (results.saved_objects.length > 0) {
-                results.saved_objects[0].attributes.status = 'failed';
-                results.saved_objects[0].attributes.lastFailureAt = failingRule.executionStatus.lastExecutionDate.toISOString();
-              }
-            }
-            return results;
-          })
-        );
-        const ruleActions = await Promise.all(
-          rules.data.map(async (rule) => {
-            const results = await getRuleActionsSavedObject({
-              savedObjectsClient,
-              ruleAlertId: rule.id,
-            });
-
-            return results;
-          })
-        );
+        const alertIds = rules.data.map((rule) => rule.id);
+        const [ruleStatuses, ruleActions] = await Promise.all([
+          ruleStatusClient.findBulk(alertIds, 1),
+          getBulkRuleActionsSavedObject({ alertIds, savedObjectsClient }),
+        ]);
         const transformed = transformFindAlerts(rules, ruleActions, ruleStatuses);
         if (transformed == null) {
           return siemResponse.error({ statusCode: 500, body: 'Internal error transforming' });

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.ts

@@ -8,13 +8,13 @@
 import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
 import type { SecuritySolutionPluginRouter } from '../../../../types';
 import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
-import { RuleStatusResponse } from '../../rules/types';
 import { transformError, buildSiemResponse, mergeStatuses, getFailingRules } from '../utils';
 import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_saved_objects_client';
 import {
   findRulesStatusesSchema,
   FindRulesStatusesSchemaDecoded,
 } from '../../../../../common/detection_engine/schemas/request/find_rule_statuses_schema';
+import { mergeAlertWithSidecarStatus } from '../../schemas/rule_converters';
 
 /**
  * Given a list of rule ids, return the current status and
@@ -49,45 +49,27 @@ export const findRulesStatusesRoute = (router: SecuritySolutionPluginRouter) =>
       const ids = body.ids;
       try {
         const ruleStatusClient = ruleStatusSavedObjectsClientFactory(savedObjectsClient);
-        const failingRules = await getFailingRules(ids, alertsClient);
+        const [statusesById, failingRules] = await Promise.all([
+          ruleStatusClient.findBulk(ids, 6),
+          getFailingRules(ids, alertsClient),
+        ]);
 
-        const statuses = await ids.reduce(async (acc, id) => {
-          const accumulated = await acc;
-          const lastFiveErrorsForId = await ruleStatusClient.find({
-            perPage: 6,
-            sortField: 'statusDate',
-            sortOrder: 'desc',
-            search: id,
-            searchFields: ['alertId'],
-          });
+        const statuses = ids.reduce((acc, id) => {
+          const lastFiveErrorsForId = statusesById[id];
 
-          if (lastFiveErrorsForId.saved_objects.length === 0) {
-            return accumulated;
+          if (lastFiveErrorsForId == null || lastFiveErrorsForId.length === 0) {
+            return acc;
           }
 
           const failingRule = failingRules[id];
-          const lastFailureAt = lastFiveErrorsForId.saved_objects[0].attributes.lastFailureAt;
 
-          if (
-            failingRule != null &&
-            (lastFailureAt == null ||
-              new Date(failingRule.executionStatus.lastExecutionDate) > new Date(lastFailureAt))
-          ) {
-            const currentStatus = lastFiveErrorsForId.saved_objects[0];
-            currentStatus.attributes.lastFailureMessage = `Reason: ${failingRule.executionStatus.error?.reason} Message: ${failingRule.executionStatus.error?.message}`;
-            currentStatus.attributes.lastFailureAt = failingRule.executionStatus.lastExecutionDate.toISOString();
-            currentStatus.attributes.statusDate = failingRule.executionStatus.lastExecutionDate.toISOString();
-            currentStatus.attributes.status = 'failed';
-            const updatedLastFiveErrorsSO = [
-              currentStatus,
-              ...lastFiveErrorsForId.saved_objects.slice(1),
-            ];
-
-            return mergeStatuses(id, updatedLastFiveErrorsSO, accumulated);
+          if (failingRule != null) {
+            const currentStatus = mergeAlertWithSidecarStatus(failingRule, lastFiveErrorsForId[0]);
+            const updatedLastFiveErrorsSO = [currentStatus, ...lastFiveErrorsForId.slice(1)];
+            return mergeStatuses(id, updatedLastFiveErrorsSO, acc);
           }
-          return mergeStatuses(id, [...lastFiveErrorsForId.saved_objects], accumulated);
-        }, Promise.resolve<RuleStatusResponse>({}));
-
+          return mergeStatuses(id, [...lastFiveErrorsForId], acc);
+        }, {});
         return response.ok({ body: statuses });
       } catch (err) {
         const error = transformError(err);

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts

@@ -256,7 +256,7 @@ describe('utils', () => {
 
   describe('transformFindAlerts', () => {
     test('outputs empty data set when data set is empty correct', () => {
-      const output = transformFindAlerts({ data: [], page: 1, perPage: 0, total: 0 }, []);
+      const output = transformFindAlerts({ data: [], page: 1, perPage: 0, total: 0 }, {}, {});
       expect(output).toEqual({ data: [], page: 1, perPage: 0, total: 0 });
     });
 
@@ -268,7 +268,8 @@ describe('utils', () => {
           total: 0,
           data: [getAlertMock(getQueryRuleParams())],
         },
-        []
+        {},
+        {}
       );
       const expected = getOutputRuleAlertForRest();
       expect(output).toEqual({
@@ -288,7 +289,8 @@ describe('utils', () => {
           perPage: 1,
           total: 1,
         },
-        []
+        {},
+        {}
       );
       expect(output).toBeNull();
     });

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts

@@ -6,7 +6,7 @@
  */
 
 import { countBy } from 'lodash/fp';
-import { SavedObject, SavedObjectsFindResponse } from 'kibana/server';
+import { SavedObject } from 'kibana/server';
 import uuid from 'uuid';
 
 import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema';
@@ -17,11 +17,10 @@ import { INTERNAL_IDENTIFIER } from '../../../../../common/constants';
 import {
   RuleAlertType,
   isAlertType,
-  isAlertTypes,
   IRuleSavedAttributesSavedObjectAttributes,
   isRuleStatusFindType,
-  isRuleStatusFindTypes,
   isRuleStatusSavedObjectType,
+  IRuleStatusSOAttributes,
 } from '../../rules/types';
 import {
   createBulkErrorObject,
@@ -34,6 +33,7 @@ import {
 import { RuleActions } from '../../rule_actions/types';
 import { internalRuleToAPIResponse } from '../../schemas/rule_converters';
 import { RuleParams } from '../../schemas/rule_schemas';
+import { SanitizedAlert } from '../../../../../../alerting/common';
 
 type PromiseFromStreams = ImportRulesSchemaDecoded | Error;
 
@@ -103,11 +103,11 @@ export const transformTags = (tags: string[]): string[] => {
 // Transforms the data but will remove any null or undefined it encounters and not include
 // those on the export
 export const transformAlertToRule = (
-  alert: RuleAlertType,
+  alert: SanitizedAlert<RuleParams>,
   ruleActions?: RuleActions | null,
   ruleStatus?: SavedObject<IRuleSavedAttributesSavedObjectAttributes>
 ): Partial<RulesSchema> => {
-  return internalRuleToAPIResponse(alert, ruleActions, ruleStatus);
+  return internalRuleToAPIResponse(alert, ruleActions, ruleStatus?.attributes);
 };
 
 export const transformAlertsToRules = (alerts: RuleAlertType[]): Array<Partial<RulesSchema>> => {
@@ -116,33 +116,24 @@ export const transformAlertsToRules = (alerts: RuleAlertType[]): Array<Partial<R
 
 export const transformFindAlerts = (
   findResults: FindResult<RuleParams>,
-  ruleActions: Array<RuleActions | null>,
-  ruleStatuses?: Array<SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes>>
+  ruleActions: { [key: string]: RuleActions | undefined },
+  ruleStatuses: { [key: string]: IRuleStatusSOAttributes[] | undefined }
 ): {
   page: number;
   perPage: number;
   total: number;
   data: Array<Partial<RulesSchema>>;
 } | null => {
-  if (!ruleStatuses && isAlertTypes(findResults.data)) {
-    return {
-      page: findResults.page,
-      perPage: findResults.perPage,
-      total: findResults.total,
-      data: findResults.data.map((alert, idx) => transformAlertToRule(alert, ruleActions[idx])),
-    };
-  } else if (isAlertTypes(findResults.data) && isRuleStatusFindTypes(ruleStatuses)) {
-    return {
-      page: findResults.page,
-      perPage: findResults.perPage,
-      total: findResults.total,
-      data: findResults.data.map((alert, idx) =>
-        transformAlertToRule(alert, ruleActions[idx], ruleStatuses[idx].saved_objects[0])
-      ),
-    };
-  } else {
-    return null;
-  }
+  return {
+    page: findResults.page,
+    perPage: findResults.perPage,
+    total: findResults.total,
+    data: findResults.data.map((alert) => {
+      const statuses = ruleStatuses[alert.id];
+      const status = statuses ? statuses[0] : undefined;
+      return internalRuleToAPIResponse(alert, ruleActions[alert.id], status);
+    }),
+  };
 };
 
 export const transform = (

x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts

@@ -15,12 +15,13 @@ import {
   RouteValidationFunction,
   KibanaResponseFactory,
   CustomHttpResponseOptions,
-  SavedObjectsFindResult,
 } from '../../../../../../../src/core/server';
 import { AlertsClient } from '../../../../../alerting/server';
 import { BadRequestError } from '../errors/bad_request_error';
 import { RuleStatusResponse, IRuleStatusSOAttributes } from '../rules/types';
 
+import { RuleParams } from '../schemas/rule_schemas';
+
 export interface OutputError {
   message: string;
   statusCode: number;
@@ -311,7 +312,7 @@ export const convertToSnakeCase = <T extends Record<string, unknown>>(
  */
 export const mergeStatuses = (
   id: string,
-  currentStatusAndFailures: Array<SavedObjectsFindResult<IRuleStatusSOAttributes>>,
+  currentStatusAndFailures: IRuleStatusSOAttributes[],
   acc: RuleStatusResponse
 ): RuleStatusResponse => {
   if (currentStatusAndFailures.length === 0) {
@@ -320,20 +321,20 @@ export const mergeStatuses = (
     };
   }
   const convertedCurrentStatus = convertToSnakeCase<IRuleStatusSOAttributes>(
-    currentStatusAndFailures[0].attributes
+    currentStatusAndFailures[0]
   );
   return {
     ...acc,
     [id]: {
       current_status: convertedCurrentStatus,
       failures: currentStatusAndFailures
         .slice(1)
-        .map((errorItem) => convertToSnakeCase<IRuleStatusSOAttributes>(errorItem.attributes)),
+        .map((errorItem) => convertToSnakeCase<IRuleStatusSOAttributes>(errorItem)),
     },
   } as RuleStatusResponse;
 };
 
-export type GetFailingRulesResult = Record<string, SanitizedAlert>;
+export type GetFailingRulesResult = Record<string, SanitizedAlert<RuleParams>>;
 
 export const getFailingRules = async (
   ids: string[],
@@ -350,13 +351,11 @@ export const getFailingRules = async (
     return errorRules
       .filter((rule) => rule.executionStatus.status === 'error')
       .reduce<GetFailingRulesResult>((acc, failingRule) => {
-        const accum = acc;
-        const theRule = failingRule;
         return {
-          [theRule.id]: {
-            ...theRule,
+          [failingRule.id]: {
+            ...failingRule,
           },
-          ...accum,
+          ...acc,
         };
       }, {});
   } catch (exc) {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_actions/get_bulk_rule_actions_saved_object.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AlertServices } from '../../../../../alerting/server';
+import { ruleActionsSavedObjectType } from './saved_object_mappings';
+import { IRuleActionsAttributesSavedObjectAttributes } from './types';
+import { getRuleActionsFromSavedObject } from './utils';
+import { RulesActionsSavedObject } from './get_rule_actions_saved_object';
+import { buildChunkedOrFilter } from '../signals/utils';
+
+interface GetBulkRuleActionsSavedObject {
+  alertIds: string[];
+  savedObjectsClient: AlertServices['savedObjectsClient'];
+}
+
+export const getBulkRuleActionsSavedObject = async ({
+  alertIds,
+  savedObjectsClient,
+}: GetBulkRuleActionsSavedObject): Promise<Record<string, RulesActionsSavedObject>> => {
+  const filter = buildChunkedOrFilter(
+    `${ruleActionsSavedObjectType}.attributes.ruleAlertId`,
+    alertIds
+  );
+  const {
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    saved_objects,
+  } = await savedObjectsClient.find<IRuleActionsAttributesSavedObjectAttributes>({
+    type: ruleActionsSavedObjectType,
+    perPage: 10000,
+    filter,
+  });
+  return saved_objects.reduce((acc: { [key: string]: RulesActionsSavedObject }, savedObject) => {
+    acc[savedObject.attributes.ruleAlertId] = getRuleActionsFromSavedObject(savedObject);
+    return acc;
+  }, {});
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts

@@ -211,12 +211,6 @@ export const isRuleStatusFindType = (
   return get('saved_objects', obj) != null;
 };
 
-export const isRuleStatusFindTypes = (
-  obj: unknown[] | undefined
-): obj is Array<SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes>> => {
-  return obj ? obj.every((ruleStatus) => isRuleStatusFindType(ruleStatus)) : false;
-};
-
 export interface CreateRulesOptions {
   alertsClient: AlertsClient;
   anomalyThreshold: AnomalyThresholdOrUndefined;