[RAC][Security Solution] Register Security Detection Rules with Rule …

…Registry (#96015) (#100940)

## Summary

This PR starts the migration of the Security Solution rules to use the rule-registry introduced in #95903. This is a pathfinding effort in porting over the existing Security Solution rules, and may include some temporary reference rules for testing out different paradigms as we move the rules over. See #95735 for details

Enable via the following feature flags in your `kibana.dev.yml`:

```
# Security Solution Rules on Rule Registry
xpack.ruleRegistry.index: '.kibana-[USERNAME]-alerts' # Only necessary to scope from other devs testing, if not specified defaults to `.alerts-security-solution`
xpack.securitySolution.enableExperimental: ['ruleRegistryEnabled']
```

> Note: if setting a custom `xpack.ruleRegistry.index`, for the time being you must also update the [DEFAULT_ALERTS_INDEX](https://github.com/elastic/kibana/blob/9e213fb7a5a0337591a50a0567924ebe950b9791/x-pack/plugins/security_solution/common/constants.ts#L28) in order for the UI to display alerts within the alerts table.

---

Three reference rule types have been added (`query`, `eql`, `threshold`), along with scripts for creating them located in:

```
x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/scripts/
```

Main Detection page TGrid queries have been short-circuited to query `.alerts-security-solution*` for displaying alerts from the new alerts as data indices.

To test, checkout, enable the above feature flag(s), and run one of the scripts from the above directory, e.g.  `./create_reference_rule_query.sh` (ensure your ENV vars as set! :)

Alerts as data within the main Detection Page 🎉
<p align="center">
  <img width="500" src="https://user-images.githubusercontent.com/2946766/119911768-39cfba00-bf17-11eb-8996-63c0b813fdcc.png" />
</p>

cc @madirey @dgieselaar @pmuellr @yctercero @dhurley14 @marshallmain

# Conflicts:
#	x-pack/plugins/security_solution/server/plugin.ts

x-pack/plugins/rule_registry/README.md

@@ -145,3 +145,6 @@ The following fields are defined in the technical field component template and s
 - `kibana.rac.alert.severity.value`: the severity of the alert, as a numerical value, which allows sorting.
 - `kibana.rac.alert.evaluation.value`: The measured (numerical value).
 - `kibana.rac.alert.threshold.value`: The threshold that was defined (or, in case of multiple thresholds, the one that was exceeded).
+- `kibana.rac.alert.ancestors`: the array of ancestors (if any) for the alert.
+- `kibana.rac.alert.depth`: the depth of the alert in the ancestral tree (default 0).
+- `kibana.rac.alert.building_block_type`: the building block type of the alert (default undefined).

x-pack/plugins/rule_registry/server/index.ts

@@ -14,6 +14,7 @@ export { RuleDataClient } from './rule_data_client';
 export { IRuleDataClient } from './rule_data_client/types';
 export { getRuleExecutorData, RuleExecutorData } from './utils/get_rule_executor_data';
 export { createLifecycleRuleTypeFactory } from './utils/create_lifecycle_rule_type_factory';
+export { createPersistenceRuleTypeFactory } from './utils/create_persistence_rule_type_factory';
 
 export const plugin = (initContext: PluginInitializerContext) =>
   new RuleRegistryPlugin(initContext);

x-pack/plugins/rule_registry/server/rule_data_client/index.ts

@@ -73,8 +73,8 @@ export class RuleDataClient implements IRuleDataClient {
         return clusterClient.bulk(requestWithDefaultParameters).then((response) => {
           if (response.body.errors) {
             if (
-              response.body.items.length === 1 &&
-              response.body.items[0]?.index?.error?.type === 'index_not_found_exception'
+              response.body.items.length > 0 &&
+              response.body.items?.[0]?.index?.error?.type === 'index_not_found_exception'
             ) {
               return this.createOrUpdateWriteTarget({ namespace }).then(() => {
                 return clusterClient.bulk(requestWithDefaultParameters);

x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { ESSearchRequest } from 'typings/elasticsearch';
+import v4 from 'uuid/v4';
+import { Logger } from '@kbn/logging';
+
+import { AlertInstance } from '../../../alerting/server';
+import {
+  AlertInstanceContext,
+  AlertInstanceState,
+  AlertTypeParams,
+} from '../../../alerting/common';
+import { RuleDataClient } from '../rule_data_client';
+import { AlertTypeWithExecutor } from '../types';
+
+type PersistenceAlertService<TAlertInstanceContext extends Record<string, unknown>> = (
+  alerts: Array<Record<string, unknown>>
+) => Array<AlertInstance<AlertInstanceState, TAlertInstanceContext, string>>;
+
+type PersistenceAlertQueryService = (
+  query: ESSearchRequest
+) => Promise<Array<Record<string, unknown>>>;
+
+type CreatePersistenceRuleTypeFactory = (options: {
+  ruleDataClient: RuleDataClient;
+  logger: Logger;
+}) => <
+  TParams extends AlertTypeParams,
+  TAlertInstanceContext extends AlertInstanceContext,
+  TServices extends {
+    alertWithPersistence: PersistenceAlertService<TAlertInstanceContext>;
+    findAlerts: PersistenceAlertQueryService;
+  }
+>(
+  type: AlertTypeWithExecutor<TParams, TAlertInstanceContext, TServices>
+) => AlertTypeWithExecutor<TParams, TAlertInstanceContext, any>;
+
+export const createPersistenceRuleTypeFactory: CreatePersistenceRuleTypeFactory = ({
+  logger,
+  ruleDataClient,
+}) => (type) => {
+  return {
+    ...type,
+    executor: async (options) => {
+      const {
+        services: { alertInstanceFactory, scopedClusterClient },
+      } = options;
+
+      const currentAlerts: Array<Record<string, unknown>> = [];
+      const timestamp = options.startedAt.toISOString();
+
+      const state = await type.executor({
+        ...options,
+        services: {
+          ...options.services,
+          alertWithPersistence: (alerts) => {
+            alerts.forEach((alert) => currentAlerts.push(alert));
+            return alerts.map((alert) =>
+              alertInstanceFactory(alert['kibana.rac.alert.uuid']! as string)
+            );
+          },
+          findAlerts: async (query) => {
+            const { body } = await scopedClusterClient.asCurrentUser.search({
+              ...query,
+              body: {
+                ...query.body,
+              },
+              ignore_unavailable: true,
+            });
+            return body.hits.hits
+              .map((event: { _source: any }) => event._source!)
+              .map((event: { [x: string]: any }) => {
+                const alertUuid = event['kibana.rac.alert.uuid'];
+                const isAlert = alertUuid != null;
+                return {
+                  ...event,
+                  'event.kind': 'signal',
+                  'kibana.rac.alert.id': '???',
+                  'kibana.rac.alert.status': 'open',
+                  'kibana.rac.alert.uuid': v4(),
+                  'kibana.rac.alert.ancestors': isAlert
+                    ? ((event['kibana.rac.alert.ancestors'] as string[]) ?? []).concat([
+                        alertUuid!,
+                      ] as string[])
+                    : [],
+                  'kibana.rac.alert.depth': isAlert
+                    ? ((event['kibana.rac.alert.depth'] as number) ?? 0) + 1
+                    : 0,
+                  '@timestamp': timestamp,
+                };
+              });
+          },
+        },
+      });
+
+      const numAlerts = currentAlerts.length;
+      logger.debug(`Found ${numAlerts} alerts.`);
+
+      if (ruleDataClient && numAlerts) {
+        await ruleDataClient.getWriter().bulk({
+          body: currentAlerts.flatMap((event) => [{ index: {} }, event]),
+        });
+      }
+
+      return state;
+    },
+  };
+};

x-pack/plugins/security_solution/common/constants.ts

@@ -25,6 +25,7 @@ export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults';
 export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults';
 export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults';
 export const DEFAULT_APP_REFRESH_INTERVAL = 'securitySolution:refreshIntervalDefaults';
+export const DEFAULT_ALERTS_INDEX = '.alerts-security-solution';
 export const DEFAULT_SIGNALS_INDEX = '.siem-signals';
 export const DEFAULT_LISTS_INDEX = '.lists';
 export const DEFAULT_ITEMS_INDEX = '.items';
@@ -148,6 +149,18 @@ export const DEFAULT_TRANSFORMS_SETTING = JSON.stringify(defaultTransformsSettin
  */
 export const SIGNALS_ID = `siem.signals`;
 
+/**
+ * Id's for reference rule types
+ */
+export const REFERENCE_RULE_ALERT_TYPE_ID = `siem.referenceRule`;
+export const REFERENCE_RULE_PERSISTENCE_ALERT_TYPE_ID = `siem.referenceRulePersistence`;
+
+export const CUSTOM_ALERT_TYPE_ID = `siem.customRule`;
+export const EQL_ALERT_TYPE_ID = `siem.eqlRule`;
+export const INDICATOR_ALERT_TYPE_ID = `siem.indicatorRule`;
+export const ML_ALERT_TYPE_ID = `siem.mlRule`;
+export const THRESHOLD_ALERT_TYPE_ID = `siem.thresholdRule`;
+
 /**
  * Id for the notifications alerting type
  */

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -15,6 +15,7 @@ const allowedExperimentalValues = Object.freeze({
   trustedAppsByPolicyEnabled: false,
   metricsEntitiesEnabled: false,
   hostIsolationEnabled: false,
+  ruleRegistryEnabled: false,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

x-pack/plugins/security_solution/kibana.json

@@ -8,6 +8,7 @@
     "actions",
     "alerting",
     "cases",
+    "ruleRegistry",
     "data",
     "dataEnhanced",
     "embeddable",

x-pack/plugins/security_solution/public/common/components/exceptions/use_add_exception.test.tsx

@@ -21,7 +21,7 @@ import type {
   CreateExceptionListItemSchema,
   UpdateExceptionListItemSchema,
 } from '@kbn/securitysolution-io-ts-list-types';
-
+import { TestProviders } from '../../mock';
 import {
   useAddOrUpdateException,
   UseAddOrUpdateExceptionProps,
@@ -134,12 +134,16 @@ describe('useAddOrUpdateException', () => {
 
     addOrUpdateItemsArgs = [ruleId, itemsToAddOrUpdate];
     render = () =>
-      renderHook<UseAddOrUpdateExceptionProps, ReturnUseAddOrUpdateException>(() =>
-        useAddOrUpdateException({
-          http: mockKibanaHttpService,
-          onError,
-          onSuccess,
-        })
+      renderHook<UseAddOrUpdateExceptionProps, ReturnUseAddOrUpdateException>(
+        () =>
+          useAddOrUpdateException({
+            http: mockKibanaHttpService,
+            onError,
+            onSuccess,
+          }),
+        {
+          wrapper: TestProviders,
+        }
       );
   });
 

x-pack/plugins/security_solution/public/common/components/exceptions/use_add_exception.tsx

@@ -19,9 +19,11 @@ import { getUpdateAlertsQuery } from '../../../detections/components/alerts_tabl
 import {
   buildAlertStatusFilter,
   buildAlertsRuleIdFilter,
+  buildAlertStatusFilterRuleRegistry,
 } from '../../../detections/components/alerts_table/default_config';
 import { getQueryFilter } from '../../../../common/detection_engine/get_query_filter';
 import { Index } from '../../../../common/detection_engine/schemas/common/schemas';
+import { useIsExperimentalFeatureEnabled } from '../../hooks/use_experimental_features';
 import { formatExceptionItemForUpdate, prepareExceptionItemsForBulkClose } from './helpers';
 import { useKibana } from '../../lib/kibana';
 
@@ -82,6 +84,8 @@ export const useAddOrUpdateException = ({
     },
     []
   );
+  // TODO: Once we are past experimental phase this code should be removed
+  const ruleRegistryEnabled = useIsExperimentalFeatureEnabled('ruleRegistryEnabled');
 
   useEffect(() => {
     let isSubscribed = true;
@@ -127,10 +131,15 @@ export const useAddOrUpdateException = ({
         }
 
         if (bulkCloseIndex != null) {
+          // TODO: Once we are past experimental phase this code should be removed
+          const alertStatusFilter = ruleRegistryEnabled
+            ? buildAlertStatusFilterRuleRegistry('open')
+            : buildAlertStatusFilter('open');
+
           const filter = getQueryFilter(
             '',
             'kuery',
-            [...buildAlertsRuleIdFilter(ruleId), ...buildAlertStatusFilter('open')],
+            [...buildAlertsRuleIdFilter(ruleId), ...alertStatusFilter],
             bulkCloseIndex,
             prepareExceptionItemsForBulkClose(exceptionItemsToAddOrUpdate),
             false
@@ -176,7 +185,14 @@ export const useAddOrUpdateException = ({
       isSubscribed = false;
       abortCtrl.abort();
     };
-  }, [http, onSuccess, onError, updateExceptionListItem, addExceptionListItem]);
+  }, [
+    addExceptionListItem,
+    http,
+    onSuccess,
+    onError,
+    ruleRegistryEnabled,
+    updateExceptionListItem,
+  ]);
 
   return [{ isLoading }, addOrUpdateException];
 };

x-pack/plugins/security_solution/public/common/mock/global_state.ts

@@ -43,6 +43,7 @@ export const mockGlobalState: State = {
       trustedAppsByPolicyEnabled: false,
       metricsEntitiesEnabled: false,
       hostIsolationEnabled: false,
+      ruleRegistryEnabled: false,
     },
   },
   hosts: {

x-pack/plugins/security_solution/public/common/mock/test_providers.tsx

@@ -24,11 +24,12 @@ import {
 import { FieldHook } from '../../shared_imports';
 import { SUB_PLUGINS_REDUCER } from './utils';
 import { createSecuritySolutionStorageMock, localStorageMock } from './mock_local_storage';
+import { UserPrivilegesProvider } from '../../detections/components/user_privileges';
 
 const state: State = mockGlobalState;
 
 interface Props {
-  children: React.ReactNode;
+  children?: React.ReactNode;
   store?: Store;
   onDragEnd?: (result: DropResult, provided: ResponderProvided) => void;
 }
@@ -59,7 +60,30 @@ const TestProvidersComponent: React.FC<Props> = ({
   </I18nProvider>
 );
 
+/**
+ * A utility for wrapping children in the providers required to run most tests
+ * WITH user privileges provider.
+ */
+const TestProvidersWithPrivilegesComponent: React.FC<Props> = ({
+  children,
+  store = createStore(state, SUB_PLUGINS_REDUCER, kibanaObservable, storage),
+  onDragEnd = jest.fn(),
+}) => (
+  <I18nProvider>
+    <MockKibanaContextProvider>
+      <ReduxStoreProvider store={store}>
+        <ThemeProvider theme={() => ({ eui: euiDarkVars, darkMode: true })}>
+          <UserPrivilegesProvider>
+            <DragDropContext onDragEnd={onDragEnd}>{children}</DragDropContext>
+          </UserPrivilegesProvider>
+        </ThemeProvider>
+      </ReduxStoreProvider>
+    </MockKibanaContextProvider>
+  </I18nProvider>
+);
+
 export const TestProviders = React.memo(TestProvidersComponent);
+export const TestProvidersWithPrivileges = React.memo(TestProvidersWithPrivilegesComponent);
 
 export const useFormFieldMock = <T,>(options?: Partial<FieldHook<T>>): FieldHook<T> => {
   return {