[Security Solutions][Detection Engine] Adds exception lists to the saved object references when created or modified (part 1)

x-pack/plugins/security_solution/server/lib/detection_engine/signals/saved_object_references/README.md

@@ -0,0 +1,144 @@
+This is where you add code when you have rules which contain saved object references. Saved object references are for
+when you have "joins" in the saved objects between one saved object and another one. This can be a 1 to M (1 to many)
+relationship for example where you have a rule which contains the "id" of another saved object.
+
+Examples are the `exceptionsList` on a rule which contains a saved object reference from the rule to another set of
+saved objects of the type `exception-list`
+
+## Useful queries
+How to get all your alerts to see if you have `exceptionsList` on it or not in dev tools:
+
+```json
+GET .kibana/_search
+{
+  "query": {
+    "term": {
+      "type": {
+        "value": "alert"
+      }
+    }
+  }
+}
+```
+
+## Structure on disk
+Run a query in dev tools and you should see this code that adds the following savedObject references
+to any newly saved rule:
+
+```json
+      {
+        "_index" : ".kibana-hassanabad19_8.0.0_001",
+        "_id" : "alert:38482620-ef1b-11eb-ad71-7de7959be71c",
+        "_score" : 6.2607274,
+        "_source" : {
+          "alert" : {
+            "name" : "kql test rule 1",
+            "tags" : [
+              "__internal_rule_id:4ec223b9-77fa-4895-8539-6b3e586a2858",
+              "__internal_immutable:false"
+            ],
+            "alertTypeId" : "siem.signals",
+            "other data... other data": "other data...other data",
+              "exceptionsList" : [
+                {
+                  "id" : "endpoint_list",
+                  "list_id" : "endpoint_list",
+                  "namespace_type" : "agnostic",
+                  "type" : "endpoint"
+                },
+                {
+                  "id" : "50e3bd70-ef1b-11eb-ad71-7de7959be71c",
+                  "list_id" : "cd152d0d-3590-4a45-a478-eed04da7936b",
+                  "type" : "detection",
+                  "namespace_type" : "single"
+                }
+              ],
+          "other data... other data": "other data...other data",
+          "references" : [
+            {
+              "name" : "param:exceptionsList_0",
+              "id" : "endpoint_list",
+              "type" : "exception-list"
+            },
+            {
+              "name" : "param:exceptionsList_1",
+              "id" : "50e3bd70-ef1b-11eb-ad71-7de7959be71c",
+              "type" : "exception-list"
+            }
+          ],
+         "other data... other data": "other data...other data"
+          }
+        }
+      }
+```
+
+The structure is that the alerting framework in conjunction with this code will make an array of saved object references which are going to be:
+```json
+{
+  "references" : [
+    {
+      "name" : "param:exceptionsList_1",
+      "id" : "50e3bd70-ef1b-11eb-ad71-7de7959be71c",
+      "type" : "exception-list"
+    }
+  ]
+}
+```
+
+`name` is the pattern of `param:${name}_${index}`. See the functions and constants in `utils.ts` of:
+
+* EXCEPTIONS_LIST_NAME
+* getSavedObjectNamePattern
+* getSavedObjectNamePatternForExceptionsList
+* getSavedObjectReference
+* getSavedObjectReferenceForExceptionsList
+
+For how it is constructed and retrieved. If you need to add more types, you should copy and create your own versions or use the generic
+utilities/helpers if possible.
+
+`id` is the saved object id and should always be the same value as the `"exceptionsList" : [ "id" : "50e3bd70-ef1b-11eb-ad71-7de7959be71c" ...`.
+If for some reason the saved object id changes or is different, then on the next save/persist the `exceptionsList.id` will update to that within
+its saved object. Note though, that the references id replaces _always_ the `exceptionsList.id` at all times through `inject_references.ts`. If
+for some reason the `references` id is deleted, then on the next `inject_references` it will prefer to use the last good known reference and log
+a warning.
+
+Within the rule parameters you can still keep the last known good saved object reference id as above it is shown
+```json
+{
+  "exceptionsList" : [
+    {
+      "id" : "endpoint_list",
+      "list_id" : "endpoint_list",
+      "namespace_type" : "agnostic",
+      "type" : "endpoint"
+    },
+    {
+      "id" : "50e3bd70-ef1b-11eb-ad71-7de7959be71c",
+      "list_id" : "cd152d0d-3590-4a45-a478-eed04da7936b",
+      "type" : "detection",
+      "namespace_type" : "single"
+    }
+  ],
+}
+```
+
+## How to add a new saved object id reference to a rule
+
+See the files of:
+* extract_references.ts
+* inject_references.ts
+
+And their top level comments for how to wire up new instances. It's best to create a new file per saved object reference and push only the needed data
+per file.
+
+Good examples and utilities can be found in the folder of `utils` such as:
+* EXCEPTIONS_LIST_NAME
+* getSavedObjectNamePattern
+* getSavedObjectNamePatternForExceptionsList
+* getSavedObjectReference
+* getSavedObjectReferenceForExceptionsList
+
+You can follow those patterns but if it doesn't fit your use case it's fine to just create a new file and wire up your new saved object references
+
+## End to end tests
+At this moment there are none.

x-pack/plugins/security_solution/server/lib/detection_engine/signals/saved_object_references/extract_exceptions_list.test.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { extractExceptionsList } from './extract_exceptions_list';
+import { loggingSystemMock } from 'src/core/server/mocks';
+import { RuleParams } from '../../schemas/rule_schemas';
+import { EXCEPTION_LIST_NAMESPACE } from '@kbn/securitysolution-list-constants';
+import { EXCEPTIONS_SAVED_OBJECT_REFERENCE_NAME } from './utils';
+
+describe('extract_exceptions_list', () => {
+  type FuncReturn = ReturnType<typeof extractExceptionsList>;
+  let logger = loggingSystemMock.create().get('security_solution');
+  const mockExceptionsList = (): RuleParams['exceptionsList'] => [
+    {
+      id: '123',
+      list_id: '456',
+      type: 'detection',
+      namespace_type: 'agnostic',
+    },
+  ];
+
+  beforeEach(() => {
+    logger = loggingSystemMock.create().get('security_solution');
+  });
+
+  test('it returns an empty array given an empty array for exceptionsList', () => {
+    expect(extractExceptionsList({ logger, exceptionsList: [] })).toEqual<FuncReturn>([]);
+  });
+
+  test('logs expect error message if the exceptionsList is undefined', () => {
+    extractExceptionsList({
+      logger,
+      exceptionsList: (undefined as unknown) as RuleParams['exceptionsList'],
+    });
+    expect(logger.error).toBeCalledWith(
+      'Exception list is null when it never should be. This indicates potentially that saved object migrations did not run correctly. Returning empty saved object reference'
+    );
+  });
+
+  test('It returns exception list transformed into a saved object references', () => {
+    expect(
+      extractExceptionsList({ logger, exceptionsList: mockExceptionsList() })
+    ).toEqual<FuncReturn>([
+      {
+        id: '123',
+        name: `${EXCEPTIONS_SAVED_OBJECT_REFERENCE_NAME}_0`,
+        type: EXCEPTION_LIST_NAMESPACE,
+      },
+    ]);
+  });
+
+  test('It returns two exception lists transformed into a saved object references', () => {
+    const twoInputs: RuleParams['exceptionsList'] = [
+      mockExceptionsList()[0],
+      { ...mockExceptionsList()[0], id: '976' },
+    ];
+    expect(extractExceptionsList({ logger, exceptionsList: twoInputs })).toEqual<FuncReturn>([
+      {
+        id: '123',
+        name: `${EXCEPTIONS_SAVED_OBJECT_REFERENCE_NAME}_0`,
+        type: EXCEPTION_LIST_NAMESPACE,
+      },
+      {
+        id: '976',
+        name: `${EXCEPTIONS_SAVED_OBJECT_REFERENCE_NAME}_1`,
+        type: EXCEPTION_LIST_NAMESPACE,
+      },
+    ]);
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/signals/saved_object_references/extract_exceptions_list.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger, SavedObjectReference } from 'src/core/server';
+import { EXCEPTION_LIST_NAMESPACE } from '@kbn/securitysolution-list-constants';
+import { RuleParams } from '../../schemas/rule_schemas';
+import { getSavedObjectNamePatternForExceptionsList } from './utils';
+
+/**
+ * This extracts the "exceptionsList" "id" and returns it as a saved object reference.
+ * NOTE: Due to rolling upgrades with migrations and a few bugs with migrations, I do an additional check for if "exceptionsList" exists or not. Once
+ * those bugs are fixed, we can remove the "if (exceptionsList == null) {" check, but for the time being it is there to keep things running even
+ * if exceptionsList has not been migrated.
+ * @param logger The kibana injected logger
+ * @param exceptionsList The exceptions list to get the id from and return it as a saved object reference.
+ * @returns The saved object references from the exceptions list
+ */
+export const extractExceptionsList = ({
+  logger,
+  exceptionsList,
+}: {
+  logger: Logger;
+  exceptionsList: RuleParams['exceptionsList'];
+}): SavedObjectReference[] => {
+  if (exceptionsList == null) {
+    logger.error(
+      'Exception list is null when it never should be. This indicates potentially that saved object migrations did not run correctly. Returning empty saved object reference'
+    );
+    return [];
+  } else {
+    return exceptionsList.map((exceptionItem, index) => ({
+      name: getSavedObjectNamePatternForExceptionsList(index),
+      id: exceptionItem.id,
+      type: EXCEPTION_LIST_NAMESPACE,
+    }));
+  }
+};

x-pack/plugins/security_solution/server/lib/detection_engine/signals/saved_object_references/extract_references.test.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { loggingSystemMock } from 'src/core/server/mocks';
+import { extractReferences } from './extract_references';
+import { RuleParams } from '../../schemas/rule_schemas';
+import { EXCEPTION_LIST_NAMESPACE } from '@kbn/securitysolution-list-constants';
+import { EXCEPTIONS_SAVED_OBJECT_REFERENCE_NAME } from './utils';
+
+describe('extract_references', () => {
+  type FuncReturn = ReturnType<typeof extractReferences>;
+  let logger = loggingSystemMock.create().get('security_solution');
+  const mockExceptionsList = (): RuleParams['exceptionsList'] => [
+    {
+      id: '123',
+      list_id: '456',
+      type: 'detection',
+      namespace_type: 'agnostic',
+    },
+  ];
+
+  beforeEach(() => {
+    logger = loggingSystemMock.create().get('security_solution');
+  });
+
+  test('It returns params untouched and the references extracted as exception list saved object references', () => {
+    const params: Partial<RuleParams> = {
+      note: 'some note',
+      exceptionsList: mockExceptionsList(),
+    };
+    expect(
+      extractReferences({
+        logger,
+        params: params as RuleParams,
+      })
+    ).toEqual<FuncReturn>({
+      params: params as RuleParams,
+      references: [
+        {
+          id: '123',
+          name: `${EXCEPTIONS_SAVED_OBJECT_REFERENCE_NAME}_0`,
+          type: EXCEPTION_LIST_NAMESPACE,
+        },
+      ],
+    });
+  });
+
+  test('It returns params untouched and the references an empty array if the exceptionsList is an empty array', () => {
+    const params: Partial<RuleParams> = {
+      note: 'some note',
+      exceptionsList: [],
+    };
+    expect(
+      extractReferences({
+        logger,
+        params: params as RuleParams,
+      })
+    ).toEqual<FuncReturn>({
+      params: params as RuleParams,
+      references: [],
+    });
+  });
+
+  test('It returns params untouched and the references an empty array if the exceptionsList is missing for any reason', () => {
+    const params: Partial<RuleParams> = {
+      note: 'some note',
+    };
+    expect(
+      extractReferences({
+        logger,
+        params: params as RuleParams,
+      })
+    ).toEqual<FuncReturn>({
+      params: params as RuleParams,
+      references: [],
+    });
+  });
+});