[SIEM][Detection Engine] Adds lists feature flag and list values to t…

…he REST interfaces ## Summary * #60022 * Adds the feature flag for simple list values * Adds the boolean filters of "and", "and not" to further filter based on simple values * Adds unit tests and e2e tests for the values. * Most tests can include the simple list values but some have to be skipped until we move those to more functions or just enable simple list values as a permanent feature. * DOES NOT FILTER ON THE VALUES JUST YET (That will be a follow on PR) ## Testing: To turn on/off the feature flag do this with an env variable (set this in your .bashrc/.zshrc): ```ts export ELASTIC_XPACK_SIEM_LISTS_FEATURE=true ``` Expect to see this error in the console when the environment variable is set: ```ts server log [11:41:16.245] [error][plugins][siem] You have activated the lists feature flag which is NOT currently supported for SIEM! You should turn this feature flag off immediately by un-setting the environment variable: ELASTIC_XPACK_SIEM_LISTS_FEATURE and restarting Kibana ``` Expect create and update to work when the environment variable is set and look like this: ```ts ./update_rule.sh ./rules/updates/update_list.json { "created_at": "2020-03-15T17:42:37.074Z", "updated_at": "2020-03-15T17:54:22.427Z", "created_by": "yo", "description": "Query with a list", "enabled": true, "false_positives": [], "from": "now-6m", "id": "c602e3f6-713b-4f43-9bdd-b60fbfead1c5", "immutable": false, "interval": "5m", "rule_id": "query-with-list", "language": "kuery", "output_index": ".siem-signals-hassanabad-frank-default", "max_signals": 100, "risk_score": 1, "name": "Query with a list", "query": "user.name: root or user.name: admin", "references": [], "severity": "high", "updated_by": "yo", "tags": [], "to": "now", "type": "query", "threat": [], "version": 6, "lists": [ { "field": "source.ip", "boolean_operator": "and", "values": [ { "name": "127.0.0.1", "type": "value" } ] }, { "field": "host.name", "boolean_operator": "and not", "values": [ { "name": "rock01", "type": "value" } ] } ], "status": "succeeded", "status_date": "2020-03-15T17:42:40.718Z", "last_success_at": "2020-03-15T17:42:40.718Z", "last_success_message": "succeeded" } ``` ```ts ./post_rule.sh ./rules/queries/query_with_list.json { "created_at": "2020-03-15T17:42:37.074Z", "updated_at": "2020-03-15T17:42:37.116Z", "created_by": "yo", "description": "Query with a list", "enabled": true, "false_positives": [], "from": "now-6m", "id": "c602e3f6-713b-4f43-9bdd-b60fbfead1c5", "immutable": false, "interval": "5m", "rule_id": "query-with-list", "language": "kuery", "output_index": ".siem-signals-hassanabad-frank-default", "max_signals": 100, "risk_score": 1, "name": "Query with a list", "query": "user.name: root or user.name: admin", "references": [], "severity": "high", "updated_by": "yo", "tags": [], "to": "now", "type": "query", "threat": [], "version": 1, "lists": [ { "field": "source.ip", "boolean_operator": "and", "values": [ { "name": "127.0.0.1", "type": "value" } ] }, { "field": "host.name", "boolean_operator": "and not", "values": [ { "name": "rock01", "type": "value" }, { "name": "mothra", "type": "value" } ] } ] } ``` ```ts ./patch_rule.sh ./rules/patches/update_list.json { "created_at": "2020-03-15T18:02:52.434Z", "updated_at": "2020-03-15T18:02:57.675Z", "created_by": "yo", "description": "Query with a list", "enabled": true, "false_positives": [], "from": "now-6m", "id": "40b7c2fb-83b4-4820-bf7c-056f3a631126", "immutable": false, "interval": "5m", "rule_id": "query-with-list", "language": "kuery", "output_index": ".siem-signals-hassanabad-frank-default", "max_signals": 100, "risk_score": 1, "name": "Query with a list", "query": "user.name: root or user.name: admin", "references": [], "severity": "high", "updated_by": "yo", "tags": [], "to": "now", "type": "query", "threat": [], "version": 1, "lists": [ { "field": "source.ip", "boolean_operator": "and", "values": [ { "name": "127.0.0.1", "type": "value" } ] }, { "field": "host.name", "boolean_operator": "and not", "values": [ { "name": "rock01", "type": "value" }, { "name": "mothra", "type": "value" } ] } ], "status": "succeeded", "status_date": "2020-03-15T18:02:56.426Z", "last_success_at": "2020-03-15T18:02:56.426Z", "last_success_message": "succeeded" } ``` ```ts ./get_rule_by_rule_id.sh query-with-list { "created_at": "2020-03-15T18:10:07.657Z", "updated_at": "2020-03-15T18:10:08.479Z", "created_by": "yo", "description": "Query with a list", "enabled": true, "false_positives": [], "from": "now-6m", "id": "9854162b-003c-47be-af59-8c3c9545aafa", "immutable": false, "interval": "5m", "rule_id": "query-with-list", "language": "kuery", "output_index": ".siem-signals-hassanabad-frank-default", "max_signals": 100, "risk_score": 1, "name": "Query with a list", "query": "user.name: root or user.name: admin", "references": [], "severity": "high", "updated_by": "yo", "tags": [], "to": "now", "type": "query", "threat": [], "version": 1, "lists": [ { "field": "source.ip", "boolean_operator": "and", "values": [ { "name": "127.0.0.1", "type": "value" } ] }, { "field": "host.name", "boolean_operator": "and not", "values": [ { "name": "rock01", "type": "value" }, { "name": "mothra", "type": "value" } ] } ], "status": "going to run", "status_date": "2020-03-15T18:10:10.738Z" } ``` Expect these errors when the environment variable is not set: ```ts ./post_rule.sh ./rules/queries/query_with_list.json { "statusCode": 400, "error": "Bad Request", "message": "[request body]: child \"lists\" fails because [\"lists\" is not allowed]" } ``` ```ts ./update_rule.sh ./rules/queries/query_with_list.json { "statusCode": 400, "error": "Bad Request", "message": "[request body]: child \"lists\" fails because [\"lists\" is not allowed]" } ``` ```ts ./patch_rule.sh ./rules/patches/update_list.json { "statusCode": 400, "error": "Bad Request", "message": "[request body]: child \"lists\" fails because [\"lists\" is not allowed]" } ``` Expect that this is _backwards_ compatible with the feature flag but not necessarily _forwards_ compatible. This means: * You can have older data that never had lists and it will show up as an empty list when you query it. (backwards compatible) * You _might_ have lists and remove the env. variable and get back items as if the list was not there for (forwards compatible) * You can export without lists, flip on the env flag and import with newer lists feature (backwards compatible) * You can export lists and it will _not_ work with an older system (not forwards compatible) ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios
elastic · Mar 19, 2020 · 01571b6 · 01571b6
1 parent cf08850
commit 01571b6
Show file tree

Hide file tree

Showing 85 changed files with 2,172 additions and 810 deletions.
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/feature_flags.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/feature_flags.test.ts
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  listsEnvFeatureFlagName,
+  hasListsFeature,
+  unSetFeatureFlagsForTestsOnly,
+  setFeatureFlagsForTestsOnly,
+} from './feature_flags';
+
+describe('feature_flags', () => {
+  beforeAll(() => {
+    delete process.env[listsEnvFeatureFlagName];
+  });
+
+  afterEach(() => {
+    delete process.env[listsEnvFeatureFlagName];
+  });
+
+  describe('hasListsFeature', () => {
+    test('hasListsFeature should return false if process.env is not set', () => {
+      expect(hasListsFeature()).toEqual(false);
+    });
+
+    test('hasListsFeature should return true if process.env is set to true', () => {
+      process.env[listsEnvFeatureFlagName] = 'true';
+      expect(hasListsFeature()).toEqual(true);
+    });
+
+    test('hasListsFeature should return false if process.env is set to false', () => {
+      process.env[listsEnvFeatureFlagName] = 'false';
+      expect(hasListsFeature()).toEqual(false);
+    });
+
+    test('hasListsFeature should return false if process.env is set to a non true value', () => {
+      process.env[listsEnvFeatureFlagName] = 'something else';
+      expect(hasListsFeature()).toEqual(false);
+    });
+  });
+
+  describe('setFeatureFlagsForTestsOnly', () => {
+    test('it can be called once and sets the environment variable for tests', () => {
+      setFeatureFlagsForTestsOnly();
+      expect(process.env[listsEnvFeatureFlagName]).toEqual('true');
+      unSetFeatureFlagsForTestsOnly(); // This is needed to not pollute other tests since this has to be paired
+    });
+
+    test('if it is called twice it throws an exception', () => {
+      setFeatureFlagsForTestsOnly();
+      expect(() => setFeatureFlagsForTestsOnly()).toThrow(
+        'In your tests you need to ensure in your afterEach/afterAll blocks you are calling unSetFeatureFlagsForTestsOnly'
+      );
+      unSetFeatureFlagsForTestsOnly(); // This is needed to not pollute other tests since this has to be paired
+    });
+
+    test('it can be called twice as long as unSetFeatureFlagsForTestsOnly is called in-between', () => {
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      setFeatureFlagsForTestsOnly();
+      expect(process.env[listsEnvFeatureFlagName]).toEqual('true');
+      unSetFeatureFlagsForTestsOnly(); // This is needed to not pollute other tests since this has to be paired
+    });
+  });
+
+  describe('unSetFeatureFlagsForTestsOnly', () => {
+    test('it can sets the value to undefined', () => {
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      expect(process.env[listsEnvFeatureFlagName]).toEqual(undefined);
+    });
+
+    test('it can not be be called before setFeatureFlagsForTestsOnly without throwing', () => {
+      expect(() => unSetFeatureFlagsForTestsOnly()).toThrow(
+        'In your tests you need to ensure in your beforeEach/beforeAll blocks you are calling setFeatureFlagsForTestsOnly'
+      );
+    });
+
+    test('if it is called twice it throws an exception', () => {
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      expect(() => unSetFeatureFlagsForTestsOnly()).toThrow(
+        'In your tests you need to ensure in your beforeEach/beforeAll blocks you are calling setFeatureFlagsForTestsOnly'
+      );
+    });
+
+    test('it can be called twice as long as setFeatureFlagsForTestsOnly is called in-between', () => {
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      expect(process.env[listsEnvFeatureFlagName]).toEqual(undefined);
+    });
+  });
+});
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/feature_flags.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/feature_flags.ts
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// TODO: (LIST-FEATURE) Delete this file once the lists features are within the product and in a particular version
+
+// Very temporary file where we put our feature flags for detection lists.
+// We need to use an environment variable and CANNOT use a kibana.dev.yml setting because some definitions
+// of things are global in the modules are are initialized before the init of the server has a chance to start.
+// Set this in your .bashrc/.zshrc to turn on lists feature, export ELASTIC_XPACK_SIEM_LISTS_FEATURE=true
+
+// NOTE: This feature is forwards and backwards compatible but forwards compatible is not guaranteed.
+// Once you enable this and begin using it you might not be able to easily go back back.
+// So it's best to not turn it on unless you are developing code.
+export const listsEnvFeatureFlagName = 'ELASTIC_XPACK_SIEM_LISTS_FEATURE';
+
+// This is for setFeatureFlagsForTestsOnly and unSetFeatureFlagsForTestsOnly only to use
+let setFeatureFlagsForTestsOnlyCalled = false;
+
+// Use this to detect if the lists feature is enabled or not
+export const hasListsFeature = (): boolean => {
+  return process.env[listsEnvFeatureFlagName]?.trim().toLowerCase() === 'true';
+};
+
+// This is for tests only to use in your beforeAll() calls
+export const setFeatureFlagsForTestsOnly = (): void => {
+  if (setFeatureFlagsForTestsOnlyCalled) {
+    throw new Error(
+      'In your tests you need to ensure in your afterEach/afterAll blocks you are calling unSetFeatureFlagsForTestsOnly'
+    );
+  } else {
+    setFeatureFlagsForTestsOnlyCalled = true;
+    process.env[listsEnvFeatureFlagName] = 'true';
+  }
+};
+
+// This is for tests only to use in your afterAll() calls
+export const unSetFeatureFlagsForTestsOnly = (): void => {
+  if (!setFeatureFlagsForTestsOnlyCalled) {
+    throw new Error(
+      'In your tests you need to ensure in your beforeEach/beforeAll blocks you are calling setFeatureFlagsForTestsOnly'
+    );
+  } else {
+    delete process.env[listsEnvFeatureFlagName];
+    setFeatureFlagsForTestsOnlyCalled = false;
+  }
+};
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_index_exists.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_index_exists.test.ts
@@ -5,6 +5,7 @@
  */
 
 import { getIndexExists } from './get_index_exists';
+import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../feature_flags';
 
 class StatusCode extends Error {
   status: number = -1;
@@ -15,6 +16,14 @@ class StatusCode extends Error {
 }
 
 describe('get_index_exists', () => {
+  beforeAll(() => {
+    setFeatureFlagsForTestsOnly();
+  });
+
+  afterAll(() => {
+    unSetFeatureFlagsForTestsOnly();
+  });
+
   test('it should return a true if you have _shards', async () => {
     const callWithRequest = jest.fn().mockResolvedValue({ _shards: { total: 1 } });
     const indexExists = await getIndexExists(callWithRequest, 'some-index');

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
@@ -412,6 +412,32 @@ export const getResult = (): RuleAlertType => ({
     references: ['http://www.example.com', 'https://ww.example.com'],
     note: '# Investigative notes',
     version: 1,
+    lists: [
+      {
+        field: 'source.ip',
+        boolean_operator: 'and',
+        values: [
+          {
+            name: '127.0.0.1',
+            type: 'value',
+          },
+        ],
+      },
+      {
+        field: 'host.name',
+        boolean_operator: 'and not',
+        values: [
+          {
+            name: 'rock01',
+            type: 'value',
+          },
+          {
+            name: 'mothra',
+            type: 'value',
+          },
+        ],
+      },
+    ],
   },
   createdAt: new Date('2019-12-13T16:40:33.400Z'),
   updatedAt: new Date('2019-12-13T16:40:33.400Z'),

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/utils.ts
@@ -77,3 +77,92 @@ export const buildHapiStream = (string: string, filename = 'file.ndjson'): HapiR
 
   return stream;
 };
+
+export const getOutputRuleAlertForRest = (): Omit<
+  OutputRuleAlertRest,
+  'machine_learning_job_id' | 'anomaly_threshold'
+> => ({
+  created_by: 'elastic',
+  created_at: '2019-12-13T16:40:33.400Z',
+  updated_at: '2019-12-13T16:40:33.400Z',
+  description: 'Detecting root and admin users',
+  enabled: true,
+  false_positives: [],
+  from: 'now-6m',
+  id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+  immutable: false,
+  index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+  interval: '5m',
+  risk_score: 50,
+  rule_id: 'rule-1',
+  language: 'kuery',
+  max_signals: 100,
+  name: 'Detect Root/Admin Users',
+  output_index: '.siem-signals',
+  query: 'user.name: root or user.name: admin',
+  references: ['http://www.example.com', 'https://ww.example.com'],
+  severity: 'high',
+  updated_by: 'elastic',
+  tags: [],
+  threat: [
+    {
+      framework: 'MITRE ATT&CK',
+      tactic: {
+        id: 'TA0040',
+        name: 'impact',
+        reference: 'https://attack.mitre.org/tactics/TA0040/',
+      },
+      technique: [
+        {
+          id: 'T1499',
+          name: 'endpoint denial of service',
+          reference: 'https://attack.mitre.org/techniques/T1499/',
+        },
+      ],
+    },
+  ],
+  lists: [
+    {
+      field: 'source.ip',
+      boolean_operator: 'and',
+      values: [
+        {
+          name: '127.0.0.1',
+          type: 'value',
+        },
+      ],
+    },
+    {
+      field: 'host.name',
+      boolean_operator: 'and not',
+      values: [
+        {
+          name: 'rock01',
+          type: 'value',
+        },
+        {
+          name: 'mothra',
+          type: 'value',
+        },
+      ],
+    },
+  ],
+  filters: [
+    {
+      query: {
+        match_phrase: {
+          'host.name': 'some-host',
+        },
+      },
+    },
+  ],
+  meta: {
+    someMeta: 'someField',
+  },
+  timeline_id: 'some-timeline-id',
+  timeline_title: 'some-timeline-title',
+  to: 'now',
+  type: 'query',
+  note: '# Investigative notes',
+  version: 1,
+});
diff --git a/...plugins/siem/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts b/...plugins/siem/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts
@@ -14,6 +14,7 @@ import {
 import { requestContextMock, serverMock } from '../__mocks__';
 import { addPrepackedRulesRoute } from './add_prepackaged_rules_route';
 import { PrepackagedRules } from '../../types';
+import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
 
 jest.mock('../../rules/get_prepackaged_rules', () => {
   return {
@@ -44,6 +45,14 @@ describe('add_prepackaged_rules_route', () => {
   let server: ReturnType<typeof serverMock.create>;
   let { clients, context } = requestContextMock.createTools();
 
+  beforeAll(() => {
+    setFeatureFlagsForTestsOnly();
+  });
+
+  afterAll(() => {
+    unSetFeatureFlagsForTestsOnly();
+  });
+
   beforeEach(() => {
     server = serverMock.create();
     ({ clients, context } = requestContextMock.createTools());

diff --git a/...acy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts b/...acy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts
@@ -16,11 +16,20 @@ import {
 } from '../__mocks__/request_responses';
 import { requestContextMock, serverMock, requestMock } from '../__mocks__';
 import { createRulesBulkRoute } from './create_rules_bulk_route';
+import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
 
 describe('create_rules_bulk', () => {
   let server: ReturnType<typeof serverMock.create>;
   let { clients, context } = requestContextMock.createTools();
 
+  beforeAll(() => {
+    setFeatureFlagsForTestsOnly();
+  });
+
+  afterAll(() => {
+    unSetFeatureFlagsForTestsOnly();
+  });
+
   beforeEach(() => {
     server = serverMock.create();
     ({ clients, context } = requestContextMock.createTools());

diff --git a/...k/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/...k/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts
@@ -84,6 +84,7 @@ export const createRulesBulkRoute = (router: IRouter) => {
               timeline_id: timelineId,
               timeline_title: timelineTitle,
               version,
+              lists,
             } = payloadRule;
             const ruleIdOrUuid = ruleId ?? uuid.v4();
             try {
@@ -138,6 +139,7 @@ export const createRulesBulkRoute = (router: IRouter) => {
                 references,
                 note,
                 version,
+                lists,
               });
               return transformValidateBulkError(ruleIdOrUuid, createdRule);
             } catch (err) {

diff --git a/...k/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts b/...k/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts
@@ -18,11 +18,20 @@ import {
 } from '../__mocks__/request_responses';
 import { requestContextMock, serverMock, requestMock } from '../__mocks__';
 import { createRulesRoute } from './create_rules_route';
+import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
 
 describe('create_rules', () => {
   let server: ReturnType<typeof serverMock.create>;
   let { clients, context } = requestContextMock.createTools();
 
+  beforeAll(() => {
+    setFeatureFlagsForTestsOnly();
+  });
+
+  afterAll(() => {
+    unSetFeatureFlagsForTestsOnly();
+  });
+
   beforeEach(() => {
     server = serverMock.create();
     ({ clients, context } = requestContextMock.createTools());

diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
@@ -58,6 +58,7 @@ export const createRulesRoute = (router: IRouter): void => {
         type,
         references,
         note,
+        lists,
       } = request.body;
       const siemResponse = buildSiemResponse(response);
 
@@ -124,6 +125,7 @@ export const createRulesRoute = (router: IRouter): void => {
           references,
           note,
           version: 1,
+          lists,
         });
         const ruleStatuses = await savedObjectsClient.find<
           IRuleSavedAttributesSavedObjectAttributes