[SIEM][Detection Engine] Adds lists feature flag and list values to the REST interfaces

x-pack/legacy/plugins/siem/server/lib/detection_engine/feature_flags.test.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  listsEnvFeatureFlagName,
+  hasListsFeature,
+  unSetFeatureFlagsForTestsOnly,
+  setFeatureFlagsForTestsOnly,
+} from './feature_flags';
+
+describe('feature_flags', () => {
+  beforeAll(() => {
+    delete process.env[listsEnvFeatureFlagName];
+  });
+
+  afterEach(() => {
+    delete process.env[listsEnvFeatureFlagName];
+  });
+
+  describe('hasListsFeature', () => {
+    test('hasListsFeature should return false if process.env is not set', () => {
+      expect(hasListsFeature()).toEqual(false);
+    });
+
+    test('hasListsFeature should return true if process.env is set to true', () => {
+      process.env[listsEnvFeatureFlagName] = 'true';
+      expect(hasListsFeature()).toEqual(true);
+    });
+
+    test('hasListsFeature should return false if process.env is set to false', () => {
+      process.env[listsEnvFeatureFlagName] = 'false';
+      expect(hasListsFeature()).toEqual(false);
+    });
+
+    test('hasListsFeature should return false if process.env is set to a non true value', () => {
+      process.env[listsEnvFeatureFlagName] = 'something else';
+      expect(hasListsFeature()).toEqual(false);
+    });
+  });
+
+  describe('setFeatureFlagsForTestsOnly', () => {
+    test('it can be called once and sets the environment variable for tests', () => {
+      setFeatureFlagsForTestsOnly();
+      expect(process.env[listsEnvFeatureFlagName]).toEqual('true');
+      unSetFeatureFlagsForTestsOnly(); // This is needed to not pollute other tests since this has to be paired
+    });
+
+    test('if it is called twice it throws an exception', () => {
+      setFeatureFlagsForTestsOnly();
+      expect(() => setFeatureFlagsForTestsOnly()).toThrow(
+        'In your tests you need to ensure in your afterEach/afterAll blocks you are calling unSetFeatureFlagsForTestsOnly'
+      );
+      unSetFeatureFlagsForTestsOnly(); // This is needed to not pollute other tests since this has to be paired
+    });
+
+    test('it can be called twice as long as unSetFeatureFlagsForTestsOnly is called in-between', () => {
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      setFeatureFlagsForTestsOnly();
+      expect(process.env[listsEnvFeatureFlagName]).toEqual('true');
+      unSetFeatureFlagsForTestsOnly(); // This is needed to not pollute other tests since this has to be paired
+    });
+  });
+
+  describe('unSetFeatureFlagsForTestsOnly', () => {
+    test('it can sets the value to undefined', () => {
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      expect(process.env[listsEnvFeatureFlagName]).toEqual(undefined);
+    });
+
+    test('it can not be be called before setFeatureFlagsForTestsOnly without throwing', () => {
+      expect(() => unSetFeatureFlagsForTestsOnly()).toThrow(
+        'In your tests you need to ensure in your beforeEach/beforeAll blocks you are calling setFeatureFlagsForTestsOnly'
+      );
+    });
+
+    test('if it is called twice it throws an exception', () => {
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      expect(() => unSetFeatureFlagsForTestsOnly()).toThrow(
+        'In your tests you need to ensure in your beforeEach/beforeAll blocks you are calling setFeatureFlagsForTestsOnly'
+      );
+    });
+
+    test('it can be called twice as long as setFeatureFlagsForTestsOnly is called in-between', () => {
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      setFeatureFlagsForTestsOnly();
+      unSetFeatureFlagsForTestsOnly();
+      expect(process.env[listsEnvFeatureFlagName]).toEqual(undefined);
+    });
+  });
+});

x-pack/legacy/plugins/siem/server/lib/detection_engine/feature_flags.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// TODO: (LIST-FEATURE) Delete this file once the lists features are within the product and in a particular version
+
+// Very temporary file where we put our feature flags for detection lists.
+// We need to use an environment variable and CANNOT use a kibana.dev.yml setting because some definitions
+// of things are global in the modules are are initialized before the init of the server has a chance to start.
+// Set this in your .bashrc/.zshrc to turn on lists feature, export ELASTIC_XPACK_SIEM_LISTS_FEATURE=true
+
+// NOTE: This feature is forwards and backwards compatible but forwards compatible is not guaranteed.
+// Once you enable this and begin using it you might not be able to easily go back back.
+// So it's best to not turn it on unless you are developing code.
+export const listsEnvFeatureFlagName = 'ELASTIC_XPACK_SIEM_LISTS_FEATURE';
+
+// This is for setFeatureFlagsForTestsOnly and unSetFeatureFlagsForTestsOnly only to use
+let setFeatureFlagsForTestsOnlyCalled = false;
+
+// Use this to detect if the lists feature is enabled or not
+export const hasListsFeature = (): boolean => {
+  return process.env[listsEnvFeatureFlagName]?.trim().toLowerCase() === 'true';
+};
+
+// This is for tests only to use in your beforeAll() calls
+export const setFeatureFlagsForTestsOnly = (): void => {
+  if (setFeatureFlagsForTestsOnlyCalled) {
+    throw new Error(
+      'In your tests you need to ensure in your afterEach/afterAll blocks you are calling unSetFeatureFlagsForTestsOnly'
+    );
+  } else {
+    setFeatureFlagsForTestsOnlyCalled = true;
+    process.env[listsEnvFeatureFlagName] = 'true';
+  }
+};
+
+// This is for tests only to use in your afterAll() calls
+export const unSetFeatureFlagsForTestsOnly = (): void => {
+  if (!setFeatureFlagsForTestsOnlyCalled) {
+    throw new Error(
+      'In your tests you need to ensure in your beforeEach/beforeAll blocks you are calling setFeatureFlagsForTestsOnly'
+    );
+  } else {
+    delete process.env[listsEnvFeatureFlagName];
+    setFeatureFlagsForTestsOnlyCalled = false;
+  }
+};

x-pack/legacy/plugins/siem/server/lib/detection_engine/index/get_index_exists.test.ts

@@ -5,6 +5,7 @@
  */
 
 import { getIndexExists } from './get_index_exists';
+import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../feature_flags';
 
 class StatusCode extends Error {
   status: number = -1;
@@ -15,6 +16,14 @@ class StatusCode extends Error {
 }
 
 describe('get_index_exists', () => {
+  beforeAll(() => {
+    setFeatureFlagsForTestsOnly();
+  });
+
+  afterAll(() => {
+    unSetFeatureFlagsForTestsOnly();
+  });
+
   test('it should return a true if you have _shards', async () => {
     const callWithRequest = jest.fn().mockResolvedValue({ _shards: { total: 1 } });
     const indexExists = await getIndexExists(callWithRequest, 'some-index');

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts

@@ -395,6 +395,32 @@ export const getResult = (): RuleAlertType => ({
     references: ['http://www.example.com', 'https://ww.example.com'],
     note: '# Investigative notes',
     version: 1,
+    lists: [
+      {
+        field: 'source.ip',
+        boolean_operator: 'and',
+        values: [
+          {
+            name: '127.0.0.1',
+            type: 'value',
+          },
+        ],
+      },
+      {
+        field: 'host.name',
+        boolean_operator: 'and not',
+        values: [
+          {
+            name: 'rock01',
+            type: 'value',
+          },
+          {
+            name: 'mothra',
+            type: 'value',
+          },
+        ],
+      },
+    ],
   },
   createdAt: new Date('2019-12-13T16:40:33.400Z'),
   updatedAt: new Date('2019-12-13T16:40:33.400Z'),

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/utils.ts

@@ -77,3 +77,89 @@ export const buildHapiStream = (string: string, filename = 'file.ndjson'): HapiR
 
   return stream;
 };
+
+export const getOutputRuleAlertForRest = (): OutputRuleAlertRest => ({
+  created_by: 'elastic',
+  created_at: '2019-12-13T16:40:33.400Z',
+  updated_at: '2019-12-13T16:40:33.400Z',
+  description: 'Detecting root and admin users',
+  enabled: true,
+  false_positives: [],
+  from: 'now-6m',
+  id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+  immutable: false,
+  index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+  interval: '5m',
+  risk_score: 50,
+  rule_id: 'rule-1',
+  language: 'kuery',
+  max_signals: 100,
+  name: 'Detect Root/Admin Users',
+  output_index: '.siem-signals',
+  query: 'user.name: root or user.name: admin',
+  references: ['http://www.example.com', 'https://ww.example.com'],
+  severity: 'high',
+  updated_by: 'elastic',
+  tags: [],
+  threat: [
+    {
+      framework: 'MITRE ATT&CK',
+      tactic: {
+        id: 'TA0040',
+        name: 'impact',
+        reference: 'https://attack.mitre.org/tactics/TA0040/',
+      },
+      technique: [
+        {
+          id: 'T1499',
+          name: 'endpoint denial of service',
+          reference: 'https://attack.mitre.org/techniques/T1499/',
+        },
+      ],
+    },
+  ],
+  lists: [
+    {
+      field: 'source.ip',
+      boolean_operator: 'and',
+      values: [
+        {
+          name: '127.0.0.1',
+          type: 'value',
+        },
+      ],
+    },
+    {
+      field: 'host.name',
+      boolean_operator: 'and not',
+      values: [
+        {
+          name: 'rock01',
+          type: 'value',
+        },
+        {
+          name: 'mothra',
+          type: 'value',
+        },
+      ],
+    },
+  ],
+  filters: [
+    {
+      query: {
+        match_phrase: {
+          'host.name': 'some-host',
+        },
+      },
+    },
+  ],
+  meta: {
+    someMeta: 'someField',
+  },
+  timeline_id: 'some-timeline-id',
+  timeline_title: 'some-timeline-title',
+  to: 'now',
+  type: 'query',
+  note: '# Investigative notes',
+  version: 1,
+});

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts

@@ -14,6 +14,7 @@ import {
 import { requestContextMock, serverMock } from '../__mocks__';
 import { addPrepackedRulesRoute } from './add_prepackaged_rules_route';
 import { PrepackagedRules } from '../../types';
+import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
 
 jest.mock('../../rules/get_prepackaged_rules', () => {
   return {
@@ -44,6 +45,14 @@ describe('add_prepackaged_rules_route', () => {
   let server: ReturnType<typeof serverMock.create>;
   let { clients, context } = requestContextMock.createTools();
 
+  beforeAll(() => {
+    setFeatureFlagsForTestsOnly();
+  });
+
+  afterAll(() => {
+    unSetFeatureFlagsForTestsOnly();
+  });
+
   beforeEach(() => {
     server = serverMock.create();
     ({ clients, context } = requestContextMock.createTools());

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts

@@ -16,11 +16,20 @@ import {
 } from '../__mocks__/request_responses';
 import { requestContextMock, serverMock, requestMock } from '../__mocks__';
 import { createRulesBulkRoute } from './create_rules_bulk_route';
+import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
 
 describe('create_rules_bulk', () => {
   let server: ReturnType<typeof serverMock.create>;
   let { clients, context } = requestContextMock.createTools();
 
+  beforeAll(() => {
+    setFeatureFlagsForTestsOnly();
+  });
+
+  afterAll(() => {
+    unSetFeatureFlagsForTestsOnly();
+  });
+
   beforeEach(() => {
     server = serverMock.create();
     ({ clients, context } = requestContextMock.createTools());

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts

@@ -82,6 +82,7 @@ export const createRulesBulkRoute = (router: IRouter) => {
               timeline_id: timelineId,
               timeline_title: timelineTitle,
               version,
+              lists,
             } = payloadRule;
             const ruleIdOrUuid = ruleId ?? uuid.v4();
             try {
@@ -134,6 +135,7 @@ export const createRulesBulkRoute = (router: IRouter) => {
                 references,
                 note,
                 version,
+                lists,
               });
               return transformValidateBulkError(ruleIdOrUuid, createdRule);
             } catch (err) {

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts

@@ -17,11 +17,20 @@ import {
 } from '../__mocks__/request_responses';
 import { requestContextMock, serverMock, requestMock } from '../__mocks__';
 import { createRulesRoute } from './create_rules_route';
+import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
 
 describe('create_rules', () => {
   let server: ReturnType<typeof serverMock.create>;
   let { clients, context } = requestContextMock.createTools();
 
+  beforeAll(() => {
+    setFeatureFlagsForTestsOnly();
+  });
+
+  afterAll(() => {
+    unSetFeatureFlagsForTestsOnly();
+  });
+
   beforeEach(() => {
     server = serverMock.create();
     ({ clients, context } = requestContextMock.createTools());

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts

@@ -56,6 +56,7 @@ export const createRulesRoute = (router: IRouter): void => {
         type,
         references,
         note,
+        lists,
       } = request.body;
       const siemResponse = buildSiemResponse(response);
 
@@ -120,6 +121,7 @@ export const createRulesRoute = (router: IRouter): void => {
           references,
           note,
           version: 1,
+          lists,
         });
         const ruleStatuses = await savedObjectsClient.find<
           IRuleSavedAttributesSavedObjectAttributes