Merge branch 'master' into fix/form-lib-memoize

.ci/Jenkinsfile_visual_baseline

@@ -21,5 +21,6 @@ kibanaPipeline(timeoutMinutes: 120) {
     }
 
     kibanaPipeline.sendMail()
+    slackNotifications.onFailure()
   }
 }

test/functional/services/remote/webdriver.ts

@@ -28,7 +28,7 @@ import { delay } from 'bluebird';
 import chromeDriver from 'chromedriver';
 // @ts-ignore types not available
 import geckoDriver from 'geckodriver';
-import { Builder, Capabilities, logging } from 'selenium-webdriver';
+import { Builder, logging } from 'selenium-webdriver';
 import chrome from 'selenium-webdriver/chrome';
 import firefox from 'selenium-webdriver/firefox';
 import edge from 'selenium-webdriver/edge';

test/scripts/jenkins_xpack_visual_regression.sh

@@ -5,7 +5,7 @@ source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"
 
 echo " -> building and extracting default Kibana distributable"
 cd "$KIBANA_DIR"
-node scripts/build --debug
+node scripts/build --debug --no-oss
 linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
 installDir="$PARENT_DIR/install/kibana"
 mkdir -p "$installDir"
@@ -22,5 +22,5 @@ yarn percy exec -t 10000 -- -- \
 # cd "$KIBANA_DIR"
 # source "test/scripts/jenkins_xpack_page_load_metrics.sh"
 
-cd "$XPACK_DIR"
-source "$KIBANA_DIR/test/scripts/jenkins_xpack_saved_objects_field_metrics.sh"
+cd "$KIBANA_DIR"
+source "test/scripts/jenkins_xpack_saved_objects_field_metrics.sh"

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts

@@ -139,6 +139,139 @@ describe('buildEventTypeSignal', () => {
     });
   });
 
+  test('it should deduplicate exception entries', async () => {
+    const testEntries: EntriesArray = [
+      { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
+      { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
+      { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
+      { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' },
+      {
+        field: 'host.hostname.text',
+        operator: 'included',
+        type: 'match_any',
+        value: ['estc', 'kibana'],
+      },
+    ];
+
+    const expectedEndpointExceptions = {
+      type: 'simple',
+      entries: [
+        {
+          field: 'server.domain',
+          operator: 'included',
+          type: 'exact_caseless',
+          value: 'DOMAIN',
+        },
+        {
+          field: 'server.ip',
+          operator: 'included',
+          type: 'exact_cased',
+          value: '192.168.1.1',
+        },
+        {
+          field: 'host.hostname',
+          operator: 'included',
+          type: 'exact_caseless_any',
+          value: ['estc', 'kibana'],
+        },
+      ],
+    };
+
+    const first = getFoundExceptionListItemSchemaMock();
+    first.data[0].entries = testEntries;
+    mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
+
+    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
+    expect(resp).toEqual({
+      entries: [expectedEndpointExceptions],
+    });
+  });
+
+  test('it should not deduplicate exception entries across nested boundaries', async () => {
+    const testEntries: EntriesArray = [
+      {
+        entries: [
+          { field: 'nested.field', operator: 'included', type: 'match', value: 'some value' },
+        ],
+        field: 'some.parentField',
+        type: 'nested',
+      },
+      // Same as above but not inside the nest
+      { field: 'nested.field', operator: 'included', type: 'match', value: 'some value' },
+    ];
+
+    const expectedEndpointExceptions = {
+      type: 'simple',
+      entries: [
+        {
+          entries: [
+            {
+              field: 'nested.field',
+              operator: 'included',
+              type: 'exact_cased',
+              value: 'some value',
+            },
+          ],
+          field: 'some.parentField',
+          type: 'nested',
+        },
+        {
+          field: 'nested.field',
+          operator: 'included',
+          type: 'exact_cased',
+          value: 'some value',
+        },
+      ],
+    };
+
+    const first = getFoundExceptionListItemSchemaMock();
+    first.data[0].entries = testEntries;
+    mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
+
+    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
+    expect(resp).toEqual({
+      entries: [expectedEndpointExceptions],
+    });
+  });
+
+  test('it should deduplicate exception items', async () => {
+    const testEntries: EntriesArray = [
+      { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
+      { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' },
+    ];
+
+    const expectedEndpointExceptions = {
+      type: 'simple',
+      entries: [
+        {
+          field: 'server.domain',
+          operator: 'included',
+          type: 'exact_caseless',
+          value: 'DOMAIN',
+        },
+        {
+          field: 'server.ip',
+          operator: 'included',
+          type: 'exact_cased',
+          value: '192.168.1.1',
+        },
+      ],
+    };
+
+    const first = getFoundExceptionListItemSchemaMock();
+    first.data[0].entries = testEntries;
+
+    // Create a second exception item with the same entries
+    first.data[1] = getExceptionListItemSchemaMock();
+    first.data[1].entries = testEntries;
+    mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
+
+    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
+    expect(resp).toEqual({
+      entries: [expectedEndpointExceptions],
+    });
+  });
+
   test('it should ignore unsupported entries', async () => {
     // Lists and exists are not supported by the Endpoint
     const testEntries: EntriesArray = [
@@ -178,8 +311,9 @@ describe('buildEventTypeSignal', () => {
   });
 
   test('it should convert the exception lists response to the proper endpoint format while paging', async () => {
-    // The first call returns one exception
+    // The first call returns two exceptions
     const first = getFoundExceptionListItemSchemaMock();
+    first.data.push(getExceptionListItemSchemaMock());
 
     // The second call returns two exceptions
     const second = getFoundExceptionListItemSchemaMock();
@@ -194,7 +328,8 @@ describe('buildEventTypeSignal', () => {
       .mockReturnValueOnce(second)
       .mockReturnValueOnce(third);
     const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
-    expect(resp.entries.length).toEqual(3);
+    // Expect 2 exceptions, the first two calls returned the same exception list items
+    expect(resp.entries.length).toEqual(2);
   });
 
   test('it should handle no exceptions', async () => {

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts

@@ -97,10 +97,18 @@ export function translateToEndpointExceptions(
   exc: FoundExceptionListItemSchema,
   schemaVersion: string
 ): TranslatedExceptionListItem[] {
+  const entrySet = new Set();
+  const entriesFiltered: TranslatedExceptionListItem[] = [];
   if (schemaVersion === 'v1') {
-    return exc.data.map((item) => {
-      return translateItem(schemaVersion, item);
+    exc.data.forEach((entry) => {
+      const translatedItem = translateItem(schemaVersion, entry);
+      const entryHash = createHash('sha256').update(JSON.stringify(translatedItem)).digest('hex');
+      if (!entrySet.has(entryHash)) {
+        entriesFiltered.push(translatedItem);
+        entrySet.add(entryHash);
+      }
     });
+    return entriesFiltered;
   } else {
     throw new Error('unsupported schemaVersion');
   }
@@ -124,12 +132,17 @@ function translateItem(
   schemaVersion: string,
   item: ExceptionListItemSchema
 ): TranslatedExceptionListItem {
+  const itemSet = new Set();
   return {
     type: item.type,
     entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => {
       const translatedEntry = translateEntry(schemaVersion, entry);
       if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) {
-        translatedEntries.push(translatedEntry);
+        const itemHash = createHash('sha256').update(JSON.stringify(translatedEntry)).digest('hex');
+        if (!itemSet.has(itemHash)) {
+          translatedEntries.push(translatedEntry);
+          itemSet.add(itemHash);
+        }
       }
       return translatedEntries;
     }, []),