[SIEM] Tests for search_after and bulk index (#50129)

* tests for detection engine get/put utils

* increases unit test code statement coverage to 100% for search_after / bulk index reindexer

* removes mockLogger declaration from individual test cases - clears mock counts before each test case runs so as to not accumulate method calls after each test case

* resets default paging size to 1000 - typo from when I was working through my tests

* updates tests after rebase with master

* fixes type check after fixing test from rebase with master

* removes undefined from maxSignals in type definition, updates tests with pure jest function implementations of logger and services - modifying only the return values or creating a mock implementation when necessary, removed some overlapping test cases

* fixes type issue

* replaces mock implementation with mock return value for unit test

* removes mock logger expected counts, just check if error logs are called, don't care about debug / warn etc.

* fixes more type checks after rebase with master

x-pack/legacy/plugins/siem/common/constants.ts

@@ -15,6 +15,7 @@ export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults';
 export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults';
 export const DEFAULT_SIEM_TIME_RANGE = 'siem:timeDefaults';
 export const DEFAULT_SIEM_REFRESH_INTERVAL = 'siem:refreshIntervalDefaults';
+export const DEFAULT_SIGNALS_INDEX = '.siem-signals';
 export const DEFAULT_ANOMALY_SCORE = 'siem:defaultAnomalyScore';
 export const DEFAULT_MAX_TABLE_QUERY_SIZE = 10000;
 export const DEFAULT_SCALE_DATE_FORMAT = 'dateFormat:scaled';

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts

@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SignalSourceHit, SignalSearchResponse, SignalAlertParams } from '../types';
+
+export const sampleSignalAlertParams = (maxSignals: number | undefined): SignalAlertParams => ({
+  id: 'rule-1',
+  description: 'Detecting root and admin users',
+  falsePositives: [],
+  immutable: false,
+  index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
+  interval: '5m',
+  name: 'Detect Root/Admin Users',
+  type: 'query',
+  from: 'now-6m',
+  tags: ['some fake tag'],
+  to: 'now',
+  severity: 'high',
+  query: 'user.name: root or user.name: admin',
+  language: 'kuery',
+  references: ['http://google.com'],
+  maxSignals: maxSignals ? maxSignals : 10000,
+  enabled: true,
+  filter: undefined,
+  filters: undefined,
+  savedId: undefined,
+  size: 1000,
+});
+
+export const sampleDocNoSortId: SignalSourceHit = {
+  _index: 'myFakeSignalIndex',
+  _type: 'doc',
+  _score: 100,
+  _version: 1,
+  _id: 'someFakeId',
+  _source: {
+    someKey: 'someValue',
+    '@timestamp': 'someTimeStamp',
+  },
+};
+
+export const sampleDocWithSortId: SignalSourceHit = {
+  _index: 'myFakeSignalIndex',
+  _type: 'doc',
+  _score: 100,
+  _version: 1,
+  _id: 'someFakeId',
+  _source: {
+    someKey: 'someValue',
+    '@timestamp': 'someTimeStamp',
+  },
+  sort: ['1234567891111'],
+};
+
+export const sampleEmptyDocSearchResults: SignalSearchResponse = {
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: 0,
+    max_score: 100,
+    hits: [],
+  },
+};
+
+export const sampleDocSearchResultsNoSortId: SignalSearchResponse = {
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: 100,
+    max_score: 100,
+    hits: [
+      {
+        ...sampleDocNoSortId,
+      },
+    ],
+  },
+};
+
+export const sampleDocSearchResultsNoSortIdNoHits: SignalSearchResponse = {
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: 0,
+    max_score: 100,
+    hits: [
+      {
+        ...sampleDocNoSortId,
+      },
+    ],
+  },
+};
+
+export const repeatedSearchResultsWithSortId = (repeat: number) => ({
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: repeat,
+    max_score: 100,
+    hits: Array.from({ length: repeat }).map(x => ({
+      ...sampleDocWithSortId,
+    })),
+  },
+});
+
+export const sampleDocSearchResultsWithSortId: SignalSearchResponse = {
+  took: 10,
+  timed_out: false,
+  _shards: {
+    total: 10,
+    successful: 10,
+    failed: 0,
+    skipped: 0,
+  },
+  hits: {
+    total: 1,
+    max_score: 100,
+    hits: [
+      {
+        ...sampleDocWithSortId,
+      },
+    ],
+  },
+};

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts

@@ -66,7 +66,7 @@ describe('create_signals', () => {
             ],
           },
         },
-        track_total_hits: true,
+
         sort: [
           {
             '@timestamp': {
@@ -136,14 +136,158 @@ describe('create_signals', () => {
             ],
           },
         },
-        track_total_hits: true,
+
+        sort: [
+          {
+            '@timestamp': {
+              order: 'asc',
+            },
+          },
+        ],
+      },
+    });
+  });
+  test('if searchAfterSortId is a valid sortId string', () => {
+    const fakeSortId = '123456789012';
+    const query = buildEventsSearchQuery({
+      index: ['auditbeat-*'],
+      from: 'now-5m',
+      to: 'today',
+      filter: {},
+      size: 100,
+      searchAfterSortId: fakeSortId,
+    });
+    expect(query).toEqual({
+      allowNoIndices: true,
+      index: ['auditbeat-*'],
+      size: 100,
+      ignoreUnavailable: true,
+      body: {
+        query: {
+          bool: {
+            filter: [
+              {},
+              {
+                bool: {
+                  filter: [
+                    {
+                      bool: {
+                        should: [
+                          {
+                            range: {
+                              '@timestamp': {
+                                gte: 'now-5m',
+                              },
+                            },
+                          },
+                        ],
+                        minimum_should_match: 1,
+                      },
+                    },
+                    {
+                      bool: {
+                        should: [
+                          {
+                            range: {
+                              '@timestamp': {
+                                lte: 'today',
+                              },
+                            },
+                          },
+                        ],
+                        minimum_should_match: 1,
+                      },
+                    },
+                  ],
+                },
+              },
+              {
+                match_all: {},
+              },
+            ],
+          },
+        },
+
+        sort: [
+          {
+            '@timestamp': {
+              order: 'asc',
+            },
+          },
+        ],
+        search_after: [fakeSortId],
+      },
+    });
+  });
+  test('if searchAfterSortId is a valid sortId number', () => {
+    const fakeSortIdNumber = 123456789012;
+    const query = buildEventsSearchQuery({
+      index: ['auditbeat-*'],
+      from: 'now-5m',
+      to: 'today',
+      filter: {},
+      size: 100,
+      searchAfterSortId: fakeSortIdNumber,
+    });
+    expect(query).toEqual({
+      allowNoIndices: true,
+      index: ['auditbeat-*'],
+      size: 100,
+      ignoreUnavailable: true,
+      body: {
+        query: {
+          bool: {
+            filter: [
+              {},
+              {
+                bool: {
+                  filter: [
+                    {
+                      bool: {
+                        should: [
+                          {
+                            range: {
+                              '@timestamp': {
+                                gte: 'now-5m',
+                              },
+                            },
+                          },
+                        ],
+                        minimum_should_match: 1,
+                      },
+                    },
+                    {
+                      bool: {
+                        should: [
+                          {
+                            range: {
+                              '@timestamp': {
+                                lte: 'today',
+                              },
+                            },
+                          },
+                        ],
+                        minimum_should_match: 1,
+                      },
+                    },
+                  ],
+                },
+              },
+              {
+                match_all: {},
+              },
+            ],
+          },
+        },
+
         sort: [
           {
             '@timestamp': {
               order: 'asc',
             },
           },
         ],
+        search_after: [fakeSortIdNumber],
       },
     });
   });

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts

@@ -10,7 +10,7 @@ interface BuildEventsSearchQuery {
   to: string;
   filter: unknown;
   size: number;
-  searchAfterSortId?: string;
+  searchAfterSortId: string | number | undefined;
 }
 
 export const buildEventsSearchQuery = ({
@@ -74,7 +74,6 @@ export const buildEventsSearchQuery = ({
           ],
         },
       },
-      track_total_hits: true,
       sort: [
         {
           '@timestamp': {

x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts

@@ -5,9 +5,8 @@
  */
 
 import { schema } from '@kbn/config-schema';
-import { SIGNALS_ID } from '../../../../common/constants';
+import { SIGNALS_ID, DEFAULT_SIGNALS_INDEX } from '../../../../common/constants';
 import { Logger } from '../../../../../../../../src/core/server';
-
 // TODO: Remove this for the build_events_query call eventually
 import { buildEventsReIndex } from './build_events_reindex';
 
@@ -34,7 +33,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
         savedId: schema.nullable(schema.string()),
         query: schema.nullable(schema.string()),
         filters: schema.nullable(schema.arrayOf(schema.object({}, { allowUnknowns: true }))),
-        maxSignals: schema.number({ defaultValue: 100 }),
+        maxSignals: schema.number({ defaultValue: 10000 }),
         severity: schema.string(),
         tags: schema.arrayOf(schema.string(), { defaultValue: [] }),
         to: schema.string(),
@@ -82,6 +81,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
         to,
         filter: esFilter,
         size: searchAfterSize,
+        searchAfterSortId: undefined,
       });
 
       try {
@@ -93,7 +93,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
             to,
             // TODO: Change this out once we have solved
             // https://github.com/elastic/kibana/issues/47002
-            signalsIndex: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019',
+            signalsIndex: process.env.SIGNALS_INDEX || DEFAULT_SIGNALS_INDEX,
             severity,
             description,
             name,