From 5aa6eecff7be1deb3b9288791a3cc361cb04075f Mon Sep 17 00:00:00 2001 From: "Devin W. Hurley" Date: Sat, 16 Nov 2019 11:27:19 -0500 Subject: [PATCH] [SIEM] Tests for search_after and bulk index (#50129) (#50825) * tests for detection engine get/put utils * increases unit test code statement coverage to 100% for search_after / bulk index reindexer * removes mockLogger declaration from individual test cases - clears mock counts before each test case runs so as to not accumulate method calls after each test case * resets default paging size to 1000 - typo from when I was working through my tests * updates tests after rebase with master * fixes type check after fixing test from rebase with master * removes undefined from maxSignals in type definition, updates tests with pure jest function implementations of logger and services - modifying only the return values or creating a mock implementation when necessary, removed some overlapping test cases * fixes type issue * replaces mock implementation with mock return value for unit test * removes mock logger expected counts, just check if error logs are called, don't care about debug / warn etc. * fixes more type checks after rebase with master --- .../legacy/plugins/siem/common/constants.ts | 1 + .../alerts/__mocks__/es_results.ts | 150 ++++++++ .../alerts/build_events_query.test.ts | 148 +++++++- .../alerts/build_events_query.ts | 3 +- .../alerts/signals_alert_type.ts | 8 +- .../lib/detection_engine/alerts/utils.test.ts | 330 ++++++++++++++++++ .../lib/detection_engine/alerts/utils.ts | 37 +- 7 files changed, 656 insertions(+), 21 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts diff --git a/x-pack/legacy/plugins/siem/common/constants.ts b/x-pack/legacy/plugins/siem/common/constants.ts index 6845648ee921d..5c3bc8ab5b309 100644 --- a/x-pack/legacy/plugins/siem/common/constants.ts +++ b/x-pack/legacy/plugins/siem/common/constants.ts @@ -15,6 +15,7 @@ export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults'; export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults'; export const DEFAULT_SIEM_TIME_RANGE = 'siem:timeDefaults'; export const DEFAULT_SIEM_REFRESH_INTERVAL = 'siem:refreshIntervalDefaults'; +export const DEFAULT_SIGNALS_INDEX = '.siem-signals'; export const DEFAULT_ANOMALY_SCORE = 'siem:defaultAnomalyScore'; export const DEFAULT_MAX_TABLE_QUERY_SIZE = 10000; export const DEFAULT_SCALE_DATE_FORMAT = 'dateFormat:scaled'; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts new file mode 100644 index 0000000000000..895af32cc7af3 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SignalSourceHit, SignalSearchResponse, SignalAlertParams } from '../types'; + +export const sampleSignalAlertParams = (maxSignals: number | undefined): SignalAlertParams => ({ + id: 'rule-1', + description: 'Detecting root and admin users', + falsePositives: [], + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + name: 'Detect Root/Admin Users', + type: 'query', + from: 'now-6m', + tags: ['some fake tag'], + to: 'now', + severity: 'high', + query: 'user.name: root or user.name: admin', + language: 'kuery', + references: ['http://google.com'], + maxSignals: maxSignals ? maxSignals : 10000, + enabled: true, + filter: undefined, + filters: undefined, + savedId: undefined, + size: 1000, +}); + +export const sampleDocNoSortId: SignalSourceHit = { + _index: 'myFakeSignalIndex', + _type: 'doc', + _score: 100, + _version: 1, + _id: 'someFakeId', + _source: { + someKey: 'someValue', + '@timestamp': 'someTimeStamp', + }, +}; + +export const sampleDocWithSortId: SignalSourceHit = { + _index: 'myFakeSignalIndex', + _type: 'doc', + _score: 100, + _version: 1, + _id: 'someFakeId', + _source: { + someKey: 'someValue', + '@timestamp': 'someTimeStamp', + }, + sort: ['1234567891111'], +}; + +export const sampleEmptyDocSearchResults: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 0, + max_score: 100, + hits: [], + }, +}; + +export const sampleDocSearchResultsNoSortId: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 100, + max_score: 100, + hits: [ + { + ...sampleDocNoSortId, + }, + ], + }, +}; + +export const sampleDocSearchResultsNoSortIdNoHits: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 0, + max_score: 100, + hits: [ + { + ...sampleDocNoSortId, + }, + ], + }, +}; + +export const repeatedSearchResultsWithSortId = (repeat: number) => ({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: repeat, + max_score: 100, + hits: Array.from({ length: repeat }).map(x => ({ + ...sampleDocWithSortId, + })), + }, +}); + +export const sampleDocSearchResultsWithSortId: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 100, + hits: [ + { + ...sampleDocWithSortId, + }, + ], + }, +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts index 6c95e82485e45..b368c8fe36054 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts @@ -66,7 +66,7 @@ describe('create_signals', () => { ], }, }, - track_total_hits: true, + sort: [ { '@timestamp': { @@ -136,7 +136,150 @@ describe('create_signals', () => { ], }, }, - track_total_hits: true, + + sort: [ + { + '@timestamp': { + order: 'asc', + }, + }, + ], + }, + }); + }); + test('if searchAfterSortId is a valid sortId string', () => { + const fakeSortId = '123456789012'; + const query = buildEventsSearchQuery({ + index: ['auditbeat-*'], + from: 'now-5m', + to: 'today', + filter: {}, + size: 100, + searchAfterSortId: fakeSortId, + }); + expect(query).toEqual({ + allowNoIndices: true, + index: ['auditbeat-*'], + size: 100, + ignoreUnavailable: true, + body: { + query: { + bool: { + filter: [ + {}, + { + bool: { + filter: [ + { + bool: { + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + range: { + '@timestamp': { + lte: 'today', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, + }, + { + match_all: {}, + }, + ], + }, + }, + + sort: [ + { + '@timestamp': { + order: 'asc', + }, + }, + ], + search_after: [fakeSortId], + }, + }); + }); + test('if searchAfterSortId is a valid sortId number', () => { + const fakeSortIdNumber = 123456789012; + const query = buildEventsSearchQuery({ + index: ['auditbeat-*'], + from: 'now-5m', + to: 'today', + filter: {}, + size: 100, + searchAfterSortId: fakeSortIdNumber, + }); + expect(query).toEqual({ + allowNoIndices: true, + index: ['auditbeat-*'], + size: 100, + ignoreUnavailable: true, + body: { + query: { + bool: { + filter: [ + {}, + { + bool: { + filter: [ + { + bool: { + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + range: { + '@timestamp': { + lte: 'today', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, + }, + { + match_all: {}, + }, + ], + }, + }, + sort: [ { '@timestamp': { @@ -144,6 +287,7 @@ describe('create_signals', () => { }, }, ], + search_after: [fakeSortIdNumber], }, }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts index d2fb7c21f66f5..c75dddf896fd1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts @@ -10,7 +10,7 @@ interface BuildEventsSearchQuery { to: string; filter: unknown; size: number; - searchAfterSortId?: string; + searchAfterSortId: string | number | undefined; } export const buildEventsSearchQuery = ({ @@ -74,7 +74,6 @@ export const buildEventsSearchQuery = ({ ], }, }, - track_total_hits: true, sort: [ { '@timestamp': { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts index 44175360f80b3..be763f2b5386d 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts @@ -5,9 +5,8 @@ */ import { schema } from '@kbn/config-schema'; -import { SIGNALS_ID } from '../../../../common/constants'; +import { SIGNALS_ID, DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { Logger } from '../../../../../../../../src/core/server'; - // TODO: Remove this for the build_events_query call eventually import { buildEventsReIndex } from './build_events_reindex'; @@ -34,7 +33,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp savedId: schema.nullable(schema.string()), query: schema.nullable(schema.string()), filters: schema.nullable(schema.arrayOf(schema.object({}, { allowUnknowns: true }))), - maxSignals: schema.number({ defaultValue: 100 }), + maxSignals: schema.number({ defaultValue: 10000 }), severity: schema.string(), tags: schema.arrayOf(schema.string(), { defaultValue: [] }), to: schema.string(), @@ -82,6 +81,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp to, filter: esFilter, size: searchAfterSize, + searchAfterSortId: undefined, }); try { @@ -93,7 +93,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp to, // TODO: Change this out once we have solved // https://github.com/elastic/kibana/issues/47002 - signalsIndex: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + signalsIndex: process.env.SIGNALS_INDEX || DEFAULT_SIGNALS_INDEX, severity, description, name, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts new file mode 100644 index 0000000000000..c3ffb6e8c230a --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -0,0 +1,330 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { savedObjectsClientMock } from 'src/core/server/mocks'; + +import { Logger } from '../../../../../../../../src/core/server'; +import { + buildBulkBody, + singleBulkIndex, + singleSearchAfter, + searchAfterAndBulkIndex, +} from './utils'; +import { + sampleDocNoSortId, + sampleSignalAlertParams, + sampleDocSearchResultsNoSortId, + sampleDocSearchResultsNoSortIdNoHits, + sampleDocSearchResultsWithSortId, + sampleEmptyDocSearchResults, + repeatedSearchResultsWithSortId, +} from './__mocks__/es_results'; + +const mockLogger: Logger = { + log: jest.fn(), + trace: jest.fn(), + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), +}; + +const mockService = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + savedObjectsClient: savedObjectsClientMock.create(), +}; + +describe('utils', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + describe('buildBulkBody', () => { + test('if bulk body builds well-defined body', () => { + const sampleParams = sampleSignalAlertParams(undefined); + const fakeSignalSourceHit = buildBulkBody(sampleDocNoSortId, sampleParams); + expect(fakeSignalSourceHit).toEqual({ + someKey: 'someValue', + '@timestamp': 'someTimeStamp', + signal: { + '@timestamp': fakeSignalSourceHit.signal['@timestamp'], // timestamp generated in the body + rule_revision: 1, + rule_id: sampleParams.id, + rule_type: sampleParams.type, + parent: { + id: sampleDocNoSortId._id, + type: 'event', + index: sampleDocNoSortId._index, + depth: 1, + }, + name: sampleParams.name, + severity: sampleParams.severity, + description: sampleParams.description, + original_time: sampleDocNoSortId._source['@timestamp'], + index_patterns: sampleParams.index, + references: sampleParams.references, + }, + }); + }); + }); + describe('singleBulkIndex', () => { + test('create successful bulk index', async () => { + const sampleParams = sampleSignalAlertParams(undefined); + const sampleSearchResult = sampleDocSearchResultsNoSortId; + mockService.callCluster.mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }); + const successfulSingleBulkIndex = await singleBulkIndex( + sampleSearchResult, + sampleParams, + mockService, + mockLogger + ); + expect(successfulSingleBulkIndex).toEqual(true); + }); + test('create unsuccessful bulk index due to empty search results', async () => { + const sampleParams = sampleSignalAlertParams(undefined); + const sampleSearchResult = sampleEmptyDocSearchResults; + mockService.callCluster.mockReturnValue(false); + const successfulSingleBulkIndex = await singleBulkIndex( + sampleSearchResult, + sampleParams, + mockService, + mockLogger + ); + expect(successfulSingleBulkIndex).toEqual(true); + }); + test('create unsuccessful bulk index due to bulk index errors', async () => { + // need a sample search result, sample signal params, mock service, mock logger + const sampleParams = sampleSignalAlertParams(undefined); + const sampleSearchResult = sampleDocSearchResultsNoSortId; + mockService.callCluster.mockReturnValue({ + took: 100, + errors: true, + }); + const successfulSingleBulkIndex = await singleBulkIndex( + sampleSearchResult, + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.error).toHaveBeenCalled(); + expect(successfulSingleBulkIndex).toEqual(false); + }); + }); + describe('singleSearchAfter', () => { + test('if singleSearchAfter works without a given sort id', async () => { + let searchAfterSortId; + const sampleParams = sampleSignalAlertParams(undefined); + mockService.callCluster.mockReturnValue(sampleDocSearchResultsNoSortId); + await expect( + singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger) + ).rejects.toThrow('Attempted to search after with empty sort id'); + }); + test('if singleSearchAfter works with a given sort id', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(undefined); + mockService.callCluster.mockReturnValue(sampleDocSearchResultsWithSortId); + const searchAfterResult = await singleSearchAfter( + searchAfterSortId, + sampleParams, + mockService, + mockLogger + ); + expect(searchAfterResult).toEqual(sampleDocSearchResultsWithSortId); + }); + test('if singleSearchAfter throws error', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(undefined); + mockService.callCluster.mockImplementation(async () => { + throw Error('Fake Error'); + }); + await expect( + singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger) + ).rejects.toThrow('Fake Error'); + }); + }); + describe('searchAfterAndBulkIndex', () => { + test('if successful with empty search results', async () => { + const sampleParams = sampleSignalAlertParams(undefined); + const result = await searchAfterAndBulkIndex( + sampleEmptyDocSearchResults, + sampleParams, + mockService, + mockLogger + ); + expect(mockService.callCluster).toHaveBeenCalledTimes(0); + expect(result).toEqual(true); + }); + test('if successful iteration of while loop with maxDocs', async () => { + const sampleParams = sampleSignalAlertParams(10); + mockService.callCluster + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockReturnValueOnce(repeatedSearchResultsWithSortId(4)) + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockReturnValueOnce(repeatedSearchResultsWithSortId(4)) + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }); + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(mockService.callCluster).toHaveBeenCalledTimes(5); + expect(result).toEqual(true); + }); + test('if unsuccessful first bulk index', async () => { + const sampleParams = sampleSignalAlertParams(10); + mockService.callCluster.mockReturnValue({ + took: 100, + errors: true, // will cause singleBulkIndex to return false + }); + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.error).toHaveBeenCalled(); + expect(result).toEqual(false); + }); + test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids', async () => { + const sampleParams = sampleSignalAlertParams(undefined); + + mockService.callCluster.mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }); + const result = await searchAfterAndBulkIndex( + sampleDocSearchResultsNoSortId, + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.error).toHaveBeenCalled(); + expect(result).toEqual(false); + }); + test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids and 0 total hits', async () => { + const sampleParams = sampleSignalAlertParams(undefined); + mockService.callCluster.mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }); + const result = await searchAfterAndBulkIndex( + sampleDocSearchResultsNoSortIdNoHits, + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(true); + }); + test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => { + const sampleParams = sampleSignalAlertParams(10); + mockService.callCluster + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockReturnValueOnce(sampleDocSearchResultsNoSortId); + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(true); + }); + test('if logs error when iteration is unsuccessful when bulk index results in a failure', async () => { + const sampleParams = sampleSignalAlertParams(5); + mockService.callCluster + .mockReturnValueOnce({ + // first bulk insert + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockReturnValueOnce(sampleDocSearchResultsWithSortId); // get some more docs + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.error).toHaveBeenCalled(); + expect(result).toEqual(true); + }); + test('if returns false when singleSearchAfter throws an exception', async () => { + const sampleParams = sampleSignalAlertParams(10); + mockService.callCluster + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockRejectedValueOnce(Error('Fake Error')); + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(false); + }); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index a514baa186fd2..509181f915f55 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -5,6 +5,7 @@ */ import { performance } from 'perf_hooks'; import { SignalHit } from '../../types'; +import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { Logger } from '../../../../../../../../src/core/server'; import { AlertServices } from '../../../../../alerting/server/types'; import { SignalSourceHit, SignalSearchResponse, SignalAlertParams, BulkResponse } from './types'; @@ -48,7 +49,7 @@ export const singleBulkIndex = async ( const bulkBody = sr.hits.hits.flatMap(doc => [ { index: { - _index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + _index: process.env.SIGNALS_INDEX || DEFAULT_SIGNALS_INDEX, _id: doc._id, }, }, @@ -56,7 +57,7 @@ export const singleBulkIndex = async ( ]); const time1 = performance.now(); const firstResult: BulkResponse = await service.callCluster('bulk', { - index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + index: process.env.SIGNALS_INDEX || DEFAULT_SIGNALS_INDEX, refresh: false, body: bulkBody, }); @@ -64,7 +65,7 @@ export const singleBulkIndex = async ( logger.debug(`individual bulk process time took: ${time2 - time1} milliseconds`); logger.debug(`took property says bulk took: ${firstResult.took} milliseconds`); if (firstResult.errors) { - logger.error(`[-] bulkResponse had errors: ${JSON.stringify(firstResult.errors, null, 2)}}`); + logger.error(`[-] bulkResponse had errors: ${JSON.stringify(firstResult.errors, null, 2)}`); return false; } return true; @@ -89,7 +90,10 @@ export const singleSearchAfter = async ( size: params.size ? params.size : 1000, searchAfterSortId, }); - const nextSearchAfterResult = await service.callCluster('search', searchAfterQuery); + const nextSearchAfterResult: SignalSearchResponse = await service.callCluster( + 'search', + searchAfterQuery + ); return nextSearchAfterResult; } catch (exc) { logger.error(`[-] nextSearchAfter threw an error ${exc}`); @@ -117,11 +121,18 @@ export const searchAfterAndBulkIndex = async ( const totalHits = typeof someResult.hits.total === 'number' ? someResult.hits.total : someResult.hits.total.value; - let size = someResult.hits.hits.length - 1; - logger.debug(`first size: ${size}`); + // maxTotalHitsSize represents the total number of docs to + // query for. If maxSignals is present we will only query + // up to max signals - otherwise use the value + // from track_total_hits. + const maxTotalHitsSize = params.maxSignals ? params.maxSignals : totalHits; + + // number of docs in the current search result + let hitsSize = someResult.hits.hits.length; + logger.debug(`first size: ${hitsSize}`); let sortIds = someResult.hits.hits[0].sort; if (sortIds == null && totalHits > 0) { - logger.error(`sortIds was empty on first search when encountering ${totalHits}`); + logger.error('sortIds was empty on first search but expected more'); return false; } else if (sortIds == null && totalHits === 0) { return true; @@ -130,8 +141,7 @@ export const searchAfterAndBulkIndex = async ( if (sortIds != null) { sortId = sortIds[0]; } - while (size < totalHits) { - // utilize track_total_hits instead of true + while (hitsSize < maxTotalHitsSize && hitsSize !== 0) { try { logger.debug(`sortIds: ${sortIds}`); const searchAfterResult: SignalSearchResponse = await singleSearchAfter( @@ -140,12 +150,13 @@ export const searchAfterAndBulkIndex = async ( service, logger ); - size += searchAfterResult.hits.hits.length - 1; - logger.debug(`size adjusted: ${size}`); + sortIds = searchAfterResult.hits.hits[0].sort; + hitsSize += searchAfterResult.hits.hits.length; + logger.debug(`size adjusted: ${hitsSize}`); sortIds = searchAfterResult.hits.hits[0].sort; if (sortIds == null) { - logger.error('sortIds was empty search when running a signal rule'); - return false; + logger.debug('sortIds was empty on search'); + return true; // no more search results } sortId = sortIds[0]; logger.debug('next bulk index');