diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts index 1c7a2a5de6594..2cebaacc67681 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts @@ -5,14 +5,13 @@ */ import { - buildQueryExceptions, - buildExceptionItemEntries, + buildExceptionListQueries, + buildExceptionItem, operatorBuilder, buildExists, buildMatch, buildMatchAny, - evaluateValues, - formatQuery, + buildEntry, getLanguageBooleanOperator, buildNested, } from './build_exceptions_query'; @@ -30,7 +29,6 @@ import { getEntryMatchAnyMock } from '../../../lists/common/schemas/types/entry_ import { getEntryExistsMock } from '../../../lists/common/schemas/types/entry_exists.mock'; describe('build_exceptions_query', () => { - let exclude: boolean; const makeMatchEntry = ({ field, value = 'value-1', @@ -97,10 +95,6 @@ describe('build_exceptions_query', () => { operator: 'excluded', }); - beforeEach(() => { - exclude = true; - }); - describe('getLanguageBooleanOperator', () => { test('it returns value as uppercase if language is "lucene"', () => { const result = getLanguageBooleanOperator({ language: 'lucene', value: 'not' }); @@ -143,14 +137,14 @@ describe('build_exceptions_query', () => { describe('kuery', () => { test('it returns formatted wildcard string when operator is "excluded"', () => { const query = buildExists({ - item: existsEntryWithExcluded, + entry: existsEntryWithExcluded, language: 'kuery', }); expect(query).toEqual('not host.name:*'); }); test('it returns formatted wildcard string when operator is "included"', () => { const query = buildExists({ - item: existsEntryWithIncluded, + entry: existsEntryWithIncluded, language: 'kuery', }); expect(query).toEqual('host.name:*'); @@ -160,14 +154,14 @@ describe('build_exceptions_query', () => { describe('lucene', () => { test('it returns formatted wildcard string when operator is "excluded"', () => { const query = buildExists({ - item: existsEntryWithExcluded, + entry: existsEntryWithExcluded, language: 'lucene', }); expect(query).toEqual('NOT _exists_host.name'); }); test('it returns formatted wildcard string when operator is "included"', () => { const query = buildExists({ - item: existsEntryWithIncluded, + entry: existsEntryWithIncluded, language: 'lucene', }); expect(query).toEqual('_exists_host.name'); @@ -179,14 +173,14 @@ describe('build_exceptions_query', () => { describe('kuery', () => { test('it returns formatted string when operator is "included"', () => { const query = buildMatch({ - item: matchEntryWithIncluded, + entry: matchEntryWithIncluded, language: 'kuery', }); expect(query).toEqual('host.name:"suricata"'); }); test('it returns formatted string when operator is "excluded"', () => { const query = buildMatch({ - item: matchEntryWithExcluded, + entry: matchEntryWithExcluded, language: 'kuery', }); expect(query).toEqual('not host.name:"suricata"'); @@ -196,14 +190,14 @@ describe('build_exceptions_query', () => { describe('lucene', () => { test('it returns formatted string when operator is "included"', () => { const query = buildMatch({ - item: matchEntryWithIncluded, + entry: matchEntryWithIncluded, language: 'lucene', }); expect(query).toEqual('host.name:"suricata"'); }); test('it returns formatted string when operator is "excluded"', () => { const query = buildMatch({ - item: matchEntryWithExcluded, + entry: matchEntryWithExcluded, language: 'lucene', }); expect(query).toEqual('NOT host.name:"suricata"'); @@ -229,7 +223,7 @@ describe('build_exceptions_query', () => { describe('kuery', () => { test('it returns empty string if given an empty array for "values"', () => { const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndNoValues, + entry: entryWithIncludedAndNoValues, language: 'kuery', }); expect(exceptionSegment).toEqual(''); @@ -237,7 +231,7 @@ describe('build_exceptions_query', () => { test('it returns formatted string when "values" includes only one item', () => { const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, + entry: entryWithIncludedAndOneValue, language: 'kuery', }); @@ -246,7 +240,7 @@ describe('build_exceptions_query', () => { test('it returns formatted string when operator is "included"', () => { const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, + entry: matchAnyEntryWithIncludedAndTwoValues, language: 'kuery', }); @@ -255,7 +249,7 @@ describe('build_exceptions_query', () => { test('it returns formatted string when operator is "excluded"', () => { const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, + entry: entryWithExcludedAndTwoValues, language: 'kuery', }); @@ -266,7 +260,7 @@ describe('build_exceptions_query', () => { describe('lucene', () => { test('it returns formatted string when operator is "included"', () => { const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, + entry: matchAnyEntryWithIncludedAndTwoValues, language: 'lucene', }); @@ -274,7 +268,7 @@ describe('build_exceptions_query', () => { }); test('it returns formatted string when operator is "excluded"', () => { const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, + entry: entryWithExcludedAndTwoValues, language: 'lucene', }); @@ -282,7 +276,7 @@ describe('build_exceptions_query', () => { }); test('it returns formatted string when "values" includes only one item', () => { const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, + entry: entryWithIncludedAndOneValue, language: 'lucene', }); @@ -295,7 +289,7 @@ describe('build_exceptions_query', () => { // NOTE: Only KQL supports nested describe('kuery', () => { test('it returns formatted query when one item in nested entry', () => { - const item: EntryNested = { + const entry: EntryNested = { field: 'parent', type: 'nested', entries: [ @@ -307,35 +301,35 @@ describe('build_exceptions_query', () => { }, ], }; - const result = buildNested({ item, language: 'kuery' }); + const result = buildNested({ entry, language: 'kuery' }); expect(result).toEqual('parent:{ nestedField:"value-1" }'); }); test('it returns formatted query when entry item is "exists"', () => { - const item: EntryNested = { + const entry: EntryNested = { field: 'parent', type: 'nested', entries: [{ ...getEntryExistsMock(), field: 'nestedField', operator: 'included' }], }; - const result = buildNested({ item, language: 'kuery' }); + const result = buildNested({ entry, language: 'kuery' }); expect(result).toEqual('parent:{ nestedField:* }'); }); test('it returns formatted query when entry item is "exists" and operator is "excluded"', () => { - const item: EntryNested = { + const entry: EntryNested = { field: 'parent', type: 'nested', entries: [{ ...getEntryExistsMock(), field: 'nestedField', operator: 'excluded' }], }; - const result = buildNested({ item, language: 'kuery' }); + const result = buildNested({ entry, language: 'kuery' }); expect(result).toEqual('parent:{ not nestedField:* }'); }); test('it returns formatted query when entry item is "match_any"', () => { - const item: EntryNested = { + const entry: EntryNested = { field: 'parent', type: 'nested', entries: [ @@ -347,13 +341,13 @@ describe('build_exceptions_query', () => { }, ], }; - const result = buildNested({ item, language: 'kuery' }); + const result = buildNested({ entry, language: 'kuery' }); expect(result).toEqual('parent:{ nestedField:("value1" or "value2") }'); }); test('it returns formatted query when entry item is "match_any" and operator is "excluded"', () => { - const item: EntryNested = { + const entry: EntryNested = { field: 'parent', type: 'nested', entries: [ @@ -365,13 +359,13 @@ describe('build_exceptions_query', () => { }, ], }; - const result = buildNested({ item, language: 'kuery' }); + const result = buildNested({ entry, language: 'kuery' }); expect(result).toEqual('parent:{ not nestedField:("value1" or "value2") }'); }); test('it returns formatted query when multiple items in nested entry', () => { - const item: EntryNested = { + const entry: EntryNested = { field: 'parent', type: 'nested', entries: [ @@ -389,34 +383,34 @@ describe('build_exceptions_query', () => { }, ], }; - const result = buildNested({ item, language: 'kuery' }); + const result = buildNested({ entry, language: 'kuery' }); expect(result).toEqual('parent:{ nestedField:"value-1" and nestedFieldB:"value-2" }'); }); }); }); - describe('evaluateValues', () => { + describe('buildEntry', () => { describe('kuery', () => { test('it returns formatted wildcard string when "type" is "exists"', () => { - const result = evaluateValues({ - item: existsEntryWithIncluded, + const result = buildEntry({ + entry: existsEntryWithIncluded, language: 'kuery', }); expect(result).toEqual('host.name:*'); }); test('it returns formatted string when "type" is "match"', () => { - const result = evaluateValues({ - item: matchEntryWithIncluded, + const result = buildEntry({ + entry: matchEntryWithIncluded, language: 'kuery', }); expect(result).toEqual('host.name:"suricata"'); }); test('it returns formatted string when "type" is "match_any"', () => { - const result = evaluateValues({ - item: matchAnyEntryWithIncludedAndTwoValues, + const result = buildEntry({ + entry: matchAnyEntryWithIncludedAndTwoValues, language: 'kuery', }); expect(result).toEqual('host.name:("suricata" or "auditd")'); @@ -424,95 +418,35 @@ describe('build_exceptions_query', () => { }); describe('lucene', () => { - describe('kuery', () => { - test('it returns formatted wildcard string when "type" is "exists"', () => { - const result = evaluateValues({ - item: existsEntryWithIncluded, - language: 'lucene', - }); - expect(result).toEqual('_exists_host.name'); - }); - - test('it returns formatted string when "type" is "match"', () => { - const result = evaluateValues({ - item: matchEntryWithIncluded, - language: 'lucene', - }); - expect(result).toEqual('host.name:"suricata"'); - }); - - test('it returns formatted string when "type" is "match_any"', () => { - const result = evaluateValues({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'lucene', - }); - expect(result).toEqual('host.name:("suricata" OR "auditd")'); - }); - }); - }); - }); - - describe('formatQuery', () => { - describe('exclude is true', () => { - describe('when query is empty string', () => { - test('it returns empty string if "exceptions" is empty array', () => { - const formattedQuery = formatQuery({ exceptions: [], language: 'kuery', exclude: true }); - expect(formattedQuery).toEqual(''); - }); - - test('it returns expected query string when single exception in array', () => { - const formattedQuery = formatQuery({ - exceptions: ['b:("value-1" or "value-2") and not c:*'], - language: 'kuery', - exclude: true, - }); - expect(formattedQuery).toEqual('not ((b:("value-1" or "value-2") and not c:*))'); - }); - }); - - test('it returns expected query string when multiple exceptions in array', () => { - const formattedQuery = formatQuery({ - exceptions: ['b:("value-1" or "value-2") and not c:*', 'not d:*'], - language: 'kuery', - exclude: true, + test('it returns formatted wildcard string when "type" is "exists"', () => { + const result = buildEntry({ + entry: existsEntryWithIncluded, + language: 'lucene', }); - expect(formattedQuery).toEqual( - 'not ((b:("value-1" or "value-2") and not c:*) or (not d:*))' - ); + expect(result).toEqual('_exists_host.name'); }); - }); - describe('exclude is false', () => { - describe('when query is empty string', () => { - test('it returns empty string if "exceptions" is empty array', () => { - const formattedQuery = formatQuery({ exceptions: [], language: 'kuery', exclude: false }); - expect(formattedQuery).toEqual(''); - }); - - test('it returns expected query string when single exception in array', () => { - const formattedQuery = formatQuery({ - exceptions: ['b:("value-1" or "value-2") and not c:*'], - language: 'kuery', - exclude: false, - }); - expect(formattedQuery).toEqual('(b:("value-1" or "value-2") and not c:*)'); + test('it returns formatted string when "type" is "match"', () => { + const result = buildEntry({ + entry: matchEntryWithIncluded, + language: 'lucene', }); + expect(result).toEqual('host.name:"suricata"'); }); - test('it returns expected query string when multiple exceptions in array', () => { - const formattedQuery = formatQuery({ - exceptions: ['b:("value-1" or "value-2") and not c:*', 'not d:*'], - language: 'kuery', - exclude: false, + test('it returns formatted string when "type" is "match_any"', () => { + const result = buildEntry({ + entry: matchAnyEntryWithIncludedAndTwoValues, + language: 'lucene', }); - expect(formattedQuery).toEqual('(b:("value-1" or "value-2") and not c:*) or (not d:*)'); + expect(result).toEqual('host.name:("suricata" OR "auditd")'); }); }); }); - describe('buildExceptionItemEntries', () => { + describe('buildExceptionItem', () => { test('it returns empty string if empty lists array passed in', () => { - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries: [], }); @@ -525,7 +459,7 @@ describe('build_exceptions_query', () => { makeMatchAnyEntry({ field: 'b' }), makeMatchEntry({ field: 'c', operator: 'excluded', value: 'value-3' }), ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries: payload, }); @@ -545,7 +479,7 @@ describe('build_exceptions_query', () => { ], }, ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -566,7 +500,7 @@ describe('build_exceptions_query', () => { }, makeExistsEntry({ field: 'd' }), ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -587,7 +521,7 @@ describe('build_exceptions_query', () => { }, makeExistsEntry({ field: 'e', operator: 'excluded' }), ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'lucene', entries, }); @@ -599,7 +533,7 @@ describe('build_exceptions_query', () => { describe('exists', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { const entries: EntriesArray = [makeExistsEntry({ field: 'b' })]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -610,7 +544,7 @@ describe('build_exceptions_query', () => { test('it returns expected query when list includes single list item with operator of "excluded"', () => { const entries: EntriesArray = [makeExistsEntry({ field: 'b', operator: 'excluded' })]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -628,7 +562,7 @@ describe('build_exceptions_query', () => { entries: [makeMatchEntry({ field: 'c', operator: 'included', value: 'value-1' })], }, ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -650,7 +584,7 @@ describe('build_exceptions_query', () => { }, makeExistsEntry({ field: 'e' }), ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -663,7 +597,7 @@ describe('build_exceptions_query', () => { describe('match', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { const entries: EntriesArray = [makeMatchEntry({ field: 'b', value: 'value' })]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -676,7 +610,7 @@ describe('build_exceptions_query', () => { const entries: EntriesArray = [ makeMatchEntry({ field: 'b', operator: 'excluded', value: 'value' }), ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -694,7 +628,7 @@ describe('build_exceptions_query', () => { entries: [makeMatchEntry({ field: 'c', operator: 'included', value: 'valueC' })], }, ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -716,7 +650,7 @@ describe('build_exceptions_query', () => { }, makeMatchEntry({ field: 'e', value: 'valueE' }), ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -730,7 +664,7 @@ describe('build_exceptions_query', () => { describe('match_any', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { const entries: EntriesArray = [makeMatchAnyEntry({ field: 'b' })]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -741,7 +675,7 @@ describe('build_exceptions_query', () => { test('it returns expected query when list includes single list item with operator of "excluded"', () => { const entries: EntriesArray = [makeMatchAnyEntry({ field: 'b', operator: 'excluded' })]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -759,7 +693,7 @@ describe('build_exceptions_query', () => { entries: [makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' })], }, ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -773,7 +707,7 @@ describe('build_exceptions_query', () => { makeMatchAnyEntry({ field: 'b' }), makeMatchAnyEntry({ field: 'c' }), ]; - const query = buildExceptionItemEntries({ + const query = buildExceptionItem({ language: 'kuery', entries, }); @@ -784,15 +718,15 @@ describe('build_exceptions_query', () => { }); }); - describe('buildQueryExceptions', () => { + describe('buildExceptionListQueries', () => { test('it returns empty array if lists is empty array', () => { - const query = buildQueryExceptions({ language: 'kuery', lists: [] }); + const query = buildExceptionListQueries({ language: 'kuery', lists: [] }); expect(query).toEqual([]); }); test('it returns empty array if lists is undefined', () => { - const query = buildQueryExceptions({ language: 'kuery', lists: undefined }); + const query = buildExceptionListQueries({ language: 'kuery', lists: undefined }); expect(query).toEqual([]); }); @@ -812,14 +746,24 @@ describe('build_exceptions_query', () => { }, makeMatchAnyEntry({ field: 'e', operator: 'excluded' }), ]; - const query = buildQueryExceptions({ + const queries = buildExceptionListQueries({ language: 'kuery', lists: [payload, payload2], }); - const expectedQuery = - 'not ((some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value") or (b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and not e:("value-1" or "value-2")))'; + const expectedQueries = [ + { + query: + 'some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value"', + language: 'kuery', + }, + { + query: + 'b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and not e:("value-1" or "value-2")', + language: 'kuery', + }, + ]; - expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); + expect(queries).toEqual(expectedQueries); }); test('it returns expected query when lists exist and language is "lucene"', () => { @@ -827,78 +771,58 @@ describe('build_exceptions_query', () => { payload.entries = [makeMatchAnyEntry({ field: 'a' }), makeMatchAnyEntry({ field: 'b' })]; const payload2 = getExceptionListItemSchemaMock(); payload2.entries = [makeMatchAnyEntry({ field: 'c' }), makeMatchAnyEntry({ field: 'd' })]; - const query = buildQueryExceptions({ + const queries = buildExceptionListQueries({ language: 'lucene', lists: [payload, payload2], }); - const expectedQuery = - 'NOT ((a:("value-1" OR "value-2") AND b:("value-1" OR "value-2")) OR (c:("value-1" OR "value-2") AND d:("value-1" OR "value-2")))'; + const expectedQueries = [ + { + query: 'a:("value-1" OR "value-2") AND b:("value-1" OR "value-2")', + language: 'lucene', + }, + { + query: 'c:("value-1" OR "value-2") AND d:("value-1" OR "value-2")', + language: 'lucene', + }, + ]; - expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]); + expect(queries).toEqual(expectedQueries); }); - describe('when "exclude" is false', () => { - beforeEach(() => { - exclude = false; + test('it builds correct queries for nested excluded fields', () => { + const payload = getExceptionListItemSchemaMock(); + const payload2 = getExceptionListItemSchemaMock(); + payload2.entries = [ + makeMatchAnyEntry({ field: 'b' }), + { + field: 'parent', + type: 'nested', + entries: [ + // TODO: these operators are not being respected. buildNested needs to be updated + makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }), + makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }), + ], + }, + makeMatchAnyEntry({ field: 'e' }), + ]; + const queries = buildExceptionListQueries({ + language: 'kuery', + lists: [payload, payload2], }); - - test('it returns empty array if lists is empty array', () => { - const query = buildQueryExceptions({ + const expectedQueries = [ + { + query: + 'some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value"', language: 'kuery', - lists: [], - exclude, - }); - - expect(query).toEqual([]); - }); - - test('it returns empty array if lists is undefined', () => { - const query = buildQueryExceptions({ language: 'kuery', lists: undefined, exclude }); - - expect(query).toEqual([]); - }); - - test('it returns expected query when lists exist and language is "kuery"', () => { - const payload = getExceptionListItemSchemaMock(); - const payload2 = getExceptionListItemSchemaMock(); - payload2.entries = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }), - makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }), - ], - }, - makeMatchAnyEntry({ field: 'e' }), - ]; - const query = buildQueryExceptions({ + }, + { + query: + 'b:("value-1" or "value-2") and parent:{ not c:"valueC" and not d:"valueD" } and e:("value-1" or "value-2")', language: 'kuery', - lists: [payload, payload2], - exclude, - }); - const expectedQuery = - '(some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value") or (b:("value-1" or "value-2") and parent:{ not c:"valueC" and not d:"valueD" } and e:("value-1" or "value-2"))'; - - expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); - }); - - test('it returns expected query when lists exist and language is "lucene"', () => { - const payload = getExceptionListItemSchemaMock(); - payload.entries = [makeMatchAnyEntry({ field: 'a' }), makeMatchAnyEntry({ field: 'b' })]; - const payload2 = getExceptionListItemSchemaMock(); - payload2.entries = [makeMatchAnyEntry({ field: 'c' }), makeMatchAnyEntry({ field: 'd' })]; - const query = buildQueryExceptions({ - language: 'lucene', - lists: [payload, payload2], - exclude, - }); - const expectedQuery = - '(a:("value-1" OR "value-2") AND b:("value-1" OR "value-2")) OR (c:("value-1" OR "value-2") AND d:("value-1" OR "value-2"))'; + }, + ]; - expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]); - }); + expect(queries).toEqual(expectedQueries); }); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts index ff492dcda3b66..c64d0b124b67a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts @@ -64,13 +64,13 @@ export const operatorBuilder = ({ }; export const buildExists = ({ - item, + entry, language, }: { - item: EntryExists; + entry: EntryExists; language: Language; }): string => { - const { operator, field } = item; + const { operator, field } = entry; const exceptionOperator = operatorBuilder({ operator, language }); switch (language) { @@ -84,26 +84,26 @@ export const buildExists = ({ }; export const buildMatch = ({ - item, + entry, language, }: { - item: EntryMatch; + entry: EntryMatch; language: Language; }): string => { - const { value, operator, field } = item; + const { value, operator, field } = entry; const exceptionOperator = operatorBuilder({ operator, language }); return `${exceptionOperator}${field}:"${value}"`; }; export const buildMatchAny = ({ - item, + entry, language, }: { - item: EntryMatchAny; + entry: EntryMatchAny; language: Language; }): string => { - const { value, operator, field } = item; + const { value, operator, field } = entry; switch (value.length) { case 0: @@ -118,67 +118,40 @@ export const buildMatchAny = ({ }; export const buildNested = ({ - item, + entry, language, }: { - item: EntryNested; + entry: EntryNested; language: Language; }): string => { - const { field, entries } = item; + const { field, entries: subentries } = entry; const and = getLanguageBooleanOperator({ language, value: 'and' }); - const values = entries.map((entry) => evaluateValues({ item: entry, language })); + const values = subentries.map((subentry) => buildEntry({ entry: subentry, language })); return `${field}:{ ${values.join(` ${and} `)} }`; }; -export const evaluateValues = ({ - item, +export const buildEntry = ({ + entry, language, }: { - item: Entry | EntryNested; + entry: Entry | EntryNested; language: Language; }): string => { - if (entriesExists.is(item)) { - return buildExists({ item, language }); - } else if (entriesMatch.is(item)) { - return buildMatch({ item, language }); - } else if (entriesMatchAny.is(item)) { - return buildMatchAny({ item, language }); - } else if (entriesNested.is(item)) { - return buildNested({ item, language }); + if (entriesExists.is(entry)) { + return buildExists({ entry, language }); + } else if (entriesMatch.is(entry)) { + return buildMatch({ entry, language }); + } else if (entriesMatchAny.is(entry)) { + return buildMatchAny({ entry, language }); + } else if (entriesNested.is(entry)) { + return buildNested({ entry, language }); } else { return ''; } }; -export const formatQuery = ({ - exceptions, - language, - exclude, -}: { - exceptions: string[]; - language: Language; - exclude: boolean; -}): string => { - if (exceptions == null || (exceptions != null && exceptions.length === 0)) { - return ''; - } - - const or = getLanguageBooleanOperator({ language, value: 'or' }); - const not = getLanguageBooleanOperator({ language, value: 'not' }); - const formattedExceptionItems = exceptions.map((exceptionItem, index) => { - if (index === 0) { - return `(${exceptionItem})`; - } - - return `${or} (${exceptionItem})`; - }); - - const exceptionItemsQuery = formattedExceptionItems.join(' '); - return exclude ? `${not} (${exceptionItemsQuery})` : exceptionItemsQuery; -}; - -export const buildExceptionItemEntries = ({ +export const buildExceptionItem = ({ entries, language, }: { @@ -186,22 +159,19 @@ export const buildExceptionItemEntries = ({ language: Language; }): string => { const and = getLanguageBooleanOperator({ language, value: 'and' }); - const exceptionItemEntries = entries.reduce((accum, listItem) => { - const exceptionSegment = evaluateValues({ item: listItem, language }); - return [...accum, exceptionSegment]; - }, []); + const exceptionItemEntries = entries.map((entry) => { + return buildEntry({ entry, language }); + }); return exceptionItemEntries.join(` ${and} `); }; -export const buildQueryExceptions = ({ +export const buildExceptionListQueries = ({ language, lists, - exclude = true, }: { language: Language; lists: Array | undefined; - exclude?: boolean; }): DataQuery[] => { if (lists == null || (lists != null && lists.length === 0)) { return []; @@ -211,7 +181,7 @@ export const buildQueryExceptions = ({ const { entries } = exceptionItem; if (entries != null && entries.length > 0 && !hasLargeValueList(entries)) { - return [...acc, buildExceptionItemEntries({ entries, language })]; + return [...acc, buildExceptionItem({ entries, language })]; } else { return acc; } @@ -220,12 +190,11 @@ export const buildQueryExceptions = ({ if (exceptionItems.length === 0) { return []; } else { - const formattedQuery = formatQuery({ exceptions: exceptionItems, language, exclude }); - return [ - { - query: formattedQuery, + return exceptionItems.map((exceptionItem) => { + return { + query: exceptionItem, language, - }, - ]; + }; + }); } }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index a8eb4e7bbb15b..72ef230a42342 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getQueryFilter } from './get_query_filter'; -import { Filter } from 'src/plugins/data/public'; +import { getQueryFilter, buildExceptionFilter } from './get_query_filter'; +import { Filter, EsQueryConfig } from 'src/plugins/data/public'; import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock'; describe('get_filter', () => { @@ -363,49 +363,151 @@ describe('get_filter', () => { bool: { filter: [ { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } }, + ], + must: [], + must_not: [ { bool: { - must_not: { - bool: { - filter: [ - { - nested: { - path: 'some.parentField', - query: { - bool: { - minimum_should_match: 1, - should: [ - { - match_phrase: { - 'some.parentField.nested.field': 'some value', + should: [ + { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, }, + ], + }, + }, + score_mode: 'none', + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', }, - ], + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + should: [], + }, + }); + }); + + test('it should work with a list with multiple items', () => { + const esQuery = getQueryFilter( + 'host.name: linux', + 'kuery', + [], + ['auditbeat-*'], + [getExceptionListItemSchemaMock(), getExceptionListItemSchemaMock()] + ); + expect(esQuery).toEqual({ + bool: { + filter: [ + { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } }, + ], + must: [], + must_not: [ + { + bool: { + should: [ + { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], + }, }, + score_mode: 'none', }, - score_mode: 'none', }, - }, - { - bool: { - minimum_should_match: 1, - should: [ - { - match_phrase: { - 'some.not.nested.field': 'some value', + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', + }, + }, + ], + }, + }, + ], + }, + }, + { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], }, }, - ], + score_mode: 'none', + }, }, - }, - ], + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', + }, + }, + ], + }, + }, + ], + }, }, - }, + ], }, }, ], - must: [], - must_not: [], should: [], }, }); @@ -455,32 +557,137 @@ describe('get_filter', () => { { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } }, { bool: { - filter: [ + should: [ { - nested: { - path: 'some.parentField', - query: { - bool: { - minimum_should_match: 1, - should: [ - { - match_phrase: { - 'some.parentField.nested.field': 'some value', + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], }, }, - ], + score_mode: 'none', + }, }, - }, - score_mode: 'none', + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + }); + }); + + test('it should work with a list with multiple items', () => { + const esQuery = getQueryFilter( + 'host.name: linux', + 'kuery', + [], + ['auditbeat-*'], + [getExceptionListItemSchemaMock(), getExceptionListItemSchemaMock()], + false + ); + expect(esQuery).toEqual({ + bool: { + filter: [ + { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } }, + { + bool: { + should: [ + { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], + }, + }, + score_mode: 'none', + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', + }, + }, + ], + }, + }, + ], }, }, { bool: { - minimum_should_match: 1, - should: [ + filter: [ { - match_phrase: { - 'some.not.nested.field': 'some value', + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], + }, + }, + score_mode: 'none', + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', + }, + }, + ], }, }, ], @@ -703,4 +910,179 @@ describe('get_filter', () => { }); }); }); + + describe('buildExceptionFilter', () => { + const config: EsQueryConfig = { + allowLeadingWildcards: true, + queryStringOptions: { analyze_wildcard: true }, + ignoreFilterIfFieldNotInIndex: false, + dateFormatTZ: 'Zulu', + }; + test('it should build a filter without chunking exception items', () => { + const exceptionFilter = buildExceptionFilter( + [ + { language: 'kuery', query: 'host.name: linux and some.field: value' }, + { language: 'kuery', query: 'user.name: name' }, + ], + { + fields: [], + title: 'auditbeat-*', + }, + config, + true, + 2 + ); + expect(exceptionFilter).toEqual({ + meta: { + alias: null, + negate: true, + disabled: false, + }, + query: { + bool: { + should: [ + { + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'host.name': 'linux', + }, + }, + ], + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'some.field': 'value', + }, + }, + ], + }, + }, + ], + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'user.name': 'name', + }, + }, + ], + }, + }, + ], + }, + }, + }); + }); + + test('it should properly chunk exception items', () => { + const exceptionFilter = buildExceptionFilter( + [ + { language: 'kuery', query: 'host.name: linux and some.field: value' }, + { language: 'kuery', query: 'user.name: name' }, + { language: 'kuery', query: 'file.path: /safe/path' }, + ], + { + fields: [], + title: 'auditbeat-*', + }, + config, + true, + 2 + ); + expect(exceptionFilter).toEqual({ + meta: { + alias: null, + negate: true, + disabled: false, + }, + query: { + bool: { + should: [ + { + bool: { + should: [ + { + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'host.name': 'linux', + }, + }, + ], + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'some.field': 'value', + }, + }, + ], + }, + }, + ], + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'user.name': 'name', + }, + }, + ], + }, + }, + ], + }, + }, + { + bool: { + should: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'file.path': '/safe/path', + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index a41589b5d0231..466a004c14c66 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -6,20 +6,21 @@ import { Filter, + Query, IIndexPattern, isFilterDisabled, buildEsQuery, - Query as DataQuery, + EsQueryConfig, } from '../../../../../src/plugins/data/common'; import { ExceptionListItemSchema, CreateExceptionListItemSchema, } from '../../../lists/common/schemas'; -import { buildQueryExceptions } from './build_exceptions_query'; -import { Query, Language, Index } from './schemas/common/schemas'; +import { buildExceptionListQueries } from './build_exceptions_query'; +import { Query as QueryString, Language, Index } from './schemas/common/schemas'; export const getQueryFilter = ( - query: Query, + query: QueryString, language: Language, filters: Array>, index: Index, @@ -31,7 +32,14 @@ export const getQueryFilter = ( title: index.join(), }; - const initialQuery = [{ query, language }]; + const config: EsQueryConfig = { + allowLeadingWildcards: true, + queryStringOptions: { analyze_wildcard: true }, + ignoreFilterIfFieldNotInIndex: false, + dateFormatTZ: 'Zulu', + }; + + const enabledFilters = ((filters as unknown) as Filter[]).filter((f) => !isFilterDisabled(f)); /* * Pinning exceptions to 'kuery' because lucene * does not support nested queries, while our exceptions @@ -39,16 +47,78 @@ export const getQueryFilter = ( * buildEsQuery, this allows us to offer nested queries * regardless */ - const exceptions = buildQueryExceptions({ language: 'kuery', lists, exclude: excludeExceptions }); - const queries: DataQuery[] = [...initialQuery, ...exceptions]; + const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists }); + if (exceptionQueries.length > 0) { + // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), + // allowing us to make 1024-item chunks of exception list items. + // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a + // very conservative value. + const exceptionFilter = buildExceptionFilter( + exceptionQueries, + indexPattern, + config, + excludeExceptions, + 1024 + ); + enabledFilters.push(exceptionFilter); + } + const initialQuery = { query, language }; - const config = { - allowLeadingWildcards: true, - queryStringOptions: { analyze_wildcard: true }, - ignoreFilterIfFieldNotInIndex: false, - dateFormatTZ: 'Zulu', - }; + return buildEsQuery(indexPattern, initialQuery, enabledFilters, config); +}; - const enabledFilters = ((filters as unknown) as Filter[]).filter((f) => !isFilterDisabled(f)); - return buildEsQuery(indexPattern, queries, enabledFilters, config); +export const buildExceptionFilter = ( + exceptionQueries: Query[], + indexPattern: IIndexPattern, + config: EsQueryConfig, + excludeExceptions: boolean, + chunkSize: number +) => { + const exceptionFilter: Filter = { + meta: { + alias: null, + negate: excludeExceptions, + disabled: false, + }, + query: { + bool: { + should: undefined, + }, + }, + }; + if (exceptionQueries.length <= chunkSize) { + const query = buildEsQuery(indexPattern, exceptionQueries, [], config); + exceptionFilter.query.bool.should = query.bool.filter; + } else { + const chunkedFilters: Filter[] = []; + for (let index = 0; index < exceptionQueries.length; index += chunkSize) { + const exceptionQueriesChunk = exceptionQueries.slice(index, index + chunkSize); + const esQueryChunk = buildEsQuery(indexPattern, exceptionQueriesChunk, [], config); + const filterChunk: Filter = { + meta: { + alias: null, + negate: false, + disabled: false, + }, + query: { + bool: { + should: esQueryChunk.bool.filter, + }, + }, + }; + chunkedFilters.push(filterChunk); + } + // Here we build a query with only the exceptions: it will put them all in the `filter` array + // of the resulting object, which would AND the exceptions together. When creating exceptionFilter, + // we move the `filter` array to `should` so they are OR'd together instead. + // This gets around the problem with buildEsQuery not allowing callers to specify whether queries passed in + // should be ANDed or ORed together. + exceptionFilter.query.bool.should = buildEsQuery( + indexPattern, + [], + chunkedFilters, + config + ).bool.filter; + } + return exceptionFilter; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts index a5740d7719f47..56768ec78745d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts @@ -209,49 +209,52 @@ describe('get_filter', () => { minimum_should_match: 1, }, }, + ], + must_not: [ { bool: { - must_not: { - bool: { - filter: [ - { - nested: { - path: 'some.parentField', - query: { - bool: { - should: [ - { - match_phrase: { - 'some.parentField.nested.field': 'some value', + should: [ + { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, }, - }, - ], - minimum_should_match: 1, + ], + minimum_should_match: 1, + }, }, + score_mode: 'none', }, - score_mode: 'none', }, - }, - { - bool: { - should: [ - { - match_phrase: { - 'some.not.nested.field': 'some value', + { + bool: { + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', + }, }, - }, - ], - minimum_should_match: 1, + ], + minimum_should_match: 1, + }, }, - }, - ], + ], + }, }, - }, + ], }, }, ], should: [], - must_not: [], }, }); });