diff --git a/packages/kbn-securitysolution-list-utils/src/build_exception_filter/index.ts b/packages/kbn-securitysolution-list-utils/src/build_exception_filter/index.ts index 080bd0a311d7e..72db4991a49a4 100644 --- a/packages/kbn-securitysolution-list-utils/src/build_exception_filter/index.ts +++ b/packages/kbn-securitysolution-list-utils/src/build_exception_filter/index.ts @@ -19,6 +19,7 @@ import { entriesMatch, entriesMatchAny, entriesNested, + OsTypeArray, } from '@kbn/securitysolution-io-ts-list-types'; import { hasLargeValueList } from '../has_large_value_list'; @@ -69,26 +70,87 @@ export const chunkExceptions = ( return chunk(chunkSize, exceptions); }; -export const buildExceptionItemFilter = ( - exceptionItem: ExceptionItemSansLargeValueLists -): BooleanFilter | NestedFilter => { - const { entries } = exceptionItem; +/** + * Transforms the os_type into a regular filter as if the user had created it + * from the fields for the next state of transforms which will create the elastic filters + * from it. + * + * Note: We use two types of fields, the "host.os.type" and "host.os.name.caseless" + * The endpoint/endgame agent has been using "host.os.name.caseless" as the same value as the ECS + * value of "host.os.type" where the auditbeat, winlogbeat, etc... (other agents) are all using + * "host.os.type". In order to be compatible with both, I create an "OR" between these two data types + * where if either has a match then we will exclude it as part of the match. This should also be + * forwards compatible for endpoints/endgame agents when/if they upgrade to using "host.os.type" + * rather than using "host.os.name.caseless" values. + * + * Also we create another "OR" from the osType names so that if there are multiples such as ['windows', 'linux'] + * this will exclude anything with either 'windows' or with 'linux' + * @param osTypes The os_type array from the REST interface that is an array such as ['windows', 'linux'] + * @param entries The entries to join the OR's with before the elastic filter change out + */ +export const transformOsType = ( + osTypes: OsTypeArray, + entries: NonListEntry[] +): NonListEntry[][] => { + const hostTypeTransformed = osTypes.map((osType) => { + return [ + { field: 'host.os.type', operator: 'included', type: 'match', value: osType }, + ...entries, + ]; + }); + const caseLessTransformed = osTypes.map((osType) => { + return [ + { field: 'host.os.name.caseless', operator: 'included', type: 'match', value: osType }, + ...entries, + ]; + }); + return [...hostTypeTransformed, ...caseLessTransformed]; +}; - if (entries.length === 1) { - return createInnerAndClauses(entries[0]); - } else { +/** + * This builds an exception item filter with the os type + * @param osTypes The os_type array from the REST interface that is an array such as ['windows', 'linux'] + * @param entries The entries to join the OR's with before the elastic filter change out + */ +export const buildExceptionItemFilterWithOsType = ( + osTypes: OsTypeArray, + entries: NonListEntry[] +): BooleanFilter[] => { + const entriesWithOsTypes = transformOsType(osTypes, entries); + return entriesWithOsTypes.map((entryWithOsType) => { return { bool: { - filter: entries.map((entry) => createInnerAndClauses(entry)), + filter: entryWithOsType.map((entry) => createInnerAndClauses(entry)), }, }; + }); +}; + +export const buildExceptionItemFilter = ( + exceptionItem: ExceptionItemSansLargeValueLists +): Array => { + const { entries, os_types: osTypes } = exceptionItem; + if (osTypes != null && osTypes.length > 0) { + return buildExceptionItemFilterWithOsType(osTypes, entries); + } else { + if (entries.length === 1) { + return [createInnerAndClauses(entries[0])]; + } else { + return [ + { + bool: { + filter: entries.map((entry) => createInnerAndClauses(entry)), + }, + }, + ]; + } } }; export const createOrClauses = ( exceptionItems: ExceptionItemSansLargeValueLists[] ): Array => { - return exceptionItems.map((exceptionItem) => buildExceptionItemFilter(exceptionItem)); + return exceptionItems.flatMap((exceptionItem) => buildExceptionItemFilter(exceptionItem)); }; export const buildExceptionFilter = ({ diff --git a/x-pack/plugins/lists/common/exceptions/build_exceptions_filter.test.ts b/x-pack/plugins/lists/common/exceptions/build_exceptions_filter.test.ts index 9a996f8f1ac46..feee231f232b0 100644 --- a/x-pack/plugins/lists/common/exceptions/build_exceptions_filter.test.ts +++ b/x-pack/plugins/lists/common/exceptions/build_exceptions_filter.test.ts @@ -611,114 +611,115 @@ describe('build_exceptions_filter', () => { getEntryExistsExcludedMock(), ], }); - - expect(exceptionItemFilter).toEqual({ - bool: { - filter: [ - { - nested: { - path: 'parent.field', - query: { - bool: { - filter: [ - { - bool: { - minimum_should_match: 1, - should: [ - { - match_phrase: { - 'parent.field.host.name': 'some host name', + expect(exceptionItemFilter).toEqual([ + { + bool: { + filter: [ + { + nested: { + path: 'parent.field', + query: { + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'parent.field.host.name': 'some host name', + }, }, - }, - ], + ], + }, }, - }, - { - bool: { - must_not: { - bool: { - minimum_should_match: 1, - should: [ - { - bool: { - minimum_should_match: 1, - should: [ - { - match_phrase: { - 'parent.field.host.name': 'some host name', + { + bool: { + must_not: { + bool: { + minimum_should_match: 1, + should: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'parent.field.host.name': 'some host name', + }, }, - }, - ], + ], + }, }, - }, - { - bool: { - minimum_should_match: 1, - should: [ - { - match_phrase: { - 'parent.field.host.name': 'some other host name', + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'parent.field.host.name': 'some other host name', + }, }, - }, - ], + ], + }, }, - }, - ], + ], + }, }, }, }, - }, - { - bool: { - minimum_should_match: 1, - should: [{ exists: { field: 'parent.field.host.name' } }], + { + bool: { + minimum_should_match: 1, + should: [{ exists: { field: 'parent.field.host.name' } }], + }, }, - }, - ], + ], + }, }, + score_mode: 'none', }, - score_mode: 'none', }, - }, - { - bool: { - minimum_should_match: 1, - should: [ - { - bool: { - minimum_should_match: 1, - should: [{ match_phrase: { 'host.name': 'some "host" name' } }], + { + bool: { + minimum_should_match: 1, + should: [ + { + bool: { + minimum_should_match: 1, + should: [{ match_phrase: { 'host.name': 'some "host" name' } }], + }, }, - }, - { + { + bool: { + minimum_should_match: 1, + should: [{ match_phrase: { 'host.name': 'some other host name' } }], + }, + }, + ], + }, + }, + { + bool: { + must_not: { bool: { minimum_should_match: 1, - should: [{ match_phrase: { 'host.name': 'some other host name' } }], + should: [{ match_phrase: { 'host.name': 'some host name' } }], }, }, - ], - }, - }, - { - bool: { - must_not: { - bool: { - minimum_should_match: 1, - should: [{ match_phrase: { 'host.name': 'some host name' } }], - }, }, }, - }, - { - bool: { - must_not: { - bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] }, + { + bool: { + must_not: { + bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] }, + }, }, }, - }, - ], + ], + }, }, - }); + ]); }); }); diff --git a/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts index 22a176da222d6..d04080e8a56c0 100644 --- a/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts @@ -39,7 +39,7 @@ export const getExceptionListItemSchemaMock = ( meta: META, name: NAME, namespace_type: NAMESPACE_TYPE, - os_types: ['linux'], + os_types: [], tags: ['user added string for a tag', 'malware'], tie_breaker_id: TIE_BREAKER, type: ITEM_TYPE, diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx index 37f368ebb88ea..14d4635eec528 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx @@ -185,7 +185,7 @@ describe('Exception helpers', () => { meta: {}, name: 'some name', namespace_type: 'single', - os_types: ['linux'], + os_types: [], tags: ['user added string for a tag', 'malware'], type: 'simple', }; diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/exception_item/exception_details.test.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/exception_item/exception_details.test.tsx index e3e9ba1bfa132..1d5094021c3d3 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/exception_item/exception_details.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/exception_item/exception_details.test.tsx @@ -157,7 +157,7 @@ describe('ExceptionDetails', () => { }); test('it renders the operating system if one is specified in the exception item', () => { - const exceptionItem = getExceptionListItemSchemaMock(); + const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); const wrapper = mount( { }); test('it renders the exception item creator', () => { - const exceptionItem = getExceptionListItemSchemaMock(); + const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); const wrapper = mount( { }); test('it renders the exception item creation timestamp', () => { - const exceptionItem = getExceptionListItemSchemaMock(); + const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); const wrapper = mount( { }); test('it renders the description if one is included on the exception item', () => { - const exceptionItem = getExceptionListItemSchemaMock(); + const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); const wrapper = mount( { }); test('it renders with Name and Modified info when showName and showModified props are true', () => { - const exceptionItem = getExceptionListItemSchemaMock(); + const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); exceptionItem.comments = []; const wrapper = mount( diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/helpers.test.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/helpers.test.tsx index 634c7975a13a9..d67f526fa9bdc 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/helpers.test.tsx @@ -154,7 +154,7 @@ describe('Exception viewer helpers', () => { describe('#getDescriptionListContent', () => { test('it returns formatted description list with os if one is specified', () => { - const payload = getExceptionListItemSchemaMock(); + const payload = getExceptionListItemSchemaMock({ os_types: ['linux'] }); payload.description = ''; const result = getDescriptionListContent(payload); const expected: DescriptionListItem[] = [ @@ -176,7 +176,7 @@ describe('Exception viewer helpers', () => { }); test('it returns formatted description list with a description if one specified', () => { - const payload = getExceptionListItemSchemaMock(); + const payload = getExceptionListItemSchemaMock({ os_types: ['linux'] }); payload.description = 'Im a description'; const result = getDescriptionListContent(payload); const expected: DescriptionListItem[] = [ @@ -202,7 +202,7 @@ describe('Exception viewer helpers', () => { }); test('it returns just user and date created if no other fields specified', () => { - const payload = getExceptionListItemSchemaMock(); + const payload = getExceptionListItemSchemaMock({ os_types: ['linux'] }); payload.description = ''; const result = getDescriptionListContent(payload); const expected: DescriptionListItem[] = [ @@ -224,7 +224,10 @@ describe('Exception viewer helpers', () => { }); test('it returns Modified By/On info. when `includeModified` is true', () => { - const result = getDescriptionListContent(getExceptionListItemSchemaMock(), true); + const result = getDescriptionListContent( + getExceptionListItemSchemaMock({ os_types: ['linux'] }), + true + ); expect(result).toEqual([ { description: 'Linux', @@ -254,7 +257,11 @@ describe('Exception viewer helpers', () => { }); test('it returns Name when `includeName` is true', () => { - const result = getDescriptionListContent(getExceptionListItemSchemaMock(), false, true); + const result = getDescriptionListContent( + getExceptionListItemSchemaMock({ os_types: ['linux'] }), + false, + true + ); expect(result).toEqual([ { description: 'some name', diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.test.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.test.tsx index d44ce7a136fdf..b974dfebd4eb1 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.test.tsx @@ -115,7 +115,6 @@ describe('When on the Event Filters List Page', () => { expect(eventMeta).toEqual([ 'some name', - 'Linux', 'April 20th 2020 @ 11:25:31', 'some user', 'April 20th 2020 @ 11:25:31', diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts new file mode 100644 index 0000000000000..4a50a146421f6 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts @@ -0,0 +1,862 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { + createListsIndex, + deleteAllExceptions, + deleteListsIndex, +} from '../../../lists_api_integration/utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + createRule, + createRuleWithExceptionEntries, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getRuleForSignalTesting, + getSignalsById, + waitForRuleSuccessOrStatus, + waitForSignalsToBePresent, +} from '../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + describe('Rule exception operators for endpoints', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await createListsIndex(supertest); + await esArchiver.load( + 'x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type' + ); + await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/agent'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await deleteAllExceptions(es); + await deleteListsIndex(supertest); + await esArchiver.unload( + 'x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type' + ); + await esArchiver.unload('x-pack/test/functional/es_archives/rule_exceptions/agent'); + }); + + describe('no exceptions set', () => { + it('should find all the "hosts" from a "agent" index when no exceptions are set on the rule', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host).sort(); + expect(hits).to.eql([ + { + os: { type: 'linux' }, + }, + { + os: { type: 'windows' }, + }, + { + os: { type: 'macos' }, + }, + { + os: { type: 'linux' }, + }, + ]); + }); + + it('should find all the "hosts" from a "endpoint_without_host_type" index when no exceptions are set on the rule', async () => { + const rule = getRuleForSignalTesting(['endpoint_without_host_type']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host).sort(); + expect(hits).to.eql([ + { + os: { name: 'Linux' }, + }, + { + os: { name: 'Windows' }, + }, + { + os: { name: 'Macos' }, + }, + { + os: { name: 'Linux' }, + }, + ]); + }); + }); + + describe('operating system types (os_types)', () => { + describe('endpoints', () => { + it('should filter 1 operating system types (os_type) if it is set as part of an endpoint exception', async () => { + const rule = getRuleForSignalTesting(['endpoint_without_host_type']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { name: 'Windows' }, + }, + { + os: { name: 'Macos' }, + }, + { + os: { name: 'Linux' }, + }, + ]); + }); + + it('should filter 2 operating system types as an "OR" (os_type) if it is set as part of an endpoint exception', async () => { + const rule = getRuleForSignalTesting(['endpoint_without_host_type']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['linux', 'macos'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { name: 'Windows' }, + }, + { + os: { name: 'Macos' }, + }, + { + os: { name: 'Linux' }, + }, + ]); + }); + + it('should filter multiple operating system types if it is set as part of an endpoint exception', async () => { + const rule = getRuleForSignalTesting(['endpoint_without_host_type']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + { + osTypes: ['windows'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '2', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { name: 'Macos' }, + }, + { + os: { name: 'Linux' }, + }, + ]); + }); + + it('should filter multiple operating system types (os_type) with multiple filter items for an endpoint', async () => { + const rule = getRuleForSignalTesting(['endpoint_without_host_type']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['macos', 'linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + { + osTypes: ['windows', 'linux', 'macos'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '2', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { name: 'Macos' }, + }, + { + os: { name: 'Linux' }, + }, + ]); + }); + }); + + describe('agent', () => { + it('should filter 1 operating system types (os_type) if it is set as part of an endpoint exception', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'windows' }, + }, + { + os: { type: 'macos' }, + }, + { + os: { type: 'linux' }, + }, + ]); + }); + + it('should filter 1 operating system type as an "OR" (os_type) if it is set as part of an endpoint exception', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['linux', 'macos'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'windows' }, + }, + { + os: { type: 'macos' }, + }, + { + os: { type: 'linux' }, + }, + ]); + }); + + it('should filter multiple operating system types if it is set as part of an endpoint exception', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + { + osTypes: ['windows'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '2', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'macos' }, + }, + { + os: { type: 'linux' }, + }, + ]); + }); + + it('should filter multiple operating system types (os_type) with multiple filter items for an endpoint', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['macos', 'linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + { + osTypes: ['windows', 'linux', 'macos'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '2', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'macos' }, + }, + { + os: { type: 'linux' }, + }, + ]); + }); + }); + + describe('agent and endpoint', () => { + it('should filter 2 operating system types (os_type) if it is set as part of an endpoint exception', async () => { + const rule = getRuleForSignalTesting(['agent', 'endpoint_without_host_type']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 6, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'windows' }, + }, + { + os: { name: 'Windows' }, + }, + { + os: { type: 'macos' }, + }, + { + os: { name: 'Macos' }, + }, + { + os: { type: 'linux' }, + }, + { + os: { name: 'Linux' }, + }, + ]); + }); + + it('should filter 2 operating system types as an "OR" (os_type) if it is set as part of an endpoint exception', async () => { + const rule = getRuleForSignalTesting(['agent', 'endpoint_without_host_type']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['linux', 'macos'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 6, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'windows' }, + }, + { + os: { name: 'Windows' }, + }, + { + os: { type: 'macos' }, + }, + { + os: { name: 'Macos' }, + }, + { + os: { type: 'linux' }, + }, + { + os: { name: 'Linux' }, + }, + ]); + }); + + it('should filter multiple operating system types if it is set as part of an endpoint exception', async () => { + const rule = getRuleForSignalTesting(['agent', 'endpoint_without_host_type']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + { + osTypes: ['windows'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '2', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'macos' }, + }, + { + os: { name: 'Macos' }, + }, + { + os: { type: 'linux' }, + }, + { + os: { name: 'Linux' }, + }, + ]); + }); + + it('should filter multiple operating system types (os_type) with multiple filter items for an endpoint', async () => { + const rule = getRuleForSignalTesting(['agent', 'endpoint_without_host_type']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['macos', 'linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '1', + }, + ], + }, + { + osTypes: ['windows', 'linux', 'macos'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match', + value: '2', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'macos' }, + }, + { + os: { name: 'Macos' }, + }, + { + os: { type: 'linux' }, + }, + { + os: { name: 'Linux' }, + }, + ]); + }); + }); + }); + + describe('"is" operator', () => { + it('should filter 1 value set as an endpoint exception and 1 value set as a normal rule exception ', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [ + [ + { + field: 'host.os.type', + operator: 'included', + type: 'match', + value: 'linux', + }, + ], + ], + [ + { + osTypes: undefined, // This "undefined" is not possible through the user interface but is possible in the REST API + entries: [ + { + field: 'host.os.type', + operator: 'included', + type: 'match', + value: 'windows', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'macos' }, + }, + ]); + }); + + it('should filter 1 value set as an endpoint exception and 1 value set as a normal rule exception with os_type set', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [ + [ + { + field: 'host.os.type', + operator: 'included', + type: 'match', + value: 'linux', + }, + ], + ], + [ + { + osTypes: ['windows'], + entries: [ + { + field: 'host.os.type', + operator: 'included', + type: 'match', + value: 'windows', + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'macos' }, + }, + ]); + }); + }); + + describe('"is one of" operator', () => { + it('should filter 1 single value if it is set as an exception and the os_type is set to only 1 value', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['windows'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match_any', + value: ['1', '2'], + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'linux' }, + }, + { + os: { type: 'macos' }, + }, + { + os: { type: 'linux' }, + }, + ]); + }); + + it('should filter 2 values if it is set as an exception and the os_type is set to 2 values', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['windows', 'linux'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match_any', + value: ['1', '2'], + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'macos' }, + }, + { + os: { type: 'linux' }, + }, + ]); + }); + + it('should filter 2 values if it is set as an exception and the os_type is set to undefined', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: undefined, // This is only possible through the REST API + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match_any', + value: ['1', '2'], + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'macos' }, + }, + { + os: { type: 'linux' }, + }, + ]); + }); + + it('should filter no values if they are set as an exception but the os_type is set to something not within the documents', async () => { + const rule = getRuleForSignalTesting(['agent']); + const { id } = await createRuleWithExceptionEntries( + supertest, + rule, + [], + [ + { + osTypes: ['macos'], + entries: [ + { + field: 'event.code', + operator: 'included', + type: 'match_any', + value: ['1', '2'], + }, + ], + }, + ] + ); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.host); + expect(hits).to.eql([ + { + os: { type: 'linux' }, + }, + { + os: { type: 'windows' }, + }, + { + os: { type: 'macos' }, + }, + { + os: { type: 'linux' }, + }, + ]); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 3c5e04ee1f64e..44e6023bf366a 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -14,6 +14,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { this.tags('ciGroup11'); loadTestFile(require.resolve('./aliases')); + loadTestFile(require.resolve('./create_endpoint_exceptions')); loadTestFile(require.resolve('./add_actions')); loadTestFile(require.resolve('./update_actions')); loadTestFile(require.resolve('./add_prepackaged_rules')); diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index ac11dd31c15e8..f8989c685c82c 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -12,7 +12,11 @@ import { SuperTest } from 'supertest'; import supertestAsPromised from 'supertest-as-promised'; import { Context } from '@elastic/elasticsearch/lib/Transport'; import { SearchResponse } from 'elasticsearch'; -import type { NonEmptyEntriesArray } from '@kbn/securitysolution-io-ts-list-types'; +import type { + ListArray, + NonEmptyEntriesArray, + OsTypeArray, +} from '@kbn/securitysolution-io-ts-list-types'; import type { CreateExceptionListItemSchema, CreateExceptionListSchema, @@ -21,7 +25,6 @@ import type { } from '@kbn/securitysolution-io-ts-list-types'; import { EXCEPTION_LIST_ITEM_URL, EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants'; import { PrePackagedRulesAndTimelinesStatusSchema } from '../../plugins/security_solution/common/detection_engine/schemas/response'; -import { getCreateExceptionListDetectionSchemaMock } from '../../plugins/lists/common/schemas/request/create_exception_list_schema.mock'; import { CreateRulesSchema, UpdateRulesSchema, @@ -45,7 +48,6 @@ import { INTERNAL_IMMUTABLE_KEY, INTERNAL_RULE_ID_KEY, } from '../../plugins/security_solution/common/constants'; -import { getCreateExceptionListItemMinimalSchemaMockWithoutId } from '../../plugins/lists/common/schemas/request/create_exception_list_item_schema.mock'; /** * This will remove server generated properties such as date times, etc... @@ -1149,28 +1151,97 @@ export const installPrePackagedRules = async ( }; /** - * Convenience testing function where you can pass in just the entries and you will - * get a rule created with the entries added to an exception list and exception list item - * all auto-created at once. + * Convenience testing function where you can pass in just the endpoint entries and you will + * get a container created with the entries. + * @param supertest super test agent + * @param endpointEntries The endpoint entries to create the rule and exception list from + * @param osTypes The os types to optionally add or not to add to the container + */ +export const createContainerWithEndpointEntries = async ( + supertest: SuperTest, + endpointEntries: Array<{ + entries: NonEmptyEntriesArray; + osTypes: OsTypeArray | undefined; + }> +): Promise => { + // If not given any endpoint entries, return without any + if (endpointEntries.length === 0) { + return []; + } + + // create the endpoint exception list container + // eslint-disable-next-line @typescript-eslint/naming-convention + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, { + description: 'endpoint description', + list_id: 'endpoint_list', + name: 'endpoint_list', + type: 'endpoint', + }); + + // Add the endpoint exception list container to the backend + await Promise.all( + endpointEntries.map((endpointEntry) => { + const exceptionListItem: CreateExceptionListItemSchema = { + description: 'endpoint description', + entries: endpointEntry.entries, + list_id: 'endpoint_list', + name: 'endpoint_list', + os_types: endpointEntry.osTypes, + type: 'simple', + }; + return createExceptionListItem(supertest, exceptionListItem); + }) + ); + + // To reduce the odds of in-determinism and/or bugs we ensure we have + // the same length of entries before continuing. + await waitFor(async () => { + const { body } = await supertest.get(`${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${list_id}`); + return body.data.length === endpointEntries.length; + }, `within createContainerWithEndpointEntries ${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${list_id}`); + + return [ + { + id, + list_id, + namespace_type, + type, + }, + ]; +}; + +/** + * Convenience testing function where you can pass in just the endpoint entries and you will + * get a container created with the entries. * @param supertest super test agent - * @param rule The rule to create and attach an exception list to * @param entries The entries to create the rule and exception list from + * @param osTypes The os types to optionally add or not to add to the container */ -export const createRuleWithExceptionEntries = async ( +export const createContainerWithEntries = async ( supertest: SuperTest, - rule: CreateRulesSchema, entries: NonEmptyEntriesArray[] -): Promise => { +): Promise => { + // If not given any endpoint entries, return without any + if (entries.length === 0) { + return []; + } + // Create the rule exception list container // eslint-disable-next-line @typescript-eslint/naming-convention - const { id, list_id, namespace_type, type } = await createExceptionList( - supertest, - getCreateExceptionListDetectionSchemaMock() - ); + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, { + description: 'some description', + list_id: 'some-list-id', + name: 'some name', + type: 'detection', + }); + // Add the rule exception list container to the backend await Promise.all( entries.map((entry) => { const exceptionListItem: CreateExceptionListItemSchema = { - ...getCreateExceptionListItemMinimalSchemaMockWithoutId(), + description: 'some description', + list_id: 'some-list-id', + name: 'some name', + type: 'simple', entries: entry, }; return createExceptionListItem(supertest, exceptionListItem); @@ -1180,13 +1251,44 @@ export const createRuleWithExceptionEntries = async ( // To reduce the odds of in-determinism and/or bugs we ensure we have // the same length of entries before continuing. await waitFor(async () => { - const { body } = await supertest.get( - `${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${ - getCreateExceptionListDetectionSchemaMock().list_id - }` - ); + const { body } = await supertest.get(`${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${list_id}`); return body.data.length === entries.length; - }, `within createRuleWithExceptionEntries ${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${getCreateExceptionListDetectionSchemaMock().list_id}`); + }, `within createContainerWithEntries ${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${list_id}`); + + return [ + { + id, + list_id, + namespace_type, + type, + }, + ]; +}; + +/** + * Convenience testing function where you can pass in just the entries and you will + * get a rule created with the entries added to an exception list and exception list item + * all auto-created at once. + * @param supertest super test agent + * @param rule The rule to create and attach an exception list to + * @param entries The entries to create the rule and exception list from + * @param endpointEntries The endpoint entries to create the rule and exception list from + * @param osTypes The os types to optionally add or not to add to the container + */ +export const createRuleWithExceptionEntries = async ( + supertest: SuperTest, + rule: CreateRulesSchema, + entries: NonEmptyEntriesArray[], + endpointEntries?: Array<{ + entries: NonEmptyEntriesArray; + osTypes: OsTypeArray | undefined; + }> +): Promise => { + const maybeExceptionList = await createContainerWithEntries(supertest, entries); + const maybeEndpointList = await createContainerWithEndpointEntries( + supertest, + endpointEntries ?? [] + ); // create the rule but don't run it immediately as running it immediately can cause // the rule to sometimes not filter correctly the first time with an exception list @@ -1195,14 +1297,7 @@ export const createRuleWithExceptionEntries = async ( const ruleWithException: CreateRulesSchema = { ...rule, enabled: false, - exceptions_list: [ - { - id, - list_id, - namespace_type, - type, - }, - ], + exceptions_list: [...maybeExceptionList, ...maybeEndpointList], }; const ruleResponse = await createRule(supertest, ruleWithException); await supertest diff --git a/x-pack/test/functional/es_archives/rule_exceptions/README.md b/x-pack/test/functional/es_archives/rule_exceptions/README.md index 1fbf4962d55fe..a7c5aebe8a7e2 100644 --- a/x-pack/test/functional/es_archives/rule_exceptions/README.md +++ b/x-pack/test/functional/es_archives/rule_exceptions/README.md @@ -1,7 +1,8 @@ Within this folder is input test data for tests such as: ```ts -security_and_spaces/tests/rule_exceptions.ts +security_and_spaces/tests/operator_data_types +security_and_spaces/tests/create_endpoint_exceptions.ts ``` where these are small ECS compliant input indexes that try to express tests that exercise different parts of diff --git a/x-pack/test/functional/es_archives/rule_exceptions/agent/data.json b/x-pack/test/functional/es_archives/rule_exceptions/agent/data.json new file mode 100644 index 0000000000000..e20e7de50cfeb --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/agent/data.json @@ -0,0 +1,79 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "agent", + "source": { + "@timestamp": "2020-10-28T05:00:53.000Z", + "host": { + "os": { + "type": "linux" + } + }, + "event": { + "code": 1 + } + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "agent", + "source": { + "@timestamp": "2020-10-28T05:01:53.000Z", + "host": { + "os": { + "type": "windows" + } + }, + "event": { + "code": 2 + } + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "agent", + "source": { + "@timestamp": "2020-10-28T05:02:53.000Z", + "host": { + "os": { + "type": "macos" + } + }, + "event": { + "code": 3 + } + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "agent", + "source": { + "@timestamp": "2020-10-28T05:03:53.000Z", + "host": { + "os": { + "type": "linux" + } + }, + "event": { + "code": 4 + } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/rule_exceptions/agent/mappings.json b/x-pack/test/functional/es_archives/rule_exceptions/agent/mappings.json new file mode 100644 index 0000000000000..028e417ae34c0 --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/agent/mappings.json @@ -0,0 +1,40 @@ +{ + "type": "index", + "value": { + "index": "agent", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "properties": { + "os": { + "properties": { + "type": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "event": { + "properties": { + "code": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type/data.json b/x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type/data.json new file mode 100644 index 0000000000000..d0e69259a861f --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type/data.json @@ -0,0 +1,79 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "endpoint_without_host_type", + "source": { + "@timestamp": "2020-10-28T05:00:53.000Z", + "host": { + "os": { + "name": "Linux" + } + }, + "event": { + "code": 1 + } + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "endpoint_without_host_type", + "source": { + "@timestamp": "2020-10-28T05:01:53.000Z", + "host": { + "os": { + "name": "Windows" + } + }, + "event": { + "code": 2 + } + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "endpoint_without_host_type", + "source": { + "@timestamp": "2020-10-28T05:02:53.000Z", + "host": { + "os": { + "name": "Macos" + } + }, + "event": { + "code": 3 + } + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "endpoint_without_host_type", + "source": { + "@timestamp": "2020-10-28T05:03:53.000Z", + "host": { + "os": { + "name": "Linux" + } + }, + "event": { + "code": 4 + } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type/mappings.json b/x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type/mappings.json new file mode 100644 index 0000000000000..7775d5e20e305 --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type/mappings.json @@ -0,0 +1,64 @@ +{ + "type": "index", + "value": { + "index": "endpoint_without_host_type", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "properties": { + "os": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + }, + "text": { + "type": "text" + } + } + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 1024, + "fields": { + "caseless": { + "type": "keyword", + "ignore_above": 1024, + "normalizer": "lowercase" + }, + "text": { + "type": "text" + } + } + } + } + }, + "event": { + "properties": { + "code": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}