From 84fa97f89141ab01ba9f237ee6388c73ec8502df Mon Sep 17 00:00:00 2001 From: nnamdifrankie <56440728+nnamdifrankie@users.noreply.github.com> Date: Tue, 4 Feb 2020 14:22:30 -0500 Subject: [PATCH] [Endpoint] EMT-67: add kql support for endpoint list (#56328) [Endpoint] EMT-67: add kql support for endpoint list --- .../endpoint/server/routes/endpoints.test.ts | 65 ++++++++++++++++++- .../endpoint/server/routes/endpoints.ts | 28 +++++--- .../endpoint/endpoint_query_builders.test.ts | 59 +++++++++++++++++ .../endpoint/endpoint_query_builders.ts | 14 +++- .../apis/endpoint/endpoints.ts | 42 +++++++++++- 5 files changed, 194 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/endpoint/server/routes/endpoints.test.ts b/x-pack/plugins/endpoint/server/routes/endpoints.test.ts index be14554f128c3..25c4225495a41 100644 --- a/x-pack/plugins/endpoint/server/routes/endpoints.test.ts +++ b/x-pack/plugins/endpoint/server/routes/endpoints.test.ts @@ -79,7 +79,7 @@ describe('test endpoint route', () => { expect(endpointResultList.request_page_size).toEqual(10); }); - it('test find the latest of all endpoints with params', async () => { + it('test find the latest of all endpoints with paging properties', async () => { const mockRequest = httpServerMock.createKibanaRequest({ body: { paging_properties: [ @@ -112,6 +112,69 @@ describe('test endpoint route', () => { ); expect(mockScopedClient.callAsCurrentUser).toBeCalled(); + expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({ + match_all: {}, + }); + expect(routeConfig.options).toEqual({ authRequired: true }); + expect(mockResponse.ok).toBeCalled(); + const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as EndpointResultList; + expect(endpointResultList.endpoints.length).toEqual(2); + expect(endpointResultList.total).toEqual(2); + expect(endpointResultList.request_page_index).toEqual(10); + expect(endpointResultList.request_page_size).toEqual(10); + }); + + it('test find the latest of all endpoints with paging and filters properties', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + body: { + paging_properties: [ + { + page_size: 10, + }, + { + page_index: 1, + }, + ], + + filter: 'not host.ip:10.140.73.246', + }, + }); + mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => + Promise.resolve((data as unknown) as SearchResponse) + ); + [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/endpoints') + )!; + + await routeHandler( + ({ + core: { + elasticsearch: { + dataClient: mockScopedClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + + expect(mockScopedClient.callAsCurrentUser).toBeCalled(); + expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({ + bool: { + must_not: { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'host.ip': '10.140.73.246', + }, + }, + ], + }, + }, + }, + }); expect(routeConfig.options).toEqual({ authRequired: true }); expect(mockResponse.ok).toBeCalled(); const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as EndpointResultList; diff --git a/x-pack/plugins/endpoint/server/routes/endpoints.ts b/x-pack/plugins/endpoint/server/routes/endpoints.ts index 24ad8e3941f5e..054172a7f258a 100644 --- a/x-pack/plugins/endpoint/server/routes/endpoints.ts +++ b/x-pack/plugins/endpoint/server/routes/endpoints.ts @@ -26,16 +26,26 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp validate: { body: schema.nullable( schema.object({ - paging_properties: schema.arrayOf( - schema.oneOf([ - // the number of results to return for this request per page - schema.object({ - page_size: schema.number({ defaultValue: 10, min: 1, max: 10000 }), - }), - // the index of the page to return - schema.object({ page_index: schema.number({ defaultValue: 0, min: 0 }) }), - ]) + paging_properties: schema.nullable( + schema.arrayOf( + schema.oneOf([ + /** + * the number of results to return for this request per page + */ + schema.object({ + page_size: schema.number({ defaultValue: 10, min: 1, max: 10000 }), + }), + /** + * the zero based page index of the the total number of pages of page size + */ + schema.object({ page_index: schema.number({ defaultValue: 0, min: 0 }) }), + ]) + ) ), + /** + * filter to be applied, it could be a kql expression or discrete filter to be implemented + */ + filter: schema.nullable(schema.oneOf([schema.string()])), }) ), }, diff --git a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts index e453f777fbd50..bd9986ecf1f97 100644 --- a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts +++ b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts @@ -55,6 +55,65 @@ describe('query builder', () => { }); }); + describe('test query builder with kql filter', () => { + it('test default query params for all endpoints when no params or body is provided', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + body: { + filter: 'not host.ip:10.140.73.246', + }, + }); + const query = await kibanaRequestToEndpointListQuery(mockRequest, { + logFactory: loggingServiceMock.create(), + config: () => Promise.resolve(EndpointConfigSchema.validate({})), + }); + expect(query).toEqual({ + body: { + query: { + bool: { + must_not: { + bool: { + minimum_should_match: 1, + should: [ + { + match: { + 'host.ip': '10.140.73.246', + }, + }, + ], + }, + }, + }, + }, + collapse: { + field: 'host.id.keyword', + inner_hits: { + name: 'most_recent', + size: 1, + sort: [{ 'event.created': 'desc' }], + }, + }, + aggs: { + total: { + cardinality: { + field: 'host.id.keyword', + }, + }, + }, + sort: [ + { + 'event.created': { + order: 'desc', + }, + }, + ], + }, + from: 0, + size: 10, + index: 'endpoint-agent*', + } as Record); + }); + }); + describe('EndpointFetchQuery', () => { it('searches for the correct ID', () => { const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899'; diff --git a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts index b4f295a64b6ea..c143b09ec453c 100644 --- a/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts +++ b/x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts @@ -6,6 +6,7 @@ import { KibanaRequest } from 'kibana/server'; import { EndpointAppConstants } from '../../../common/types'; import { EndpointAppContext } from '../../types'; +import { esKuery } from '../../../../../../src/plugins/data/server'; export const kibanaRequestToEndpointListQuery = async ( request: KibanaRequest, @@ -14,9 +15,7 @@ export const kibanaRequestToEndpointListQuery = async ( const pagingProperties = await getPagingProperties(request, endpointAppContext); return { body: { - query: { - match_all: {}, - }, + query: buildQueryBody(request), collapse: { field: 'host.id.keyword', inner_hits: { @@ -66,6 +65,15 @@ async function getPagingProperties( }; } +function buildQueryBody(request: KibanaRequest): Record { + if (typeof request?.body?.filter === 'string') { + return esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(request.body.filter)); + } + return { + match_all: {}, + }; +} + export const kibanaRequestToEndpointFetchQuery = ( request: KibanaRequest, endpointAppContext: EndpointAppContext diff --git a/x-pack/test/api_integration/apis/endpoint/endpoints.ts b/x-pack/test/api_integration/apis/endpoint/endpoints.ts index 1c520fe92e38e..210e9d78d7e18 100644 --- a/x-pack/test/api_integration/apis/endpoint/endpoints.ts +++ b/x-pack/test/api_integration/apis/endpoint/endpoints.ts @@ -40,7 +40,7 @@ export default function({ getService }: FtrProviderContext) { expect(body.request_page_index).to.eql(0); }); - it('endpoints api should return page based on params passed.', async () => { + it('endpoints api should return page based on paging properties passed.', async () => { const { body } = await supertest .post('/api/endpoint/endpoints') .set('kbn-xsrf', 'xxx') @@ -102,6 +102,46 @@ export default function({ getService }: FtrProviderContext) { .expect(400); expect(body.message).to.contain('Value is [0] but it must be equal to or greater than [1]'); }); + + it('endpoints api should return page based on filters passed.', async () => { + const { body } = await supertest + .post('/api/endpoint/endpoints') + .set('kbn-xsrf', 'xxx') + .send({ filter: 'not host.ip:10.101.149.26' }) + .expect(200); + expect(body.total).to.eql(2); + expect(body.endpoints.length).to.eql(2); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); + + it('endpoints api should return page based on filters and paging passed.', async () => { + const notIncludedIp = '10.101.149.26'; + const { body } = await supertest + .post('/api/endpoint/endpoints') + .set('kbn-xsrf', 'xxx') + .send({ + paging_properties: [ + { + page_size: 10, + }, + { + page_index: 0, + }, + ], + filter: `not host.ip:${notIncludedIp}`, + }) + .expect(200); + expect(body.total).to.eql(2); + const resultIps: string[] = [].concat( + ...body.endpoints.map((metadata: Record) => metadata.host.ip) + ); + expect(resultIps).to.eql(['10.192.213.130', '10.70.28.129', '10.46.229.234']); + expect(resultIps).not.include.eql(notIncludedIp); + expect(body.endpoints.length).to.eql(2); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); }); }); }