[Endpoint] EMT-67: add kql support for endpoint list (#56328)

[Endpoint] EMT-67: add kql support for endpoint list

x-pack/plugins/endpoint/server/routes/endpoints.test.ts

@@ -79,7 +79,7 @@ describe('test endpoint route', () => {
     expect(endpointResultList.request_page_size).toEqual(10);
   });
 
-  it('test find the latest of all endpoints with params', async () => {
+  it('test find the latest of all endpoints with paging properties', async () => {
     const mockRequest = httpServerMock.createKibanaRequest({
       body: {
         paging_properties: [
@@ -112,6 +112,69 @@ describe('test endpoint route', () => {
     );
 
     expect(mockScopedClient.callAsCurrentUser).toBeCalled();
+    expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({
+      match_all: {},
+    });
+    expect(routeConfig.options).toEqual({ authRequired: true });
+    expect(mockResponse.ok).toBeCalled();
+    const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as EndpointResultList;
+    expect(endpointResultList.endpoints.length).toEqual(2);
+    expect(endpointResultList.total).toEqual(2);
+    expect(endpointResultList.request_page_index).toEqual(10);
+    expect(endpointResultList.request_page_size).toEqual(10);
+  });
+
+  it('test find the latest of all endpoints with paging and filters properties', async () => {
+    const mockRequest = httpServerMock.createKibanaRequest({
+      body: {
+        paging_properties: [
+          {
+            page_size: 10,
+          },
+          {
+            page_index: 1,
+          },
+        ],
+
+        filter: 'not host.ip:10.140.73.246',
+      },
+    });
+    mockScopedClient.callAsCurrentUser.mockImplementationOnce(() =>
+      Promise.resolve((data as unknown) as SearchResponse<EndpointMetadata>)
+    );
+    [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
+      path.startsWith('/api/endpoint/endpoints')
+    )!;
+
+    await routeHandler(
+      ({
+        core: {
+          elasticsearch: {
+            dataClient: mockScopedClient,
+          },
+        },
+      } as unknown) as RequestHandlerContext,
+      mockRequest,
+      mockResponse
+    );
+
+    expect(mockScopedClient.callAsCurrentUser).toBeCalled();
+    expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({
+      bool: {
+        must_not: {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                match: {
+                  'host.ip': '10.140.73.246',
+                },
+              },
+            ],
+          },
+        },
+      },
+    });
     expect(routeConfig.options).toEqual({ authRequired: true });
     expect(mockResponse.ok).toBeCalled();
     const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as EndpointResultList;

x-pack/plugins/endpoint/server/routes/endpoints.ts

@@ -26,16 +26,26 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp
       validate: {
         body: schema.nullable(
           schema.object({
-            paging_properties: schema.arrayOf(
-              schema.oneOf([
-                // the number of results to return for this request per page
-                schema.object({
-                  page_size: schema.number({ defaultValue: 10, min: 1, max: 10000 }),
-                }),
-                // the index of the page to return
-                schema.object({ page_index: schema.number({ defaultValue: 0, min: 0 }) }),
-              ])
+            paging_properties: schema.nullable(
+              schema.arrayOf(
+                schema.oneOf([
+                  /**
+                   * the number of results to return for this request per page
+                   */
+                  schema.object({
+                    page_size: schema.number({ defaultValue: 10, min: 1, max: 10000 }),
+                  }),
+                  /**
+                   * the zero based page index of the the total number of pages of page size
+                   */
+                  schema.object({ page_index: schema.number({ defaultValue: 0, min: 0 }) }),
+                ])
+              )
             ),
+            /**
+             * filter to be applied, it could be a kql expression or discrete filter to be implemented
+             */
+            filter: schema.nullable(schema.oneOf([schema.string()])),
           })
         ),
       },

x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.test.ts

@@ -55,6 +55,65 @@ describe('query builder', () => {
     });
   });
 
+  describe('test query builder with kql filter', () => {
+    it('test default query params for all endpoints when no params or body is provided', async () => {
+      const mockRequest = httpServerMock.createKibanaRequest({
+        body: {
+          filter: 'not host.ip:10.140.73.246',
+        },
+      });
+      const query = await kibanaRequestToEndpointListQuery(mockRequest, {
+        logFactory: loggingServiceMock.create(),
+        config: () => Promise.resolve(EndpointConfigSchema.validate({})),
+      });
+      expect(query).toEqual({
+        body: {
+          query: {
+            bool: {
+              must_not: {
+                bool: {
+                  minimum_should_match: 1,
+                  should: [
+                    {
+                      match: {
+                        'host.ip': '10.140.73.246',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+          },
+          collapse: {
+            field: 'host.id.keyword',
+            inner_hits: {
+              name: 'most_recent',
+              size: 1,
+              sort: [{ 'event.created': 'desc' }],
+            },
+          },
+          aggs: {
+            total: {
+              cardinality: {
+                field: 'host.id.keyword',
+              },
+            },
+          },
+          sort: [
+            {
+              'event.created': {
+                order: 'desc',
+              },
+            },
+          ],
+        },
+        from: 0,
+        size: 10,
+        index: 'endpoint-agent*',
+      } as Record<string, any>);
+    });
+  });
+
   describe('EndpointFetchQuery', () => {
     it('searches for the correct ID', () => {
       const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899';

x-pack/plugins/endpoint/server/services/endpoint/endpoint_query_builders.ts

@@ -6,6 +6,7 @@
 import { KibanaRequest } from 'kibana/server';
 import { EndpointAppConstants } from '../../../common/types';
 import { EndpointAppContext } from '../../types';
+import { esKuery } from '../../../../../../src/plugins/data/server';
 
 export const kibanaRequestToEndpointListQuery = async (
   request: KibanaRequest<any, any, any>,
@@ -14,9 +15,7 @@ export const kibanaRequestToEndpointListQuery = async (
   const pagingProperties = await getPagingProperties(request, endpointAppContext);
   return {
     body: {
-      query: {
-        match_all: {},
-      },
+      query: buildQueryBody(request),
       collapse: {
         field: 'host.id.keyword',
         inner_hits: {
@@ -66,6 +65,15 @@ async function getPagingProperties(
   };
 }
 
+function buildQueryBody(request: KibanaRequest<any, any, any>): Record<string, any> {
+  if (typeof request?.body?.filter === 'string') {
+    return esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(request.body.filter));
+  }
+  return {
+    match_all: {},
+  };
+}
+
 export const kibanaRequestToEndpointFetchQuery = (
   request: KibanaRequest<any, any, any>,
   endpointAppContext: EndpointAppContext

x-pack/test/api_integration/apis/endpoint/endpoints.ts

@@ -40,7 +40,7 @@ export default function({ getService }: FtrProviderContext) {
         expect(body.request_page_index).to.eql(0);
       });
 
-      it('endpoints api should return page based on params passed.', async () => {
+      it('endpoints api should return page based on paging properties passed.', async () => {
         const { body } = await supertest
           .post('/api/endpoint/endpoints')
           .set('kbn-xsrf', 'xxx')
@@ -102,6 +102,46 @@ export default function({ getService }: FtrProviderContext) {
           .expect(400);
         expect(body.message).to.contain('Value is [0] but it must be equal to or greater than [1]');
       });
+
+      it('endpoints api should return page based on filters passed.', async () => {
+        const { body } = await supertest
+          .post('/api/endpoint/endpoints')
+          .set('kbn-xsrf', 'xxx')
+          .send({ filter: 'not host.ip:10.101.149.26' })
+          .expect(200);
+        expect(body.total).to.eql(2);
+        expect(body.endpoints.length).to.eql(2);
+        expect(body.request_page_size).to.eql(10);
+        expect(body.request_page_index).to.eql(0);
+      });
+
+      it('endpoints api should return page based on filters and paging passed.', async () => {
+        const notIncludedIp = '10.101.149.26';
+        const { body } = await supertest
+          .post('/api/endpoint/endpoints')
+          .set('kbn-xsrf', 'xxx')
+          .send({
+            paging_properties: [
+              {
+                page_size: 10,
+              },
+              {
+                page_index: 0,
+              },
+            ],
+            filter: `not host.ip:${notIncludedIp}`,
+          })
+          .expect(200);
+        expect(body.total).to.eql(2);
+        const resultIps: string[] = [].concat(
+          ...body.endpoints.map((metadata: Record<string, any>) => metadata.host.ip)
+        );
+        expect(resultIps).to.eql(['10.192.213.130', '10.70.28.129', '10.46.229.234']);
+        expect(resultIps).not.include.eql(notIncludedIp);
+        expect(body.endpoints.length).to.eql(2);
+        expect(body.request_page_size).to.eql(10);
+        expect(body.request_page_index).to.eql(0);
+      });
     });
   });
 }