From fd741b47e79bea44d8269c27d79d0d59007bc725 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Fri, 3 Dec 2021 12:01:37 +0100 Subject: [PATCH] [Watcher] Prevent expensive queries on ES by using PointInTimeFinder to get indexPatterns (#119717) * Use PointInTimeFinder to paginate through indexPatterns in a more performant way * commit using @elastic.co * Start working on tests * Add tests * Add missing types * Fix tests * Update tests responses * Remove unnecessary mocks * Fix type Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../watch_create_threshold.test.tsx | 8 ---- .../client_integration/watch_edit.test.ts | 8 ---- .../watcher/public/application/lib/api.ts | 8 ++-- .../threshold_watch_edit.tsx | 5 +- .../register_get_index_patterns_route.ts | 47 +++++++++++++++++++ .../api/indices/register_indices_routes.ts | 2 + x-pack/test/api_integration/apis/index.ts | 1 + .../api_integration/apis/watcher/index.ts | 14 ++++++ .../api_integration/apis/watcher/watcher.ts | 47 +++++++++++++++++++ 9 files changed, 116 insertions(+), 24 deletions(-) create mode 100644 x-pack/plugins/watcher/server/routes/api/indices/register_get_index_patterns_route.ts create mode 100644 x-pack/test/api_integration/apis/watcher/index.ts create mode 100644 x-pack/test/api_integration/apis/watcher/watcher.ts diff --git a/x-pack/plugins/watcher/__jest__/client_integration/watch_create_threshold.test.tsx b/x-pack/plugins/watcher/__jest__/client_integration/watch_create_threshold.test.tsx index 481f59093d7dc..52c3a69938d74 100644 --- a/x-pack/plugins/watcher/__jest__/client_integration/watch_create_threshold.test.tsx +++ b/x-pack/plugins/watcher/__jest__/client_integration/watch_create_threshold.test.tsx @@ -50,14 +50,6 @@ jest.mock('../../public/application/lib/api', () => { return { ...original, - loadIndexPatterns: async () => { - const INDEX_PATTERNS = [ - { attributes: { title: 'index1' } }, - { attributes: { title: 'index2' } }, - { attributes: { title: 'index3' } }, - ]; - return await INDEX_PATTERNS; - }, getHttpClient: () => mockHttpClient, }; }); diff --git a/x-pack/plugins/watcher/__jest__/client_integration/watch_edit.test.ts b/x-pack/plugins/watcher/__jest__/client_integration/watch_edit.test.ts index 1188cc8469a58..b5fb2aa9d915a 100644 --- a/x-pack/plugins/watcher/__jest__/client_integration/watch_edit.test.ts +++ b/x-pack/plugins/watcher/__jest__/client_integration/watch_edit.test.ts @@ -23,14 +23,6 @@ jest.mock('../../public/application/lib/api', () => { return { ...original, - loadIndexPatterns: async () => { - const INDEX_PATTERNS = [ - { attributes: { title: 'index1' } }, - { attributes: { title: 'index2' } }, - { attributes: { title: 'index3' } }, - ]; - return await INDEX_PATTERNS; - }, getHttpClient: () => mockHttpClient, }; }); diff --git a/x-pack/plugins/watcher/public/application/lib/api.ts b/x-pack/plugins/watcher/public/application/lib/api.ts index 61ea124695cb9..0971371f7949b 100644 --- a/x-pack/plugins/watcher/public/application/lib/api.ts +++ b/x-pack/plugins/watcher/public/application/lib/api.ts @@ -151,12 +151,10 @@ export const executeWatch = async (executeWatchDetails: ExecutedWatchDetails, wa }; export const loadIndexPatterns = async () => { - const { savedObjects } = await getSavedObjectsClient().find({ - type: 'index-pattern', - fields: ['title'], - perPage: 10000, + return sendRequest({ + path: `${basePath}/indices/index_patterns`, + method: 'get', }); - return savedObjects; }; const getWatchVisualizationDataDeserializer = (data: { visualizeData: any }) => data?.visualizeData; diff --git a/x-pack/plugins/watcher/public/application/sections/watch_edit/components/threshold_watch_edit/threshold_watch_edit.tsx b/x-pack/plugins/watcher/public/application/sections/watch_edit/components/threshold_watch_edit/threshold_watch_edit.tsx index a4c19827c902a..a885c86bc8817 100644 --- a/x-pack/plugins/watcher/public/application/sections/watch_edit/components/threshold_watch_edit/threshold_watch_edit.tsx +++ b/x-pack/plugins/watcher/public/application/sections/watch_edit/components/threshold_watch_edit/threshold_watch_edit.tsx @@ -182,9 +182,8 @@ export const ThresholdWatchEdit = ({ pageTitle }: { pageTitle: string }) => { useEffect(() => { const getIndexPatterns = async () => { - const indexPatternObjects = await loadIndexPatterns(); - const titles = indexPatternObjects.map((indexPattern: any) => indexPattern.attributes.title); - setIndexPatterns(titles); + const { data: indexPatternTitles } = await loadIndexPatterns(); + setIndexPatterns(indexPatternTitles); }; const loadData = async () => { diff --git a/x-pack/plugins/watcher/server/routes/api/indices/register_get_index_patterns_route.ts b/x-pack/plugins/watcher/server/routes/api/indices/register_get_index_patterns_route.ts new file mode 100644 index 0000000000000..237f6dac6a254 --- /dev/null +++ b/x-pack/plugins/watcher/server/routes/api/indices/register_get_index_patterns_route.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsFindResult } from 'src/core/server'; +import { RouteDependencies } from '../../../types'; + +export function registerGetIndexPatternsRoute({ + router, + license, + lib: { handleEsError }, +}: RouteDependencies) { + router.get( + { + path: '/api/watcher/indices/index_patterns', + validate: false, + }, + license.guardApiRoute(async ({ core: { savedObjects } }, request, response) => { + try { + const finder = savedObjects.client.createPointInTimeFinder({ + type: 'index-pattern', + fields: ['title'], + perPage: 1000, + }); + + const responses: string[] = []; + + for await (const result of finder.find()) { + responses.push( + ...result.saved_objects.map( + (indexPattern: SavedObjectsFindResult) => indexPattern.attributes.title + ) + ); + } + + await finder.close(); + + return response.ok({ body: responses }); + } catch (error) { + return handleEsError({ error, response }); + } + }) + ); +} diff --git a/x-pack/plugins/watcher/server/routes/api/indices/register_indices_routes.ts b/x-pack/plugins/watcher/server/routes/api/indices/register_indices_routes.ts index 041457aadf8eb..6e7003f1cafed 100644 --- a/x-pack/plugins/watcher/server/routes/api/indices/register_indices_routes.ts +++ b/x-pack/plugins/watcher/server/routes/api/indices/register_indices_routes.ts @@ -6,8 +6,10 @@ */ import { registerGetRoute } from './register_get_route'; +import { registerGetIndexPatternsRoute } from './register_get_index_patterns_route'; import { RouteDependencies } from '../../../types'; export function registerIndicesRoutes(deps: RouteDependencies) { registerGetRoute(deps); + registerGetIndexPatternsRoute(deps); } diff --git a/x-pack/test/api_integration/apis/index.ts b/x-pack/test/api_integration/apis/index.ts index 56b2042dc4854..b37d88a5dc426 100644 --- a/x-pack/test/api_integration/apis/index.ts +++ b/x-pack/test/api_integration/apis/index.ts @@ -34,5 +34,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./painless_lab')); loadTestFile(require.resolve('./file_upload')); loadTestFile(require.resolve('./ml')); + loadTestFile(require.resolve('./watcher')); }); } diff --git a/x-pack/test/api_integration/apis/watcher/index.ts b/x-pack/test/api_integration/apis/watcher/index.ts new file mode 100644 index 0000000000000..964b7fa0099af --- /dev/null +++ b/x-pack/test/api_integration/apis/watcher/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Watcher', () => { + loadTestFile(require.resolve('./watcher')); + }); +} diff --git a/x-pack/test/api_integration/apis/watcher/watcher.ts b/x-pack/test/api_integration/apis/watcher/watcher.ts new file mode 100644 index 0000000000000..734b6c8c6212d --- /dev/null +++ b/x-pack/test/api_integration/apis/watcher/watcher.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const log = getService('log'); + const supertest = getService('supertest'); + const transform = getService('transform'); + + describe('watcher', () => { + before(async () => { + try { + await transform.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date'); + } catch (error) { + log.debug('[Setup error] Error creating index pattern'); + throw error; + } + }); + + after(async () => { + try { + await transform.testResources.deleteIndexPatternByTitle('ft_ecommerce'); + } catch (error) { + log.debug('[Cleanup error] Error deleting index pattern'); + throw error; + } + }); + + describe('POST /api/watcher/indices/index_patterns', () => { + it('returns list of index patterns', async () => { + const response = await supertest + .get('/api/watcher/indices/index_patterns') + .set('kbn-xsrf', 'kibana') + .expect(200); + + expect(response.body).to.contain('ft_ecommerce'); + }); + }); + }); +}