From 3bfabb0fabb969948380bffef3637f3722b55c1a Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Fri, 20 Mar 2020 02:20:44 -0700 Subject: [PATCH] [Search service] Asynchronous ES search strategy (#53538) (#60469) * Add async search strategy * Add async search * Fix async strategy and add tests * Move types to separate file * Revert changes to demo search * Update demo search strategy to use async * Add async es search strategy * Return response as rawResponse * Poll after initial request * Add cancellation to search strategies * Add tests * Simplify async search strategy * Move loadingCount to search strategy * Update abort controller library * Bootstrap * Abort when the request is aborted * Add utility and update value suggestions route * Fix bad merge conflict * Update tests * Move to data_enhanced plugin * Remove bad merge * Revert switching abort controller libraries * Revert package.json in lib * Move to previous abort controller * Add support for frozen indices * Fix test to use fake timers to run debounced handlers * Revert changes to example plugin * Fix loading bar not going away when cancelling * Call getSearchStrategy instead of passing directly * Add async demo search strategy * Fix error with setting state * Update how aborting works * Fix type checks * Add test for loading count * Attempt to fix broken example test * Revert changes to test * Fix test * Update name to camelCase * Fix failing test * Don't require data_enhanced in example plugin * Actually send DELETE request * Use waitForCompletion parameter * Use default search params * Add support for rollups * Only make changes needed for frozen indices/rollups * Only make changes needed for frozen indices/rollups * Add back in async functionality * Fix tests/types * Fix issue with sending empty body in GET * Don't include skipped in loaded/total * Don't wait before polling the next time * Simplify search logic * Fix merge error * Review feedback * Fix issue with hits.total Co-authored-by: Elastic Machine Co-authored-by: Elastic Machine --- ...bana-plugin-plugins-data-server.icancel.md | 11 ---- ...lugin-plugins-data-server.isearchcancel.md | 11 ++++ .../kibana-plugin-plugins-data-server.md | 2 +- src/plugins/data/server/index.ts | 2 +- .../search/i_route_handler_search_context.ts | 4 +- src/plugins/data/server/search/i_search.ts | 4 +- .../data/server/search/i_search_strategy.ts | 4 +- src/plugins/data/server/search/index.ts | 8 ++- src/plugins/data/server/server.api.md | 12 ++-- .../public/search/es_search_strategy.ts | 16 ++++-- .../server/search/es_search_strategy.ts | 56 ++++++++++++++++--- 11 files changed, 90 insertions(+), 40 deletions(-) delete mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.icancel.md create mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchcancel.md diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.icancel.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.icancel.md deleted file mode 100644 index 27141c68ae1a7..0000000000000 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.icancel.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [ICancel](./kibana-plugin-plugins-data-server.icancel.md) - -## ICancel type - -Signature: - -```typescript -export declare type ICancel = (id: string) => Promise; -``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchcancel.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchcancel.md new file mode 100644 index 0000000000000..99c30515e8da6 --- /dev/null +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchcancel.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [ISearchCancel](./kibana-plugin-plugins-data-server.isearchcancel.md) + +## ISearchCancel type + +Signature: + +```typescript +export declare type ISearchCancel = (id: string) => Promise; +``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md index 12d53f1a35ea0..e756eb9b72905 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md @@ -67,9 +67,9 @@ | Type Alias | Description | | --- | --- | | [FieldFormatsGetConfigFn](./kibana-plugin-plugins-data-server.fieldformatsgetconfigfn.md) | | -| [ICancel](./kibana-plugin-plugins-data-server.icancel.md) | | | [IFieldFormatsRegistry](./kibana-plugin-plugins-data-server.ifieldformatsregistry.md) | | | [ISearch](./kibana-plugin-plugins-data-server.isearch.md) | | +| [ISearchCancel](./kibana-plugin-plugins-data-server.isearchcancel.md) | | | [ParsedInterval](./kibana-plugin-plugins-data-server.parsedinterval.md) | | | [TSearchStrategyProvider](./kibana-plugin-plugins-data-server.tsearchstrategyprovider.md) | Search strategy provider creates an instance of a search strategy with the request handler context bound to it. This way every search strategy can use whatever information they require from the request context. | diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index 0165486fc2de7..5038b4226fad8 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -166,7 +166,7 @@ export { ParsedInterval } from '../common'; export { ISearch, - ICancel, + ISearchCancel, ISearchOptions, IRequestTypesMap, IResponseTypesMap, diff --git a/src/plugins/data/server/search/i_route_handler_search_context.ts b/src/plugins/data/server/search/i_route_handler_search_context.ts index 89862781b826e..9888c774ea104 100644 --- a/src/plugins/data/server/search/i_route_handler_search_context.ts +++ b/src/plugins/data/server/search/i_route_handler_search_context.ts @@ -17,9 +17,9 @@ * under the License. */ -import { ISearchGeneric, ICancelGeneric } from './i_search'; +import { ISearchGeneric, ISearchCancelGeneric } from './i_search'; export interface IRouteHandlerSearchContext { search: ISearchGeneric; - cancel: ICancelGeneric; + cancel: ISearchCancelGeneric; } diff --git a/src/plugins/data/server/search/i_search.ts b/src/plugins/data/server/search/i_search.ts index ea014c5e136d9..fa4aa72ac7287 100644 --- a/src/plugins/data/server/search/i_search.ts +++ b/src/plugins/data/server/search/i_search.ts @@ -42,7 +42,7 @@ export type ISearchGeneric = Promise; -export type ICancelGeneric = ( +export type ISearchCancelGeneric = ( id: string, strategy?: T ) => Promise; @@ -52,4 +52,4 @@ export type ISearch = ( options?: ISearchOptions ) => Promise; -export type ICancel = (id: string) => Promise; +export type ISearchCancel = (id: string) => Promise; diff --git a/src/plugins/data/server/search/i_search_strategy.ts b/src/plugins/data/server/search/i_search_strategy.ts index 4cfc9608383a9..9b405034f883f 100644 --- a/src/plugins/data/server/search/i_search_strategy.ts +++ b/src/plugins/data/server/search/i_search_strategy.ts @@ -18,7 +18,7 @@ */ import { APICaller } from 'kibana/server'; -import { ISearch, ICancel, ISearchGeneric } from './i_search'; +import { ISearch, ISearchCancel, ISearchGeneric } from './i_search'; import { TStrategyTypes } from './strategy_types'; import { ISearchContext } from './i_search_context'; @@ -28,7 +28,7 @@ import { ISearchContext } from './i_search_context'; */ export interface ISearchStrategy { search: ISearch; - cancel?: ICancel; + cancel?: ISearchCancel; } /** diff --git a/src/plugins/data/server/search/index.ts b/src/plugins/data/server/search/index.ts index 385e96ee803b6..15738a3befb27 100644 --- a/src/plugins/data/server/search/index.ts +++ b/src/plugins/data/server/search/index.ts @@ -21,7 +21,13 @@ export { ISearchSetup } from './i_search_setup'; export { ISearchContext } from './i_search_context'; -export { ISearch, ICancel, ISearchOptions, IRequestTypesMap, IResponseTypesMap } from './i_search'; +export { + ISearch, + ISearchCancel, + ISearchOptions, + IRequestTypesMap, + IResponseTypesMap, +} from './i_search'; export { TStrategyTypes } from './strategy_types'; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index af802bcd65f67..627c7d9849b71 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -329,12 +329,6 @@ export function getDefaultSearchParams(config: SharedGlobalConfig): { restTotalHitsAsInt: boolean; }; -// Warning: (ae-forgotten-export) The symbol "TStrategyTypes" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "ICancel" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export type ICancel = (id: string) => Promise; - // Warning: (ae-missing-release-tag) "IFieldFormatsRegistry" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -509,11 +503,17 @@ export interface IResponseTypesMap { [ES_SEARCH_STRATEGY]: IEsSearchResponse; } +// Warning: (ae-forgotten-export) The symbol "TStrategyTypes" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "ISearch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) export type ISearch = (request: IRequestTypesMap[T], options?: ISearchOptions) => Promise; +// Warning: (ae-missing-release-tag) "ISearchCancel" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ISearchCancel = (id: string) => Promise; + // Warning: (ae-missing-release-tag) "ISearchContext" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/x-pack/plugins/data_enhanced/public/search/es_search_strategy.ts b/x-pack/plugins/data_enhanced/public/search/es_search_strategy.ts index 25c6a789cca93..c493e8ce86781 100644 --- a/x-pack/plugins/data_enhanced/public/search/es_search_strategy.ts +++ b/x-pack/plugins/data_enhanced/public/search/es_search_strategy.ts @@ -10,16 +10,17 @@ import { TSearchStrategyProvider, ISearchContext, ISearch, - SYNC_SEARCH_STRATEGY, getEsPreference, } from '../../../../../src/plugins/data/public'; import { IEnhancedEsSearchRequest, EnhancedSearchParams } from '../../common'; +import { ASYNC_SEARCH_STRATEGY } from './async_search_strategy'; +import { IAsyncSearchOptions } from './types'; export const enhancedEsSearchStrategyProvider: TSearchStrategyProvider = ( context: ISearchContext ) => { - const syncStrategyProvider = context.getSearchStrategy(SYNC_SEARCH_STRATEGY); - const { search: syncSearch } = syncStrategyProvider(context); + const asyncStrategyProvider = context.getSearchStrategy(ASYNC_SEARCH_STRATEGY); + const { search: asyncSearch } = asyncStrategyProvider(context); const search: ISearch = ( request: IEnhancedEsSearchRequest, @@ -32,9 +33,12 @@ export const enhancedEsSearchStrategyProvider: TSearchStrategyProvider; + const asyncOptions: IAsyncSearchOptions = { pollInterval: 0, ...options }; + + return asyncSearch( + { ...request, serverStrategy: ES_SEARCH_STRATEGY }, + asyncOptions + ) as Observable; }; return { search }; diff --git a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts index 69b357196dc32..11f0b9a0dc83c 100644 --- a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts +++ b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts @@ -14,10 +14,16 @@ import { TSearchStrategyProvider, ISearch, ISearchOptions, + ISearchCancel, getDefaultSearchParams, } from '../../../../../src/plugins/data/server'; import { IEnhancedEsSearchRequest } from '../../common'; +export interface AsyncSearchResponse { + id: string; + response: SearchResponse; +} + export const enhancedEsSearchStrategyProvider: TSearchStrategyProvider = ( context: ISearchContext, caller: APICaller @@ -28,28 +34,62 @@ export const enhancedEsSearchStrategyProvider: TSearchStrategyProvider { const config = await context.config$.pipe(first()).toPromise(); const defaultParams = getDefaultSearchParams(config); - const params = { ...defaultParams, ...request.params }; + const params = { ...defaultParams, trackTotalHits: true, ...request.params }; - const rawResponse = (await (request.indexType === 'rollup' + const response = await (request.indexType === 'rollup' ? rollupSearch(caller, { ...request, params }, options) - : caller('search', params, options))) as SearchResponse; + : asyncSearch(caller, { ...request, params }, options)); + + const rawResponse = + request.indexType === 'rollup' + ? (response as SearchResponse) + : (response as AsyncSearchResponse).response; + + if (typeof rawResponse.hits.total !== 'number') { + // @ts-ignore This should be fixed as part of https://github.com/elastic/kibana/issues/26356 + rawResponse.hits.total = rawResponse.hits.total.value; + } + const id = (response as AsyncSearchResponse).id; const { total, failed, successful } = rawResponse._shards; const loaded = failed + successful; - return { total, loaded, rawResponse }; + return { id, total, loaded, rawResponse }; }; - return { search }; + const cancel: ISearchCancel = async id => { + const method = 'DELETE'; + const path = `_async_search/${id}`; + await caller('transport.request', { method, path }); + }; + + return { search, cancel }; }; -function rollupSearch( +function asyncSearch( + caller: APICaller, + request: IEnhancedEsSearchRequest, + options?: ISearchOptions +) { + const { body = undefined, index = undefined, ...params } = request.id ? {} : request.params; + + // If we have an ID, then just poll for that ID, otherwise send the entire request body + const method = request.id ? 'GET' : 'POST'; + const path = request.id ? `_async_search/${request.id}` : `${index}/_async_search`; + + // Wait up to 1s for the response to return + const query = toSnakeCase({ waitForCompletion: '1s', ...params }); + + return caller('transport.request', { method, path, body, query }, options); +} + +async function rollupSearch( caller: APICaller, request: IEnhancedEsSearchRequest, options?: ISearchOptions ) { + const { body, index, ...params } = request.params; const method = 'POST'; - const path = `${request.params.index}/_rollup_search`; - const { body, ...params } = request.params; + const path = `${index}/_rollup_search`; const query = toSnakeCase(params); return caller('transport.request', { method, path, body, query }, options); }