From 5d8de7a162a28dd987804e15dd33eac185df2070 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 9 Jun 2020 21:40:05 +0300 Subject: [PATCH] Use Search API in Vega (#68257) * Use Search API in Vega * fix PR comments * fix PR comments --- ...-data-public.getsearchparamsfromrequest.md | 44 ++++++++++++ .../kibana-plugin-plugins-data-public.md | 1 + .../vis_type_vega/vega_visualization.js | 44 ++++++++++-- src/plugins/data/public/index.ts | 1 + src/plugins/data/public/public.api.md | 46 ++++++++---- .../public/search/fetch/get_search_params.ts | 18 ++++- src/plugins/data/public/search/fetch/index.ts | 1 + src/plugins/data/public/search/index.ts | 1 + .../search/search_source/search_source.ts | 15 ++-- .../public/__mocks__/services.ts | 10 +-- .../public/data_model/es_query_parser.js | 41 ++++------- .../public/data_model/es_query_parser.test.js | 26 ++++--- .../public/data_model/search_api.ts | 60 ++++++++++++++++ .../public/data_model/search_cache.js | 48 ------------- .../public/data_model/search_cache.test.js | 71 ------------------- .../public/data_model/vega_parser.js | 4 +- .../public/data_model/vega_parser.test.js | 18 ++++- src/plugins/vis_type_vega/public/plugin.ts | 2 + src/plugins/vis_type_vega/public/services.ts | 14 +++- src/plugins/vis_type_vega/public/vega_fn.ts | 4 +- .../public/vega_request_handler.ts | 33 +++++---- 21 files changed, 289 insertions(+), 213 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.getsearchparamsfromrequest.md create mode 100644 src/plugins/vis_type_vega/public/data_model/search_api.ts delete mode 100644 src/plugins/vis_type_vega/public/data_model/search_cache.js delete mode 100644 src/plugins/vis_type_vega/public/data_model/search_cache.test.js diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.getsearchparamsfromrequest.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.getsearchparamsfromrequest.md new file mode 100644 index 0000000000000..1923f0e2e4ea1 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.getsearchparamsfromrequest.md @@ -0,0 +1,44 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [getSearchParamsFromRequest](./kibana-plugin-plugins-data-public.getsearchparamsfromrequest.md) + +## getSearchParamsFromRequest() function + +Signature: + +```typescript +export declare function getSearchParamsFromRequest(searchRequest: SearchRequest, dependencies: { + injectedMetadata: CoreStart['injectedMetadata']; + uiSettings: IUiSettingsClient; +}): { + rest_total_hits_as_int: boolean; + ignore_unavailable: boolean; + ignore_throttled: boolean; + max_concurrent_shard_requests: any; + preference: any; + timeout: string | undefined; + index: any; + body: any; +}; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| searchRequest | SearchRequest | | +| dependencies | {
injectedMetadata: CoreStart['injectedMetadata'];
uiSettings: IUiSettingsClient;
} | | + +Returns: + +`{ + rest_total_hits_as_int: boolean; + ignore_unavailable: boolean; + ignore_throttled: boolean; + max_concurrent_shard_requests: any; + preference: any; + timeout: string | undefined; + index: any; + body: any; +}` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md index e818fb009fb19..bc1eb9100e85c 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md @@ -40,6 +40,7 @@ | [getEsPreference(uiSettings, sessionId)](./kibana-plugin-plugins-data-public.getespreference.md) | | | [getQueryLog(uiSettings, storage, appName, language)](./kibana-plugin-plugins-data-public.getquerylog.md) | | | [getSearchErrorType({ message })](./kibana-plugin-plugins-data-public.getsearcherrortype.md) | | +| [getSearchParamsFromRequest(searchRequest, dependencies)](./kibana-plugin-plugins-data-public.getsearchparamsfromrequest.md) | | | [getTime(indexPattern, timeRange, options)](./kibana-plugin-plugins-data-public.gettime.md) | | | [plugin(initializerContext)](./kibana-plugin-plugins-data-public.plugin.md) | | diff --git a/src/legacy/core_plugins/kibana/public/__tests__/vis_type_vega/vega_visualization.js b/src/legacy/core_plugins/kibana/public/__tests__/vis_type_vega/vega_visualization.js index 6d6eb69e66792..485390dc50a79 100644 --- a/src/legacy/core_plugins/kibana/public/__tests__/vis_type_vega/vega_visualization.js +++ b/src/legacy/core_plugins/kibana/public/__tests__/vis_type_vega/vega_visualization.js @@ -44,7 +44,7 @@ import vegaMapImage256 from './vega_map_image_256.png'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { VegaParser } from '../../../../../../plugins/vis_type_vega/public/data_model/vega_parser'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { SearchCache } from '../../../../../../plugins/vis_type_vega/public/data_model/search_cache'; +import { SearchAPI } from '../../../../../../plugins/vis_type_vega/public/data_model/search_api'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { createVegaTypeDefinition } from '../../../../../../plugins/vis_type_vega/public/vega_type'; @@ -205,7 +205,14 @@ describe('VegaVisualizations', () => { try { vegaVis = new VegaVisualization(domNode, vis); - const vegaParser = new VegaParser(vegaliteGraph, new SearchCache()); + const vegaParser = new VegaParser( + vegaliteGraph, + new SearchAPI({ + search: npStart.plugins.data.search, + uiSettings: npStart.core.uiSettings, + injectedMetadata: npStart.core.injectedMetadata, + }) + ); await vegaParser.parseAsync(); await vegaVis.render(vegaParser, vis.params, { data: true }); @@ -227,7 +234,14 @@ describe('VegaVisualizations', () => { let vegaVis; try { vegaVis = new VegaVisualization(domNode, vis); - const vegaParser = new VegaParser(vegaGraph, new SearchCache()); + const vegaParser = new VegaParser( + vegaGraph, + new SearchAPI({ + search: npStart.plugins.data.search, + uiSettings: npStart.core.uiSettings, + injectedMetadata: npStart.core.injectedMetadata, + }) + ); await vegaParser.parseAsync(); await vegaVis.render(vegaParser, vis.params, { data: true }); @@ -243,7 +257,14 @@ describe('VegaVisualizations', () => { let vegaVis; try { vegaVis = new VegaVisualization(domNode, vis); - const vegaParser = new VegaParser(vegaTooltipGraph, new SearchCache()); + const vegaParser = new VegaParser( + vegaTooltipGraph, + new SearchAPI({ + search: npStart.plugins.data.search, + uiSettings: npStart.core.uiSettings, + injectedMetadata: npStart.core.injectedMetadata, + }) + ); await vegaParser.parseAsync(); await vegaVis.render(vegaParser, vis.params, { data: true }); @@ -285,7 +306,14 @@ describe('VegaVisualizations', () => { let vegaVis; try { vegaVis = new VegaVisualization(domNode, vis); - const vegaParser = new VegaParser(vegaMapGraph, new SearchCache()); + const vegaParser = new VegaParser( + vegaMapGraph, + new SearchAPI({ + search: npStart.plugins.data.search, + uiSettings: npStart.core.uiSettings, + injectedMetadata: npStart.core.injectedMetadata, + }) + ); await vegaParser.parseAsync(); domNode.style.width = '256px'; @@ -324,7 +352,11 @@ describe('VegaVisualizations', () => { } ] }`, - new SearchCache() + new SearchAPI({ + search: npStart.plugins.data.search, + uiSettings: npStart.core.uiSettings, + injectedMetadata: npStart.core.injectedMetadata, + }) ); await vegaParser.parseAsync(); diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index eb3f937a4168b..301ff8d3f67d8 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -358,6 +358,7 @@ export { ISearchSource, parseSearchSourceJSON, injectSearchSourceReferences, + getSearchParamsFromRequest, extractSearchSourceReferences, SearchSourceFields, EsQuerySortValue, diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 7054575e8ef9e..bd3ec0d3f2294 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -30,6 +30,7 @@ import { IconType } from '@elastic/eui'; import { InjectedIntl } from '@kbn/i18n/react'; import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { IUiSettingsClient } from 'src/core/public'; +import { IUiSettingsClient as IUiSettingsClient_3 } from 'kibana/public'; import { Location } from 'history'; import { LocationDescriptorObject } from 'history'; import { MaybePromise } from '@kbn/utility-types'; @@ -641,6 +642,23 @@ export function getQueryLog(uiSettings: IUiSettingsClient, storage: IStorageWrap // @public (undocumented) export function getSearchErrorType({ message }: Pick): "UNSUPPORTED_QUERY" | undefined; +// Warning: (ae-missing-release-tag) "getSearchParamsFromRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export function getSearchParamsFromRequest(searchRequest: SearchRequest, dependencies: { + injectedMetadata: CoreStart['injectedMetadata']; + uiSettings: IUiSettingsClient_3; +}): { + rest_total_hits_as_int: boolean; + ignore_unavailable: boolean; + ignore_throttled: boolean; + max_concurrent_shard_requests: any; + preference: any; + timeout: string | undefined; + index: any; + body: any; +}; + // Warning: (ae-missing-release-tag) "getTime" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -1851,20 +1869,20 @@ export const UI_SETTINGS: { // src/plugins/data/public/index.ts:236:27 - (ae-forgotten-export) The symbol "getFromSavedObject" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:236:27 - (ae-forgotten-export) The symbol "flattenHitWrapper" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:236:27 - (ae-forgotten-export) The symbol "formatHitProvider" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:375:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:375:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:375:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:375:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:377:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:378:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:387:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:388:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:389:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:393:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:394:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:397:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:398:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:401:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:376:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:376:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:376:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:376:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:378:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:379:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:388:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:389:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:390:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:394:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:395:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:398:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:399:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:402:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts // src/plugins/data/public/query/state_sync/connect_to_query_state.ts:33:33 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts // src/plugins/data/public/query/state_sync/connect_to_query_state.ts:37:1 - (ae-forgotten-export) The symbol "QueryStateChange" needs to be exported by the entry point index.d.ts // src/plugins/data/public/types.ts:52:5 - (ae-forgotten-export) The symbol "createFiltersFromValueClickAction" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/data/public/search/fetch/get_search_params.ts b/src/plugins/data/public/search/fetch/get_search_params.ts index 60bdc9ed6473a..f2ad243ce72d0 100644 --- a/src/plugins/data/public/search/fetch/get_search_params.ts +++ b/src/plugins/data/public/search/fetch/get_search_params.ts @@ -17,8 +17,9 @@ * under the License. */ -import { IUiSettingsClient } from 'kibana/public'; +import { IUiSettingsClient, CoreStart } from 'kibana/public'; import { UI_SETTINGS } from '../../../common'; +import { SearchRequest } from './types'; const sessionId = Date.now(); @@ -53,3 +54,18 @@ export function getPreference(config: IUiSettingsClient) { export function getTimeout(esShardTimeout: number) { return esShardTimeout > 0 ? `${esShardTimeout}ms` : undefined; } + +export function getSearchParamsFromRequest( + searchRequest: SearchRequest, + dependencies: { injectedMetadata: CoreStart['injectedMetadata']; uiSettings: IUiSettingsClient } +) { + const { injectedMetadata, uiSettings } = dependencies; + const esShardTimeout = injectedMetadata.getInjectedVar('esShardTimeout') as number; + const searchParams = getSearchParams(uiSettings, esShardTimeout); + + return { + index: searchRequest.index.title || searchRequest.index, + body: searchRequest.body, + ...searchParams, + }; +} diff --git a/src/plugins/data/public/search/fetch/index.ts b/src/plugins/data/public/search/fetch/index.ts index 39845ec31bfaa..ab856d681ba12 100644 --- a/src/plugins/data/public/search/fetch/index.ts +++ b/src/plugins/data/public/search/fetch/index.ts @@ -20,6 +20,7 @@ export * from './types'; export { getSearchParams, + getSearchParamsFromRequest, getPreference, getTimeout, getIgnoreThrottled, diff --git a/src/plugins/data/public/search/index.ts b/src/plugins/data/public/search/index.ts index 53686f9be9b4d..1b5395e1071c5 100644 --- a/src/plugins/data/public/search/index.ts +++ b/src/plugins/data/public/search/index.ts @@ -44,6 +44,7 @@ export { SearchRequest, SearchResponse, getSearchErrorType, + getSearchParamsFromRequest, } from './fetch'; export { diff --git a/src/plugins/data/public/search/search_source/search_source.ts b/src/plugins/data/public/search/search_source/search_source.ts index b926739112e0e..a33cda964bd1d 100644 --- a/src/plugins/data/public/search/search_source/search_source.ts +++ b/src/plugins/data/public/search/search_source/search_source.ts @@ -77,7 +77,7 @@ import { filterDocvalueFields } from './filter_docvalue_fields'; import { fieldWildcardFilter } from '../../../../kibana_utils/public'; import { IIndexPattern, ISearchGeneric, SearchRequest } from '../..'; import { SearchSourceOptions, SearchSourceFields } from './types'; -import { FetchOptions, RequestFailure, getSearchParams, handleResponse } from '../fetch'; +import { FetchOptions, RequestFailure, handleResponse, getSearchParamsFromRequest } from '../fetch'; import { getEsQueryConfig, buildEsQuery, Filter, UI_SETTINGS } from '../../../common'; import { getHighlightRequest } from '../../../common/field_formats'; @@ -204,13 +204,12 @@ export class SearchSource { */ private fetch$(searchRequest: SearchRequest, signal?: AbortSignal) { const { search, injectedMetadata, uiSettings } = this.dependencies; - const esShardTimeout = injectedMetadata.getInjectedVar('esShardTimeout') as number; - const searchParams = getSearchParams(uiSettings, esShardTimeout); - const params = { - index: searchRequest.index.title || searchRequest.index, - body: searchRequest.body, - ...searchParams, - }; + + const params = getSearchParamsFromRequest(searchRequest, { + injectedMetadata, + uiSettings, + }); + return search({ params, indexType: searchRequest.indexType }, { signal }).pipe( map(({ rawResponse }) => handleResponse(searchRequest, rawResponse)) ); diff --git a/src/plugins/vis_type_vega/public/__mocks__/services.ts b/src/plugins/vis_type_vega/public/__mocks__/services.ts index 1bf051232e4c9..4775241a66d50 100644 --- a/src/plugins/vis_type_vega/public/__mocks__/services.ts +++ b/src/plugins/vis_type_vega/public/__mocks__/services.ts @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +import { CoreStart, IUiSettingsClient, NotificationsStart, SavedObjectsStart } from 'kibana/public'; import { createGetterSetter } from '../../../kibana_utils/public'; import { DataPublicPluginStart } from '../../../data/public'; -import { IUiSettingsClient, NotificationsStart, SavedObjectsStart } from 'kibana/public'; import { dataPluginMock } from '../../../data/public/mocks'; import { coreMock } from '../../../../core/public/mocks'; @@ -34,22 +34,24 @@ setNotifications(coreMock.createStart().notifications); export const [getUISettings, setUISettings] = createGetterSetter('UISettings'); setUISettings(coreMock.createStart().uiSettings); +export const [getInjectedMetadata, setInjectedMetadata] = createGetterSetter< + CoreStart['injectedMetadata'] +>('InjectedMetadata'); +setInjectedMetadata(coreMock.createStart().injectedMetadata); + export const [getSavedObjects, setSavedObjects] = createGetterSetter( 'SavedObjects' ); setSavedObjects(coreMock.createStart().savedObjects); export const [getInjectedVars, setInjectedVars] = createGetterSetter<{ - esShardTimeout: number; enableExternalUrls: boolean; emsTileLayerId: unknown; }>('InjectedVars'); setInjectedVars({ emsTileLayerId: {}, enableExternalUrls: true, - esShardTimeout: 10000, }); -export const getEsShardTimeout = () => getInjectedVars().esShardTimeout; export const getEnableExternalUrls = () => getInjectedVars().enableExternalUrls; export const getEmsTileLayerId = () => getInjectedVars().emsTileLayerId; diff --git a/src/plugins/vis_type_vega/public/data_model/es_query_parser.js b/src/plugins/vis_type_vega/public/data_model/es_query_parser.js index 066c9f06fc109..387301c2c7de9 100644 --- a/src/plugins/vis_type_vega/public/data_model/es_query_parser.js +++ b/src/plugins/vis_type_vega/public/data_model/es_query_parser.js @@ -17,11 +17,9 @@ * under the License. */ -import _ from 'lodash'; import moment from 'moment'; import { i18n } from '@kbn/i18n'; - -import { getEsShardTimeout } from '../services'; +import { isPlainObject, cloneDeep } from 'lodash'; const TIMEFILTER = '%timefilter%'; const AUTOINTERVAL = '%autointerval%'; @@ -37,12 +35,11 @@ const TIMEFIELD = '%timefield%'; * This class parses ES requests specified in the data.url objects. */ export class EsQueryParser { - constructor(timeCache, searchCache, filters, onWarning) { + constructor(timeCache, searchAPI, filters, onWarning) { this._timeCache = timeCache; - this._searchCache = searchCache; + this._searchAPI = searchAPI; this._filters = filters; this._onWarning = onWarning; - this._esShardTimeout = getEsShardTimeout(); } // noinspection JSMethodCanBeStatic @@ -59,7 +56,7 @@ export class EsQueryParser { if (body === undefined) { url.body = body = {}; - } else if (!_.isPlainObject(body)) { + } else if (!isPlainObject(body)) { throw new Error( i18n.translate('visTypeVega.esQueryParser.urlBodyValueTypeErrorMessage', { defaultMessage: '{configName} must be an object', @@ -167,7 +164,7 @@ export class EsQueryParser { if (context) { // Use dashboard context - const newQuery = _.cloneDeep(this._filters); + const newQuery = cloneDeep(this._filters); if (timefield) { newQuery.bool.must.push(body.query); } @@ -179,34 +176,20 @@ export class EsQueryParser { return { dataObject, url }; } - mapRequest = (request) => { - const esRequest = request.url; - if (this._esShardTimeout) { - // remove possible timeout query param to prevent two conflicting timeout parameters - const { body = {}, timeout, ...rest } = esRequest; //eslint-disable-line no-unused-vars - body.timeout = `${this._esShardTimeout}ms`; - return { - body, - ...rest, - }; - } else { - return esRequest; - } - }; - /** * Process items generated by parseUrl() * @param {object[]} requests each object is generated by parseUrl() * @returns {Promise} */ async populateData(requests) { - const esSearches = requests.map(this.mapRequest); + const esSearches = requests.map((r) => r.url); + const data$ = this._searchAPI.search(esSearches); - const results = await this._searchCache.search(esSearches); + const results = await data$.toPromise(); - for (let i = 0; i < requests.length; i++) { - requests[i].dataObject.values = results[i]; - } + results.forEach((data) => { + requests[data.id].dataObject.values = data.rawResponse; + }); } /** @@ -222,7 +205,7 @@ export class EsQueryParser { const item = obj[pos]; if (isQuery && (item === MUST_CLAUSE || item === MUST_NOT_CLAUSE)) { const ctxTag = item === MUST_CLAUSE ? 'must' : 'must_not'; - const ctx = _.cloneDeep(this._filters); + const ctx = cloneDeep(this._filters); if (ctx && ctx.bool && ctx.bool[ctxTag]) { if (Array.isArray(ctx.bool[ctxTag])) { // replace one value with an array of values diff --git a/src/plugins/vis_type_vega/public/data_model/es_query_parser.test.js b/src/plugins/vis_type_vega/public/data_model/es_query_parser.test.js index c519da33ab1c9..fd474bef73b8c 100644 --- a/src/plugins/vis_type_vega/public/data_model/es_query_parser.test.js +++ b/src/plugins/vis_type_vega/public/data_model/es_query_parser.test.js @@ -94,28 +94,36 @@ describe(`EsQueryParser time`, () => { }); describe('EsQueryParser.populateData', () => { - let searchStub; + let searchApiStub; + let data; let parser; beforeEach(() => { - searchStub = jest.fn(() => Promise.resolve([{}, {}])); - parser = new EsQueryParser({}, { search: searchStub }, undefined, undefined); + searchApiStub = { + search: jest.fn(() => ({ + toPromise: jest.fn(() => Promise.resolve(data)), + })), + }; + parser = new EsQueryParser({}, searchApiStub, undefined, undefined); }); test('should set the timeout for each request', async () => { + data = [ + { id: 0, rawResponse: {} }, + { id: 1, rawResponse: {} }, + ]; await parser.populateData([ { url: { body: {} }, dataObject: {} }, { url: { body: {} }, dataObject: {} }, ]); - expect(searchStub.mock.calls[0][0][0].body.timeout).toBe.defined; + + expect(searchApiStub.search.mock.calls[0][0][0].body).toBeDefined(); }); test('should remove possible timeout parameters on a request', async () => { - await parser.populateData([ - { url: { timeout: '500h', body: { timeout: '500h' } }, dataObject: {} }, - ]); - expect(searchStub.mock.calls[0][0][0].body.timeout).toBe.defined; - expect(searchStub.mock.calls[0][0][0].timeout).toBe(undefined); + data = [{ id: 0, rawResponse: {} }]; + await parser.populateData([{ url: { body: { timeout: '500h' } }, dataObject: {} }]); + expect(searchApiStub.search.mock.calls[0][0][0].body.timeout).toBeDefined(); }); }); diff --git a/src/plugins/vis_type_vega/public/data_model/search_api.ts b/src/plugins/vis_type_vega/public/data_model/search_api.ts new file mode 100644 index 0000000000000..c2eecf13c2d51 --- /dev/null +++ b/src/plugins/vis_type_vega/public/data_model/search_api.ts @@ -0,0 +1,60 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { combineLatest } from 'rxjs'; +import { map } from 'rxjs/operators'; +import { CoreStart, IUiSettingsClient } from 'kibana/public'; +import { + getSearchParamsFromRequest, + SearchRequest, + DataPublicPluginStart, +} from '../../../data/public'; + +export interface SearchAPIDependencies { + uiSettings: IUiSettingsClient; + injectedMetadata: CoreStart['injectedMetadata']; + search: DataPublicPluginStart['search']; +} + +export class SearchAPI { + constructor( + private readonly dependencies: SearchAPIDependencies, + private readonly abortSignal?: AbortSignal + ) {} + + search(searchRequests: SearchRequest[]) { + const { search } = this.dependencies.search; + + return combineLatest( + searchRequests.map((request, index) => { + const params = getSearchParamsFromRequest(request, { + uiSettings: this.dependencies.uiSettings, + injectedMetadata: this.dependencies.injectedMetadata, + }); + + return search({ params }, { signal: this.abortSignal }).pipe( + map((data) => ({ + id: index, + rawResponse: data.rawResponse, + })) + ); + }) + ); + } +} diff --git a/src/plugins/vis_type_vega/public/data_model/search_cache.js b/src/plugins/vis_type_vega/public/data_model/search_cache.js deleted file mode 100644 index 41e4c67c3b2ad..0000000000000 --- a/src/plugins/vis_type_vega/public/data_model/search_cache.js +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import LruCache from 'lru-cache'; - -export class SearchCache { - constructor(es, cacheOpts) { - this._es = es; - this._cache = new LruCache(cacheOpts); - } - - /** - * Execute multiple searches, possibly combining the results of the cached searches - * with the new ones already in cache - * @param {object[]} requests array of search requests - */ - search(requests) { - const promises = []; - - for (const request of requests) { - const key = JSON.stringify(request); - let pending = this._cache.get(key); - if (pending === undefined) { - pending = this._es.search(request); - this._cache.set(key, pending); - } - promises.push(pending); - } - - return Promise.all(promises); - } -} diff --git a/src/plugins/vis_type_vega/public/data_model/search_cache.test.js b/src/plugins/vis_type_vega/public/data_model/search_cache.test.js deleted file mode 100644 index 92f80545ce1b5..0000000000000 --- a/src/plugins/vis_type_vega/public/data_model/search_cache.test.js +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { SearchCache } from './search_cache'; -jest.mock('../services'); - -describe(`SearchCache`, () => { - class FauxEs { - constructor() { - // contains all request batches, separated by 0 - this.searches = []; - } - - async search(request) { - this.searches.push(request); - return { req: request }; - } - } - - const request1 = { body: 'b1' }; - const expected1 = { req: { body: 'b1' } }; - const request2 = { body: 'b2' }; - const expected2 = { req: { body: 'b2' } }; - const request3 = { body: 'b3' }; - const expected3 = { req: { body: 'b3' } }; - - it(`sequence`, async () => { - const sc = new SearchCache(new FauxEs()); - - // empty request - let res = await sc.search([]); - expect(res).toEqual([]); - expect(sc._es.searches).toEqual([]); - - // single request - res = await sc.search([request1]); - expect(res).toEqual([expected1]); - expect(sc._es.searches).toEqual([request1]); - - // repeat the same search, use array notation - res = await sc.search([request1]); - expect(res).toEqual([expected1]); - expect(sc._es.searches).toEqual([request1]); // no new entries - - // new single search - res = await sc.search([request2]); - expect(res).toEqual([expected2]); - expect(sc._es.searches).toEqual([request1, request2]); - - // multiple search, some new, some old - res = await sc.search([request1, request3, request2]); - expect(res).toEqual([expected1, expected3, expected2]); - expect(sc._es.searches).toEqual([request1, request2, request3]); - }); -}); diff --git a/src/plugins/vis_type_vega/public/data_model/vega_parser.js b/src/plugins/vis_type_vega/public/data_model/vega_parser.js index f541b9f104adc..cbfe2a6ede4f2 100644 --- a/src/plugins/vis_type_vega/public/data_model/vega_parser.js +++ b/src/plugins/vis_type_vega/public/data_model/vega_parser.js @@ -46,7 +46,7 @@ const locToDirMap = { const DEFAULT_PARSER = 'elasticsearch'; export class VegaParser { - constructor(spec, searchCache, timeCache, filters, serviceSettings) { + constructor(spec, searchAPI, timeCache, filters, serviceSettings) { this.spec = spec; this.hideWarnings = false; this.error = undefined; @@ -54,7 +54,7 @@ export class VegaParser { const onWarn = this._onWarning.bind(this); this._urlParsers = { - elasticsearch: new EsQueryParser(timeCache, searchCache, filters, onWarn), + elasticsearch: new EsQueryParser(timeCache, searchAPI, filters, onWarn), emsfile: new EmsFileParser(serviceSettings), url: new UrlParser(onWarn), }; diff --git a/src/plugins/vis_type_vega/public/data_model/vega_parser.test.js b/src/plugins/vis_type_vega/public/data_model/vega_parser.test.js index 1bd26b8713044..a40ef31260b6f 100644 --- a/src/plugins/vis_type_vega/public/data_model/vega_parser.test.js +++ b/src/plugins/vis_type_vega/public/data_model/vega_parser.test.js @@ -78,9 +78,25 @@ describe(`VegaParser._setDefaultColors`, () => { }); describe('VegaParser._resolveEsQueries', () => { + let searchApiStub; + const data = [ + { + id: 0, + rawResponse: [42], + }, + ]; + + beforeEach(() => { + searchApiStub = { + search: jest.fn(() => ({ + toPromise: jest.fn(() => Promise.resolve(data)), + })), + }; + }); + function check(spec, expected, warnCount) { return async () => { - const vp = new VegaParser(spec, { search: async () => [[42]] }, 0, 0, { + const vp = new VegaParser(spec, searchApiStub, 0, 0, { getFileLayers: async () => [{ name: 'file1', url: 'url1' }], getUrlForRegionLayer: async (layer) => { return layer.url; diff --git a/src/plugins/vis_type_vega/public/plugin.ts b/src/plugins/vis_type_vega/public/plugin.ts index 1bce7ac92e564..b3e35dac3711f 100644 --- a/src/plugins/vis_type_vega/public/plugin.ts +++ b/src/plugins/vis_type_vega/public/plugin.ts @@ -28,6 +28,7 @@ import { setUISettings, setKibanaMapFactory, setMapsLegacyConfig, + setInjectedMetadata, } from './services'; import { createVegaFn } from './vega_fn'; @@ -96,5 +97,6 @@ export class VegaPlugin implements Plugin, void> { setNotifications(core.notifications); setSavedObjects(core.savedObjects); setData(data); + setInjectedMetadata(core.injectedMetadata); } } diff --git a/src/plugins/vis_type_vega/public/services.ts b/src/plugins/vis_type_vega/public/services.ts index f2fddb41cf72b..7d988d464b52b 100644 --- a/src/plugins/vis_type_vega/public/services.ts +++ b/src/plugins/vis_type_vega/public/services.ts @@ -17,8 +17,13 @@ * under the License. */ -import { SavedObjectsStart } from 'kibana/public'; -import { NotificationsStart, IUiSettingsClient } from 'src/core/public'; +import { + CoreStart, + SavedObjectsStart, + NotificationsStart, + IUiSettingsClient, +} from 'src/core/public'; + import { DataPublicPluginStart } from '../../data/public'; import { createGetterSetter } from '../../kibana_utils/public'; import { MapsLegacyConfigType } from '../../maps_legacy/public'; @@ -34,6 +39,10 @@ export const [getKibanaMapFactory, setKibanaMapFactory] = createGetterSetter('UISettings'); +export const [getInjectedMetadata, setInjectedMetadata] = createGetterSetter< + CoreStart['injectedMetadata'] +>('InjectedMetadata'); + export const [getSavedObjects, setSavedObjects] = createGetterSetter( 'SavedObjects' ); @@ -48,6 +57,5 @@ export const [getMapsLegacyConfig, setMapsLegacyConfig] = createGetterSetter getInjectedVars().esShardTimeout; export const getEnableExternalUrls = () => getInjectedVars().enableExternalUrls; export const getEmsTileLayerId = () => getMapsLegacyConfig().emsTileLayerId; diff --git a/src/plugins/vis_type_vega/public/vega_fn.ts b/src/plugins/vis_type_vega/public/vega_fn.ts index 6d45e043f7cee..a9c915fcfb636 100644 --- a/src/plugins/vis_type_vega/public/vega_fn.ts +++ b/src/plugins/vis_type_vega/public/vega_fn.ts @@ -54,8 +54,8 @@ export const createVegaFn = ( help: '', }, }, - async fn(input, args) { - const vegaRequestHandler = createVegaRequestHandler(dependencies); + async fn(input, args, context) { + const vegaRequestHandler = createVegaRequestHandler(dependencies, context.abortSignal); const response = await vegaRequestHandler({ timeRange: get(input, 'timeRange'), diff --git a/src/plugins/vis_type_vega/public/vega_request_handler.ts b/src/plugins/vis_type_vega/public/vega_request_handler.ts index efc02e368efa8..ac28f0b3782b2 100644 --- a/src/plugins/vis_type_vega/public/vega_request_handler.ts +++ b/src/plugins/vis_type_vega/public/vega_request_handler.ts @@ -19,14 +19,14 @@ import { Filter, esQuery, TimeRange, Query } from '../../data/public'; -// @ts-ignore -import { SearchCache } from './data_model/search_cache'; +import { SearchAPI } from './data_model/search_api'; + // @ts-ignore import { TimeCache } from './data_model/time_cache'; import { VegaVisualizationDependencies } from './plugin'; import { VisParams } from './vega_fn'; -import { getData } from './services'; +import { getData, getInjectedMetadata } from './services'; interface VegaRequestHandlerParams { query: Query; @@ -35,12 +35,11 @@ interface VegaRequestHandlerParams { visParams: VisParams; } -export function createVegaRequestHandler({ - plugins: { data }, - core: { uiSettings }, - serviceSettings, -}: VegaVisualizationDependencies) { - let searchCache: SearchCache | undefined; +export function createVegaRequestHandler( + { plugins: { data }, core: { uiSettings }, serviceSettings }: VegaVisualizationDependencies, + abortSignal?: AbortSignal +) { + let searchAPI: SearchAPI; const { timefilter } = data.query.timefilter; const timeCache = new TimeCache(timefilter, 3 * 1000); @@ -50,11 +49,15 @@ export function createVegaRequestHandler({ query, visParams, }: VegaRequestHandlerParams) { - if (!searchCache) { - searchCache = new SearchCache(getData().search.__LEGACY.esClient, { - max: 10, - maxAge: 4 * 1000, - }); + if (!searchAPI) { + searchAPI = new SearchAPI( + { + uiSettings, + search: getData().search, + injectedMetadata: getInjectedMetadata(), + }, + abortSignal + ); } timeCache.setTimeRange(timeRange); @@ -63,7 +66,7 @@ export function createVegaRequestHandler({ const filtersDsl = esQuery.buildEsQuery(undefined, query, filters, esQueryConfigs); // @ts-ignore const { VegaParser } = await import('./data_model/vega_parser'); - const vp = new VegaParser(visParams.spec, searchCache, timeCache, filtersDsl, serviceSettings); + const vp = new VegaParser(visParams.spec, searchAPI, timeCache, filtersDsl, serviceSettings); return await vp.parseAsync(); };