diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker index 9a73f1f295492..e0a9171e52169 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker @@ -41,6 +41,9 @@ kibana_vars=( csp.report_to data.autocomplete.valueSuggestions.terminateAfter data.autocomplete.valueSuggestions.timeout + data.search.asyncSearch.waitForCompletion + data.search.asyncSearch.keepAlive + data.search.asyncSearch.batchedReduceSize data.search.sessions.defaultExpiration data.search.sessions.enabled data.search.sessions.maxUpdateRetries diff --git a/src/plugins/data/config.mock.ts b/src/plugins/data/config.mock.ts new file mode 100644 index 0000000000000..ab1d02cb63b31 --- /dev/null +++ b/src/plugins/data/config.mock.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment/moment'; +import { SearchConfigSchema, SearchSessionsConfigSchema } from './config'; + +export const getMockSearchConfig = ({ + sessions: { enabled = true, defaultExpiration = moment.duration(7, 'd') } = { + enabled: true, + defaultExpiration: moment.duration(7, 'd'), + }, + asyncSearch: { + waitForCompletion = moment.duration(100, 'ms'), + keepAlive = moment.duration(1, 'm'), + batchedReduceSize = 64, + } = { + waitForCompletion: moment.duration(100, 'ms'), + keepAlive: moment.duration(1, 'm'), + batchedReduceSize: 64, + }, +}: Partial<{ + sessions: Partial; + asyncSearch: Partial; +}>): SearchConfigSchema => + ({ + asyncSearch: { + waitForCompletion, + keepAlive, + batchedReduceSize, + } as SearchConfigSchema['asyncSearch'], + sessions: { + enabled, + defaultExpiration, + } as SearchSessionsConfigSchema, + } as SearchConfigSchema); diff --git a/src/plugins/data/config.ts b/src/plugins/data/config.ts index d4331ecd760b4..d981621087f98 100644 --- a/src/plugins/data/config.ts +++ b/src/plugins/data/config.ts @@ -46,20 +46,29 @@ export const searchSessionsConfigSchema = schema.object({ }), }); -export const configSchema = schema.object({ - search: schema.object({ - aggs: schema.object({ - shardDelay: schema.object({ - // Whether or not to register the shard_delay (which is only available in snapshot versions - // of Elasticsearch) agg type/expression function to make it available in the UI for either - // functional or manual testing - enabled: schema.boolean({ defaultValue: false }), - }), +export const searchConfigSchema = schema.object({ + asyncSearch: schema.object({ + waitForCompletion: schema.duration({ defaultValue: '100ms' }), + keepAlive: schema.duration({ defaultValue: '1m' }), + batchedReduceSize: schema.number({ defaultValue: 64 }), + }), + aggs: schema.object({ + shardDelay: schema.object({ + // Whether or not to register the shard_delay (which is only available in snapshot versions + // of Elasticsearch) agg type/expression function to make it available in the UI for either + // functional or manual testing + enabled: schema.boolean({ defaultValue: false }), }), - sessions: searchSessionsConfigSchema, }), + sessions: searchSessionsConfigSchema, +}); + +export const configSchema = schema.object({ + search: searchConfigSchema, }); export type ConfigSchema = TypeOf; +export type SearchConfigSchema = TypeOf; + export type SearchSessionsConfigSchema = TypeOf; diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index 7523da61752b4..eba7ea22acf4f 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -178,6 +178,7 @@ export class SearchService implements Plugin { ENHANCED_ES_SEARCH_STRATEGY, enhancedEsSearchStrategyProvider( this.initializerContext.config.legacy.globalConfig$, + this.initializerContext.config.get().search, this.logger, usage ) @@ -189,13 +190,20 @@ export class SearchService implements Plugin { // for example use case this.searchAsInternalUser = enhancedEsSearchStrategyProvider( this.initializerContext.config.legacy.globalConfig$, + this.initializerContext.config.get().search, this.logger, usage, true ); - this.registerSearchStrategy(EQL_SEARCH_STRATEGY, eqlSearchStrategyProvider(this.logger)); - this.registerSearchStrategy(SQL_SEARCH_STRATEGY, sqlSearchStrategyProvider(this.logger)); + this.registerSearchStrategy( + EQL_SEARCH_STRATEGY, + eqlSearchStrategyProvider(this.initializerContext.config.get().search, this.logger) + ); + this.registerSearchStrategy( + SQL_SEARCH_STRATEGY, + sqlSearchStrategyProvider(this.initializerContext.config.get().search, this.logger) + ); registerBsearchRoute( bfetch, diff --git a/src/plugins/data/server/search/session/types.ts b/src/plugins/data/server/search/session/types.ts index 76d5f30028736..88ad5f21e2755 100644 --- a/src/plugins/data/server/search/session/types.ts +++ b/src/plugins/data/server/search/session/types.ts @@ -49,7 +49,7 @@ export interface IScopedSearchSessionsClient { expires: Date ) => Promise>; status: (sessionId: string) => Promise; - getConfig: () => SearchSessionsConfigSchema | null; + getConfig: () => SearchSessionsConfigSchema; } export interface ISearchSessionService { diff --git a/src/plugins/data/server/search/strategies/common/async_utils.test.ts b/src/plugins/data/server/search/strategies/common/async_utils.test.ts index 9771a042f1872..f8cabdae04923 100644 --- a/src/plugins/data/server/search/strategies/common/async_utils.test.ts +++ b/src/plugins/data/server/search/strategies/common/async_utils.test.ts @@ -6,42 +6,39 @@ * Side Public License, v 1. */ -import { getCommonDefaultAsyncSubmitParams, getCommonDefaultAsyncGetParams } from './async_utils'; +import { getCommonDefaultAsyncGetParams, getCommonDefaultAsyncSubmitParams } from './async_utils'; import moment from 'moment'; -import { SearchSessionsConfigSchema } from '../../../../config'; - -const getMockSearchSessionsConfig = ({ - enabled = true, - defaultExpiration = moment.duration(7, 'd'), -} = {}) => - ({ - enabled, - defaultExpiration, - } as SearchSessionsConfigSchema); +import { getMockSearchConfig } from '../../../../config.mock'; describe('request utils', () => { describe('getCommonDefaultAsyncSubmitParams', () => { - test('Uses short `keep_alive` if no `sessionId` is provided', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), + test('Uses `keep_alive` from asyncSearch config if no `sessionId` is provided', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + }, }); const params = getCommonDefaultAsyncSubmitParams(mockConfig, {}); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); test('Uses short `keep_alive` if sessions enabled but no yet saved', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + }, }); const params = getCommonDefaultAsyncSubmitParams(mockConfig, { sessionId: 'foo', }); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); test('Uses `keep_alive` from config if sessions enabled and session is saved', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + }, }); const params = getCommonDefaultAsyncSubmitParams(mockConfig, { sessionId: 'foo', @@ -50,29 +47,33 @@ describe('request utils', () => { expect(params).toHaveProperty('keep_alive', '259200000ms'); }); - test('Uses `keepAlive` of `1m` if disabled', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: false, + test('Uses `keepAlive` from asyncSearch config if sessions disabled', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: false, + }, }); const params = getCommonDefaultAsyncSubmitParams(mockConfig, { sessionId: 'foo', }); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); - test('Uses `keep_on_completion` if enabled', async () => { - const mockConfig = getMockSearchSessionsConfig({}); + test('Uses `keep_on_completion` if sessions enabled', async () => { + const mockConfig = getMockSearchConfig({}); const params = getCommonDefaultAsyncSubmitParams(mockConfig, { sessionId: 'foo', }); expect(params).toHaveProperty('keep_on_completion', true); }); - test('Does not use `keep_on_completion` if disabled', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: false, + test('Does not use `keep_on_completion` if sessions disabled', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: false, + }, }); const params = getCommonDefaultAsyncSubmitParams(mockConfig, { sessionId: 'foo', @@ -83,36 +84,44 @@ describe('request utils', () => { describe('getCommonDefaultAsyncGetParams', () => { test('Uses `wait_for_completion_timeout`', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getCommonDefaultAsyncGetParams(mockConfig, {}); expect(params).toHaveProperty('wait_for_completion_timeout'); }); test('Uses `keep_alive` if `sessionId` is not provided', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getCommonDefaultAsyncGetParams(mockConfig, {}); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); - test('Has short `keep_alive` if `sessionId` is provided', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + test('Has `keep_alive` from asyncSearch config if `sessionId` is provided', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getCommonDefaultAsyncGetParams(mockConfig, { sessionId: 'foo' }); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); test('Has `keep_alive` from config if `sessionId` is provided and session is stored', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getCommonDefaultAsyncGetParams(mockConfig, { sessionId: 'foo', @@ -122,9 +131,11 @@ describe('request utils', () => { }); test("Don't extend keepAlive if search has already been extended", async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getCommonDefaultAsyncGetParams(mockConfig, { sessionId: 'foo', @@ -135,9 +146,11 @@ describe('request utils', () => { }); test("Don't extend keepAlive if search is being restored", async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getCommonDefaultAsyncGetParams(mockConfig, { sessionId: 'foo', @@ -149,12 +162,68 @@ describe('request utils', () => { }); test('Uses `keep_alive` if `sessionId` is provided but sessions disabled', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: false, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: false, + }, }); const params = getCommonDefaultAsyncGetParams(mockConfig, { sessionId: 'foo' }); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); + }); + }); + + describe('overrides: force disable sessions', () => { + test('Does not use `keep_on_completion` if sessions disabled through overrides', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, + }); + const params = getCommonDefaultAsyncSubmitParams( + mockConfig, + { + sessionId: 'foo', + }, + { disableSearchSessions: true } + ); + expect(params).toHaveProperty('keep_on_completion', false); + }); + + test('Uses `keepAlive` from asyncSearch config if sessions disabled through overrides', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, + }); + const params = getCommonDefaultAsyncSubmitParams( + mockConfig, + { + sessionId: 'foo', + }, + { disableSearchSessions: true } + ); + expect(params).toHaveProperty('keep_alive', '60000ms'); + }); + + test('Uses `keep_alive` from asyncSearch config if sessions disabled through overrides and session is saved', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, + }); + const params = getCommonDefaultAsyncSubmitParams( + mockConfig, + { + sessionId: 'foo', + isStored: true, + }, + { disableSearchSessions: true } + ); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); }); }); diff --git a/src/plugins/data/server/search/strategies/common/async_utils.ts b/src/plugins/data/server/search/strategies/common/async_utils.ts index c0af68f915bd8..cbc1e1acd3cd0 100644 --- a/src/plugins/data/server/search/strategies/common/async_utils.ts +++ b/src/plugins/data/server/search/strategies/common/async_utils.ts @@ -10,29 +10,35 @@ import { AsyncSearchSubmitRequest, AsyncSearchGetRequest, } from '@elastic/elasticsearch/lib/api/types'; -import { SearchSessionsConfigSchema } from '../../../../config'; import { ISearchOptions } from '../../../../common'; +import { SearchConfigSchema } from '../../../../config'; /** @internal */ export function getCommonDefaultAsyncSubmitParams( - searchSessionsConfig: SearchSessionsConfigSchema | null, - options: ISearchOptions + config: SearchConfigSchema, + options: ISearchOptions, + /** + * Allows to override some of internal logic (e.g. eql / sql searches don't fully support search sessions yet) + */ + overrides?: { + disableSearchSessions?: true; + } ): Pick< AsyncSearchSubmitRequest, 'keep_alive' | 'wait_for_completion_timeout' | 'keep_on_completion' > { - const useSearchSessions = searchSessionsConfig?.enabled && !!options.sessionId; - + const useSearchSessions = + config.sessions.enabled && !!options.sessionId && !overrides?.disableSearchSessions; const keepAlive = useSearchSessions && options.isStored - ? `${searchSessionsConfig!.defaultExpiration.asMilliseconds()}ms` - : '1m'; + ? `${config.sessions.defaultExpiration.asMilliseconds()}ms` + : `${config.asyncSearch.keepAlive.asMilliseconds()}ms`; return { - // Wait up to 100ms for the response to return - wait_for_completion_timeout: '100ms', + // Wait up to the timeout for the response to return + wait_for_completion_timeout: `${config.asyncSearch.waitForCompletion.asMilliseconds()}ms`, // If search sessions are used, store and get an async ID even for short running requests. keep_on_completion: useSearchSessions, // The initial keepalive is as defined in defaultExpiration if search sessions are used or 1m otherwise. @@ -44,24 +50,31 @@ export function getCommonDefaultAsyncSubmitParams( @internal */ export function getCommonDefaultAsyncGetParams( - searchSessionsConfig: SearchSessionsConfigSchema | null, - options: ISearchOptions + config: SearchConfigSchema, + options: ISearchOptions, + /** + * Allows to override some of internal logic (e.g. eql / sql searches don't fully support search sessions yet) + */ + overrides?: { + disableSearchSessions?: true; + } ): Pick { - const useSearchSessions = searchSessionsConfig?.enabled && !!options.sessionId; + const useSearchSessions = + config.sessions.enabled && !!options.sessionId && !overrides?.disableSearchSessions; return { - // Wait up to 100ms for the response to return - wait_for_completion_timeout: '100ms', + // Wait up to the timeout for the response to return + wait_for_completion_timeout: `${config.asyncSearch.waitForCompletion.asMilliseconds()}ms`, ...(useSearchSessions && options.isStored ? // Use session's keep_alive if search belongs to a stored session options.isSearchStored || options.isRestore // if search was already stored and extended, then no need to extend keepAlive ? {} : { - keep_alive: `${searchSessionsConfig!.defaultExpiration.asMilliseconds()}ms`, + keep_alive: `${config.sessions.defaultExpiration.asMilliseconds()}ms`, } : { // We still need to do polling for searches not within the context of a search session or when search session disabled - keep_alive: '1m', + keep_alive: `${config.asyncSearch.keepAlive.asMilliseconds()}ms`, }), }; } diff --git a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts index 7394f51f861ef..d6015c23058b1 100644 --- a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts +++ b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts @@ -11,6 +11,7 @@ import { eqlSearchStrategyProvider } from './eql_search_strategy'; import { SearchStrategyDependencies } from '../../types'; import { EqlSearchStrategyRequest } from '../../../../common'; import { firstValueFrom } from 'rxjs'; +import { getMockSearchConfig } from '../../../../config.mock'; const getMockEqlResponse = () => ({ body: { @@ -32,6 +33,7 @@ const getMockEqlResponse = () => ({ describe('EQL search strategy', () => { let mockLogger: Logger; + const mockSearchConfig = getMockSearchConfig({}); beforeEach(() => { mockLogger = { debug: jest.fn() } as unknown as Logger; @@ -39,12 +41,12 @@ describe('EQL search strategy', () => { describe('strategy interface', () => { it('returns a strategy with a `search` function', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); expect(typeof eqlSearch.search).toBe('function'); }); it('returns a strategy with a `cancel` function', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); expect(typeof eqlSearch.cancel).toBe('function'); }); }); @@ -81,7 +83,7 @@ describe('EQL search strategy', () => { describe('async functionality', () => { it('performs an eql client search with params when no ID is provided', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); await eqlSearch.search({ options, params }, {}, mockDeps).toPromise(); const [[request, requestOptions]] = mockEqlSearch.mock.calls; @@ -89,7 +91,7 @@ describe('EQL search strategy', () => { body: { query: 'process where 1 == 1' }, ignore_unavailable: true, index: 'logstash-*', - keep_alive: '1m', + keep_alive: '60000ms', max_concurrent_shard_requests: undefined, wait_for_completion_timeout: '100ms', }); @@ -97,14 +99,14 @@ describe('EQL search strategy', () => { }); it('retrieves the current request if an id is provided', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); await eqlSearch.search({ id: 'my-search-id' }, {}, mockDeps).toPromise(); const [[requestParams]] = mockEqlGet.mock.calls; expect(mockEqlSearch).not.toHaveBeenCalled(); expect(requestParams).toEqual({ id: 'my-search-id', - keep_alive: '1m', + keep_alive: '60000ms', wait_for_completion_timeout: '100ms', }); }); @@ -112,7 +114,7 @@ describe('EQL search strategy', () => { it('emits an error if the client throws', async () => { expect.assertions(1); mockEqlSearch.mockReset().mockRejectedValueOnce(new Error('client error')); - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); eqlSearch.search({ options, params }, {}, mockDeps).subscribe( () => {}, (err) => { @@ -124,7 +126,7 @@ describe('EQL search strategy', () => { describe('arguments', () => { it('sends along async search options', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); await eqlSearch.search({ options, params }, {}, mockDeps).toPromise(); const [[request]] = mockEqlSearch.mock.calls; @@ -136,7 +138,7 @@ describe('EQL search strategy', () => { }); it('sends along default search parameters', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); await eqlSearch.search({ options, params }, {}, mockDeps).toPromise(); const [[request]] = mockEqlSearch.mock.calls; @@ -148,7 +150,7 @@ describe('EQL search strategy', () => { }); it('allows search parameters to be overridden', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); await eqlSearch .search( { @@ -174,7 +176,7 @@ describe('EQL search strategy', () => { }); it('allows search options to be overridden', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); await eqlSearch .search( { @@ -196,7 +198,7 @@ describe('EQL search strategy', () => { }); it('passes (deprecated) transport options for an existing request', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); await eqlSearch .search({ id: 'my-search-id', options: { ignore: [400] } }, {}, mockDeps) .toPromise(); @@ -207,7 +209,7 @@ describe('EQL search strategy', () => { }); it('passes abort signal', async () => { - const eqlSearch = eqlSearchStrategyProvider(mockLogger); + const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger); const eql: EqlSearchStrategyRequest = { id: 'my-search-id' }; const abortController = new AbortController(); await firstValueFrom( @@ -219,7 +221,7 @@ describe('EQL search strategy', () => { }); it('passes transport options for search with id', async () => { - const eqlSearch = eqlSearchStrategyProvider(mockLogger); + const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger); const eql: EqlSearchStrategyRequest = { id: 'my-search-id' }; await firstValueFrom( eqlSearch.search(eql, { transport: { maxResponseSize: 13131313 } }, mockDeps) @@ -234,7 +236,7 @@ describe('EQL search strategy', () => { }); it('passes transport options for search without id', async () => { - const eqlSearch = eqlSearchStrategyProvider(mockLogger); + const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger); const eql: EqlSearchStrategyRequest = { params: { index: 'all' } }; await firstValueFrom(eqlSearch.search(eql, { transport: { ignore: [400] } }, mockDeps)); const [[_params, requestOptions]] = mockEqlSearch.mock.calls; @@ -245,7 +247,7 @@ describe('EQL search strategy', () => { describe('response', () => { it('contains a rawResponse field containing the full search response', async () => { - const eqlSearch = await eqlSearchStrategyProvider(mockLogger); + const eqlSearch = await eqlSearchStrategyProvider(mockSearchConfig, mockLogger); const response = await eqlSearch .search({ id: 'my-search-id', options: { ignore: [400] } }, {}, mockDeps) .toPromise(); diff --git a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts index aab1341f9dbfa..14b5c76c67f46 100644 --- a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts @@ -9,6 +9,7 @@ import type { TransportResult } from '@elastic/elasticsearch'; import { tap } from 'rxjs/operators'; import type { IScopedClusterClient, Logger } from '@kbn/core/server'; +import { SearchConfigSchema } from '../../../../config'; import { EqlSearchStrategyRequest, EqlSearchStrategyResponse, @@ -23,6 +24,7 @@ import { getIgnoreThrottled } from '../ese_search/request_utils'; import { getCommonDefaultAsyncGetParams } from '../common/async_utils'; export const eqlSearchStrategyProvider = ( + searchConfig: SearchConfigSchema, logger: Logger ): ISearchStrategy => { async function cancelAsyncSearch(id: string, esClient: IScopedClusterClient) { @@ -46,11 +48,15 @@ export const eqlSearchStrategyProvider = ( uiSettingsClient ); const params = id - ? getCommonDefaultAsyncGetParams(null, options) + ? getCommonDefaultAsyncGetParams(searchConfig, options, { + /* disable until full eql support */ disableSearchSessions: true, + }) : { ...(await getIgnoreThrottled(uiSettingsClient)), ...defaultParams, - ...getCommonDefaultAsyncGetParams(null, options), + ...getCommonDefaultAsyncGetParams(searchConfig, options, { + /* disable until full eql support */ disableSearchSessions: true, + }), ...request.params, }; const response = id diff --git a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.test.ts b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.test.ts index c2c42f1ff8963..070d07c07c956 100644 --- a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.test.ts +++ b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.test.ts @@ -14,6 +14,7 @@ import * as xContentParseException from '../../../../common/search/test_data/x_c import { SearchStrategyDependencies } from '../../types'; import { enhancedEsSearchStrategyProvider } from './ese_search_strategy'; import { createSearchSessionsClientMock } from '../../mocks'; +import { getMockSearchConfig } from '../../../../config.mock'; const mockAsyncResponse = { body: { @@ -74,6 +75,8 @@ describe('ES search strategy', () => { }, }); + const mockSearchConfig = getMockSearchConfig({}); + beforeEach(() => { mockApiCaller.mockClear(); mockGetCaller.mockClear(); @@ -82,7 +85,11 @@ describe('ES search strategy', () => { }); it('returns a strategy with `search and `cancel`', async () => { - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); expect(typeof esSearch.search).toBe('function'); }); @@ -93,7 +100,11 @@ describe('ES search strategy', () => { mockSubmitCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch.search({ params }, {}, mockDeps).toPromise(); @@ -101,14 +112,18 @@ describe('ES search strategy', () => { const request = mockSubmitCaller.mock.calls[0][0]; expect(request.index).toEqual(params.index); expect(request.body).toEqual(params.body); - expect(request).toHaveProperty('keep_alive', '1m'); + expect(request).toHaveProperty('keep_alive', '60000ms'); }); it('makes a GET request to async search with ID', async () => { mockGetCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch.search({ id: 'foo', params }, {}, mockDeps).toPromise(); @@ -116,14 +131,18 @@ describe('ES search strategy', () => { const request = mockGetCaller.mock.calls[0][0]; expect(request.id).toEqual('foo'); expect(request).toHaveProperty('wait_for_completion_timeout'); - expect(request).toHaveProperty('keep_alive', '1m'); + expect(request).toHaveProperty('keep_alive', '60000ms'); }); it('sets transport options on POST requests', async () => { const transportOptions = { maxRetries: 1 }; mockSubmitCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await firstValueFrom( esSearch.search({ params }, { transport: transportOptions }, mockDeps) @@ -136,7 +155,7 @@ describe('ES search strategy', () => { body: { query: {} }, ignore_unavailable: true, index: 'logstash-*', - keep_alive: '1m', + keep_alive: '60000ms', keep_on_completion: false, max_concurrent_shard_requests: undefined, track_total_hits: true, @@ -149,7 +168,11 @@ describe('ES search strategy', () => { it('sets transport options on GET requests', async () => { mockGetCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await firstValueFrom( esSearch.search({ id: 'foo', params }, { transport: { maxRetries: 1 } }, mockDeps) @@ -159,7 +182,7 @@ describe('ES search strategy', () => { 1, expect.objectContaining({ id: 'foo', - keep_alive: '1m', + keep_alive: '60000ms', wait_for_completion_timeout: '100ms', }), expect.objectContaining({ maxRetries: 1, meta: true, signal: undefined }) @@ -170,7 +193,11 @@ describe('ES search strategy', () => { mockSubmitCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'foo-*', body: {} }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch.search({ params }, {}, mockDeps).toPromise(); @@ -184,7 +211,11 @@ describe('ES search strategy', () => { mockApiCaller.mockResolvedValueOnce(mockRollupResponse); const params = { index: 'foo-程', body: {} }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch .search( @@ -209,7 +240,11 @@ describe('ES search strategy', () => { mockSubmitCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch.search({ params }, { sessionId: '1' }, mockDeps).toPromise(); @@ -218,14 +253,18 @@ describe('ES search strategy', () => { expect(request.index).toEqual(params.index); expect(request.body).toEqual(params.body); - expect(request).toHaveProperty('keep_alive', '1m'); + expect(request).toHaveProperty('keep_alive', '60000ms'); }); it('Submit search with session id and session is saved creates a search with long keep_alive', async () => { mockSubmitCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch.search({ params }, { sessionId: '1', isStored: true }, mockDeps).toPromise(); @@ -241,7 +280,11 @@ describe('ES search strategy', () => { mockGetCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch.search({ id: 'foo', params }, { sessionId: '1' }, mockDeps).toPromise(); @@ -249,14 +292,18 @@ describe('ES search strategy', () => { const request = mockGetCaller.mock.calls[0][0]; expect(request.id).toEqual('foo'); expect(request).toHaveProperty('wait_for_completion_timeout'); - expect(request).toHaveProperty('keep_alive', '1m'); + expect(request).toHaveProperty('keep_alive', '60000ms'); }); it('makes a GET request to async search with long keepalive, if session is saved', async () => { mockGetCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch .search({ id: 'foo', params }, { sessionId: '1', isStored: true }, mockDeps) @@ -273,7 +320,11 @@ describe('ES search strategy', () => { mockGetCaller.mockResolvedValueOnce(mockAsyncResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch .search( @@ -303,7 +354,11 @@ describe('ES search strategy', () => { mockSubmitCaller.mockRejectedValue(errResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); let err: KbnServerError | undefined; try { @@ -324,7 +379,11 @@ describe('ES search strategy', () => { mockSubmitCaller.mockRejectedValue(errResponse); const params = { index: 'logstash-*', body: { query: {} } }; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); let err: KbnServerError | undefined; try { @@ -345,7 +404,11 @@ describe('ES search strategy', () => { mockDeleteCaller.mockResolvedValueOnce(200); const id = 'some_id'; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch.cancel!(id, {}, mockDeps); @@ -365,7 +428,11 @@ describe('ES search strategy', () => { mockDeleteCaller.mockRejectedValue(errResponse); const id = 'some_id'; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); let err: KbnServerError | undefined; try { @@ -388,7 +455,11 @@ describe('ES search strategy', () => { const id = 'some_other_id'; const keepAlive = '1d'; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); await esSearch.extend!(id, keepAlive, {}, mockDeps); @@ -403,7 +474,11 @@ describe('ES search strategy', () => { const id = 'some_other_id'; const keepAlive = '1d'; - const esSearch = await enhancedEsSearchStrategyProvider(mockLegacyConfig$, mockLogger); + const esSearch = await enhancedEsSearchStrategyProvider( + mockLegacyConfig$, + mockSearchConfig, + mockLogger + ); let err: KbnServerError | undefined; try { diff --git a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts index 33234d7c65730..3c9d6a40c58c1 100644 --- a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts @@ -33,9 +33,11 @@ import { getTotalLoaded, shimHitsTotal, } from '../es_search'; +import { SearchConfigSchema } from '../../../../config'; export const enhancedEsSearchStrategyProvider = ( legacyConfig$: Observable, + searchConfig: SearchConfigSchema, logger: Logger, usage?: SearchUsage, useInternalUser: boolean = false @@ -52,19 +54,15 @@ export const enhancedEsSearchStrategyProvider = ( function asyncSearch( { id, ...request }: IEsSearchRequest, options: IAsyncSearchOptions, - { esClient, uiSettingsClient, searchSessionsClient }: SearchStrategyDependencies + { esClient, uiSettingsClient }: SearchStrategyDependencies ) { const client = useInternalUser ? esClient.asInternalUser : esClient.asCurrentUser; const search = async () => { const params = id - ? getDefaultAsyncGetParams(searchSessionsClient.getConfig(), options) + ? getDefaultAsyncGetParams(searchConfig, options) : { - ...(await getDefaultAsyncSubmitParams( - uiSettingsClient, - searchSessionsClient.getConfig(), - options - )), + ...(await getDefaultAsyncSubmitParams(uiSettingsClient, searchConfig, options)), ...request.params, }; const { body, headers } = id diff --git a/src/plugins/data/server/search/strategies/ese_search/request_utils.test.ts b/src/plugins/data/server/search/strategies/ese_search/request_utils.test.ts index b908a1df4f0ec..e14be555535d3 100644 --- a/src/plugins/data/server/search/strategies/ese_search/request_utils.test.ts +++ b/src/plugins/data/server/search/strategies/ese_search/request_utils.test.ts @@ -14,21 +14,12 @@ import { import { IUiSettingsClient } from '@kbn/core/server'; import { UI_SETTINGS } from '../../../../common'; import moment from 'moment'; -import { SearchSessionsConfigSchema } from '../../../../config'; +import { getMockSearchConfig } from '../../../../config.mock'; const getMockUiSettingsClient = (config: Record) => { return { get: async (key: string) => config[key] } as IUiSettingsClient; }; -const getMockSearchSessionsConfig = ({ - enabled = true, - defaultExpiration = moment.duration(7, 'd'), -} = {}) => - ({ - enabled, - defaultExpiration, - } as SearchSessionsConfigSchema); - describe('request utils', () => { describe('getIgnoreThrottled', () => { test('does not return `ignore_throttled` when `includeFrozen` is `false`', async () => { @@ -53,19 +44,23 @@ describe('request utils', () => { const mockUiSettingsClient = getMockUiSettingsClient({ [UI_SETTINGS.SEARCH_INCLUDE_FROZEN]: false, }); - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + }, }); const params = await getDefaultAsyncSubmitParams(mockUiSettingsClient, mockConfig, {}); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); test('Uses `keep_alive` from config if enabled and session is stored', async () => { const mockUiSettingsClient = getMockUiSettingsClient({ [UI_SETTINGS.SEARCH_INCLUDE_FROZEN]: false, }); - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + }, }); const params = await getDefaultAsyncSubmitParams(mockUiSettingsClient, mockConfig, { sessionId: 'foo', @@ -78,21 +73,23 @@ describe('request utils', () => { const mockUiSettingsClient = getMockUiSettingsClient({ [UI_SETTINGS.SEARCH_INCLUDE_FROZEN]: false, }); - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: false, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: false, + }, }); const params = await getDefaultAsyncSubmitParams(mockUiSettingsClient, mockConfig, { sessionId: 'foo', }); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); test('Uses `keep_on_completion` if enabled', async () => { const mockUiSettingsClient = getMockUiSettingsClient({ [UI_SETTINGS.SEARCH_INCLUDE_FROZEN]: false, }); - const mockConfig = getMockSearchSessionsConfig({}); + const mockConfig = getMockSearchConfig({}); const params = await getDefaultAsyncSubmitParams(mockUiSettingsClient, mockConfig, { sessionId: 'foo', }); @@ -103,9 +100,11 @@ describe('request utils', () => { const mockUiSettingsClient = getMockUiSettingsClient({ [UI_SETTINGS.SEARCH_INCLUDE_FROZEN]: false, }); - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: false, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: false, + }, }); const params = await getDefaultAsyncSubmitParams(mockUiSettingsClient, mockConfig, { sessionId: 'foo', @@ -116,27 +115,33 @@ describe('request utils', () => { describe('getDefaultAsyncGetParams', () => { test('Uses `wait_for_completion_timeout`', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getDefaultAsyncGetParams(mockConfig, {}); expect(params).toHaveProperty('wait_for_completion_timeout'); }); test('Uses `keep_alive` if `sessionId` is not provided', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getDefaultAsyncGetParams(mockConfig, {}); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); test('Has no `keep_alive` if `sessionId` is provided and search already stored', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getDefaultAsyncGetParams(mockConfig, { sessionId: 'foo', @@ -147,12 +152,14 @@ describe('request utils', () => { }); test('Uses `keep_alive` if `sessionId` is provided but sessions disabled', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: false, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: false, + }, }); const params = getDefaultAsyncGetParams(mockConfig, { sessionId: 'foo' }); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); }); }); diff --git a/src/plugins/data/server/search/strategies/ese_search/request_utils.ts b/src/plugins/data/server/search/strategies/ese_search/request_utils.ts index 07f1c9d1ae9a5..e9bd7a54c0071 100644 --- a/src/plugins/data/server/search/strategies/ese_search/request_utils.ts +++ b/src/plugins/data/server/search/strategies/ese_search/request_utils.ts @@ -11,7 +11,7 @@ import { AsyncSearchGetRequest } from '@elastic/elasticsearch/lib/api/typesWithB import { AsyncSearchSubmitRequest } from '@elastic/elasticsearch/lib/api/types'; import { ISearchOptions, UI_SETTINGS } from '../../../../common'; import { getDefaultSearchParams } from '../es_search'; -import { SearchSessionsConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../../config'; import { getCommonDefaultAsyncGetParams, getCommonDefaultAsyncSubmitParams, @@ -32,7 +32,7 @@ export async function getIgnoreThrottled( */ export async function getDefaultAsyncSubmitParams( uiSettingsClient: Pick, - searchSessionsConfig: SearchSessionsConfigSchema | null, + searchConfig: SearchConfigSchema, options: ISearchOptions ): Promise< Pick< @@ -49,11 +49,10 @@ export async function getDefaultAsyncSubmitParams( > { return { // TODO: adjust for partial results - batched_reduce_size: 64, - ...getCommonDefaultAsyncSubmitParams(searchSessionsConfig, options), + batched_reduce_size: searchConfig.asyncSearch.batchedReduceSize, + ...getCommonDefaultAsyncSubmitParams(searchConfig, options), ...(await getIgnoreThrottled(uiSettingsClient)), ...(await getDefaultSearchParams(uiSettingsClient)), - // If search sessions are used, set the initial expiration time. }; } @@ -61,10 +60,10 @@ export async function getDefaultAsyncSubmitParams( @internal */ export function getDefaultAsyncGetParams( - searchSessionsConfig: SearchSessionsConfigSchema | null, + searchConfig: SearchConfigSchema, options: ISearchOptions ): Pick { return { - ...getCommonDefaultAsyncGetParams(searchSessionsConfig, options), + ...getCommonDefaultAsyncGetParams(searchConfig, options), }; } diff --git a/src/plugins/data/server/search/strategies/sql_search/request_utils.test.ts b/src/plugins/data/server/search/strategies/sql_search/request_utils.test.ts index d37bcfb0655f8..f21b766db3e1d 100644 --- a/src/plugins/data/server/search/strategies/sql_search/request_utils.test.ts +++ b/src/plugins/data/server/search/strategies/sql_search/request_utils.test.ts @@ -8,30 +8,26 @@ import { getDefaultAsyncSubmitParams, getDefaultAsyncGetParams } from './request_utils'; import moment from 'moment'; -import { SearchSessionsConfigSchema } from '../../../../config'; - -const getMockSearchSessionsConfig = ({ - enabled = true, - defaultExpiration = moment.duration(7, 'd'), -} = {}) => - ({ - enabled, - defaultExpiration, - } as SearchSessionsConfigSchema); +import { getMockSearchConfig } from '../../../../config.mock'; describe('request utils', () => { describe('getDefaultAsyncSubmitParams', () => { test('Uses `keep_alive` from default params if no `sessionId` is provided', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + }, }); const params = getDefaultAsyncSubmitParams(mockConfig, {}); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); - test('Uses `keep_alive` from config if enabled and session is stored', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), + // unskip when SQL has full session support https://github.com/elastic/kibana/issues/127880 + test.skip('Uses `keep_alive` from config if enabled and session is stored', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + }, }); const params = getDefaultAsyncSubmitParams(mockConfig, { sessionId: 'foo', @@ -40,29 +36,57 @@ describe('request utils', () => { expect(params).toHaveProperty('keep_alive', '259200000ms'); }); + // remove when SQL has full session support https://github.com/elastic/kibana/issues/127880 + test('Uses `keep_alive` from asyncSearch config if sessions enabled and session is stored', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + }, + }); + const params = getDefaultAsyncSubmitParams(mockConfig, { + sessionId: 'foo', + isStored: true, + }); + expect(params).toHaveProperty('keep_alive', '60000ms'); + }); + test('Uses `keepAlive` of `1m` if disabled', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: false, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: false, + }, }); const params = getDefaultAsyncSubmitParams(mockConfig, { sessionId: 'foo', }); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); - test('Uses `keep_on_completion` if enabled', async () => { - const mockConfig = getMockSearchSessionsConfig({}); + // unskip when SQL has full session support https://github.com/elastic/kibana/issues/127880 + test.skip('Uses `keep_on_completion` if sessions enabled', async () => { + const mockConfig = getMockSearchConfig({}); const params = getDefaultAsyncSubmitParams(mockConfig, { sessionId: 'foo', }); expect(params).toHaveProperty('keep_on_completion', true); }); + // remove when SQL has full session support https://github.com/elastic/kibana/issues/127880 + test("Don't use `keep_on_completion` if sessions enabled", async () => { + const mockConfig = getMockSearchConfig({}); + const params = getDefaultAsyncSubmitParams(mockConfig, { + sessionId: 'foo', + }); + expect(params).toHaveProperty('keep_on_completion', false); + }); + test('Does not use `keep_on_completion` if disabled', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: false, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: false, + }, }); const params = getDefaultAsyncSubmitParams(mockConfig, { sessionId: 'foo', @@ -73,27 +97,34 @@ describe('request utils', () => { describe('getDefaultAsyncGetParams', () => { test('Uses `wait_for_completion_timeout`', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getDefaultAsyncGetParams(mockConfig, {}); expect(params).toHaveProperty('wait_for_completion_timeout'); }); test('Uses `keep_alive` if `sessionId` is not provided', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getDefaultAsyncGetParams(mockConfig, {}); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); - test('Has no `keep_alive` if `sessionId` is provided, search and session are stored', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: true, + // remove when SQL has full session support https://github.com/elastic/kibana/issues/127880 + test.skip('Has no `keep_alive` if `sessionId` is provided, search and session are stored', async () => { + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: true, + }, }); const params = getDefaultAsyncGetParams(mockConfig, { sessionId: 'foo', @@ -104,12 +135,14 @@ describe('request utils', () => { }); test('Uses `keep_alive` if `sessionId` is provided but sessions disabled', async () => { - const mockConfig = getMockSearchSessionsConfig({ - defaultExpiration: moment.duration(3, 'd'), - enabled: false, + const mockConfig = getMockSearchConfig({ + sessions: { + defaultExpiration: moment.duration(3, 'd'), + enabled: false, + }, }); const params = getDefaultAsyncGetParams(mockConfig, { sessionId: 'foo' }); - expect(params).toHaveProperty('keep_alive', '1m'); + expect(params).toHaveProperty('keep_alive', '60000ms'); }); }); }); diff --git a/src/plugins/data/server/search/strategies/sql_search/request_utils.ts b/src/plugins/data/server/search/strategies/sql_search/request_utils.ts index de8ced65d16c6..e170b506b537d 100644 --- a/src/plugins/data/server/search/strategies/sql_search/request_utils.ts +++ b/src/plugins/data/server/search/strategies/sql_search/request_utils.ts @@ -8,7 +8,7 @@ import { SqlGetAsyncRequest, SqlQueryRequest } from '@elastic/elasticsearch/lib/api/types'; import { ISearchOptions } from '../../../../common'; -import { SearchSessionsConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../../config'; import { getCommonDefaultAsyncGetParams, getCommonDefaultAsyncSubmitParams, @@ -18,11 +18,17 @@ import { @internal */ export function getDefaultAsyncSubmitParams( - searchSessionsConfig: SearchSessionsConfigSchema | null, + searchConfig: SearchConfigSchema, options: ISearchOptions ): Pick { return { - ...getCommonDefaultAsyncSubmitParams(searchSessionsConfig, options), + ...getCommonDefaultAsyncSubmitParams(searchConfig, options, { + /** + * force disable search sessions until sessions support SQL + * https://github.com/elastic/kibana/issues/127880 + */ + disableSearchSessions: true, + }), }; } @@ -30,10 +36,16 @@ export function getDefaultAsyncSubmitParams( @internal */ export function getDefaultAsyncGetParams( - searchSessionsConfig: SearchSessionsConfigSchema | null, + searchConfig: SearchConfigSchema, options: ISearchOptions ): Pick { return { - ...getCommonDefaultAsyncGetParams(searchSessionsConfig, options), + ...getCommonDefaultAsyncGetParams(searchConfig, options, { + /** + * force disable search sessions until sessions support SQL + * https://github.com/elastic/kibana/issues/127880 + */ + disableSearchSessions: true, + }), }; } diff --git a/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.test.ts b/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.test.ts index e36b66a31c017..f4cfcabf0737b 100644 --- a/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.test.ts +++ b/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.test.ts @@ -14,6 +14,7 @@ import { SearchStrategyDependencies } from '../../types'; import { sqlSearchStrategyProvider } from './sql_search_strategy'; import { createSearchSessionsClientMock } from '../../mocks'; import { SqlSearchStrategyRequest } from '../../../../common'; +import { getMockSearchConfig } from '../../../../config.mock'; const mockSqlResponse = { body: { @@ -46,6 +47,8 @@ describe('SQL search strategy', () => { searchSessionsClient: createSearchSessionsClientMock(), } as unknown as SearchStrategyDependencies; + const mockSearchConfig = getMockSearchConfig({}); + beforeEach(() => { mockSqlGetAsync.mockClear(); mockSqlQuery.mockClear(); @@ -54,7 +57,7 @@ describe('SQL search strategy', () => { }); it('returns a strategy with `search and `cancel`, `extend`', async () => { - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); expect(typeof esSearch.search).toBe('function'); expect(typeof esSearch.cancel).toBe('function'); @@ -70,7 +73,7 @@ describe('SQL search strategy', () => { query: 'SELECT customer_first_name FROM kibana_sample_data_ecommerce ORDER BY order_date DESC', }; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch .search({ params }, { transport: { requestTimeout: 30000 } }, mockDeps) @@ -80,8 +83,8 @@ describe('SQL search strategy', () => { const [request, searchOptions] = mockSqlQuery.mock.calls[0]; expect(request).toEqual({ format: 'json', - keep_alive: '1m', - keep_on_completion: undefined, + keep_alive: '60000ms', + keep_on_completion: false, query: 'SELECT customer_first_name FROM kibana_sample_data_ecommerce ORDER BY order_date DESC', wait_for_completion_timeout: '100ms', @@ -101,7 +104,7 @@ describe('SQL search strategy', () => { 'SELECT customer_first_name FROM kibana_sample_data_ecommerce ORDER BY order_date DESC', }; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch .search({ id: 'foo', params }, { transport: { requestTimeout: 30000 } }, mockDeps) @@ -112,7 +115,7 @@ describe('SQL search strategy', () => { expect(request).toEqual({ format: 'json', id: 'foo', - keep_alive: '1m', + keep_alive: '60000ms', wait_for_completion_timeout: '100ms', }); expect(searchOptions).toEqual({ @@ -131,7 +134,7 @@ describe('SQL search strategy', () => { query: 'SELECT customer_first_name FROM kibana_sample_data_ecommerce ORDER BY order_date DESC', }; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch.search({ params }, { sessionId: '1' }, mockDeps).toPromise(); @@ -150,7 +153,7 @@ describe('SQL search strategy', () => { 'SELECT customer_first_name FROM kibana_sample_data_ecommerce ORDER BY order_date DESC', }; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch.search({ id: 'foo', params }, { sessionId: '1' }, mockDeps).toPromise(); @@ -169,7 +172,7 @@ describe('SQL search strategy', () => { query: 'SELECT customer_first_name FROM kibana_sample_data_ecommerce ORDER BY order_date DESC', }; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch.search({ params }, { sessionId: '1' }, mockDeps).toPromise(); @@ -178,7 +181,7 @@ describe('SQL search strategy', () => { expect(request.query).toEqual(params.query); expect(request).toHaveProperty('wait_for_completion_timeout'); - expect(request).toHaveProperty('keep_alive', '1m'); + expect(request).toHaveProperty('keep_alive', '60000ms'); }); it('makes a GET request to async search with keepalive', async () => { @@ -189,7 +192,7 @@ describe('SQL search strategy', () => { 'SELECT customer_first_name FROM kibana_sample_data_ecommerce ORDER BY order_date DESC', }; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch.search({ id: 'foo', params }, { sessionId: '1' }, mockDeps).toPromise(); @@ -197,7 +200,7 @@ describe('SQL search strategy', () => { const request = mockSqlGetAsync.mock.calls[0][0]; expect(request.id).toEqual('foo'); expect(request).toHaveProperty('wait_for_completion_timeout'); - expect(request).toHaveProperty('keep_alive', '1m'); + expect(request).toHaveProperty('keep_alive', '60000ms'); }); }); @@ -216,7 +219,7 @@ describe('SQL search strategy', () => { query: 'SELECT customer_first_name FROM kibana_sample_data_ecommerce ORDER BY order_date DESC', }; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); let err: KbnServerError | undefined; try { @@ -240,7 +243,7 @@ describe('SQL search strategy', () => { query: 'SELECT customer_first_name FROM kibana_sample_data_ecommerce ORDER BY order_date DESC', }; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); let err: KbnServerError | undefined; try { @@ -262,7 +265,7 @@ describe('SQL search strategy', () => { }) ); - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch.search({ id: 'foo', params: { query: 'query' } }, {}, mockDeps).toPromise(); expect(mockSqlClearCursor).not.toHaveBeenCalled(); @@ -273,7 +276,7 @@ describe('SQL search strategy', () => { merge({}, mockSqlResponse, { body: { cursor: 'cursor' } }) ); - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch .search({ id: 'foo', params: { query: 'query', keep_cursor: true } }, {}, mockDeps) .toPromise(); @@ -286,7 +289,7 @@ describe('SQL search strategy', () => { merge({}, mockSqlResponse, { body: { cursor: 'cursor' } }) ); - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch.search({ id: 'foo', params: { query: 'query' } }, {}, mockDeps).toPromise(); expect(mockSqlClearCursor).toHaveBeenCalledWith({ cursor: 'cursor' }); @@ -295,7 +298,7 @@ describe('SQL search strategy', () => { it('returns the time it took to run a search', async () => { mockSqlGetAsync.mockResolvedValueOnce(mockSqlResponse); - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await expect( esSearch.search({ id: 'foo', params: { query: 'query' } }, {}, mockDeps).toPromise() ).resolves.toHaveProperty('took', expect.any(Number)); @@ -307,7 +310,7 @@ describe('SQL search strategy', () => { mockSqlDelete.mockResolvedValueOnce(200); const id = 'some_id'; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch.cancel!(id, {}, mockDeps); @@ -323,7 +326,7 @@ describe('SQL search strategy', () => { const id = 'some_other_id'; const keepAlive = '1d'; - const esSearch = await sqlSearchStrategyProvider(mockLogger); + const esSearch = await sqlSearchStrategyProvider(mockSearchConfig, mockLogger); await esSearch.extend!(id, keepAlive, {}, mockDeps); expect(mockSqlGetAsync).toBeCalled(); diff --git a/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts b/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts index b84dcfe45d2cf..64c4d6fd5ee0f 100644 --- a/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts @@ -20,8 +20,10 @@ import type { import { pollSearch } from '../../../../common'; import { getDefaultAsyncGetParams, getDefaultAsyncSubmitParams } from './request_utils'; import { toAsyncKibanaSearchResponse } from './response_utils'; +import { SearchConfigSchema } from '../../../../config'; export const sqlSearchStrategyProvider = ( + searchConfig: SearchConfigSchema, logger: Logger, useInternalUser: boolean = false ): ISearchStrategy => { @@ -42,11 +44,6 @@ export const sqlSearchStrategyProvider = ( const client = useInternalUser ? esClient.asInternalUser : esClient.asCurrentUser; const startTime = Date.now(); - // disable search sessions until session task manager supports SQL - // https://github.com/elastic/kibana/issues/127880 - // const sessionConfig = searchSessionsClient.getConfig(); - const sessionConfig = null; - const search = async () => { const { keep_cursor: keepCursor, ...params } = request.params ?? {}; let body: SqlQueryResponse; @@ -56,7 +53,7 @@ export const sqlSearchStrategyProvider = ( ({ body, headers } = await client.sql.getAsync( { format: params?.format ?? 'json', - ...getDefaultAsyncGetParams(sessionConfig, options), + ...getDefaultAsyncGetParams(searchConfig, options), id, }, { ...options.transport, signal: options.abortSignal, meta: true } @@ -65,7 +62,7 @@ export const sqlSearchStrategyProvider = ( ({ headers, body } = await client.sql.query( { format: params.format ?? 'json', - ...getDefaultAsyncSubmitParams(sessionConfig, options), + ...getDefaultAsyncSubmitParams(searchConfig, options), ...params, }, { ...options.transport, signal: options.abortSignal, meta: true } diff --git a/src/plugins/data/tsconfig.json b/src/plugins/data/tsconfig.json index 8f391fc15e871..2e9c05992cc9d 100644 --- a/src/plugins/data/tsconfig.json +++ b/src/plugins/data/tsconfig.json @@ -11,6 +11,7 @@ "public/**/*", "server/**/*", "config.ts", + "config.mock.ts", "common/**/*.json", "public/**/*.json", "../../../typings/index.d.ts" diff --git a/test/plugin_functional/test_suites/core_plugins/rendering.ts b/test/plugin_functional/test_suites/core_plugins/rendering.ts index a3968330e1425..2bfd0c7b5dda3 100644 --- a/test/plugin_functional/test_suites/core_plugins/rendering.ts +++ b/test/plugin_functional/test_suites/core_plugins/rendering.ts @@ -91,6 +91,9 @@ export default function ({ getService }: PluginFunctionalProviderContext) { 'unifiedSearch.autocomplete.valueSuggestions.tiers (array)', 'unifiedSearch.autocomplete.valueSuggestions.timeout (duration)', 'data.search.aggs.shardDelay.enabled (boolean)', + 'data.search.asyncSearch.batchedReduceSize (number)', + 'data.search.asyncSearch.keepAlive (duration)', + 'data.search.asyncSearch.waitForCompletion (duration)', 'data.search.sessions.defaultExpiration (duration)', 'data.search.sessions.enabled (boolean)', 'data.search.sessions.management.expiresSoonWarning (duration)',