[Search]Add EQL search strategy (#78645)

* Add EQL search strategy

Since EQL is an x-pack feature, this strategy will live in the
x-pack plugin data_enhanced.

* Refactor our test setup to minimize shared state

* Ensures that the same variable is not used for both test setup and
  test assertions
* Ensures that mocks are reinstantiated on every test

* Use explicit top-level exports

* Move async search options to a helper function

* Move our workaround to a helper function

This was repeated in five places, time to consolidate.

* Commit documentation changes

We export a few new helper functions.

* Mark our internal methods as such

Updates documentation accordingly.

Co-authored-by: Kibana Machine <[email protected]>

src/plugins/data/server/index.ts

@@ -214,11 +214,13 @@ export {
   ISearchSetup,
   ISearchStart,
   toSnakeCase,
+  getAsyncOptions,
   getDefaultSearchParams,
   getShardTimeout,
   getTotalLoaded,
   shimHitsTotal,
   usageProvider,
+  shimAbortSignal,
   SearchUsage,
 } from './search';
 

src/plugins/data/server/search/es_search/es_search_strategy.ts

@@ -23,7 +23,13 @@ import { Observable } from 'rxjs';
 import { ApiResponse } from '@elastic/elasticsearch';
 import { SearchUsage } from '../collectors/usage';
 import { toSnakeCase } from './to_snake_case';
-import { ISearchStrategy, getDefaultSearchParams, getTotalLoaded, getShardTimeout } from '..';
+import {
+  ISearchStrategy,
+  getDefaultSearchParams,
+  getTotalLoaded,
+  getShardTimeout,
+  shimAbortSignal,
+} from '..';
 
 export const esSearchStrategyProvider = (
   config$: Observable<SharedGlobalConfig>,
@@ -52,10 +58,10 @@ export const esSearchStrategyProvider = (
       });
 
       try {
-        // Temporary workaround until https://github.com/elastic/elasticsearch-js/issues/1297
-        const promise = context.core.elasticsearch.client.asCurrentUser.search(params);
-        if (options?.abortSignal)
-          options.abortSignal.addEventListener('abort', () => promise.abort());
+        const promise = shimAbortSignal(
+          context.core.elasticsearch.client.asCurrentUser.search(params),
+          options?.abortSignal
+        );
         const { body: rawResponse } = (await promise) as ApiResponse<SearchResponse<any>>;
 
         if (usage) usage.trackSuccess(rawResponse.took);

src/plugins/data/server/search/es_search/get_default_search_params.ts

@@ -42,3 +42,11 @@ export async function getDefaultSearchParams(uiSettingsClient: IUiSettingsClient
     trackTotalHits: true,
   };
 }
+
+/**
+ @internal
+ */
+export const getAsyncOptions = () => ({
+  waitForCompletionTimeout: '100ms', // Wait up to 100ms for the response to return
+  keepAlive: '1m', // Extend the TTL for this search request by one minute
+});

src/plugins/data/server/search/es_search/index.ts

@@ -21,5 +21,6 @@ export { esSearchStrategyProvider } from './es_search_strategy';
 export * from './get_default_search_params';
 export { getTotalLoaded } from './get_total_loaded';
 export * from './to_snake_case';
+export { shimAbortSignal } from './shim_abort_signal';
 
 export { ES_SEARCH_STRATEGY, IEsSearchRequest, IEsSearchResponse } from '../../../common';

src/plugins/data/server/search/es_search/shim_abort_signal.test.ts

@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { elasticsearchServiceMock } from '../../../../../core/server/mocks';
+import { shimAbortSignal } from '.';
+
+describe('shimAbortSignal', () => {
+  it('aborts the promise if the signal is aborted', () => {
+    const promise = elasticsearchServiceMock.createSuccessTransportRequestPromise({
+      success: true,
+    });
+    const controller = new AbortController();
+    shimAbortSignal(promise, controller.signal);
+    controller.abort();
+
+    expect(promise.abort).toHaveBeenCalled();
+  });
+
+  it('returns the original promise', async () => {
+    const promise = elasticsearchServiceMock.createSuccessTransportRequestPromise({
+      success: true,
+    });
+    const controller = new AbortController();
+    const response = await shimAbortSignal(promise, controller.signal);
+
+    expect(response).toEqual(expect.objectContaining({ body: { success: true } }));
+  });
+
+  it('allows the promise to be aborted manually', () => {
+    const promise = elasticsearchServiceMock.createSuccessTransportRequestPromise({
+      success: true,
+    });
+    const controller = new AbortController();
+    const enhancedPromise = shimAbortSignal(promise, controller.signal);
+
+    enhancedPromise.abort();
+    expect(promise.abort).toHaveBeenCalled();
+  });
+});

src/plugins/data/server/search/es_search/shim_abort_signal.ts

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { TransportRequestPromise } from '@elastic/elasticsearch/lib/Transport';
+
+/**
+ *
+ * @internal
+ * NOTE: Temporary workaround until https://github.com/elastic/elasticsearch-js/issues/1297
+ * is resolved
+ *
+ * @param promise a TransportRequestPromise
+ * @param signal optional AbortSignal
+ *
+ * @returns a TransportRequestPromise that will be aborted if the signal is aborted
+ */
+export const shimAbortSignal = <T extends TransportRequestPromise<unknown>>(
+  promise: T,
+  signal: AbortSignal | undefined
+): T => {
+  if (signal) {
+    signal.addEventListener('abort', () => promise.abort());
+  }
+  return promise;
+};

src/plugins/data/server/search/routes/call_msearch.ts

@@ -25,7 +25,7 @@ import { IUiSettingsClient, IScopedClusterClient, SharedGlobalConfig } from 'src
 
 import { MsearchRequestBody, MsearchResponse } from '../../../common/search/search_source';
 import { shimHitsTotal } from './shim_hits_total';
-import { getShardTimeout, getDefaultSearchParams, toSnakeCase } from '..';
+import { getShardTimeout, getDefaultSearchParams, toSnakeCase, shimAbortSignal } from '..';
 
 /** @internal */
 export function convertRequestBody(
@@ -74,18 +74,17 @@ export function getCallMsearch(dependencies: CallMsearchDependencies) {
 
     const body = convertRequestBody(params.body, timeout);
 
-    // Temporary workaround until https://github.com/elastic/elasticsearch-js/issues/1297
-    const promise = esClient.asCurrentUser.msearch(
-      {
-        body,
-      },
-      {
-        querystring: toSnakeCase(defaultParams),
-      }
+    const promise = shimAbortSignal(
+      esClient.asCurrentUser.msearch(
+        {
+          body,
+        },
+        {
+          querystring: toSnakeCase(defaultParams),
+        }
+      ),
+      params.signal
     );
-    if (params.signal) {
-      params.signal.addEventListener('abort', () => promise.abort());
-    }
     const response = (await promise) as ApiResponse<{ responses: Array<SearchResponse<any>> }>;
 
     return {

src/plugins/data/server/server.api.md

@@ -47,6 +47,7 @@ import { SearchResponse } from 'elasticsearch';
 import { SerializedFieldFormat as SerializedFieldFormat_2 } from 'src/plugins/expressions/common';
 import { ShardsResponse } from 'elasticsearch';
 import { ToastInputFields } from 'src/core/public/notifications';
+import { TransportRequestPromise } from '@elastic/elasticsearch/lib/Transport';
 import { Type } from '@kbn/config-schema';
 import { TypeOf } from '@kbn/config-schema';
 import { Unit } from '@elastic/datemath';
@@ -354,6 +355,12 @@ export type Filter = {
     query?: any;
 };
 
+// @internal (undocumented)
+export const getAsyncOptions: () => {
+    waitForCompletionTimeout: string;
+    keepAlive: string;
+};
+
 // Warning: (ae-forgotten-export) The symbol "IUiSettingsClient" needs to be exported by the entry point index.d.ts
 // Warning: (ae-missing-release-tag) "getDefaultSearchParams" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
@@ -980,6 +987,9 @@ export interface SearchUsage {
     trackSuccess(duration: number): Promise<void>;
 }
 
+// @internal
+export const shimAbortSignal: <T extends TransportRequestPromise<unknown>>(promise: T, signal: AbortSignal | undefined) => T;
+
 // @internal
 export function shimHitsTotal(response: SearchResponse<any>): {
     hits: {
@@ -1115,19 +1125,19 @@ export function usageProvider(core: CoreSetup_2): SearchUsage;
 // src/plugins/data/server/index.ts:101:26 - (ae-forgotten-export) The symbol "TruncateFormat" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/index.ts:127:27 - (ae-forgotten-export) The symbol "isFilterable" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/index.ts:127:27 - (ae-forgotten-export) The symbol "isNestedField" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:226:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:226:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:226:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:226:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:228:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:229:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:238:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:239:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:240:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:244:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:245:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:249:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts
-// src/plugins/data/server/index.ts:252:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:228:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:228:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:228:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:228:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:230:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:231:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:240:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:241:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:242:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:246:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:247:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:251:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts
+// src/plugins/data/server/index.ts:254:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/index_patterns/index_patterns_service.ts:50:14 - (ae-forgotten-export) The symbol "IndexPatternsService" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/plugin.ts:88:66 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts
 // src/plugins/data/server/search/types.ts:78:5 - (ae-forgotten-export) The symbol "ISearchStartSearchSource" needs to be exported by the entry point index.d.ts

x-pack/plugins/data_enhanced/common/index.ts

@@ -5,7 +5,11 @@
  */
 
 export {
-  IEnhancedEsSearchRequest,
-  IAsyncSearchRequest,
   ENHANCED_ES_SEARCH_STRATEGY,
+  EQL_SEARCH_STRATEGY,
+  EqlRequestParams,
+  EqlSearchStrategyRequest,
+  EqlSearchStrategyResponse,
+  IAsyncSearchRequest,
+  IEnhancedEsSearchRequest,
 } from './search';

x-pack/plugins/data_enhanced/common/search/index.ts

@@ -4,8 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export {
-  IEnhancedEsSearchRequest,
-  IAsyncSearchRequest,
-  ENHANCED_ES_SEARCH_STRATEGY,
-} from './types';
+export * from './types';

x-pack/plugins/data_enhanced/common/search/types.ts

@@ -4,7 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { IEsSearchRequest } from '../../../../../src/plugins/data/common';
+import { EqlSearch } from '@elastic/elasticsearch/api/requestParams';
+import { ApiResponse, TransportRequestOptions } from '@elastic/elasticsearch/lib/Transport';
+
+import {
+  IEsSearchRequest,
+  IKibanaSearchRequest,
+  IKibanaSearchResponse,
+} from '../../../../../src/plugins/data/common';
 
 export const ENHANCED_ES_SEARCH_STRATEGY = 'ese';
 
@@ -21,3 +28,13 @@ export interface IEnhancedEsSearchRequest extends IEsSearchRequest {
    */
   isRollup?: boolean;
 }
+
+export const EQL_SEARCH_STRATEGY = 'eql';
+
+export type EqlRequestParams = EqlSearch<Record<string, unknown>>;
+
+export interface EqlSearchStrategyRequest extends IKibanaSearchRequest<EqlRequestParams> {
+  options?: TransportRequestOptions;
+}
+
+export type EqlSearchStrategyResponse<T = unknown> = IKibanaSearchResponse<ApiResponse<T>>;

x-pack/plugins/data_enhanced/server/index.ts

@@ -11,6 +11,6 @@ export function plugin(initializerContext: PluginInitializerContext) {
   return new EnhancedDataServerPlugin(initializerContext);
 }
 
-export { ENHANCED_ES_SEARCH_STRATEGY } from '../common';
+export { ENHANCED_ES_SEARCH_STRATEGY, EQL_SEARCH_STRATEGY } from '../common';
 
 export { EnhancedDataServerPlugin as Plugin };

x-pack/plugins/data_enhanced/server/plugin.ts

@@ -16,10 +16,10 @@ import {
   PluginStart as DataPluginStart,
   usageProvider,
 } from '../../../../src/plugins/data/server';
-import { enhancedEsSearchStrategyProvider } from './search';
+import { enhancedEsSearchStrategyProvider, eqlSearchStrategyProvider } from './search';
 import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server';
 import { getUiSettings } from './ui_settings';
-import { ENHANCED_ES_SEARCH_STRATEGY } from '../common';
+import { ENHANCED_ES_SEARCH_STRATEGY, EQL_SEARCH_STRATEGY } from '../common';
 
 interface SetupDependencies {
   data: DataPluginSetup;
@@ -47,6 +47,11 @@ export class EnhancedDataServerPlugin implements Plugin<void, void, SetupDepende
       )
     );
 
+    deps.data.search.registerSearchStrategy(
+      EQL_SEARCH_STRATEGY,
+      eqlSearchStrategyProvider(this.logger)
+    );
+
     deps.data.__enhance({
       search: {
         defaultStrategy: ENHANCED_ES_SEARCH_STRATEGY,