Add essql search strategy and integrate in canvas (#94754)

* Add essql search strategy, new escount temporary function, and new essql temporary function

* Move old es* functions to legacy, add esdocs to use search strategy, add parameter arg for essql

* Clean up

* cleanup

* cleanup

* Move request builder files to common

* cleanup

* add comment

* PR Feedback

* Removing old types

* update type

* Add data.search to labs and fix error messages

* Fix function help type

* Add data service to usage collector types

* Update telemetry

* remove unrelated telemetry change

* Enable multi value leniency for SQL  queries

* Display data service lab project

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -440,4 +440,8 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'labs:canvas:useDataService': {
+    type: 'boolean',
+    _meta: { description: 'Non-default value of setting.' },
+  },
 };

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -119,5 +119,6 @@ export interface UsageStats {
   'banners:textColor': string;
   'banners:backgroundColor': string;
   'labs:canvas:enable_ui': boolean;
+  'labs:canvas:useDataService': boolean;
   'labs:presentation:timeToPresent': boolean;
 }

src/plugins/presentation_util/common/labs.ts

@@ -8,9 +8,10 @@
 
 import { i18n } from '@kbn/i18n';
 
+export const USE_DATA_SERVICE = 'labs:canvas:useDataService';
 export const TIME_TO_PRESENT = 'labs:presentation:timeToPresent';
 
-export const projectIDs = [TIME_TO_PRESENT] as const;
+export const projectIDs = [TIME_TO_PRESENT, USE_DATA_SERVICE] as const;
 export const environmentNames = ['kibana', 'browser', 'session'] as const;
 export const solutionNames = ['canvas', 'dashboard', 'presentation'] as const;
 
@@ -32,6 +33,22 @@ export const projects: { [ID in ProjectID]: ProjectConfig & { id: ID } } = {
     }),
     solutions: ['canvas'],
   },
+  [USE_DATA_SERVICE]: {
+    id: USE_DATA_SERVICE,
+    isActive: true,
+    isDisplayed: true,
+    environments: ['kibana', 'browser', 'session'],
+    name: i18n.translate('presentationUtil.experiments.enableUseDataServiceExperimentName', {
+      defaultMessage: 'Use data service',
+    }),
+    description: i18n.translate(
+      'presentationUtil.experiments.enableUseDataServiceExperimentDescription',
+      {
+        defaultMessage: 'An experiment of using the new data.search service for Canvas datasources',
+      }
+    ),
+    solutions: ['canvas'],
+  },
 };
 
 export type ProjectID = typeof projectIDs[number];

src/plugins/telemetry/schema/oss_plugins.json

@@ -8342,6 +8342,12 @@
           "_meta": {
             "description": "Non-default value of setting."
           }
+        },
+        "labs:canvas:useDataService": {
+          "type": "boolean",
+          "_meta": {
+            "description": "Non-default value of setting."
+          }
         }
       }
     },

x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ExpressionFunctionDefinition,
+  ExpressionValueFilter,
+} from 'src/plugins/expressions/common';
+
+// @ts-expect-error untyped local
+import { buildESRequest } from '../../../common/lib/request/build_es_request';
+
+import { searchService } from '../../../public/services';
+
+import { getFunctionHelp } from '../../../i18n';
+
+interface Arguments {
+  index: string | null;
+  query: string;
+}
+
+export function escount(): ExpressionFunctionDefinition<
+  'escount',
+  ExpressionValueFilter,
+  Arguments,
+  any
+> {
+  const { help, args: argHelp } = getFunctionHelp().escount;
+
+  return {
+    name: 'escount',
+    type: 'number',
+    context: {
+      types: ['filter'],
+    },
+    help,
+    args: {
+      query: {
+        types: ['string'],
+        aliases: ['_', 'q'],
+        help: argHelp.query,
+        default: '"-_index:.kibana"',
+      },
+      index: {
+        types: ['string'],
+        default: '_all',
+        help: argHelp.index,
+      },
+    },
+    fn: (input, args, handlers) => {
+      input.and = input.and.concat([
+        {
+          type: 'filter',
+          filterType: 'luceneQueryString',
+          query: args.query,
+          and: [],
+        },
+      ]);
+
+      const esRequest = buildESRequest(
+        {
+          index: args.index,
+          body: {
+            track_total_hits: true,
+            size: 0,
+            query: {
+              bool: {
+                must: [{ match_all: {} }],
+              },
+            },
+          },
+        },
+        input
+      );
+
+      const search = searchService.getService().search;
+      const req = {
+        params: {
+          ...esRequest,
+        },
+      };
+
+      return search
+        .search(req)
+        .toPromise()
+        .then((resp: any) => {
+          return resp.rawResponse.hits.total;
+        });
+    },
+  };
+}

x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ExpressionFunctionDefinition,
+  ExpressionValueFilter,
+} from 'src/plugins/expressions/common';
+
+// @ts-expect-error untyped local
+import { buildESRequest } from '../../../common/lib/request/build_es_request';
+
+import { searchService } from '../../../public/services';
+import { ESSQL_SEARCH_STRATEGY } from '../../../common/lib/constants';
+import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types';
+import { getFunctionHelp } from '../../../i18n';
+
+interface Arguments {
+  index: string;
+  query: string;
+  sort: string;
+  fields: string;
+  metaFields: string;
+  count: number;
+}
+
+export function esdocs(): ExpressionFunctionDefinition<
+  'esdocs',
+  ExpressionValueFilter,
+  Arguments,
+  any
+> {
+  const { help, args: argHelp } = getFunctionHelp().esdocs;
+
+  return {
+    name: 'esdocs',
+    type: 'datatable',
+    context: {
+      types: ['filter'],
+    },
+    help,
+    args: {
+      query: {
+        types: ['string'],
+        aliases: ['_', 'q'],
+        help: argHelp.query,
+        default: '-_index:.kibana',
+      },
+      count: {
+        types: ['number'],
+        default: 1000,
+        help: argHelp.count,
+      },
+      fields: {
+        help: argHelp.fields,
+        types: ['string'],
+      },
+      index: {
+        types: ['string'],
+        default: '_all',
+        help: argHelp.index,
+      },
+      // TODO: This arg isn't being used in the function.
+      // We need to restore this functionality or remove it as an arg.
+      metaFields: {
+        help: argHelp.metaFields,
+        types: ['string'],
+      },
+      sort: {
+        types: ['string'],
+        help: argHelp.sort,
+      },
+    },
+    fn: async (input, args, handlers) => {
+      const { count, index, fields, sort } = args;
+
+      input.and = input.and.concat([
+        {
+          type: 'filter',
+          filterType: 'luceneQueryString',
+          query: args.query,
+          and: [],
+        },
+      ]);
+
+      // Load ad-hoc to avoid adding to the page load bundle size
+      const squel = await import('squel');
+
+      let query = squel.select({
+        autoQuoteTableNames: true,
+        autoQuoteFieldNames: true,
+        autoQuoteAliasNames: true,
+        nameQuoteCharacter: '"',
+      });
+
+      if (index) {
+        query.from(index);
+      }
+
+      if (fields) {
+        const allFields = fields.split(',').map((field) => field.trim());
+        allFields.forEach((field) => (query = query.field(field)));
+      }
+
+      if (sort) {
+        const [sortField, sortOrder] = sort.split(',').map((str) => str.trim());
+        if (sortField) {
+          query.order(`"${sortField}"`, sortOrder === 'asc');
+        }
+      }
+
+      const search = searchService.getService().search;
+
+      const req = {
+        count,
+        query: query.toString(),
+        filter: input.and,
+      };
+
+      // We're requesting the data using the ESSQL strategy because
+      // the SQL routes return type information with the result set
+      return search
+        .search<EssqlSearchStrategyRequest, EssqlSearchStrategyResponse>(req, {
+          strategy: ESSQL_SEARCH_STRATEGY,
+        })
+        .toPromise()
+        .then((resp: EssqlSearchStrategyResponse) => {
+          return {
+            type: 'datatable',
+            meta: {
+              type: 'essql',
+            },
+            ...resp,
+          };
+        });
+    },
+  };
+}