From 7d55ce3ce304c759097aa6fb215e209fa967544c Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Tue, 16 Mar 2021 14:45:12 -0500 Subject: [PATCH 01/18] Add essql search strategy, new escount temporary function, and new essql temporary function --- .../functions/browser/escountOther.ts | 94 +++++++++++++++ .../functions/browser/essqlOther.ts | 75 ++++++++++++ .../functions/browser/index.ts | 12 +- .../canvas/i18n/functions/function_help.ts | 2 + .../canvas/public/lib/run_interpreter.ts | 8 +- x-pack/plugins/canvas/public/plugin.tsx | 3 +- .../canvas/public/services/context.tsx | 2 + .../plugins/canvas/public/services/index.ts | 5 + .../plugins/canvas/public/services/search.ts | 24 ++++ .../canvas/public/services/stubs/index.ts | 2 + .../canvas/public/services/stubs/search.ts | 14 +++ .../canvas/server/lib/es_sql_strategy.ts | 108 ++++++++++++++++++ x-pack/plugins/canvas/server/plugin.ts | 17 ++- x-pack/plugins/canvas/types/index.ts | 1 + x-pack/plugins/canvas/types/strategy.ts | 29 +++++ 15 files changed, 390 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escountOther.ts create mode 100644 x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essqlOther.ts create mode 100644 x-pack/plugins/canvas/public/services/search.ts create mode 100644 x-pack/plugins/canvas/public/services/stubs/search.ts create mode 100644 x-pack/plugins/canvas/server/lib/es_sql_strategy.ts create mode 100644 x-pack/plugins/canvas/types/strategy.ts diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escountOther.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escountOther.ts new file mode 100644 index 0000000000000..b39dbcd0a7b40 --- /dev/null +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escountOther.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ExpressionFunctionDefinition, + ExpressionValueFilter, +} from 'src/plugins/expressions/common'; +/* eslint-disable */ +// @ts-expect-error untyped local +import { buildESRequest } from '../../../server/lib/build_es_request'; + +import { searchService } from '../../../public/services'; +/* eslint-enable */ +import { getFunctionHelp } from '../../../i18n'; + +interface Arguments { + index: string | null; + query: string; +} + +export function escountOther(): ExpressionFunctionDefinition< + 'escountOther', + ExpressionValueFilter, + Arguments, + any +> { + const { help, args: argHelp } = getFunctionHelp().escount; + + return { + name: 'escountOther', + type: 'number', + context: { + types: ['filter'], + }, + help, + args: { + query: { + types: ['string'], + aliases: ['_', 'q'], + help: argHelp.query, + default: '"-_index:.kibana"', + }, + index: { + types: ['string'], + default: '_all', + help: argHelp.index, + }, + }, + fn: (input, args, handlers) => { + input.and = input.and.concat([ + { + type: 'filter', + filterType: 'luceneQueryString', + query: args.query, + and: [], + }, + ]); + + const esRequest = buildESRequest( + { + index: args.index, + body: { + track_total_hits: true, + size: 0, + query: { + bool: { + must: [{ match_all: {} }], + }, + }, + }, + }, + input + ); + + const search = searchService.getService().search; + const req = { + params: { + ...esRequest, + }, + }; + + return search + .search(req) + .toPromise() + .then((resp: any) => { + return resp.rawResponse.hits.total; + }); + }, + }; +} diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essqlOther.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essqlOther.ts new file mode 100644 index 0000000000000..77455498e61c5 --- /dev/null +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essqlOther.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ExpressionFunctionDefinition, + ExpressionValueFilter, +} from 'src/plugins/expressions/common'; +import { searchService } from '../../../public/services'; +import { getFunctionHelp } from '../../../i18n'; + +interface Arguments { + query: string; + count: number; + timezone: string; +} + +export function essqlOther(): ExpressionFunctionDefinition< + 'essqlOther', + ExpressionValueFilter, + Arguments, + any +> { + const { help, args: argHelp } = getFunctionHelp().essql; + + return { + name: 'essqlOther', + type: 'datatable', + context: { + types: ['filter'], + }, + help, + args: { + query: { + aliases: ['_', 'q'], + types: ['string'], + help: argHelp.query, + }, + count: { + types: ['number'], + help: argHelp.count, + default: 1000, + }, + timezone: { + aliases: ['tz'], + types: ['string'], + default: 'UTC', + help: argHelp.timezone, + }, + }, + fn: (input, args, handlers) => { + const search = searchService.getService().search; + const req = { + ...args, + filter: input.and, + }; + + return search + .search(req, { strategy: 'essql' }) + .toPromise() + .then((resp: any) => { + return { + type: 'datatable', + meta: { + type: 'essql', + }, + ...resp, + }; + }); + }, + }; +} diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts index 6e7c43135f414..58f5f119a5164 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts @@ -10,5 +10,15 @@ import { functions as externalFunctions } from '../external'; import { location } from './location'; import { markdown } from './markdown'; import { urlparam } from './urlparam'; +import { escountOther } from './escountOther'; +import { essqlOther } from './essqlOther'; -export const functions = [location, markdown, urlparam, ...commonFunctions, ...externalFunctions]; +export const functions = [ + location, + markdown, + urlparam, + escountOther, + essqlOther, + ...commonFunctions, + ...externalFunctions, +]; diff --git a/x-pack/plugins/canvas/i18n/functions/function_help.ts b/x-pack/plugins/canvas/i18n/functions/function_help.ts index 512ebc4ff8c93..5b09d837df2f2 100644 --- a/x-pack/plugins/canvas/i18n/functions/function_help.ts +++ b/x-pack/plugins/canvas/i18n/functions/function_help.ts @@ -189,8 +189,10 @@ export const getFunctionHelp = (): FunctionHelpDict => ({ dropdownControl, eq, escount, + escountOther: escount, esdocs, essql, + essqlOther: essql, exactly, filterrows, filters, diff --git a/x-pack/plugins/canvas/public/lib/run_interpreter.ts b/x-pack/plugins/canvas/public/lib/run_interpreter.ts index eb9be96c5367b..a6aac78e04af6 100644 --- a/x-pack/plugins/canvas/public/lib/run_interpreter.ts +++ b/x-pack/plugins/canvas/public/lib/run_interpreter.ts @@ -7,7 +7,7 @@ import { fromExpression, getType } from '@kbn/interpreter/common'; import { ExpressionValue, ExpressionAstExpression } from 'src/plugins/expressions/public'; -import { notifyService, expressionsService } from '../services'; +import { notifyService, expressionsService, searchService } from '../services'; interface Options { castToRender?: boolean; @@ -20,7 +20,8 @@ export async function interpretAst( ast: ExpressionAstExpression, variables: Record ): Promise { - const context = { variables }; + const search = searchService.getService().search; + const context = { variables, search }; return await expressionsService.getService().execute(ast, null, context).getData(); } @@ -40,7 +41,8 @@ export async function runInterpreter( variables: Record, options: Options = {} ): Promise { - const context = { variables }; + const search = searchService.getService().search; + const context = { variables, search }; try { const renderable = await expressionsService.getService().execute(ast, input, context).getData(); diff --git a/x-pack/plugins/canvas/public/plugin.tsx b/x-pack/plugins/canvas/public/plugin.tsx index 6871c8d98b8f5..dc8f22b9b12da 100644 --- a/x-pack/plugins/canvas/public/plugin.tsx +++ b/x-pack/plugins/canvas/public/plugin.tsx @@ -21,7 +21,7 @@ import { getSessionStorage } from './lib/storage'; import { SESSIONSTORAGE_LASTPATH } from '../common/lib/constants'; import { featureCatalogueEntry } from './feature_catalogue_entry'; import { ExpressionsSetup, ExpressionsStart } from '../../../../src/plugins/expressions/public'; -import { DataPublicPluginSetup } from '../../../../src/plugins/data/public'; +import { DataPublicPluginSetup, DataPublicPluginStart } from '../../../../src/plugins/data/public'; import { UiActionsStart } from '../../../../src/plugins/ui_actions/public'; import { EmbeddableStart } from '../../../../src/plugins/embeddable/public'; import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public'; @@ -51,6 +51,7 @@ export interface CanvasStartDeps { inspector: InspectorStart; uiActions: UiActionsStart; charts: ChartsPluginStart; + data: DataPublicPluginStart; } /** diff --git a/x-pack/plugins/canvas/public/services/context.tsx b/x-pack/plugins/canvas/public/services/context.tsx index 6e74b5ac98621..56f7fe9fcccab 100644 --- a/x-pack/plugins/canvas/public/services/context.tsx +++ b/x-pack/plugins/canvas/public/services/context.tsx @@ -25,6 +25,7 @@ const defaultContextValue = { notify: {}, platform: {}, navLink: {}, + search: {}, }; const context = createContext(defaultContextValue as CanvasServices); @@ -53,6 +54,7 @@ export const ServicesProvider: FC<{ notify: specifiedProviders.notify.getService(), platform: specifiedProviders.platform.getService(), navLink: specifiedProviders.navLink.getService(), + search: specifiedProviders.search.getService(), }; return {children}; }; diff --git a/x-pack/plugins/canvas/public/services/index.ts b/x-pack/plugins/canvas/public/services/index.ts index 7452352fc0ef4..e3d11541f8f60 100644 --- a/x-pack/plugins/canvas/public/services/index.ts +++ b/x-pack/plugins/canvas/public/services/index.ts @@ -13,8 +13,10 @@ import { platformServiceFactory } from './platform'; import { navLinkServiceFactory } from './nav_link'; import { embeddablesServiceFactory } from './embeddables'; import { expressionsServiceFactory } from './expressions'; +import { searchServiceFactory } from './search'; export { NotifyService } from './notify'; +export { SearchService } from './search'; export { PlatformService } from './platform'; export { NavLinkService } from './nav_link'; export { EmbeddablesService } from './embeddables'; @@ -78,6 +80,7 @@ export const services = { notify: new CanvasServiceProvider(notifyServiceFactory), platform: new CanvasServiceProvider(platformServiceFactory), navLink: new CanvasServiceProvider(navLinkServiceFactory), + search: new CanvasServiceProvider(searchServiceFactory), }; export type CanvasServiceProviders = typeof services; @@ -88,6 +91,7 @@ export interface CanvasServices { notify: ServiceFromProvider; platform: ServiceFromProvider; navLink: ServiceFromProvider; + search: ServiceFromProvider; } export const startServices = async ( @@ -114,4 +118,5 @@ export const { platform: platformService, navLink: navLinkService, expressions: expressionsService, + search: searchService, } = services; diff --git a/x-pack/plugins/canvas/public/services/search.ts b/x-pack/plugins/canvas/public/services/search.ts new file mode 100644 index 0000000000000..0fe5c89c77096 --- /dev/null +++ b/x-pack/plugins/canvas/public/services/search.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataPublicPluginStart } from 'src/plugins/data/public'; +import { CanvasServiceFactory } from '.'; + +export interface SearchService { + search: DataPublicPluginStart['search']; +} + +export const searchServiceFactory: CanvasServiceFactory = ( + setup, + start, + canvasSetup, + canvasStart +) => { + return { + search: canvasStart.data.search, + }; +}; diff --git a/x-pack/plugins/canvas/public/services/stubs/index.ts b/x-pack/plugins/canvas/public/services/stubs/index.ts index 2565445af2db2..0e9ffbe87fe84 100644 --- a/x-pack/plugins/canvas/public/services/stubs/index.ts +++ b/x-pack/plugins/canvas/public/services/stubs/index.ts @@ -11,6 +11,7 @@ import { expressionsService } from './expressions'; import { navLinkService } from './nav_link'; import { notifyService } from './notify'; import { platformService } from './platform'; +import { searchService } from './search'; export const stubs: CanvasServices = { embeddables: embeddablesService, @@ -18,6 +19,7 @@ export const stubs: CanvasServices = { navLink: navLinkService, notify: notifyService, platform: platformService, + search: searchService, }; export const startServices = async (providedServices: Partial = {}) => { diff --git a/x-pack/plugins/canvas/public/services/stubs/search.ts b/x-pack/plugins/canvas/public/services/stubs/search.ts new file mode 100644 index 0000000000000..c7cae2b57243f --- /dev/null +++ b/x-pack/plugins/canvas/public/services/stubs/search.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchService } from '../platform'; + +const noop = (..._args: any[]): any => {}; + +export const searchService: SearchService = { + search: noop, +}; diff --git a/x-pack/plugins/canvas/server/lib/es_sql_strategy.ts b/x-pack/plugins/canvas/server/lib/es_sql_strategy.ts new file mode 100644 index 0000000000000..3d4d891ac3581 --- /dev/null +++ b/x-pack/plugins/canvas/server/lib/es_sql_strategy.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { from } from 'rxjs'; +import { map, zipObject } from 'lodash'; + +import { ISearchStrategy, PluginStart } from 'src/plugins/data/server'; + +import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../types'; + +import { buildBoolArray } from './build_bool_array'; +import { sanitizeName } from './sanitize_name'; +import { normalizeType } from './normalize_type'; +interface CursorResponse { + cursor?: string; + rows: string[][]; +} + +type QueryResponse = CursorResponse & { + columns: Array<{ + name: string; + type: string; + }>; +}; + +export const ESSqlSearchStrategyProvider = ( + data: PluginStart +): ISearchStrategy => { + return { + search: (request, options, { esClient }) => { + const { count, query, filter, timezone } = request; + + const searchUntilEnd = async () => { + let response = await esClient.asCurrentUser.transport.request({ + path: '/_sql?format=json', + method: 'POST', + body: { + query, + time_zone: timezone, + fetch_size: count, + client_id: 'canvas', + filter: { + bool: { + must: [{ match_all: {} }, ...buildBoolArray(filter)], + }, + }, + }, + }); + let body = response.body as QueryResponse; + + const columns = body.columns.map(({ name, type }) => { + return { + id: sanitizeName(name), + name: sanitizeName(name), + meta: { type: normalizeType(type) }, + }; + }); + const columnNames = map(columns, 'name'); + let rows = body.rows.map((row) => zipObject(columnNames, row)); + + // If we still have rows to retrieve, continue requesting data + // using the cursor until we have everything + while (rows.length < count && body.cursor !== undefined) { + response = await esClient.asCurrentUser.transport.request({ + path: '/_sql?format=json', + method: 'POST', + body: { + cursor: body.cursor, + }, + }); + + body = response.body as QueryResponse; + + rows = [...rows, ...body.rows.map((row) => zipObject(columnNames, row))]; + } + + // If we used a cursor, clean it up + if (body.cursor !== undefined) { + await esClient.asCurrentUser.transport.request({ + path: '/_sql/close', + method: 'POST', + body: { + cursor: body.cursor, + }, + }); + } + + return { + columns, + rows, + rawResponse: response, + }; + }; + + return from(searchUntilEnd()); + }, + + cancel: async (id, options, deps) => { + // if (es.cancel) { + // await es.cancel(id, options, deps); + // } + }, + }; +}; diff --git a/x-pack/plugins/canvas/server/plugin.ts b/x-pack/plugins/canvas/server/plugin.ts index 345f6099009fc..390e1a5cfaf5c 100644 --- a/x-pack/plugins/canvas/server/plugin.ts +++ b/x-pack/plugins/canvas/server/plugin.ts @@ -6,6 +6,10 @@ */ import { CoreSetup, PluginInitializerContext, Plugin, Logger, CoreStart } from 'src/core/server'; +import { + PluginSetup as DataPluginSetup, + PluginStart as DataPluginStart, +} from 'src/plugins/data/server'; import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; import { BfetchServerSetup } from 'src/plugins/bfetch/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; @@ -18,22 +22,28 @@ import { loadSampleData } from './sample_data'; import { setupInterpreter } from './setup_interpreter'; import { customElementType, workpadType, workpadTemplateType } from './saved_objects'; import { initializeTemplates } from './templates'; +import { ESSqlSearchStrategyProvider } from './lib/es_sql_strategy'; interface PluginsSetup { expressions: ExpressionsServerSetup; features: FeaturesPluginSetup; home: HomeServerPluginSetup; bfetch: BfetchServerSetup; + data: DataPluginSetup; usageCollection?: UsageCollectionSetup; } +interface PluginsStart { + data: DataPluginStart; +} + export class CanvasPlugin implements Plugin { private readonly logger: Logger; constructor(public readonly initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); } - public setup(coreSetup: CoreSetup, plugins: PluginsSetup) { + public setup(coreSetup: CoreSetup, plugins: PluginsSetup) { coreSetup.savedObjects.registerType(customElementType); coreSetup.savedObjects.registerType(workpadType); coreSetup.savedObjects.registerType(workpadTemplateType); @@ -87,6 +97,11 @@ export class CanvasPlugin implements Plugin { registerCanvasUsageCollector(plugins.usageCollection, globalConfig.kibana.index); setupInterpreter(plugins.expressions); + + coreSetup.getStartServices().then(([_, depsStart]) => { + const strategy = ESSqlSearchStrategyProvider(depsStart.data); + plugins.data.search.registerSearchStrategy('essql', strategy); + }); } public start(coreStart: CoreStart) { diff --git a/x-pack/plugins/canvas/types/index.ts b/x-pack/plugins/canvas/types/index.ts index 80314cab06258..09ae1510be6da 100644 --- a/x-pack/plugins/canvas/types/index.ts +++ b/x-pack/plugins/canvas/types/index.ts @@ -14,5 +14,6 @@ export * from './functions'; export * from './renderers'; export * from './shortcuts'; export * from './state'; +export * from './strategy'; export * from './style'; export * from './telemetry'; diff --git a/x-pack/plugins/canvas/types/strategy.ts b/x-pack/plugins/canvas/types/strategy.ts new file mode 100644 index 0000000000000..386fd3d5d6afd --- /dev/null +++ b/x-pack/plugins/canvas/types/strategy.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IKibanaSearchRequest } from 'src/plugins/data/common'; +import { ExpressionValueFilter } from '.'; + +export interface EssqlSearchStrategyRequest extends IKibanaSearchRequest { + count: number; + query: string; + timezone?: string; + filter: ExpressionValueFilter[]; +} + +export interface EssqlSearchStrategyResponse { + columns: Array<{ + id: string; + name: string; + meta: { + type: string; + }; + }>; + rows: any; + + rawResponse: any; +} From 66789acba5855a15f68b4712e75379e89522c691 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Wed, 17 Mar 2021 13:09:24 -0500 Subject: [PATCH 02/18] Move old es* functions to legacy, add esdocs to use search strategy, add parameter arg for essql --- .../functions/{server => browser}/escount.ts | 20 ++- .../functions/{server => browser}/esdocs.ts | 32 +++-- .../browser/{essqlOther.ts => essql.ts} | 23 +++- .../functions/browser/index.ts | 10 +- .../escountLegacy.ts} | 28 ++--- .../functions/server/esdocsLegacy.ts | 114 ++++++++++++++++++ .../server/{essql.ts => essqlLegacy.ts} | 8 +- .../functions/server/index.ts | 8 +- .../canvas/i18n/functions/dict/escount.ts | 2 +- .../canvas/i18n/functions/dict/esdocs.ts | 2 +- .../canvas/i18n/functions/dict/essql.ts | 8 +- .../canvas/i18n/functions/function_help.ts | 5 +- .../canvas/server/lib/es_sql_strategy.ts | 3 +- x-pack/plugins/canvas/types/strategy.ts | 1 + 14 files changed, 208 insertions(+), 56 deletions(-) rename x-pack/plugins/canvas/canvas_plugin_src/functions/{server => browser}/escount.ts (80%) rename x-pack/plugins/canvas/canvas_plugin_src/functions/{server => browser}/esdocs.ts (76%) rename x-pack/plugins/canvas/canvas_plugin_src/functions/browser/{essqlOther.ts => essql.ts} (69%) rename x-pack/plugins/canvas/canvas_plugin_src/functions/{browser/escountOther.ts => server/escountLegacy.ts} (72%) create mode 100644 x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocsLegacy.ts rename x-pack/plugins/canvas/canvas_plugin_src/functions/server/{essql.ts => essqlLegacy.ts} (88%) diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts similarity index 80% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts rename to x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts index 206e47413ae56..aeb40e57fa361 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts @@ -12,6 +12,8 @@ import { /* eslint-disable */ // @ts-expect-error untyped local import { buildESRequest } from '../../../server/lib/build_es_request'; + +import { searchService } from '../../../public/services'; /* eslint-enable */ import { getFunctionHelp } from '../../../i18n'; @@ -62,6 +64,8 @@ export function escount(): ExpressionFunctionDefinition< { index: args.index, body: { + track_total_hits: true, + size: 0, query: { bool: { must: [{ match_all: {} }], @@ -72,9 +76,19 @@ export function escount(): ExpressionFunctionDefinition< input ); - return ((handlers as any) as { elasticsearchClient: any }) - .elasticsearchClient('count', esRequest) - .then((resp: { count: number }) => resp.count); + const search = searchService.getService().search; + const req = { + params: { + ...esRequest, + }, + }; + + return search + .search(req) + .toPromise() + .then((resp: any) => { + return resp.rawResponse.hits.total; + }); }, }; } diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocs.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts similarity index 76% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocs.ts rename to x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts index 992bceaa3390c..e6e882c2e7135 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocs.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts @@ -6,11 +6,12 @@ */ import squel from 'squel'; -import { ExpressionFunctionDefinition } from 'src/plugins/expressions'; -/* eslint-disable */ -import { queryEsSQL } from '../../../server/lib/query_es_sql'; -/* eslint-enable */ -import { ExpressionValueFilter } from '../../../types'; +import { + ExpressionFunctionDefinition, + ExpressionValueFilter, +} from 'src/plugins/expressions/common'; +import { searchService } from '../../../public/services'; +import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types'; import { getFunctionHelp } from '../../../i18n'; interface Arguments { @@ -69,7 +70,7 @@ export function esdocs(): ExpressionFunctionDefinition< help: argHelp.sort, }, }, - fn: (input, args, context) => { + fn: (input, args, handlers) => { const { count, index, fields, sort } = args; input.and = input.and.concat([ @@ -104,11 +105,26 @@ export function esdocs(): ExpressionFunctionDefinition< } } - return queryEsSQL(((context as any) as { elasticsearchClient: any }).elasticsearchClient, { + const search = searchService.getService().search; + + const req = { count, query: query.toString(), filter: input.and, - }); + }; + + return search + .search(req, { strategy: 'essql' }) + .toPromise() + .then((resp: EssqlSearchStrategyResponse) => { + return { + type: 'datatable', + meta: { + type: 'essql', + }, + ...resp, + }; + }); }, }; } diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essqlOther.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts similarity index 69% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essqlOther.ts rename to x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts index 77455498e61c5..9f2a180c0a8ae 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essqlOther.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts @@ -10,16 +10,18 @@ import { ExpressionValueFilter, } from 'src/plugins/expressions/common'; import { searchService } from '../../../public/services'; +import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types'; import { getFunctionHelp } from '../../../i18n'; interface Arguments { query: string; + parameter: Array; count: number; timezone: string; } -export function essqlOther(): ExpressionFunctionDefinition< - 'essqlOther', +export function essql(): ExpressionFunctionDefinition< + 'essql', ExpressionValueFilter, Arguments, any @@ -27,7 +29,7 @@ export function essqlOther(): ExpressionFunctionDefinition< const { help, args: argHelp } = getFunctionHelp().essql; return { - name: 'essqlOther', + name: 'essql', type: 'datatable', context: { types: ['filter'], @@ -39,6 +41,13 @@ export function essqlOther(): ExpressionFunctionDefinition< types: ['string'], help: argHelp.query, }, + parameter: { + aliases: ['param'], + types: ['string', 'number', 'boolean'], + default: '""', + multi: true, + help: argHelp.parameter, + }, count: { types: ['number'], help: argHelp.count, @@ -53,15 +62,17 @@ export function essqlOther(): ExpressionFunctionDefinition< }, fn: (input, args, handlers) => { const search = searchService.getService().search; + const { parameter, ...restOfArgs } = args; const req = { - ...args, + ...restOfArgs, + params: parameter, filter: input.and, }; return search - .search(req, { strategy: 'essql' }) + .search(req, { strategy: 'essql' }) .toPromise() - .then((resp: any) => { + .then((resp: EssqlSearchStrategyResponse) => { return { type: 'datatable', meta: { diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts index 58f5f119a5164..2cfdebafb70df 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts @@ -10,15 +10,17 @@ import { functions as externalFunctions } from '../external'; import { location } from './location'; import { markdown } from './markdown'; import { urlparam } from './urlparam'; -import { escountOther } from './escountOther'; -import { essqlOther } from './essqlOther'; +import { escount } from './escount'; +import { esdocs } from './esdocs'; +import { essql } from './essql'; export const functions = [ location, markdown, urlparam, - escountOther, - essqlOther, + escount, + esdocs, + essql, ...commonFunctions, ...externalFunctions, ]; diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escountOther.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts similarity index 72% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escountOther.ts rename to x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts index b39dbcd0a7b40..af0e46ae98a89 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escountOther.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts @@ -12,8 +12,6 @@ import { /* eslint-disable */ // @ts-expect-error untyped local import { buildESRequest } from '../../../server/lib/build_es_request'; - -import { searchService } from '../../../public/services'; /* eslint-enable */ import { getFunctionHelp } from '../../../i18n'; @@ -22,16 +20,16 @@ interface Arguments { query: string; } -export function escountOther(): ExpressionFunctionDefinition< - 'escountOther', +export function escountLegacy(): ExpressionFunctionDefinition< + 'escountLegacy', ExpressionValueFilter, Arguments, any > { - const { help, args: argHelp } = getFunctionHelp().escount; + const { help, args: argHelp } = getFunctionHelp().escountLegacy; return { - name: 'escountOther', + name: 'escountLegacy', type: 'number', context: { types: ['filter'], @@ -64,8 +62,6 @@ export function escountOther(): ExpressionFunctionDefinition< { index: args.index, body: { - track_total_hits: true, - size: 0, query: { bool: { must: [{ match_all: {} }], @@ -76,19 +72,9 @@ export function escountOther(): ExpressionFunctionDefinition< input ); - const search = searchService.getService().search; - const req = { - params: { - ...esRequest, - }, - }; - - return search - .search(req) - .toPromise() - .then((resp: any) => { - return resp.rawResponse.hits.total; - }); + return ((handlers as any) as { elasticsearchClient: any }) + .elasticsearchClient('count', esRequest) + .then((resp: { count: number }) => resp.count); }, }; } diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocsLegacy.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocsLegacy.ts new file mode 100644 index 0000000000000..260c05b4b79ed --- /dev/null +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocsLegacy.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import squel from 'squel'; +import { ExpressionFunctionDefinition } from 'src/plugins/expressions'; +/* eslint-disable */ +import { queryEsSQL } from '../../../server/lib/query_es_sql'; +/* eslint-enable */ +import { ExpressionValueFilter } from '../../../types'; +import { getFunctionHelp } from '../../../i18n'; + +interface Arguments { + index: string; + query: string; + sort: string; + fields: string; + metaFields: string; + count: number; +} + +export function esdocsLegacy(): ExpressionFunctionDefinition< + 'esdocsLegacy', + ExpressionValueFilter, + Arguments, + any +> { + const { help, args: argHelp } = getFunctionHelp().esdocsLegacy; + + return { + name: 'esdocsLegacy', + type: 'datatable', + context: { + types: ['filter'], + }, + help, + args: { + query: { + types: ['string'], + aliases: ['_', 'q'], + help: argHelp.query, + default: '-_index:.kibana', + }, + count: { + types: ['number'], + default: 1000, + help: argHelp.count, + }, + fields: { + help: argHelp.fields, + types: ['string'], + }, + index: { + types: ['string'], + default: '_all', + help: argHelp.index, + }, + // TODO: This arg isn't being used in the function. + // We need to restore this functionality or remove it as an arg. + metaFields: { + help: argHelp.metaFields, + types: ['string'], + }, + sort: { + types: ['string'], + help: argHelp.sort, + }, + }, + fn: (input, args, context) => { + const { count, index, fields, sort } = args; + + input.and = input.and.concat([ + { + type: 'filter', + filterType: 'luceneQueryString', + query: args.query, + and: [], + }, + ]); + + let query = squel.select({ + autoQuoteTableNames: true, + autoQuoteFieldNames: true, + autoQuoteAliasNames: true, + nameQuoteCharacter: '"', + }); + + if (index) { + query.from(index); + } + + if (fields) { + const allFields = fields.split(',').map((field) => field.trim()); + allFields.forEach((field) => (query = query.field(field))); + } + + if (sort) { + const [sortField, sortOrder] = sort.split(',').map((str) => str.trim()); + if (sortField) { + query.order(`"${sortField}"`, sortOrder === 'asc'); + } + } + + return queryEsSQL(((context as any) as { elasticsearchClient: any }).elasticsearchClient, { + count, + query: query.toString(), + filter: input.and, + }); + }, + }; +} diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/essql.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/essqlLegacy.ts similarity index 88% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/server/essql.ts rename to x-pack/plugins/canvas/canvas_plugin_src/functions/server/essqlLegacy.ts index d14a8c0bec762..0eb1e80edaa9c 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/essql.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/essqlLegacy.ts @@ -18,16 +18,16 @@ interface Arguments { timezone: string; } -export function essql(): ExpressionFunctionDefinition< - 'essql', +export function essqlLegacy(): ExpressionFunctionDefinition< + 'essqlLegacy', ExpressionValueFilter, Arguments, any > { - const { help, args: argHelp } = getFunctionHelp().essql; + const { help, args: argHelp } = getFunctionHelp().essqlLegacy; return { - name: 'essql', + name: 'essqlLegacy', type: 'datatable', context: { types: ['filter'], diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/index.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/index.ts index e9731448f65b7..9de9f98c48d82 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/index.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/index.ts @@ -6,9 +6,9 @@ */ import { demodata } from './demodata'; -import { escount } from './escount'; -import { esdocs } from './esdocs'; +import { escountLegacy } from './escountLegacy'; +import { esdocsLegacy } from './esdocsLegacy'; import { pointseries } from './pointseries'; -import { essql } from './essql'; +import { essqlLegacy } from './essqlLegacy'; -export const functions = [demodata, esdocs, escount, essql, pointseries]; +export const functions = [demodata, esdocsLegacy, escountLegacy, essqlLegacy, pointseries]; diff --git a/x-pack/plugins/canvas/i18n/functions/dict/escount.ts b/x-pack/plugins/canvas/i18n/functions/dict/escount.ts index d88156ba32ce4..af1337360ba6d 100644 --- a/x-pack/plugins/canvas/i18n/functions/dict/escount.ts +++ b/x-pack/plugins/canvas/i18n/functions/dict/escount.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { escount } from '../../../canvas_plugin_src/functions/server/escount'; +import { escount } from '../../../canvas_plugin_src/functions/browser/escount'; import { FunctionHelp } from '../function_help'; import { FunctionFactory } from '../../../types'; import { ELASTICSEARCH, LUCENE } from '../../constants'; diff --git a/x-pack/plugins/canvas/i18n/functions/dict/esdocs.ts b/x-pack/plugins/canvas/i18n/functions/dict/esdocs.ts index b78425de144ef..6be5acdb8bc90 100644 --- a/x-pack/plugins/canvas/i18n/functions/dict/esdocs.ts +++ b/x-pack/plugins/canvas/i18n/functions/dict/esdocs.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { esdocs } from '../../../canvas_plugin_src/functions/server/esdocs'; +import { esdocs } from '../../../canvas_plugin_src/functions/browser/esdocs'; import { FunctionHelp } from '../function_help'; import { FunctionFactory } from '../../../types'; import { ELASTICSEARCH, LUCENE } from '../../constants'; diff --git a/x-pack/plugins/canvas/i18n/functions/dict/essql.ts b/x-pack/plugins/canvas/i18n/functions/dict/essql.ts index cfe848455dc3f..700649bc3a778 100644 --- a/x-pack/plugins/canvas/i18n/functions/dict/essql.ts +++ b/x-pack/plugins/canvas/i18n/functions/dict/essql.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { essql } from '../../../canvas_plugin_src/functions/server/essql'; +import { essql } from '../../../canvas_plugin_src/functions/browser/essql'; import { FunctionHelp } from '../function_help'; import { FunctionFactory } from '../../../types'; import { ELASTICSEARCH, SQL, ISO8601, UTC } from '../../constants'; @@ -27,6 +27,12 @@ export const help: FunctionHelp> = { SQL, }, }), + parameter: i18n.translate('xpaxpack.canvas.functions.essql.args.parameterHelpText', { + defaultMessage: 'A parameter to be passed to the {SQL} query.', + values: { + SQL, + }, + }), count: i18n.translate('xpack.canvas.functions.essql.args.countHelpText', { defaultMessage: 'The number of documents to retrieve. For better performance, use a smaller data set.', diff --git a/x-pack/plugins/canvas/i18n/functions/function_help.ts b/x-pack/plugins/canvas/i18n/functions/function_help.ts index 5b09d837df2f2..3bf40ffa27d55 100644 --- a/x-pack/plugins/canvas/i18n/functions/function_help.ts +++ b/x-pack/plugins/canvas/i18n/functions/function_help.ts @@ -189,10 +189,11 @@ export const getFunctionHelp = (): FunctionHelpDict => ({ dropdownControl, eq, escount, - escountOther: escount, + escountLegacy: escount, esdocs, + esdocsLegacy: esdocs, essql, - essqlOther: essql, + essqlLegacy: essql, exactly, filterrows, filters, diff --git a/x-pack/plugins/canvas/server/lib/es_sql_strategy.ts b/x-pack/plugins/canvas/server/lib/es_sql_strategy.ts index 3d4d891ac3581..e962c4797e765 100644 --- a/x-pack/plugins/canvas/server/lib/es_sql_strategy.ts +++ b/x-pack/plugins/canvas/server/lib/es_sql_strategy.ts @@ -32,7 +32,7 @@ export const ESSqlSearchStrategyProvider = ( ): ISearchStrategy => { return { search: (request, options, { esClient }) => { - const { count, query, filter, timezone } = request; + const { count, query, filter, timezone, params } = request; const searchUntilEnd = async () => { let response = await esClient.asCurrentUser.transport.request({ @@ -40,6 +40,7 @@ export const ESSqlSearchStrategyProvider = ( method: 'POST', body: { query, + params, time_zone: timezone, fetch_size: count, client_id: 'canvas', diff --git a/x-pack/plugins/canvas/types/strategy.ts b/x-pack/plugins/canvas/types/strategy.ts index 386fd3d5d6afd..0361e32982ce8 100644 --- a/x-pack/plugins/canvas/types/strategy.ts +++ b/x-pack/plugins/canvas/types/strategy.ts @@ -11,6 +11,7 @@ import { ExpressionValueFilter } from '.'; export interface EssqlSearchStrategyRequest extends IKibanaSearchRequest { count: number; query: string; + params?: Array; timezone?: string; filter: ExpressionValueFilter[]; } From dfb102848ffa2add5d0bc09c8a462778307acd7a Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Wed, 17 Mar 2021 13:18:10 -0500 Subject: [PATCH 03/18] Clean up --- x-pack/plugins/canvas/i18n/functions/dict/essql.ts | 2 +- x-pack/plugins/canvas/public/lib/run_interpreter.ts | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/canvas/i18n/functions/dict/essql.ts b/x-pack/plugins/canvas/i18n/functions/dict/essql.ts index 700649bc3a778..6304db945fc3f 100644 --- a/x-pack/plugins/canvas/i18n/functions/dict/essql.ts +++ b/x-pack/plugins/canvas/i18n/functions/dict/essql.ts @@ -27,7 +27,7 @@ export const help: FunctionHelp> = { SQL, }, }), - parameter: i18n.translate('xpaxpack.canvas.functions.essql.args.parameterHelpText', { + parameter: i18n.translate('xpack.canvas.functions.essql.args.parameterHelpText', { defaultMessage: 'A parameter to be passed to the {SQL} query.', values: { SQL, diff --git a/x-pack/plugins/canvas/public/lib/run_interpreter.ts b/x-pack/plugins/canvas/public/lib/run_interpreter.ts index a6aac78e04af6..eb9be96c5367b 100644 --- a/x-pack/plugins/canvas/public/lib/run_interpreter.ts +++ b/x-pack/plugins/canvas/public/lib/run_interpreter.ts @@ -7,7 +7,7 @@ import { fromExpression, getType } from '@kbn/interpreter/common'; import { ExpressionValue, ExpressionAstExpression } from 'src/plugins/expressions/public'; -import { notifyService, expressionsService, searchService } from '../services'; +import { notifyService, expressionsService } from '../services'; interface Options { castToRender?: boolean; @@ -20,8 +20,7 @@ export async function interpretAst( ast: ExpressionAstExpression, variables: Record ): Promise { - const search = searchService.getService().search; - const context = { variables, search }; + const context = { variables }; return await expressionsService.getService().execute(ast, null, context).getData(); } @@ -41,8 +40,7 @@ export async function runInterpreter( variables: Record, options: Options = {} ): Promise { - const search = searchService.getService().search; - const context = { variables, search }; + const context = { variables }; try { const renderable = await expressionsService.getService().execute(ast, input, context).getData(); From ae56576b23b4617c0e321c62f16c3812420e867d Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Thu, 18 Mar 2021 10:52:26 -0500 Subject: [PATCH 04/18] cleanup --- .../canvas/canvas_plugin_src/functions/browser/esdocs.ts | 6 ++++-- x-pack/plugins/canvas/public/services/stubs/search.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts index e6e882c2e7135..6aaae08d94efe 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts @@ -5,7 +5,6 @@ * 2.0. */ -import squel from 'squel'; import { ExpressionFunctionDefinition, ExpressionValueFilter, @@ -70,7 +69,7 @@ export function esdocs(): ExpressionFunctionDefinition< help: argHelp.sort, }, }, - fn: (input, args, handlers) => { + fn: async (input, args, handlers) => { const { count, index, fields, sort } = args; input.and = input.and.concat([ @@ -82,6 +81,9 @@ export function esdocs(): ExpressionFunctionDefinition< }, ]); + // Load ad-hoc to avoid adding to the page load bundle size + const squel = await import('squel'); + let query = squel.select({ autoQuoteTableNames: true, autoQuoteFieldNames: true, diff --git a/x-pack/plugins/canvas/public/services/stubs/search.ts b/x-pack/plugins/canvas/public/services/stubs/search.ts index c7cae2b57243f..32e31e7b8ab17 100644 --- a/x-pack/plugins/canvas/public/services/stubs/search.ts +++ b/x-pack/plugins/canvas/public/services/stubs/search.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SearchService } from '../platform'; +import { SearchService } from '../search'; const noop = (..._args: any[]): any => {}; From 412cd502854e9d87e9e3c9facd4fed732589f401 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Thu, 18 Mar 2021 14:18:03 -0500 Subject: [PATCH 05/18] cleanup --- .../server/lib/{es_sql_strategy.ts => essql_strategy.ts} | 8 +------- x-pack/plugins/canvas/server/plugin.ts | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) rename x-pack/plugins/canvas/server/lib/{es_sql_strategy.ts => essql_strategy.ts} (94%) diff --git a/x-pack/plugins/canvas/server/lib/es_sql_strategy.ts b/x-pack/plugins/canvas/server/lib/essql_strategy.ts similarity index 94% rename from x-pack/plugins/canvas/server/lib/es_sql_strategy.ts rename to x-pack/plugins/canvas/server/lib/essql_strategy.ts index e962c4797e765..c2fe6c7b62b6e 100644 --- a/x-pack/plugins/canvas/server/lib/es_sql_strategy.ts +++ b/x-pack/plugins/canvas/server/lib/essql_strategy.ts @@ -27,7 +27,7 @@ type QueryResponse = CursorResponse & { }>; }; -export const ESSqlSearchStrategyProvider = ( +export const essqlSearchStrategyProvider = ( data: PluginStart ): ISearchStrategy => { return { @@ -99,11 +99,5 @@ export const ESSqlSearchStrategyProvider = ( return from(searchUntilEnd()); }, - - cancel: async (id, options, deps) => { - // if (es.cancel) { - // await es.cancel(id, options, deps); - // } - }, }; }; diff --git a/x-pack/plugins/canvas/server/plugin.ts b/x-pack/plugins/canvas/server/plugin.ts index 390e1a5cfaf5c..7712fdb495bc6 100644 --- a/x-pack/plugins/canvas/server/plugin.ts +++ b/x-pack/plugins/canvas/server/plugin.ts @@ -22,7 +22,7 @@ import { loadSampleData } from './sample_data'; import { setupInterpreter } from './setup_interpreter'; import { customElementType, workpadType, workpadTemplateType } from './saved_objects'; import { initializeTemplates } from './templates'; -import { ESSqlSearchStrategyProvider } from './lib/es_sql_strategy'; +import { essqlSearchStrategyProvider } from './lib/essql_strategy'; interface PluginsSetup { expressions: ExpressionsServerSetup; @@ -99,7 +99,7 @@ export class CanvasPlugin implements Plugin { setupInterpreter(plugins.expressions); coreSetup.getStartServices().then(([_, depsStart]) => { - const strategy = ESSqlSearchStrategyProvider(depsStart.data); + const strategy = essqlSearchStrategyProvider(depsStart.data); plugins.data.search.registerSearchStrategy('essql', strategy); }); } From dc4a147b01e767baba518071eeeef4cb9f5976d4 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Mon, 22 Mar 2021 16:26:12 -0500 Subject: [PATCH 06/18] Move request builder files to common --- .../canvas/canvas_plugin_src/functions/browser/escount.ts | 6 +++--- .../{server/lib => common/lib/request}/build_bool_array.ts | 2 +- .../{server/lib => common/lib/request}/build_es_request.js | 0 .../{server/lib => common/lib/request}/essql_strategy.ts | 2 +- .../canvas/{server/lib => common/lib/request}/filters.ts | 2 +- .../{server/lib => common/lib/request}/format_response.js | 0 .../{server/lib => common/lib/request}/get_es_filter.ts | 2 +- .../{server/lib => common/lib/request}/normalize_type.ts | 0 .../{server/lib => common/lib/request}/sanitize_name.ts | 0 x-pack/plugins/canvas/server/lib/query_es_sql.ts | 6 +++--- 10 files changed, 10 insertions(+), 10 deletions(-) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/build_bool_array.ts (92%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/build_es_request.js (100%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/essql_strategy.ts (99%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/filters.ts (98%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/format_response.js (100%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/get_es_filter.ts (93%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/normalize_type.ts (100%) rename x-pack/plugins/canvas/{server/lib => common/lib/request}/sanitize_name.ts (100%) diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts index aeb40e57fa361..97aa934280414 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts @@ -9,12 +9,12 @@ import { ExpressionFunctionDefinition, ExpressionValueFilter, } from 'src/plugins/expressions/common'; -/* eslint-disable */ + // @ts-expect-error untyped local -import { buildESRequest } from '../../../server/lib/build_es_request'; +import { buildESRequest } from '../../../common/lib/request/build_es_request'; import { searchService } from '../../../public/services'; -/* eslint-enable */ + import { getFunctionHelp } from '../../../i18n'; interface Arguments { diff --git a/x-pack/plugins/canvas/server/lib/build_bool_array.ts b/x-pack/plugins/canvas/common/lib/request/build_bool_array.ts similarity index 92% rename from x-pack/plugins/canvas/server/lib/build_bool_array.ts rename to x-pack/plugins/canvas/common/lib/request/build_bool_array.ts index 826449ca6ad39..c0d630b4c405e 100644 --- a/x-pack/plugins/canvas/server/lib/build_bool_array.ts +++ b/x-pack/plugins/canvas/common/lib/request/build_bool_array.ts @@ -6,7 +6,7 @@ */ import { getESFilter } from './get_es_filter'; -import { ExpressionValueFilter } from '../../types'; +import { ExpressionValueFilter } from '../../../types'; const compact = (arr: T[]) => (Array.isArray(arr) ? arr.filter((val) => Boolean(val)) : []); diff --git a/x-pack/plugins/canvas/server/lib/build_es_request.js b/x-pack/plugins/canvas/common/lib/request/build_es_request.js similarity index 100% rename from x-pack/plugins/canvas/server/lib/build_es_request.js rename to x-pack/plugins/canvas/common/lib/request/build_es_request.js diff --git a/x-pack/plugins/canvas/server/lib/essql_strategy.ts b/x-pack/plugins/canvas/common/lib/request/essql_strategy.ts similarity index 99% rename from x-pack/plugins/canvas/server/lib/essql_strategy.ts rename to x-pack/plugins/canvas/common/lib/request/essql_strategy.ts index c2fe6c7b62b6e..80ea6e85c8027 100644 --- a/x-pack/plugins/canvas/server/lib/essql_strategy.ts +++ b/x-pack/plugins/canvas/common/lib/request/essql_strategy.ts @@ -10,7 +10,7 @@ import { map, zipObject } from 'lodash'; import { ISearchStrategy, PluginStart } from 'src/plugins/data/server'; -import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../types'; +import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types'; import { buildBoolArray } from './build_bool_array'; import { sanitizeName } from './sanitize_name'; diff --git a/x-pack/plugins/canvas/server/lib/filters.ts b/x-pack/plugins/canvas/common/lib/request/filters.ts similarity index 98% rename from x-pack/plugins/canvas/server/lib/filters.ts rename to x-pack/plugins/canvas/common/lib/request/filters.ts index 8c6b485c4ccae..f1465fe48bdcf 100644 --- a/x-pack/plugins/canvas/server/lib/filters.ts +++ b/x-pack/plugins/canvas/common/lib/request/filters.ts @@ -11,7 +11,7 @@ import { CanvasTimeFilter, CanvasLuceneFilter, CanvasExactlyFilter, -} from '../../types'; +} from '../../../types'; /* TODO: This could be pluggable diff --git a/x-pack/plugins/canvas/server/lib/format_response.js b/x-pack/plugins/canvas/common/lib/request/format_response.js similarity index 100% rename from x-pack/plugins/canvas/server/lib/format_response.js rename to x-pack/plugins/canvas/common/lib/request/format_response.js diff --git a/x-pack/plugins/canvas/server/lib/get_es_filter.ts b/x-pack/plugins/canvas/common/lib/request/get_es_filter.ts similarity index 93% rename from x-pack/plugins/canvas/server/lib/get_es_filter.ts rename to x-pack/plugins/canvas/common/lib/request/get_es_filter.ts index 85335a4be06d7..353a793adcd17 100644 --- a/x-pack/plugins/canvas/server/lib/get_es_filter.ts +++ b/x-pack/plugins/canvas/common/lib/request/get_es_filter.ts @@ -12,7 +12,7 @@ */ import { filters } from './filters'; -import { ExpressionValueFilter } from '../../types'; +import { ExpressionValueFilter } from '../../../types'; export function getESFilter(filter: ExpressionValueFilter) { if (!filter.filterType || !filters[filter.filterType]) { diff --git a/x-pack/plugins/canvas/server/lib/normalize_type.ts b/x-pack/plugins/canvas/common/lib/request/normalize_type.ts similarity index 100% rename from x-pack/plugins/canvas/server/lib/normalize_type.ts rename to x-pack/plugins/canvas/common/lib/request/normalize_type.ts diff --git a/x-pack/plugins/canvas/server/lib/sanitize_name.ts b/x-pack/plugins/canvas/common/lib/request/sanitize_name.ts similarity index 100% rename from x-pack/plugins/canvas/server/lib/sanitize_name.ts rename to x-pack/plugins/canvas/common/lib/request/sanitize_name.ts diff --git a/x-pack/plugins/canvas/server/lib/query_es_sql.ts b/x-pack/plugins/canvas/server/lib/query_es_sql.ts index a315657dadd2b..2c4416094914d 100644 --- a/x-pack/plugins/canvas/server/lib/query_es_sql.ts +++ b/x-pack/plugins/canvas/server/lib/query_es_sql.ts @@ -6,9 +6,9 @@ */ import { map, zipObject } from 'lodash'; -import { buildBoolArray } from './build_bool_array'; -import { sanitizeName } from './sanitize_name'; -import { normalizeType } from './normalize_type'; +import { buildBoolArray } from '../../common/lib/request/build_bool_array'; +import { sanitizeName } from '../../common/lib/request/sanitize_name'; +import { normalizeType } from '../../common/lib/request/normalize_type'; import { LegacyAPICaller } from '../../../../../src/core/server'; import { ExpressionValueFilter } from '../../types'; From 39168273ee60d47c72415a5a7553c682c3c4004b Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Tue, 23 Mar 2021 10:36:26 -0500 Subject: [PATCH 07/18] cleanup --- .../canvas/canvas_plugin_src/functions/browser/esdocs.ts | 4 ++++ .../canvas_plugin_src/functions/server/escountLegacy.ts | 5 ++--- .../{common/lib/request => server/lib}/essql_strategy.ts | 8 ++++---- .../plugins/canvas/server/routes/es_fields/es_fields.ts | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) rename x-pack/plugins/canvas/{common/lib/request => server/lib}/essql_strategy.ts (92%) diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts index 6aaae08d94efe..12b6fbd6ce6a8 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts @@ -9,6 +9,10 @@ import { ExpressionFunctionDefinition, ExpressionValueFilter, } from 'src/plugins/expressions/common'; + +// @ts-expect-error untyped local +import { buildESRequest } from '../../../common/lib/request/build_es_request'; + import { searchService } from '../../../public/services'; import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types'; import { getFunctionHelp } from '../../../i18n'; diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts index af0e46ae98a89..06ef7277c103e 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts @@ -9,10 +9,9 @@ import { ExpressionFunctionDefinition, ExpressionValueFilter, } from 'src/plugins/expressions/common'; -/* eslint-disable */ // @ts-expect-error untyped local -import { buildESRequest } from '../../../server/lib/build_es_request'; -/* eslint-enable */ +import { buildESRequest } from '../../../common/lib/request/build_es_request'; + import { getFunctionHelp } from '../../../i18n'; interface Arguments { diff --git a/x-pack/plugins/canvas/common/lib/request/essql_strategy.ts b/x-pack/plugins/canvas/server/lib/essql_strategy.ts similarity index 92% rename from x-pack/plugins/canvas/common/lib/request/essql_strategy.ts rename to x-pack/plugins/canvas/server/lib/essql_strategy.ts index 80ea6e85c8027..f4b2fc891ca06 100644 --- a/x-pack/plugins/canvas/common/lib/request/essql_strategy.ts +++ b/x-pack/plugins/canvas/server/lib/essql_strategy.ts @@ -10,11 +10,11 @@ import { map, zipObject } from 'lodash'; import { ISearchStrategy, PluginStart } from 'src/plugins/data/server'; -import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types'; +import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../types'; -import { buildBoolArray } from './build_bool_array'; -import { sanitizeName } from './sanitize_name'; -import { normalizeType } from './normalize_type'; +import { buildBoolArray } from '../../common/lib/request/build_bool_array'; +import { sanitizeName } from '../../common/lib/request/sanitize_name'; +import { normalizeType } from '../../common/lib/request/normalize_type'; interface CursorResponse { cursor?: string; rows: string[][]; diff --git a/x-pack/plugins/canvas/server/routes/es_fields/es_fields.ts b/x-pack/plugins/canvas/server/routes/es_fields/es_fields.ts index 7ceace8b84dbd..20a4775847c91 100644 --- a/x-pack/plugins/canvas/server/routes/es_fields/es_fields.ts +++ b/x-pack/plugins/canvas/server/routes/es_fields/es_fields.ts @@ -9,7 +9,7 @@ import { mapValues, keys } from 'lodash'; import { schema } from '@kbn/config-schema'; import { API_ROUTE } from '../../../common/lib'; import { catchErrorHandler } from '../catch_error_handler'; -import { normalizeType } from '../../lib/normalize_type'; +import { normalizeType } from '../../../common/lib/request/normalize_type'; import { RouteInitializerDeps } from '..'; const ESFieldsRequestSchema = schema.object({ From 26925d37b47a8a6588dd1b3c596db9615aa5d477 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Fri, 26 Mar 2021 10:05:52 -0500 Subject: [PATCH 08/18] add comment --- .../canvas/canvas_plugin_src/functions/browser/esdocs.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts index 12b6fbd6ce6a8..4bd61edfcc95d 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts @@ -119,6 +119,8 @@ export function esdocs(): ExpressionFunctionDefinition< filter: input.and, }; + // We're requesting the data using the ESSQL strategy because + // the SQL routes return type information with the result set return search .search(req, { strategy: 'essql' }) .toPromise() From 10239a3c6608d8df16d4033794e4235fac444816 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Mon, 29 Mar 2021 14:21:08 -0500 Subject: [PATCH 09/18] PR Feedback --- .../functions/browser/esdocs.ts | 5 ++++- .../functions/browser/essql.ts | 5 ++++- x-pack/plugins/canvas/common/lib/constants.ts | 1 + .../plugins/canvas/server/lib/essql_strategy.ts | 17 +++++++---------- x-pack/plugins/canvas/server/plugin.ts | 3 ++- x-pack/plugins/canvas/types/strategy.ts | 1 - 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts index 4bd61edfcc95d..2b4dd9e283fb6 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts @@ -14,6 +14,7 @@ import { import { buildESRequest } from '../../../common/lib/request/build_es_request'; import { searchService } from '../../../public/services'; +import { ESSQL_SEARCH_STRATEGY } from '../../../common/lib/constants'; import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types'; import { getFunctionHelp } from '../../../i18n'; @@ -122,7 +123,9 @@ export function esdocs(): ExpressionFunctionDefinition< // We're requesting the data using the ESSQL strategy because // the SQL routes return type information with the result set return search - .search(req, { strategy: 'essql' }) + .search(req, { + strategy: ESSQL_SEARCH_STRATEGY, + }) .toPromise() .then((resp: EssqlSearchStrategyResponse) => { return { diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts index 9f2a180c0a8ae..f3802cd018554 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts @@ -10,6 +10,7 @@ import { ExpressionValueFilter, } from 'src/plugins/expressions/common'; import { searchService } from '../../../public/services'; +import { ESSQL_SEARCH_STRATEGY } from '../../../common/lib/constants'; import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types'; import { getFunctionHelp } from '../../../i18n'; @@ -70,7 +71,9 @@ export function essql(): ExpressionFunctionDefinition< }; return search - .search(req, { strategy: 'essql' }) + .search(req, { + strategy: ESSQL_SEARCH_STRATEGY, + }) .toPromise() .then((resp: EssqlSearchStrategyResponse) => { return { diff --git a/x-pack/plugins/canvas/common/lib/constants.ts b/x-pack/plugins/canvas/common/lib/constants.ts index 697389fe2ce7c..2b916033ce557 100644 --- a/x-pack/plugins/canvas/common/lib/constants.ts +++ b/x-pack/plugins/canvas/common/lib/constants.ts @@ -44,3 +44,4 @@ export const API_ROUTE_SHAREABLE_RUNTIME_DOWNLOAD = `/public/canvas/${SHAREABLE_ export const CANVAS_EMBEDDABLE_CLASSNAME = `canvasEmbeddable`; export const CONTEXT_MENU_TOP_BORDER_CLASSNAME = 'canvasContextMenu--topBorder'; export const API_ROUTE_FUNCTIONS = `${API_ROUTE}/fns`; +export const ESSQL_SEARCH_STRATEGY = 'essql'; diff --git a/x-pack/plugins/canvas/server/lib/essql_strategy.ts b/x-pack/plugins/canvas/server/lib/essql_strategy.ts index f4b2fc891ca06..516f2452510a4 100644 --- a/x-pack/plugins/canvas/server/lib/essql_strategy.ts +++ b/x-pack/plugins/canvas/server/lib/essql_strategy.ts @@ -35,9 +35,8 @@ export const essqlSearchStrategyProvider = ( const { count, query, filter, timezone, params } = request; const searchUntilEnd = async () => { - let response = await esClient.asCurrentUser.transport.request({ - path: '/_sql?format=json', - method: 'POST', + let response = await esClient.asCurrentUser.sql.query({ + format: 'json', body: { query, params, @@ -51,7 +50,8 @@ export const essqlSearchStrategyProvider = ( }, }, }); - let body = response.body as QueryResponse; + + let body = response.body; const columns = body.columns.map(({ name, type }) => { return { @@ -66,9 +66,8 @@ export const essqlSearchStrategyProvider = ( // If we still have rows to retrieve, continue requesting data // using the cursor until we have everything while (rows.length < count && body.cursor !== undefined) { - response = await esClient.asCurrentUser.transport.request({ - path: '/_sql?format=json', - method: 'POST', + response = await esClient.asCurrentUser.sql.query({ + format: 'json', body: { cursor: body.cursor, }, @@ -81,9 +80,7 @@ export const essqlSearchStrategyProvider = ( // If we used a cursor, clean it up if (body.cursor !== undefined) { - await esClient.asCurrentUser.transport.request({ - path: '/_sql/close', - method: 'POST', + await esClient.asCurrentUser.sql.clearCursor({ body: { cursor: body.cursor, }, diff --git a/x-pack/plugins/canvas/server/plugin.ts b/x-pack/plugins/canvas/server/plugin.ts index 7712fdb495bc6..4e84541b7adaf 100644 --- a/x-pack/plugins/canvas/server/plugin.ts +++ b/x-pack/plugins/canvas/server/plugin.ts @@ -15,6 +15,7 @@ import { BfetchServerSetup } from 'src/plugins/bfetch/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { HomeServerPluginSetup } from 'src/plugins/home/server'; import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server'; +import { ESSQL_SEARCH_STRATEGY } from '../common/lib/constants'; import { PluginSetupContract as FeaturesPluginSetup } from '../../features/server'; import { initRoutes } from './routes'; import { registerCanvasUsageCollector } from './collectors'; @@ -100,7 +101,7 @@ export class CanvasPlugin implements Plugin { coreSetup.getStartServices().then(([_, depsStart]) => { const strategy = essqlSearchStrategyProvider(depsStart.data); - plugins.data.search.registerSearchStrategy('essql', strategy); + plugins.data.search.registerSearchStrategy(ESSQL_SEARCH_STRATEGY, strategy); }); } diff --git a/x-pack/plugins/canvas/types/strategy.ts b/x-pack/plugins/canvas/types/strategy.ts index 0361e32982ce8..6e8e896d16a16 100644 --- a/x-pack/plugins/canvas/types/strategy.ts +++ b/x-pack/plugins/canvas/types/strategy.ts @@ -7,7 +7,6 @@ import { IKibanaSearchRequest } from 'src/plugins/data/common'; import { ExpressionValueFilter } from '.'; - export interface EssqlSearchStrategyRequest extends IKibanaSearchRequest { count: number; query: string; From 10c23ba8f54a49f32f2e9c5299b662496983efb5 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Mon, 29 Mar 2021 14:37:04 -0500 Subject: [PATCH 10/18] Removing old types --- .../canvas/server/lib/essql_strategy.ts | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/canvas/server/lib/essql_strategy.ts b/x-pack/plugins/canvas/server/lib/essql_strategy.ts index 516f2452510a4..fcd5019e6dc31 100644 --- a/x-pack/plugins/canvas/server/lib/essql_strategy.ts +++ b/x-pack/plugins/canvas/server/lib/essql_strategy.ts @@ -15,17 +15,6 @@ import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../t import { buildBoolArray } from '../../common/lib/request/build_bool_array'; import { sanitizeName } from '../../common/lib/request/sanitize_name'; import { normalizeType } from '../../common/lib/request/normalize_type'; -interface CursorResponse { - cursor?: string; - rows: string[][]; -} - -type QueryResponse = CursorResponse & { - columns: Array<{ - name: string; - type: string; - }>; -}; export const essqlSearchStrategyProvider = ( data: PluginStart @@ -35,10 +24,11 @@ export const essqlSearchStrategyProvider = ( const { count, query, filter, timezone, params } = request; const searchUntilEnd = async () => { - let response = await esClient.asCurrentUser.sql.query({ + let response = await esClient.asCurrentUser.sql.query({ format: 'json', body: { query, + // @ts-expect-error `params` missing from `QuerySqlRequest` type params, time_zone: timezone, fetch_size: count, @@ -53,7 +43,7 @@ export const essqlSearchStrategyProvider = ( let body = response.body; - const columns = body.columns.map(({ name, type }) => { + const columns = body.columns!.map(({ name, type }) => { return { id: sanitizeName(name), name: sanitizeName(name), @@ -73,7 +63,7 @@ export const essqlSearchStrategyProvider = ( }, }); - body = response.body as QueryResponse; + body = response.body; rows = [...rows, ...body.rows.map((row) => zipObject(columnNames, row))]; } From 5975bafa6b7c6860bdd864015bf3bd58a5157e94 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Wed, 31 Mar 2021 16:43:28 -0500 Subject: [PATCH 11/18] update type --- x-pack/plugins/canvas/types/strategy.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/canvas/types/strategy.ts b/x-pack/plugins/canvas/types/strategy.ts index 6e8e896d16a16..1c94059f0c9ca 100644 --- a/x-pack/plugins/canvas/types/strategy.ts +++ b/x-pack/plugins/canvas/types/strategy.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { ApiResponse } from '@elastic/elasticsearch/lib/Transport'; +import { QuerySqlResponse } from '@elastic/elasticsearch/api/types'; import { IKibanaSearchRequest } from 'src/plugins/data/common'; import { ExpressionValueFilter } from '.'; export interface EssqlSearchStrategyRequest extends IKibanaSearchRequest { @@ -23,7 +25,7 @@ export interface EssqlSearchStrategyResponse { type: string; }; }>; - rows: any; + rows: any[]; - rawResponse: any; + rawResponse: ApiResponse; } From 9b59c33b7025869cce74b333f54f8778980733db Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Tue, 13 Apr 2021 15:14:05 -0500 Subject: [PATCH 12/18] Add data.search to labs and fix error messages --- src/plugins/presentation_util/common/labs.ts | 18 ++- .../functions/browser/essql.ts | 16 ++- .../server/{escountLegacy.ts => escount.ts} | 8 +- .../server/{esdocsLegacy.ts => esdocs.ts} | 8 +- .../server/{essqlLegacy.ts => essql.ts} | 8 +- .../functions/server/index.ts | 8 +- .../canvas/public/services/expressions.ts | 14 ++- .../canvas/public/services/stubs/search.ts | 5 +- .../canvas/server/lib/essql_strategy.ts | 103 +++++++++--------- 9 files changed, 116 insertions(+), 72 deletions(-) rename x-pack/plugins/canvas/canvas_plugin_src/functions/server/{escountLegacy.ts => escount.ts} (90%) rename x-pack/plugins/canvas/canvas_plugin_src/functions/server/{esdocsLegacy.ts => esdocs.ts} (93%) rename x-pack/plugins/canvas/canvas_plugin_src/functions/server/{essqlLegacy.ts => essql.ts} (88%) diff --git a/src/plugins/presentation_util/common/labs.ts b/src/plugins/presentation_util/common/labs.ts index 65e42996ae910..985ba5cc42aa2 100644 --- a/src/plugins/presentation_util/common/labs.ts +++ b/src/plugins/presentation_util/common/labs.ts @@ -9,8 +9,9 @@ import { i18n } from '@kbn/i18n'; export const UNIFIED_TOOLBAR = 'labs:presentation:unifiedToolbar'; +export const USE_DATA_SERVICE = 'labs:canvas:useDataService'; -export const projectIDs = [UNIFIED_TOOLBAR] as const; +export const projectIDs = [UNIFIED_TOOLBAR, USE_DATA_SERVICE] as const; export const environmentNames = ['kibana', 'browser', 'session'] as const; export const solutionNames = ['canvas', 'dashboard', 'presentation'] as const; @@ -31,6 +32,21 @@ export const projects: { [ID in ProjectID]: ProjectConfig & { id: ID } } = { }), solutions: ['dashboard', 'canvas'], }, + [USE_DATA_SERVICE]: { + id: USE_DATA_SERVICE, + isActive: true, + environments: ['kibana', 'browser', 'session'], + name: i18n.translate('presentationUtil.experiments.enableUseDataServiceExperimentName', { + defaultMessage: 'Use data service', + }), + description: i18n.translate( + 'presentationUtil.experiments.enableUseDataServiceExperimentDescription', + { + defaultMessage: 'An experiment of using the new data.search service for Canvas datasources', + } + ), + solutions: ['canvas'], + }, }; export type ProjectID = typeof projectIDs[number]; diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts index f3802cd018554..1339c93032ea9 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts @@ -45,7 +45,6 @@ export function essql(): ExpressionFunctionDefinition< parameter: { aliases: ['param'], types: ['string', 'number', 'boolean'], - default: '""', multi: true, help: argHelp.parameter, }, @@ -83,6 +82,21 @@ export function essql(): ExpressionFunctionDefinition< }, ...resp, }; + }) + .catch((e) => { + let message = `Unexpected error from Elasticsearch: ${e.message}`; + if (e.err) { + const { type, reason } = e.err.attributes; + if (type === 'parsing_exception') { + message = `Couldn't parse Elasticsearch SQL query. You may need to add double quotes to names containing special characters. Check your query and try again. Error: ${reason}`; + } else { + message = `Unexpected error from Elasticsearch: ${type} - ${reason}`; + } + } + + // Re-write the error message before surfacing it up + e.message = message; + throw e; }); }, }; diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts similarity index 90% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts rename to x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts index 06ef7277c103e..95f5ef446a470 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escountLegacy.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts @@ -19,16 +19,16 @@ interface Arguments { query: string; } -export function escountLegacy(): ExpressionFunctionDefinition< - 'escountLegacy', +export function escount(): ExpressionFunctionDefinition< + 'escount', ExpressionValueFilter, Arguments, any > { - const { help, args: argHelp } = getFunctionHelp().escountLegacy; + const { help, args: argHelp } = getFunctionHelp().escount; return { - name: 'escountLegacy', + name: 'escount', type: 'number', context: { types: ['filter'], diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocsLegacy.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocs.ts similarity index 93% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocsLegacy.ts rename to x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocs.ts index 260c05b4b79ed..992bceaa3390c 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocsLegacy.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocs.ts @@ -22,16 +22,16 @@ interface Arguments { count: number; } -export function esdocsLegacy(): ExpressionFunctionDefinition< - 'esdocsLegacy', +export function esdocs(): ExpressionFunctionDefinition< + 'esdocs', ExpressionValueFilter, Arguments, any > { - const { help, args: argHelp } = getFunctionHelp().esdocsLegacy; + const { help, args: argHelp } = getFunctionHelp().esdocs; return { - name: 'esdocsLegacy', + name: 'esdocs', type: 'datatable', context: { types: ['filter'], diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/essqlLegacy.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/essql.ts similarity index 88% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/server/essqlLegacy.ts rename to x-pack/plugins/canvas/canvas_plugin_src/functions/server/essql.ts index 0eb1e80edaa9c..d14a8c0bec762 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/essqlLegacy.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/essql.ts @@ -18,16 +18,16 @@ interface Arguments { timezone: string; } -export function essqlLegacy(): ExpressionFunctionDefinition< - 'essqlLegacy', +export function essql(): ExpressionFunctionDefinition< + 'essql', ExpressionValueFilter, Arguments, any > { - const { help, args: argHelp } = getFunctionHelp().essqlLegacy; + const { help, args: argHelp } = getFunctionHelp().essql; return { - name: 'essqlLegacy', + name: 'essql', type: 'datatable', context: { types: ['filter'], diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/index.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/index.ts index 9de9f98c48d82..e9731448f65b7 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/index.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/index.ts @@ -6,9 +6,9 @@ */ import { demodata } from './demodata'; -import { escountLegacy } from './escountLegacy'; -import { esdocsLegacy } from './esdocsLegacy'; +import { escount } from './escount'; +import { esdocs } from './esdocs'; import { pointseries } from './pointseries'; -import { essqlLegacy } from './essqlLegacy'; +import { essql } from './essql'; -export const functions = [demodata, esdocsLegacy, escountLegacy, essqlLegacy, pointseries]; +export const functions = [demodata, esdocs, escount, essql, pointseries]; diff --git a/x-pack/plugins/canvas/public/services/expressions.ts b/x-pack/plugins/canvas/public/services/expressions.ts index 131919e1eefea..fd733862c4b67 100644 --- a/x-pack/plugins/canvas/public/services/expressions.ts +++ b/x-pack/plugins/canvas/public/services/expressions.ts @@ -24,6 +24,11 @@ export const expressionsServiceFactory: CanvasServiceFactory const loadServerFunctionWrappers = async () => { if (!cached) { cached = (async () => { + const labService = startPlugins.presentationUtil.labsService; + const useDataSearchProject = labService.getProject('labs:canvas:useDataService'); + const hasDataSearch = useDataSearchProject.status.isEnabled; + const dataSearchFns = ['essql', 'esdocs', 'escount']; + const serverFunctionList = await coreSetup.http.get(API_ROUTE_FUNCTIONS); const batchedFunction = bfetch.batchedFunction({ url: API_ROUTE_FUNCTIONS }); const { serialize } = serializeProvider(expressions.getTypes()); @@ -32,9 +37,16 @@ export const expressionsServiceFactory: CanvasServiceFactory // function that matches its definition, but which simply // calls the server-side function endpoint. Object.keys(serverFunctionList).forEach((functionName) => { - if (expressions.getFunction(functionName)) { + // Allow function to be overwritten if we want to use + // the server-hosted essql, esdocs, and escount functions + if (dataSearchFns.includes(functionName)) { + if (hasDataSearch && expressions.getFunction(functionName)) { + return; + } + } else if (expressions.getFunction(functionName)) { return; } + const fn = () => ({ ...serverFunctionList[functionName], fn: (input: any, args: any) => { diff --git a/x-pack/plugins/canvas/public/services/stubs/search.ts b/x-pack/plugins/canvas/public/services/stubs/search.ts index 32e31e7b8ab17..a4558a93e38a4 100644 --- a/x-pack/plugins/canvas/public/services/stubs/search.ts +++ b/x-pack/plugins/canvas/public/services/stubs/search.ts @@ -4,11 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { SearchService } from '../search'; - const noop = (..._args: any[]): any => {}; -export const searchService: SearchService = { +export const searchService: any = { search: noop, }; diff --git a/x-pack/plugins/canvas/server/lib/essql_strategy.ts b/x-pack/plugins/canvas/server/lib/essql_strategy.ts index fcd5019e6dc31..2dffe8b07529c 100644 --- a/x-pack/plugins/canvas/server/lib/essql_strategy.ts +++ b/x-pack/plugins/canvas/server/lib/essql_strategy.ts @@ -10,6 +10,7 @@ import { map, zipObject } from 'lodash'; import { ISearchStrategy, PluginStart } from 'src/plugins/data/server'; +import { getKbnServerError } from '../../../../../src/plugins/kibana_utils/server'; import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../types'; import { buildBoolArray } from '../../common/lib/request/build_bool_array'; @@ -24,64 +25,68 @@ export const essqlSearchStrategyProvider = ( const { count, query, filter, timezone, params } = request; const searchUntilEnd = async () => { - let response = await esClient.asCurrentUser.sql.query({ - format: 'json', - body: { - query, - // @ts-expect-error `params` missing from `QuerySqlRequest` type - params, - time_zone: timezone, - fetch_size: count, - client_id: 'canvas', - filter: { - bool: { - must: [{ match_all: {} }, ...buildBoolArray(filter)], + try { + let response = await esClient.asCurrentUser.sql.query({ + format: 'json', + body: { + query, + // @ts-expect-error `params` missing from `QuerySqlRequest` type + params, + time_zone: timezone, + fetch_size: count, + client_id: 'canvas', + filter: { + bool: { + must: [{ match_all: {} }, ...buildBoolArray(filter)], + }, }, }, - }, - }); - - let body = response.body; + }); - const columns = body.columns!.map(({ name, type }) => { - return { - id: sanitizeName(name), - name: sanitizeName(name), - meta: { type: normalizeType(type) }, - }; - }); - const columnNames = map(columns, 'name'); - let rows = body.rows.map((row) => zipObject(columnNames, row)); + let body = response.body; - // If we still have rows to retrieve, continue requesting data - // using the cursor until we have everything - while (rows.length < count && body.cursor !== undefined) { - response = await esClient.asCurrentUser.sql.query({ - format: 'json', - body: { - cursor: body.cursor, - }, + const columns = body.columns!.map(({ name, type }) => { + return { + id: sanitizeName(name), + name: sanitizeName(name), + meta: { type: normalizeType(type) }, + }; }); + const columnNames = map(columns, 'name'); + let rows = body.rows.map((row) => zipObject(columnNames, row)); + + // If we still have rows to retrieve, continue requesting data + // using the cursor until we have everything + while (rows.length < count && body.cursor !== undefined) { + response = await esClient.asCurrentUser.sql.query({ + format: 'json', + body: { + cursor: body.cursor, + }, + }); - body = response.body; + body = response.body; - rows = [...rows, ...body.rows.map((row) => zipObject(columnNames, row))]; - } + rows = [...rows, ...body.rows.map((row) => zipObject(columnNames, row))]; + } - // If we used a cursor, clean it up - if (body.cursor !== undefined) { - await esClient.asCurrentUser.sql.clearCursor({ - body: { - cursor: body.cursor, - }, - }); - } + // If we used a cursor, clean it up + if (body.cursor !== undefined) { + await esClient.asCurrentUser.sql.clearCursor({ + body: { + cursor: body.cursor, + }, + }); + } - return { - columns, - rows, - rawResponse: response, - }; + return { + columns, + rows, + rawResponse: response, + }; + } catch (e) { + throw getKbnServerError(e); + } }; return from(searchUntilEnd()); From 3f57cfca96469fbc10001c304c790d5b8c410fa4 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Tue, 13 Apr 2021 16:36:37 -0500 Subject: [PATCH 13/18] Fix function help type --- x-pack/plugins/canvas/i18n/functions/function_help.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugins/canvas/i18n/functions/function_help.ts b/x-pack/plugins/canvas/i18n/functions/function_help.ts index 3bf40ffa27d55..512ebc4ff8c93 100644 --- a/x-pack/plugins/canvas/i18n/functions/function_help.ts +++ b/x-pack/plugins/canvas/i18n/functions/function_help.ts @@ -189,11 +189,8 @@ export const getFunctionHelp = (): FunctionHelpDict => ({ dropdownControl, eq, escount, - escountLegacy: escount, esdocs, - esdocsLegacy: esdocs, essql, - essqlLegacy: essql, exactly, filterrows, filters, From a768b3b6aff7873b6012714a0677dc9a894e5f5f Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Wed, 14 Apr 2021 12:17:36 -0500 Subject: [PATCH 14/18] Add data service to usage collector types --- .../server/collectors/management/schema.ts | 4 ++++ .../server/collectors/management/types.ts | 1 + 2 files changed, 5 insertions(+) diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index fcdd00380755f..b5e77045df611 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -436,4 +436,8 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, + 'labs:canvas:useDataService': { + type: 'boolean', + _meta: { description: 'Non-default value of setting.' }, + }, }; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 613ada418c6e7..e5c71d81204af 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -119,4 +119,5 @@ export interface UsageStats { 'banners:textColor': string; 'banners:backgroundColor': string; 'labs:presentation:unifiedToolbar': boolean; + 'labs:canvas:useDataService': boolean; } From 06cd7bf87190938be2b3e3491902ed1498e5946b Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Wed, 14 Apr 2021 14:02:35 -0500 Subject: [PATCH 15/18] Update telemetry --- src/plugins/telemetry/schema/oss_plugins.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index d8bcf150ac167..f5245ceb12171 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -8108,6 +8108,12 @@ "description": "Non-default value of setting." } }, + "observability:enableInspectEsQueries": { + "type": "boolean", + "_meta": { + "description": "Non-default value of setting." + } + }, "banners:placement": { "type": "keyword", "_meta": { @@ -8132,13 +8138,13 @@ "description": "Non-default value of setting." } }, - "observability:enableInspectEsQueries": { + "labs:presentation:unifiedToolbar": { "type": "boolean", "_meta": { "description": "Non-default value of setting." } }, - "labs:presentation:unifiedToolbar": { + "labs:canvas:useDataService": { "type": "boolean", "_meta": { "description": "Non-default value of setting." From 1ee927766cdc51de93abbf3a34ec574db02f1b54 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Mon, 26 Apr 2021 11:06:50 -0500 Subject: [PATCH 16/18] remove unrelated telemetry change --- src/plugins/telemetry/schema/oss_plugins.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index eda58bb81c71b..2dab703a0296b 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -8331,12 +8331,6 @@ "description": "Non-default value of setting." } }, - "observability:enableAlertingExperience": { - "type": "boolean", - "_meta": { - "description": "Non-default value of setting." - } - }, "labs:presentation:unifiedToolbar": { "type": "boolean", "_meta": { From 2d948163a485074f32976416a76d0e146b2fb504 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Tue, 27 Apr 2021 15:15:32 -0500 Subject: [PATCH 17/18] Enable multi value leniency for SQL queries --- x-pack/plugins/canvas/server/lib/essql_strategy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/canvas/server/lib/essql_strategy.ts b/x-pack/plugins/canvas/server/lib/essql_strategy.ts index 2dffe8b07529c..795b4fedaaaab 100644 --- a/x-pack/plugins/canvas/server/lib/essql_strategy.ts +++ b/x-pack/plugins/canvas/server/lib/essql_strategy.ts @@ -32,6 +32,7 @@ export const essqlSearchStrategyProvider = ( query, // @ts-expect-error `params` missing from `QuerySqlRequest` type params, + field_multi_value_leniency: true, time_zone: timezone, fetch_size: count, client_id: 'canvas', From c1f5252f7edcb5a2b65a209b104d76dbcf0b1cf8 Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Tue, 27 Apr 2021 18:01:34 -0500 Subject: [PATCH 18/18] Display data service lab project --- src/plugins/presentation_util/common/labs.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/presentation_util/common/labs.ts b/src/plugins/presentation_util/common/labs.ts index ea093184ec68f..d551b733ecb8a 100644 --- a/src/plugins/presentation_util/common/labs.ts +++ b/src/plugins/presentation_util/common/labs.ts @@ -36,6 +36,7 @@ export const projects: { [ID in ProjectID]: ProjectConfig & { id: ID } } = { [USE_DATA_SERVICE]: { id: USE_DATA_SERVICE, isActive: true, + isDisplayed: true, environments: ['kibana', 'browser', 'session'], name: i18n.translate('presentationUtil.experiments.enableUseDataServiceExperimentName', { defaultMessage: 'Use data service',