[7.x] Expressions refactor (elastic#54342) (elastic#57366)

* Expressions refactor (elastic#54342)

* feat: 🎸 add UiComponent interface

* feat: 🎸 add adapter for react => ui components and back

* refactor: 💡 move registries to shared /common folder

* feat: 🎸 create expressions service state contaienr

* chore: 🤖 export some symbols

* feat: 🎸 add Executor class

* test: 💍 add simple integration test

* feat: 🎸 move registries into executor

* feat: 🎸 add initial implementation of Execution

* feat: 🎸 move Executor's state container into a signle file

* refactor: 💡 move createError() to /common folder

* feat: 🎸 use Executor in plugin definition

* refactor: 💡 rename handlers to FunctionHandlers

* feat: 🎸 improve function typings

* feat: 🎸 move types and func in sep folder, improve Execution

* refactor: 💡 cleanup expression_types folder

* refactor: 💡 improve typing names of expression types

* refactor: 💡 remove lodash from ExpressionType and improve types

* test: 💍 add ExpressionType tests

* refactor: 💡 remove function wrappers around types

* refactor: 💡 move functions to /common

* test: 💍 improve expression function tests

* feat: 🎸 create /parser folder

* refactor: 💡 move function types into /expression_functions dir

* refactor: 💡 improve parser setup

* refactor: 💡 fix export structure and move args into expr_func

* test: 💍 add ExpressionFunctionParameter tests

* fix: 🐛 fix executor types and imports

* refactor: 💡 move getByAlias to plugin, fix Execution types

* feat: 🎸 add function for argument parsing

* test: 💍 add Executor type tests

* test: 💍 add executor function and context tests

* test: 💍 check that Executor returns Execution

* test: 💍 add basic tests for Execution

* test: 💍 add basic test for execution of a chain of functions

* test: 💍 add "mult" function tot tests

* feat: 🎸 create separate expression_renderer folder

* feat: 🎸 use new executor in public plugin

* feat: 🎸 remove renderers from executor, add result to execution

* fix: 🐛 fix Kibana TypeScript errors

* test: 💍 add file to write integration tests for expr plugin

* refactor: 💡 move state_containers to /common

* refactor: 💡 move /parser to /ast and inline format() function

* refactor: 💡 remove remaining @kbn/interpreter imports

* feat: 🎸 better handling and typing for Executor context

* feat: 🎸 use Executor.run function in plugin

* fix: 🐛 fix TypeScript type errors

* test: 💍 move integration tests into one file

* feat: 🎸 create ExpressionsService

* chore: 🤖 clean up legacy code

* feat: 🎸 use ExpressionsService in /public

* refactor: 💡 move inspector adapters to /common

* feat: 🎸 improve execution

* feat: 🎸 add state to execution state and don't clone AST

* test: 💍 add tests for Execution object

* test: 💍 improve expression test helpers

* test: 💍 add Execution tests

* refactor: 💡 improve required argument checking

* fix: 🐛 fix Kibana TypeScript errors

* test: 💍 add ExpressionsService unit tests

* fix: 🐛 fix Expression plugin TypeScript types

* refactor: 💡 prefix React component with React*

* fix: 🐛 fix X-Pack TypeScript errors

* fix: 🐛 fix test TypeScript errors

* fix: 🐛 fix issues preventing loading

* feat: 🎸 remove getInitialContext() handler

* fix: 🐛 fix TypeScript errors

* chore: 🤖 remove uicomponent interface

* chore: 🤖 remove missing import

* fix: 🐛 correctly handle .query in "kibana" expression function

* refactor: 💡 call first arg in expression functions "input"

* fix: 🐛 do not free Execution state container

* test: 💍 fix tests after refactor

* test: 💍 fix more tests after refactor

* fix: 🐛 remove redundant export

* test: 💍 update intepreter_functional test shapshots

* fix: 🐛 relax "kibana" function throwing on missin gsearch ctx

* refactor: 💡 don't use ExpressionAST interface in Canvas

* docs: ✏️ improve ExpressionRenderer JSDocs

* refactor: 💡 rename context.types to inputTypes in internal fn

* refactor: 💡 replace context.types by unknown in ExprFuncDef

* refactor: 💡 improve expression function definitions in OSS

* fix: 🐛 correctly set name on metric_vis_fn

* refactor: 💡 improve Lens definitions of expression functions

* refactor: 💡 improve Canvas expression function definitions

* test: 💍 add createMockExecutionContext() helper

* refactor: 💡 add some type to events$ observable in expr handler

* feat: 🎸 add types to observables in data handler

* refactor: 💡 use inputTypes in canvas

* fix: 🐛 fix interpreter grammer generation script

* feat: 🎸 allow array in getByAlias

* test: 💍 simplify test function specs

* test: 💍 fix autocomplete tests

* fix: 🐛 use correct expression types and NP getFunctions() API

* refactor: 💡 use NP expressions to get renderer

* fix: 🐛 use context.types on server-side Canvas function defs

* refactor: 💡 use NP API to register Canvas renderers

* feat: 🎸 use NP API to get types

* style: 💄 minor formatting changes

* feat: 🎸 use NP API to get expression functions

* fix: 🐛 fix Canvas workpads

* test: 💍 add missing mock functions

* refactor: 💡 improve Lens func definition argument types

* fix: 🐛 fix Lens type error

* feat: 🎸 make lens datatable work again

* feat: 🎸 bootstrap ExpressionsService on server-side

* feat: 🎸 expose more registry related functions in contract

* feat: 🎸 add environment: server to server-side expressions

* docs: ✏️ add documentation

* test: 💍 add missing Jest mocks

* fix: 🐛 correct TypeScript type

* docs: ✏️ improve documentation

* fix: 🐛 make FunctionHelpDict type contain only Canvas functions

* fix: 🐛 fix merge conflict

* test: 💍 fix expression mocks

* fix: fix TypeScript disabled help type

Co-authored-by: Elastic Machine <[email protected]>

packages/kbn-interpreter/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "license": "Apache-2.0",
   "scripts": {
-    "interpreter:peg": "pegjs common/lib/grammar.peg",
+    "interpreter:peg": "pegjs src/common/lib/grammar.peg",
     "build": "node scripts/build",
     "kbn:bootstrap": "node scripts/build --dev",
     "kbn:watch": "node scripts/build --dev --watch"

packages/kbn-interpreter/src/common/lib/registry.js

@@ -31,9 +31,7 @@ export class Registry {
   }
 
   register(fn) {
-    if (typeof fn !== 'function') throw new Error(`Register requires an function`);
-
-    const obj = fn();
+    const obj = typeof fn === 'function' ? fn() : fn;
 
     if (typeof obj !== 'object' || !obj[this._prop]) {
       throw new Error(`Registered functions must return an object with a ${this._prop} property`);

packages/kbn-utility-types/README.md

@@ -24,3 +24,4 @@ type B = UnwrapPromise<A>; // string
 - `ShallowPromise<T>` &mdash; Same as `Promise` type, but it flat maps the wrapped type.
 - `UnwrapObservable<T>` &mdash; Returns wrapped type of an observable.
 - `UnwrapPromise<T>` &mdash; Returns wrapped type of a promise.
+- `UnwrapPromiseOrReturn<T>` &mdash; Returns wrapped type of a promise or the type itself, if it isn't a promise.

packages/kbn-utility-types/index.ts

@@ -35,6 +35,11 @@ export type ShallowPromise<T> = T extends Promise<infer U> ? Promise<U> : Promis
  */
 export type UnwrapPromise<T extends Promise<any>> = PromiseType<T>;
 
+/**
+ * Returns wrapped type of a promise, or returns type as is, if it is not a promise.
+ */
+export type UnwrapPromiseOrReturn<T> = T extends Promise<infer U> ? U : T;
+
 /**
  * Minimal interface for an object resembling an `Observable`.
  */

src/legacy/core_plugins/data/public/search/expressions/esaggs.ts

@@ -24,7 +24,7 @@ import { createFormat } from 'ui/visualize/loader/pipeline_helpers/utilities';
 import {
   KibanaContext,
   KibanaDatatable,
-  ExpressionFunction,
+  ExpressionFunctionDefinition,
   KibanaDatatableColumn,
 } from 'src/plugins/expressions/public';
 import {
@@ -66,7 +66,8 @@ export interface RequestHandlerParams {
 
 const name = 'esaggs';
 
-type Context = KibanaContext | null;
+type Input = KibanaContext | null;
+type Output = Promise<KibanaDatatable>;
 
 interface Arguments {
   index: string;
@@ -76,8 +77,6 @@ interface Arguments {
   aggConfigs: string;
 }
 
-type Return = Promise<KibanaDatatable>;
-
 const handleCourierRequest = async ({
   searchSource,
   aggs,
@@ -221,12 +220,10 @@ const handleCourierRequest = async ({
   return (searchSource as any).tabifiedResponse;
 };
 
-export const esaggs = (): ExpressionFunction<typeof name, Context, Arguments, Return> => ({
+export const esaggs = (): ExpressionFunctionDefinition<typeof name, Input, Arguments, Output> => ({
   name,
   type: 'kibana_datatable',
-  context: {
-    types: ['kibana_context', 'null'],
-  },
+  inputTypes: ['kibana_context', 'null'],
   help: i18n.translate('data.functions.esaggs.help', {
     defaultMessage: 'Run AggConfig aggregation',
   }),
@@ -256,7 +253,7 @@ export const esaggs = (): ExpressionFunction<typeof name, Context, Arguments, Re
       help: '',
     },
   },
-  async fn(context, args, { inspectorAdapters, abortSignal }) {
+  async fn(input, args, { inspectorAdapters, abortSignal }) {
     const indexPatterns = getIndexPatterns();
     const { filterManager } = getQueryService();
 
@@ -272,13 +269,13 @@ export const esaggs = (): ExpressionFunction<typeof name, Context, Arguments, Re
     const response = await handleCourierRequest({
       searchSource,
       aggs,
-      timeRange: get(context, 'timeRange', undefined),
-      query: get(context, 'query', undefined),
-      filters: get(context, 'filters', undefined),
+      timeRange: get(input, 'timeRange', undefined),
+      query: get(input, 'query', undefined),
+      filters: get(input, 'filters', undefined),
       forceFetch: true,
       metricsAtAllLevels: args.metricsAtAllLevels,
       partialRows: args.partialRows,
-      inspectorAdapters,
+      inspectorAdapters: inspectorAdapters as Adapters,
       filterManager,
       abortSignal: (abortSignal as unknown) as AbortSignal,
     });

src/legacy/core_plugins/input_control_vis/public/input_control_fn.test.ts

@@ -20,12 +20,12 @@
 import { createInputControlVisFn } from './input_control_fn';
 
 // eslint-disable-next-line
-import { functionWrapper } from '../../../../plugins/expressions/public/functions/tests/utils';
+import { functionWrapper } from '../../../../plugins/expressions/common/expression_functions/specs/tests/utils';
 
 jest.mock('./legacy_imports.ts');
 
 describe('interpreter/functions#input_control_vis', () => {
-  const fn = functionWrapper(createInputControlVisFn);
+  const fn = functionWrapper(createInputControlVisFn());
   const visConfig = {
     controls: [
       {

src/legacy/core_plugins/input_control_vis/public/input_control_fn.ts

@@ -20,15 +20,11 @@
 import { i18n } from '@kbn/i18n';
 
 import {
-  ExpressionFunction,
+  ExpressionFunctionDefinition,
   KibanaDatatable,
   Render,
 } from '../../../../plugins/expressions/public';
 
-const name = 'input_control_vis';
-
-type Context = KibanaDatatable;
-
 interface Arguments {
   visConfig: string;
 }
@@ -40,19 +36,15 @@ interface RenderValue {
   visConfig: VisParams;
 }
 
-type Return = Promise<Render<RenderValue>>;
-
-export const createInputControlVisFn = (): ExpressionFunction<
-  typeof name,
-  Context,
+export const createInputControlVisFn = (): ExpressionFunctionDefinition<
+  'input_control_vis',
+  KibanaDatatable,
   Arguments,
-  Return
+  Render<RenderValue>
 > => ({
   name: 'input_control_vis',
   type: 'render',
-  context: {
-    types: [],
-  },
+  inputTypes: [],
   help: i18n.translate('inputControl.function.help', {
     defaultMessage: 'Input control visualization',
   }),
@@ -63,7 +55,7 @@ export const createInputControlVisFn = (): ExpressionFunction<
       help: '',
     },
   },
-  async fn(context, args) {
+  fn(input, args) {
     const params = JSON.parse(args.visConfig);
     return {
       type: 'render',

src/legacy/core_plugins/interpreter/README.md

@@ -1,22 +1,2 @@
 Interpreter legacy plugin has been migrated to the New Platform. Use
 `expressions` New Platform plugin instead.
-
-In the New Platform:
-
-```ts
-class MyPlugin {
-  setup(core, { expressions }) {
-    expressions.registerFunction(myFunction);
-  }
-  start(core, { expressions }) {
-  }
-}
-```
-
-In the Legacy Platform:
-
-```ts
-import { npSetup, npStart } from 'ui/new_platform';
-
-npSetup.plugins.expressions.registerFunction(myFunction);
-```

src/legacy/core_plugins/interpreter/public/interpreter.ts

@@ -22,10 +22,7 @@ import 'uiExports/interpreter';
 import { register, registryFactory } from '@kbn/interpreter/common';
 import { npSetup } from 'ui/new_platform';
 import { registries } from './registries';
-import {
-  ExpressionInterpretWithHandlers,
-  ExpressionExecutor,
-} from '../../../../plugins/expressions/public';
+import { Executor, ExpressionExecutor } from '../../../../plugins/expressions/public';
 
 // Expose kbnInterpreter.register(specs) and kbnInterpreter.registries() globally so that plugins
 // can register without a transpile step.
@@ -46,7 +43,7 @@ export const getInterpreter = async () => {
 };
 
 // TODO: This function will be left behind in the legacy platform.
-export const interpretAst: ExpressionInterpretWithHandlers = async (ast, context, handlers) => {
+export const interpretAst: Executor['run'] = async (ast, context, handlers) => {
   const { interpreter } = await getInterpreter();
   return await interpreter.interpretAst(ast, context, handlers);
 };

src/legacy/core_plugins/region_map/public/region_map_fn.test.js

@@ -18,13 +18,13 @@
  */
 
 // eslint-disable-next-line
-import { functionWrapper } from '../../../../plugins/expressions/public/functions/tests/utils';
+import { functionWrapper } from '../../../../plugins/expressions/common/expression_functions/specs/tests/utils';
 import { createRegionMapFn } from './region_map_fn';
 
 jest.mock('ui/new_platform');
 
 describe('interpreter/functions#regionmap', () => {
-  const fn = functionWrapper(createRegionMapFn);
+  const fn = functionWrapper(createRegionMapFn());
   const context = {
     type: 'kibana_datatable',
     rows: [{ 'col-0-1': 0 }],

src/legacy/core_plugins/tile_map/public/tilemap_fn.test.js

@@ -18,7 +18,7 @@
  */
 
 // eslint-disable-next-line
-import { functionWrapper } from '../../../../plugins/expressions/public/functions/tests/utils';
+import { functionWrapper } from '../../../../plugins/expressions/common/expression_functions/specs/tests/utils';
 import { createTileMapFn } from './tile_map_fn';
 
 jest.mock('ui/new_platform');
@@ -41,7 +41,7 @@ jest.mock('ui/vis/map/convert_to_geojson', () => ({
 import { convertToGeoJson } from 'ui/vis/map/convert_to_geojson';
 
 describe('interpreter/functions#tilemap', () => {
-  const fn = functionWrapper(createTileMapFn);
+  const fn = functionWrapper(createTileMapFn());
   const context = {
     type: 'kibana_datatable',
     rows: [{ 'col-0-1': 0 }],

src/legacy/core_plugins/vis_type_markdown/public/markdown_fn.test.ts

@@ -18,11 +18,11 @@
  */
 
 // eslint-disable-next-line
-import { functionWrapper } from '../../../../plugins/expressions/public/functions/tests/utils';
+import { functionWrapper } from '../../../../plugins/expressions/common/expression_functions/specs/tests/utils';
 import { createMarkdownVisFn } from './markdown_fn';
 
 describe('interpreter/functions#markdown', () => {
-  const fn = functionWrapper(createMarkdownVisFn);
+  const fn = functionWrapper(createMarkdownVisFn());
   const args = {
     font: { spec: { fontSize: 12 } },
     openLinksInNewTab: true,

src/legacy/core_plugins/vis_type_markdown/public/markdown_fn.ts

@@ -18,31 +18,23 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { ExpressionFunction, Render } from '../../../../plugins/expressions/public';
+import { ExpressionFunctionDefinition, Render } from '../../../../plugins/expressions/public';
 import { Arguments, MarkdownVisParams } from './types';
 
-const name = 'markdownVis';
-
-type Context = undefined;
-
 interface RenderValue {
   visType: 'markdown';
   visConfig: MarkdownVisParams;
 }
 
-type Return = Promise<Render<RenderValue>>;
-
-export const createMarkdownVisFn = (): ExpressionFunction<
-  typeof name,
-  Context,
+export const createMarkdownVisFn = (): ExpressionFunctionDefinition<
+  'markdownVis',
+  unknown,
   Arguments,
-  Return
+  Render<RenderValue>
 > => ({
-  name,
+  name: 'markdownVis',
   type: 'render',
-  context: {
-    types: [],
-  },
+  inputTypes: [],
   help: i18n.translate('visTypeMarkdown.function.help', {
     defaultMessage: 'Markdown visualization',
   }),
@@ -70,7 +62,7 @@ export const createMarkdownVisFn = (): ExpressionFunction<
       }),
     },
   },
-  async fn(context, args) {
+  fn(input, args) {
     return {
       type: 'render',
       as: 'visualization',

src/legacy/core_plugins/vis_type_metric/public/components/metric_vis_component.tsx

@@ -19,21 +19,19 @@
 
 import { last, findIndex, isNaN } from 'lodash';
 import React, { Component } from 'react';
-
 import { isColorDark } from '@elastic/eui';
-
 import { getFormat } from '../legacy_imports';
 import { MetricVisValue } from './metric_vis_value';
+import { Input } from '../metric_vis_fn';
 import { FieldFormatsContentType, IFieldFormat } from '../../../../../plugins/data/public';
-import { Context } from '../metric_vis_fn';
 import { KibanaDatatable } from '../../../../../plugins/expressions/public';
 import { getHeatmapColors } from '../../../../../plugins/charts/public';
 import { VisParams, MetricVisMetric } from '../types';
 import { SchemaConfig, Vis } from '../../../visualizations/public';
 
 export interface MetricVisComponentProps {
   visParams: VisParams;
-  visData: Context;
+  visData: Input;
   vis: Vis;
   renderComplete: () => void;
 }

src/legacy/core_plugins/vis_type_metric/public/metric_vis_fn.test.ts

@@ -19,12 +19,12 @@
 
 import { createMetricVisFn } from './metric_vis_fn';
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
-import { functionWrapper } from '../../../../plugins/expressions/public/functions/tests/utils';
+import { functionWrapper } from '../../../../plugins/expressions/common/expression_functions/specs/tests/utils';
 
 jest.mock('ui/new_platform');
 
 describe('interpreter/functions#metric', () => {
-  const fn = functionWrapper(createMetricVisFn);
+  const fn = functionWrapper(createMetricVisFn());
   const context = {
     type: 'kibana_datatable',
     rows: [{ 'col-0-1': 0 }],