refactor(core): move mexp from plugins to core (#1371)

* refactor(core): move mexp from plugins to core

* remove test nest

packages/superset-ui-core/package.json

@@ -39,6 +39,7 @@
     "@types/d3-time": "^1.0.9",
     "@types/d3-time-format": "^2.1.0",
     "@types/lodash": "^4.14.149",
+    "@types/math-expression-evaluator": "^1.2.1",
     "@types/rison": "0.0.6",
     "@types/seedrandom": "^2.4.28",
     "@vx/responsive": "^0.0.199",
@@ -51,6 +52,7 @@
     "fetch-retry": "^4.0.1",
     "jed": "^1.1.1",
     "lodash": "^4.17.11",
+    "math-expression-evaluator": "^1.3.8",
     "pretty-ms": "^7.0.0",
     "react-error-boundary": "^1.2.5",
     "react-markdown": "^4.3.1",

packages/superset-ui-core/src/index.ts

@@ -32,3 +32,4 @@ export * from './validator';
 export * from './chart';
 export * from './chart-composition';
 export * from './components';
+export * from './math-expression';

packages/superset-ui-core/src/math-expression/index.ts

@@ -0,0 +1,102 @@
+import mexp from 'math-expression-evaluator';
+
+const REPLACE_OPERATORS: [RegExp, string][] = [
+  [new RegExp(/==/g), 'Eq'],
+  [new RegExp(/>=/g), 'Gte'],
+  [new RegExp(/<=/g), 'Lte'],
+  [new RegExp(/>/g), 'Gt'],
+  [new RegExp(/</g), 'Lt'],
+];
+
+const TOKENS = [
+  {
+    type: 3,
+    token: 'x',
+    show: 'x',
+    value: 'x',
+  },
+  {
+    type: 2,
+    token: '&',
+    show: '&',
+    value: (a: number, b: number): number => a & b,
+  },
+  {
+    type: 2,
+    token: '|',
+    show: '|',
+    value: (a: number, b: number): number => a | b,
+  },
+  {
+    type: 2,
+    token: 'and',
+    show: 'and',
+    value: (a: number, b: number): number => a && b,
+  },
+  {
+    type: 2,
+    token: 'xor',
+    show: 'xor',
+    value: (a: number, b: number): number => a ^ b,
+  },
+  {
+    type: 2,
+    token: 'or',
+    show: 'or',
+    value: (a: number, b: number): number => Number(a || b),
+  },
+  {
+    type: 2,
+    token: 'Eq',
+    show: 'Eq',
+    value: (a: number, b: number): number => Number(a === b),
+  },
+  {
+    type: 2,
+    token: 'Lt',
+    show: 'Lt',
+    value: (a: number, b: number): number => Number(a < b),
+  },
+  {
+    type: 2,
+    token: 'Lte',
+    show: 'Lte',
+    value: (a: number, b: number): number => Number(a <= b),
+  },
+  {
+    type: 2,
+    token: 'Gt',
+    show: 'Gt',
+    value: (a: number, b: number): number => Number(a > b),
+  },
+  {
+    type: 2,
+    token: 'Gte',
+    show: 'Gte',
+    value: (a: number, b: number): number => Number(a >= b),
+  },
+];
+
+export function evalExpression(expression: string, value: number): number {
+  let parsedExpression = expression;
+  // replace `<` with `Lt` (and others) to avoid clashes with builtin function operators
+  // that are not needed in Superset.
+  REPLACE_OPERATORS.forEach(([key, value]) => {
+    parsedExpression = parsedExpression.replace(key, value);
+  });
+  const subExpressions = String(parsedExpression).split('=');
+  parsedExpression = subExpressions[1] ?? subExpressions[0];
+  // we can ignore the type requirement on `TOKENS`, as value is always `number`
+  // and doesn't need to consider `number | underfined`.
+  // @ts-ignore
+  return Number(mexp.eval(parsedExpression, TOKENS, { x: value }));
+}
+
+export function isValidExpression(expression: string): boolean {
+  try {
+    evalExpression(expression, 0);
+  } catch (err) {
+    return false;
+  }
+  return true;
+}

packages/superset-ui-core/test/math-expression/index.test.ts

@@ -0,0 +1,79 @@
+import { evalExpression, isValidExpression } from '@superset-ui/core/src/math-expression';
+
+test('evalExpression evaluates constants correctly', () => {
+  expect(evalExpression('0', 10)).toEqual(0);
+  expect(evalExpression('0.123456', 0)).toEqual(0.123456);
+  expect(evalExpression('789', 100)).toEqual(789);
+});
+
+test('evalExpression evaluates infinities correctly', () => {
+  const formula = 'x/0';
+  expect(evalExpression(formula, 1)).toEqual(Infinity);
+  expect(evalExpression(formula, -1)).toEqual(-Infinity);
+});
+
+test('evalExpression evaluates powers correctly', () => {
+  const formula = '2^(x/2)*100';
+  expect(evalExpression(formula, 0)).toEqual(100);
+  expect(evalExpression(formula, 1)).toEqual(141.4213562373095);
+  expect(evalExpression(formula, 2)).toEqual(200);
+});
+
+test('evalExpression ignores whitespace and variables on left hand side of equals sign', () => {
+  expect(evalExpression('y=x+1', 1)).toEqual(2);
+  expect(evalExpression('y = x    - 1', 1)).toEqual(0);
+});
+
+test('evalExpression evaluates custom operators correctly', () => {
+  const equalsExpression = 'x == 10';
+  expect(evalExpression(equalsExpression, 5)).toEqual(0);
+  expect(evalExpression(equalsExpression, 10)).toEqual(1);
+  expect(evalExpression(equalsExpression, 10.1)).toEqual(0);
+
+  const closedRange = '(x > 0) and (x < 10)';
+  expect(evalExpression(closedRange, 0)).toEqual(0);
+  expect(evalExpression(closedRange, 5)).toEqual(1);
+  expect(evalExpression(closedRange, 10)).toEqual(0);
+
+  const openRange = '(x >= 0) and (x <= 10)';
+  expect(evalExpression(openRange, -0.1)).toEqual(0);
+  expect(evalExpression(openRange, 0)).toEqual(1);
+  expect(evalExpression(openRange, 5)).toEqual(1);
+  expect(evalExpression(openRange, 10)).toEqual(1);
+  expect(evalExpression(openRange, 10.1)).toEqual(0);
+
+  const orRange = '(x < 0) or (x > 10)';
+  expect(evalExpression(orRange, -0.1)).toEqual(1);
+  expect(evalExpression(orRange, 0)).toEqual(0);
+  expect(evalExpression(orRange, 5)).toEqual(0);
+  expect(evalExpression(orRange, 10)).toEqual(0);
+  expect(evalExpression(orRange, 10.1)).toEqual(1);
+
+  // other less used operators
+  expect(evalExpression('5 & x', 3)).toEqual(1);
+  expect(evalExpression('5 | x', 3)).toEqual(7);
+  expect(evalExpression('5 xor x', 2)).toEqual(7);
+
+  // complex combinations
+  const complexExpression = '20.51*(x<1577836800000)+20.2((x<15805152000000)&(x>=1577836800000))';
+  expect(evalExpression(complexExpression, 0)).toEqual(20.51);
+  expect(evalExpression(complexExpression, 1000)).toEqual(20.51);
+  expect(evalExpression(complexExpression, 1577836800000)).toEqual(20.2);
+  expect(evalExpression(complexExpression, 15805151999999)).toEqual(20.2);
+  expect(evalExpression(complexExpression, 15805152000000)).toEqual(0);
+  expect(evalExpression(complexExpression, 15805159000000)).toEqual(0);
+});
+
+test('isValidExpression correctly identifies invalid formulas', () => {
+  expect(isValidExpression('foobar')).toEqual(false);
+  expect(isValidExpression('x+')).toEqual(false);
+  expect(isValidExpression('z+1')).toEqual(false);
+});
+
+test('isValidExpression correctly identifies valid formulas', () => {
+  expect(isValidExpression('x')).toEqual(true);
+  expect(isValidExpression('x+1')).toEqual(true);
+  expect(isValidExpression('y=x-1')).toEqual(true);
+  expect(isValidExpression('y = x - 1')).toEqual(true);
+  expect(isValidExpression('y = (x < 100 and x > 0) * 100')).toEqual(true);
+});

plugins/legacy-preset-chart-nvd3/package.json

@@ -36,7 +36,6 @@
     "dompurify": "^2.0.6",
     "fast-safe-stringify": "^2.0.6",
     "lodash": "^4.17.11",
-    "math-expression-evaluator": "^1.3.8",
     "moment": "^2.20.1",
     "nvd3-fork": "^2.0.5",
     "prop-types": "^15.6.2",

plugins/legacy-preset-chart-nvd3/src/NVD3Vis.js

@@ -20,17 +20,17 @@
 import { kebabCase, throttle } from 'lodash';
 import d3 from 'd3';
 import moment from 'moment';
-import mexp from 'math-expression-evaluator';
 import nv from 'nvd3-fork';
 import PropTypes from 'prop-types';
 import {
-  t,
-  isDefined,
-  getTimeFormatter,
-  smartDateVerboseFormatter,
+  CategoricalColorNamespace,
+  evalExpression,
   getNumberFormatter,
+  getTimeFormatter,
+  isDefined,
   NumberFormats,
-  CategoricalColorNamespace,
+  smartDateVerboseFormatter,
+  t,
 } from '@superset-ui/core';
 
 import 'nvd3-fork/build/nv.d3.css';
@@ -946,13 +946,6 @@ function nvd3Vis(element, props) {
         }
 
         if (formulas.length > 0) {
-          const token = {
-            type: 3,
-            token: 'x',
-            show: 'x',
-            value: 'x',
-          };
-
           const xValues = [];
           if (vizType === 'bar') {
             // For bar-charts we want one data point evaluated for every
@@ -982,13 +975,11 @@ function nvd3Vis(element, props) {
           }
           const formulaData = formulas.map(fo => {
             const { value: expression } = fo;
-            const subExpressions = String(expression).split('=');
-
             return {
               key: fo.name,
               values: xValues.map(x => ({
                 x,
-                y: mexp.eval(subExpressions[1] ?? subExpressions[0], [token], { x }),
+                y: evalExpression(expression, x),
               })),
               color: fo.color,
               strokeWidth: fo.width,

plugins/plugin-chart-echarts/package.json

@@ -28,11 +28,9 @@
   "dependencies": {
     "@superset-ui/chart-controls": "0.18.5",
     "@superset-ui/core": "0.18.5",
-    "@types/math-expression-evaluator": "^1.2.1",
     "d3-array": "^1.2.0",
     "echarts": "^5.2.1",
-    "lodash": "^4.17.15",
-    "math-expression-evaluator": "^1.3.8"
+    "lodash": "^4.17.15"
   },
   "peerDependencies": {
     "react": "^16.13.1"

plugins/plugin-chart-echarts/src/Timeseries/transformers.ts

@@ -18,11 +18,11 @@
  */
 import {
   AnnotationData,
-  AnnotationLayer,
   AnnotationOpacity,
   CategoricalColorScale,
   EventAnnotationLayer,
   FilterState,
+  FormulaAnnotationLayer,
   getTimeFormatter,
   IntervalAnnotationLayer,
   isTimeseriesAnnotationResult,
@@ -204,7 +204,7 @@ export function transformSeries(
 }
 
 export function transformFormulaAnnotation(
-  layer: AnnotationLayer,
+  layer: FormulaAnnotationLayer,
   data: TimeseriesDataRecord[],
   colorScale: CategoricalColorScale,
 ): SeriesOption {

plugins/plugin-chart-echarts/src/utils/annotation.ts

@@ -23,29 +23,23 @@ import {
   AnnotationLayer,
   AnnotationOpacity,
   AnnotationType,
+  evalExpression,
+  FormulaAnnotationLayer,
   isRecordAnnotationResult,
   isTableAnnotationLayer,
   isTimeseriesAnnotationResult,
   TimeseriesDataRecord,
 } from '@superset-ui/core';
-import mexp from 'math-expression-evaluator';
 
 export function evalFormula(
-  formula: AnnotationLayer,
+  formula: FormulaAnnotationLayer,
   data: TimeseriesDataRecord[],
 ): [Date, number][] {
-  const token = {
-    type: 3,
-    token: 'x',
-    show: 'x',
-    value: 'x',
-  };
   const { value: expression } = formula;
-  const subExpressions = String(expression).split('=');
 
   return data.map(row => [
     new Date(Number(row.__timestamp)),
-    Number(mexp.eval(subExpressions[1] ?? subExpressions[0], [token], { x: row.__timestamp })),
+    evalExpression(expression, row.__timestamp as number),
   ]);
 }