[@kbn/interpreter] improve build/packaging (#26096)

Summary of changes:

 - move all build artifacts under `target` directory
 - run babel and webpack in parallel
 - support optional watch and sourcemaps in build
 - expose /common /public /plugin /server sub-exports as index.js
 - avoid importing deeply from `@kbn/interpreter`
 - move a couple missed dependencies from x-pack to kibana
 - remove custom babel-register implementation

.eslintignore

@@ -27,10 +27,6 @@ bower_components
 /x-pack/coverage
 /x-pack/build
 /x-pack/plugins/**/__tests__/fixtures/**
-/packages/kbn-interpreter/common
-/packages/kbn-interpreter/plugin
-/packages/kbn-interpreter/public
-/packages/kbn-interpreter/server
 /packages/kbn-interpreter/src/common/lib/grammar.js
 /x-pack/plugins/canvas/canvas_plugin
 /x-pack/plugins/canvas/canvas_plugin_src/lib/flot-charts

.eslintrc.js

@@ -102,7 +102,7 @@ module.exports = {
      * Files that ARE NOT allowed to use devDependencies
      */
     {
-      files: ['packages/kbn-ui-framework/**/*', 'x-pack/**/*'],
+      files: ['packages/kbn-ui-framework/**/*', 'x-pack/**/*', 'packages/kbn-interpreter/**/*'],
       rules: {
         'import/no-extraneous-dependencies': [
           'error',
@@ -124,6 +124,8 @@ module.exports = {
         'packages/kbn-ui-framework/generator-kui/**/*',
         'packages/kbn-ui-framework/Gruntfile.js',
         'packages/kbn-es/src/**/*',
+        'packages/kbn-interpreter/tasks/**/*',
+        'packages/kbn-interpreter/src/plugin/**/*',
         'x-pack/{dev-tools,tasks,scripts,test,build_chromium}/**/*',
         'x-pack/**/{__tests__,__test__,__jest__,__fixtures__,__mocks__}/**/*',
         'x-pack/**/*.test.js',

.gitignore

@@ -43,4 +43,3 @@ package-lock.json
 npm-debug.log*
 .tern-project
 **/public/index.css
-/packages/kbn-interpreter/plugin

package.json

@@ -199,6 +199,8 @@
     "rxjs": "^6.2.1",
     "script-loader": "0.7.2",
     "semver": "^5.5.0",
+    "socket.io": "^2.1.1",
+    "stream-stream": "^1.2.6",
     "style-loader": "0.19.0",
     "tar": "2.2.0",
     "tinygradient": "0.3.0",

packages/kbn-interpreter/.babelrc

@@ -1,7 +1,7 @@
 {
   "presets": ["@kbn/babel-preset/webpack_preset"],
   "plugins": [
-    ["transform-runtime", {
+    ["babel-plugin-transform-runtime", {
       "polyfill": false,
       "regenerator": true
     }]

packages/kbn-interpreter/.npmignore

@@ -0,0 +1,3 @@
+src
+tasks
+.babelrc

packages/kbn-interpreter/common/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "main": "../target/common/index.js",
+  "jsnext:main": "../src/common/index.js"
+}

packages/kbn-interpreter/package.json

@@ -3,9 +3,10 @@
   "version": "1.0.0",
   "license": "Apache-2.0",
   "scripts": {
-    "build": "babel src --out-dir ./ --ignore 'src/plugin' && node tasks/build.js",
     "canvas:peg": "pegjs common/lib/grammar.peg",
-    "kbn:bootstrap": "yarn build"
+    "build": "node scripts/build",
+    "kbn:bootstrap": "node scripts/build --dev",
+    "kbn:watch": "node scripts/build --dev --watch"
   },
   "dependencies": {
     "lodash": "npm:@elastic/[email protected]",
@@ -16,15 +17,19 @@
   },
   "devDependencies": {
     "@kbn/babel-preset": "1.0.0",
+    "@kbn/dev-utils": "1.0.0",
     "babel-cli": "^6.26.0",
     "babel-core": "^6.26.0",
     "babel-loader": "7.1.2",
     "babel-plugin-transform-runtime": "^6.23.0",
     "babel-polyfill": "6.20.0",
     "css-loader": "0.28.7",
+    "del": "^3.0.0",
+    "getopts": "^2.2.3",
     "pegjs": "0.9.0",
     "sass-loader": "^7.1.0",
     "style-loader": "0.19.0",
+    "supports-color": "^5.5.0",
     "url-loader": "0.5.9",
     "webpack": "3.6.0"
   }

src/core_plugins/interpreter/plugin_paths.js → packages/kbn-interpreter/plugin_paths.js

@@ -17,11 +17,9 @@
  * under the License.
  */
 
-import { resolve } from 'path';
+const { resolve } = require('path');
 
-const dir = resolve(__dirname, '..', '..', '..');
-
-export const pluginPaths = {
-  commonFunctions: resolve(dir, 'node_modules/@kbn/interpreter/plugin/functions/common'),
-  types: resolve(dir, 'node_modules/@kbn/interpreter/plugin/types'),
+exports.pluginPaths = {
+  commonFunctions: resolve(__dirname, 'target/plugin/functions/common'),
+  types: resolve(__dirname, 'target/plugin/types'),
 };

packages/kbn-interpreter/public/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "main": "../target/public/index.js",
+  "jsnext:main": "../src/public/index.js"
+}

src/core_plugins/interpreter/server/lib/route_expression/thread/polyfill.js → packages/kbn-interpreter/scripts/build.js

@@ -17,15 +17,4 @@
  * under the License.
  */
 
-// taken from kibana/src/setup_node_env/babel_register/polyfill.js
-// ...
-// `babel-preset-env` looks for and rewrites the following import
-// statement into a list of import statements based on the polyfills
-// necessary for our target environment (the current version of node)
-// but since it does that during compilation, `import 'babel-polyfill'`
-// must be in a file that is loaded with `require()` AFTER `babel-register`
-// is configured.
-//
-// This is why we have this single statement in it's own file and require
-// it from ./babeled.js
-import 'babel-polyfill';
+require('../tasks/build/cli');

packages/kbn-interpreter/server/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "main": "../target/server/index.js",
+  "jsnext:main": "../src/server/index.js"
+}

packages/kbn-interpreter/src/common/index.js

@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { pathsRegistry } from './lib/paths_registry';
+export { functionsRegistry } from './lib/functions_registry';
+export { typesRegistry } from './lib/types_registry';
+export { createError } from './interpreter/create_error';
+export { interpretProvider } from './interpreter/interpret';
+export { serializeProvider } from './lib/serialize';
+export { fromExpression, toExpression, safeElementFromExpression } from './lib/ast';
+export { Fn } from './lib/fn';
+export { getType } from './lib/get_type';
+export { castProvider } from './interpreter/cast';
+export { parse } from './lib/grammar';
+export { getByAlias } from './lib/get_by_alias';
+export { Registry } from './lib/registry';

packages/kbn-interpreter/src/public/index.js

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { populateBrowserRegistries, getBrowserRegistries } from './browser_registries';
+export { createSocket } from './socket';
+export { initializeInterpreter, interpretAst } from './interpreter';

packages/kbn-interpreter/src/public/interpreter.js

@@ -29,7 +29,7 @@ let socket;
 let resolve;
 const functionList = new Promise(_resolve => (resolve = _resolve));
 
-export async function initialize() {
+export async function initializeInterpreter() {
   socket = getSocket();
 
   // Listen for interpreter runs

packages/kbn-interpreter/src/server/index.js

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { populateServerRegistries, getServerRegistries } from './server_registries';
+export { getPluginPaths } from './get_plugin_paths';

packages/kbn-interpreter/tasks/build/cli.js

@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const { relative } = require('path');
+
+const getopts = require('getopts');
+const del = require('del');
+const supportsColor = require('supports-color');
+const { ToolingLog, withProcRunner, pickLevelFromFlags } = require('@kbn/dev-utils');
+
+const {
+  ROOT_DIR,
+  PLUGIN_SOURCE_DIR,
+  BUILD_DIR,
+  WEBPACK_CONFIG_PATH
+} = require('./paths');
+
+const unknownFlags = [];
+const flags = getopts(process.argv, {
+  boolean: [
+    'watch',
+    'dev',
+    'help',
+    'debug'
+  ],
+  unknown(name) {
+    unknownFlags.push(name);
+  }
+});
+
+const log = new ToolingLog({
+  level: pickLevelFromFlags(flags),
+  writeTo: process.stdout
+});
+
+if (unknownFlags.length) {
+  log.error(`Unknown flag(s): ${unknownFlags.join(', ')}`);
+  flags.help = true;
+  process.exitCode = 1;
+}
+
+if (flags.help) {
+  log.info(`
+    Simple build tool for @kbn/interpreter package
+
+    --dev      Build for development, include source maps
+    --watch    Run in watch mode
+    --debug    Turn on debug logging
+  `);
+  process.exit();
+}
+
+withProcRunner(log, async (proc) => {
+  log.info('Deleting old output');
+  await del(BUILD_DIR);
+
+  const cwd = ROOT_DIR;
+  const env = { ...process.env };
+  if (supportsColor.stdout) {
+    env.FORCE_COLOR = 'true';
+  }
+
+  log.info(`Starting babel and webpack${flags.watch ? ' in watch mode' : ''}`);
+  await Promise.all([
+    proc.run('babel  ', {
+      cmd: 'babel',
+      args: [
+        'src',
+        '--ignore', `${relative(cwd, PLUGIN_SOURCE_DIR)},*.test.js`,
+        '--out-dir', relative(cwd, BUILD_DIR),
+        '--copy-files',
+        ...(flags.dev ? ['--source-maps', 'inline'] : []),
+        ...(flags.watch ? ['--watch'] : ['--quiet'])
+      ],
+      wait: true,
+      env,
+      cwd
+    }),
+
+    proc.run('webpack', {
+      cmd: 'webpack',
+      args: [
+        '--config', relative(cwd, WEBPACK_CONFIG_PATH),
+        '--env.sourceMaps', String(Boolean(flags.dev)),
+        ...(flags.watch ? ['--watch'] : []),
+      ],
+      wait: true,
+      env,
+      cwd
+    }),
+  ]);
+
+  log.success('Complete');
+}).catch((error) => {
+  log.error(error);
+  process.exit(1);
+});

packages/kbn-interpreter/tasks/build.js → packages/kbn-interpreter/tasks/build/paths.js

@@ -17,24 +17,15 @@
  * under the License.
  */
 
-const webpack = require('webpack');
-const webpackConfig = require('./webpack.plugins');
+const { resolve } = require('path');
 
-const devtool = 'inline-cheap-module-source-map';
+exports.ROOT_DIR = resolve(__dirname, '../../');
+exports.SOURCE_DIR = resolve(exports.ROOT_DIR, 'src');
+exports.BUILD_DIR = resolve(exports.ROOT_DIR, 'target');
 
-const onComplete = function (done) {
-  return function (err, stats) {
-    if (err) {
-      done && done(err);
-    } else {
-      const seconds = ((stats.endTime - stats.startTime) / 1000).toFixed(2);
-      console.log(`Plugins built in ${seconds} seconds`);
-      done && done();
-    }
-  };
-};
+exports.PLUGIN_SOURCE_DIR = resolve(exports.SOURCE_DIR, 'plugin');
+exports.PLUGIN_BUILD_DIR = resolve(exports.BUILD_DIR, 'plugin');
 
-webpack({ ...webpackConfig, devtool }, onComplete(function () {
-  console.log('all done');
-}));
+exports.WEBPACK_CONFIG_PATH = require.resolve('./webpack.config');
+exports.BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset');