build immutable bundles for new platform plugins

.browserslistrc

@@ -1,3 +1,8 @@
+[production]
 last 2 versions
 > 5%
 Safari 7 # for PhantomJS support: https://github.com/elastic/kibana/issues/27136
+
+[dev]
+last 1 chrome versions
+last 1 firefox versions

.github/CODEOWNERS

@@ -92,6 +92,7 @@
 /packages/*babel*/ @elastic/kibana-operations
 /packages/kbn-dev-utils*/ @elastic/kibana-operations
 /packages/kbn-es/ @elastic/kibana-operations
+/packages/kbn-optimizer/ @elastic/kibana-operations
 /packages/kbn-pm/ @elastic/kibana-operations
 /packages/kbn-test/ @elastic/kibana-operations
 /packages/kbn-ui-shared-deps/ @elastic/kibana-operations

package.json

@@ -137,12 +137,6 @@
     "@kbn/test-subj-selector": "0.2.1",
     "@kbn/ui-framework": "1.0.0",
     "@kbn/ui-shared-deps": "1.0.0",
-    "@types/flot": "^0.0.31",
-    "@types/json-stable-stringify": "^1.0.32",
-    "@types/lodash.clonedeep": "^4.5.4",
-    "@types/node-forge": "^0.9.0",
-    "@types/react-grid-layout": "^0.16.7",
-    "@types/recompose": "^0.30.5",
     "JSONStream": "1.3.5",
     "abortcontroller-polyfill": "^1.3.0",
     "angular": "^1.7.9",
@@ -152,11 +146,12 @@
     "angular-route": "^1.7.8",
     "angular-sanitize": "^1.7.8",
     "angular-sortable-view": "^0.0.17",
-    "autoprefixer": "9.6.1",
+    "autoprefixer": "^9.7.4",
     "babel-loader": "^8.0.6",
     "bluebird": "3.5.5",
     "boom": "^7.2.0",
     "brace": "0.11.1",
+    "browserslist-useragent": "^3.0.2",
     "cache-loader": "^4.1.0",
     "chalk": "^2.4.2",
     "check-disk-space": "^2.1.0",
@@ -165,7 +160,7 @@
     "commander": "3.0.2",
     "compare-versions": "3.5.1",
     "core-js": "^3.2.1",
-    "css-loader": "2.1.1",
+    "css-loader": "^3.4.2",
     "d3": "3.5.17",
     "d3-cloud": "1.2.5",
     "deep-freeze-strict": "^1.1.1",
@@ -226,7 +221,7 @@
     "opn": "^5.5.0",
     "oppsy": "^2.0.0",
     "pegjs": "0.10.0",
-    "postcss-loader": "3.0.0",
+    "postcss-loader": "^3.0.0",
     "prop-types": "15.6.0",
     "proxy-from-env": "1.0.0",
     "pug": "^2.0.4",
@@ -259,7 +254,7 @@
     "seedrandom": "^3.0.5",
     "semver": "^5.5.0",
     "style-it": "^2.1.3",
-    "style-loader": "0.23.1",
+    "style-loader": "^1.1.3",
     "symbol-observable": "^1.2.0",
     "tar": "4.4.13",
     "terser-webpack-plugin": "^2.1.2",
@@ -279,7 +274,7 @@
     "vega-schema-url-parser": "1.0.0",
     "vega-tooltip": "^0.12.0",
     "vision": "^5.3.3",
-    "webpack": "4.41.0",
+    "webpack": "^4.41.5",
     "webpack-merge": "4.2.2",
     "whatwg-fetch": "^3.0.0",
     "wrapper-webpack-plugin": "^2.1.0",
@@ -300,6 +295,7 @@
     "@kbn/eslint-plugin-eslint": "1.0.0",
     "@kbn/expect": "1.0.0",
     "@kbn/plugin-generator": "1.0.0",
+    "@kbn/optimizer": "1.0.0",
     "@kbn/test": "1.0.0",
     "@kbn/utility-types": "1.0.0",
     "@microsoft/api-documenter": "7.7.2",
@@ -312,6 +308,7 @@
     "@types/babel__core": "^7.1.2",
     "@types/bluebird": "^3.1.1",
     "@types/boom": "^7.2.0",
+    "@types/browserslist-useragent": "^3.0.0",
     "@types/chance": "^1.0.0",
     "@types/cheerio": "^0.22.10",
     "@types/chromedriver": "^2.38.0",
@@ -324,6 +321,7 @@
     "@types/enzyme": "^3.9.0",
     "@types/eslint": "^6.1.3",
     "@types/fetch-mock": "^7.3.1",
+    "@types/flot": "^0.0.31",
     "@types/getopts": "^2.0.1",
     "@types/glob": "^7.1.1",
     "@types/globby": "^8.0.0",
@@ -337,17 +335,20 @@
     "@types/joi": "^13.4.2",
     "@types/jquery": "^3.3.31",
     "@types/js-yaml": "^3.11.1",
+    "@types/json-stable-stringify": "^1.0.32",
     "@types/json5": "^0.0.30",
     "@types/license-checker": "15.0.0",
     "@types/listr": "^0.14.0",
     "@types/lodash": "^3.10.1",
+    "@types/lodash.clonedeep": "^4.5.4",
     "@types/lru-cache": "^5.1.0",
     "@types/markdown-it": "^0.0.7",
     "@types/minimatch": "^2.0.29",
     "@types/mocha": "^5.2.7",
     "@types/moment-timezone": "^0.5.12",
     "@types/mustache": "^0.8.31",
     "@types/node": "^10.12.27",
+    "@types/node-forge": "^0.9.0",
     "@types/numeral": "^0.0.26",
     "@types/opn": "^5.1.0",
     "@types/pegjs": "^0.10.1",
@@ -357,11 +358,13 @@
     "@types/reach__router": "^1.2.6",
     "@types/react": "^16.9.11",
     "@types/react-dom": "^16.9.4",
+    "@types/react-grid-layout": "^0.16.7",
     "@types/react-redux": "^6.0.6",
     "@types/react-resize-detector": "^4.0.1",
     "@types/react-router": "^5.1.3",
     "@types/react-router-dom": "^5.1.3",
     "@types/react-virtualized": "^9.18.7",
+    "@types/recompose": "^0.30.6",
     "@types/redux": "^3.6.31",
     "@types/redux-actions": "^2.6.1",
     "@types/request": "^2.48.2",
@@ -460,7 +463,7 @@
     "pixelmatch": "^5.1.0",
     "pkg-up": "^2.0.0",
     "pngjs": "^3.4.0",
-    "postcss": "^7.0.5",
+    "postcss": "^7.0.26",
     "postcss-url": "^8.0.0",
     "prettier": "^1.19.1",
     "proxyquire": "1.8.0",

packages/kbn-dev-utils/src/serializers/absolute_path_serializer.ts

@@ -17,7 +17,9 @@
  * under the License.
  */
 
-export function createAbsolutePathSerializer(rootPath: string) {
+import { REPO_ROOT } from '../repo_root';
+
+export function createAbsolutePathSerializer(rootPath: string = REPO_ROOT) {
   return {
     print: (value: string) => value.replace(rootPath, '<absolute path>').replace(/\\/g, '/'),
     test: (value: any) => typeof value === 'string' && value.startsWith(rootPath),

packages/kbn-eslint-import-resolver-kibana/package.json

@@ -16,6 +16,6 @@
     "glob-all": "^3.1.0",
     "lru-cache": "^4.1.5",
     "resolve": "^1.7.1",
-    "webpack": "^4.41.0"
+    "webpack": "^4.41.5"
   }
 }

packages/kbn-interpreter/package.json

@@ -23,15 +23,15 @@
     "@kbn/dev-utils": "1.0.0",
     "babel-loader": "^8.0.6",
     "copy-webpack-plugin": "^5.0.4",
-    "css-loader": "2.1.1",
+    "css-loader": "^3.4.2",
     "del": "^5.1.0",
     "getopts": "^2.2.4",
     "pegjs": "0.10.0",
-    "sass-loader": "^7.3.1",
-    "style-loader": "0.23.1",
+    "sass-loader": "^8.0.2",
+    "style-loader": "^1.1.3",
     "supports-color": "^7.0.0",
     "url-loader": "2.2.0",
-    "webpack": "4.41.0",
-    "webpack-cli": "^3.3.9"
+    "webpack": "^4.41.5",
+    "webpack-cli": "^3.3.10"
   }
 }

packages/kbn-optimizer/README.md

@@ -0,0 +1,102 @@
+# @kbn/optimizer
+
+`@kbn/optimizer` is a package for building new platform UI plugins (and hopefully more soon).
+
+New Platform plugins with `"ui": true` in their `kibana.json` file will have their `public/index.ts` file (and all of its dependencies) bundled into the `target/public` directory of the plugin. The build output does not need to be updated when other plugins are updated and is included in the distributable without requiring that we ship `@kbn/optimizer` 🎉.
+
+## Webpack config
+
+The [Webpack config][WebpackConfig] is designed to provide the majority of what was available in the legacy optimizer and is the same for all plugins to promote consistency and keep things sane for the operations team. It has support for JS/TS built with babel, url imports of image and font files, and support for importing `scss` and `css` files. SCSS is pre-processed by [postcss][PostCss], built for both light and dark mode and injected automatically into the page when the parent module is loaded (page reloads are still required for switching between light/dark mode). CSS is injected into the DOM as it is written on disk when the parent module is loaded (no postcss support).
+
+Source maps are enabled execpt when building the distributable. They show the code actually being executed by the browser to strike a balance between debuggability and performance. They are not configurable at this time but will be configurable once we have a developer configuration solution that doesn't rely on the server (see [#55656](https://github.com/elastic/kibana/issues/55656)).
+
+### IE Support
+
+To make front-end code easier to debug the optimizer uses the `BROWSERSLIST_ENV=dev` environment variable (by default) to build JS and CSS that is compatible with modern browsers. In order to support older browsers like IE in development you will need to specify the `BROWSERSLIST_ENV=production` environment variable or build a distributable for testing.
+
+## Running the optimizer
+
+The `@kbn/optimizer` is automatically executed from the dev cli, the Kibana build scripts, and in CI. If you're running Kibana locally in some other way you might need to build the plugins manually, which you can do by running `node scripts/build_new_platform_plugins` (pass `--help` for options).
+
+You can limit the number of workers the optimizer uses in all of these places by setting the `KBN_OPTIMIZER_MAX_WORKERS` environment variable. You might want to do this if your system struggles to keep up while the optimizer is getting started and building all plugins as fast as possible. Setting `KBN_OPTIMIZER_MAX_WORKERS=1` will cause the optimizer to take the longest amount of time but will have the smallest impact on other components of your system.
+
+## API
+
+To run the optimizer from code, you can import the [`Optimizer`][Optimizer] and [`OptimizerConfig`][OptimizerConfig] classes. Create an [`OptimizerConfig`][OptimizerConfig] instance by calling it's static `create()` method with some options, then pass it to the [`Optimizer`][Optimizer] constructor. Calling `Optimizer#run()` will return an observable of [`OptimizerState`][Optimizer] objects, which are either bits of stdio from the workers or [`OptimizerStateSummary`][Optimizer] objects. You can use the [`logOptimizerState()`][LogOptimizerState] helper to write the relevant bits of state to a tooling log.
+
+Example:
+```ts
+import { Optimizer, OptimizerConfig, logOptimizerState } from '@kbn/optimizer';
+import { REPO_ROOT, ToolingLog } from '@kbn/dev-utils';
+
+const log = new ToolingLog({
+  level: 'verbose',
+  writeTo: process.stdout,
+})
+
+const config = OptimizerConfig.create({
+  repoRoot: Path.resolve(__dirname, '../../..'),
+  watch: false,
+  oss: true,
+  dist: true
+});
+
+const optimizer = new Optimizer(config);
+
+await optimizer
+  .run()
+  .pipe(logOptimizerState(log, config))
+  .toPromise();
+```
+
+This is essentially what we're doing in [`script/build_new_platform_plugins`][Cli] and the new [build system task][BuildTask].
+
+## Internals
+
+The optimizer runs webpack instances in worker processes. Each worker is configured to build one or more plugins via a [`WorkerConfig`][WorkerConfig] object (created by [`OptimizerConfig`][OptimizerConfig]) which is JSON serialized and passed to the worker as it's only argument.
+
+Plugins/bundles are assigned to workers based on the number of modules historically seen in each bundle in an effort to evenly distribute the load across the worker pool (see [`assignBundlesToWorkers`][AssignBundlesToWorkers]).
+
+The number of workers available is automatically chosen by dividing the number of cores available by 3 (minimum of 2).
+
+The [`WorkerConfig`][WorkerConfig] includes the location of the repo (it might be one of many builds, or the main repo), wether we are running in watch mode, wether we are building a distributable, and a list of [`BundleDefinition`][BundleDefinition] objects which include the details necessary to create a webpack config for a specific plugin's bundle (created using [`webpack.config.ts`][WebpackConfig]).
+
+Each worker communicates state back to the main process by sending [`WorkerMessage`][WorkerMessage] and [`CompilerMessage`][CompilerMessage] objects using IPC.
+
+The Optimizer captures all of these messages and produces a stream of [`OptimizerStateSummary`][Optimizer] and [`WorkerStdio`][ObserveWorker] objects. Workers shouldn't produce any data on their stdio streams, but when they do it's probably because something is wrong so make sure to show them to users somehow. All other messages represent the total state of all workers.
+
+```ts
+interface OptimizerStateSummary {
+  type: 'running' | 'compiler issue' | 'compiler success';
+  durSec: number;
+  bundles: BundleState[];
+}
+```
+
+Optimizer state summary types:
+<dl>
+  <dt><code>'running'</code></dt>
+  <dd>Emitted when any worker is in a running state. To determine which compilers are running, look for <code>BundleState</code> objects with type <code>'running'</code>.</dd>
+  <dt><code>'compiler issue'</code></dt>
+  <dd>Emitted when all workers are done running and any compiler completed with a <code>'compiler issue'</code> status. Compiler issues include things like "unable to resolve module" or syntax errors in the source modules and can be fixed by users when running in watch mode.</dd>
+  <dt><code>'compiler success'</code></dt>
+  <dd>Emitted when all workers are done running and all compilers completed with <code>'compiler success'</code>.</dd>
+</dl>
+
+Workers have several error message they may emit which indicate unrecoverable errors. When any of those messages are received the stream will error and the workers will be torn down.
+
+For an example of how to handle these states checkout the [`logOptimizerState()`][LogOptimizerState] helper.
+
+[PostCss]: https://postcss.org/
+[Cli]: src/cli.ts
+[Optimizer]: src/optimizer.ts
+[ObserveWorker]: src/observe_worker.ts
+[CompilerMessage]: src/common/compiler_messages.ts
+[WorkerMessage]: src/common/worker_messages.ts
+[WebpackConfig]: src/worker/webpack.config.ts
+[BundleDefinition]: src/common/bundle_definition.ts
+[WorkerConfig]: src/common/worker_config.ts
+[OptimizerConfig]: src/optimizer_config.ts
+[LogOptimizerState]: src/log_optimizer_state.ts
+[AssignBundlesToWorkers]: src/assign_bundles_to_workers.ts
+[BuildTask]: ../../src/dev/build/tasks/build_new_platform_plugins.js

src/cli/color.js → packages/kbn-optimizer/babel.config.js

@@ -17,8 +17,7 @@
  * under the License.
  */
 
-import chalk from 'chalk';
-
-export const green = chalk.black.bgGreen;
-export const red = chalk.white.bgRed;
-export const yellow = chalk.black.bgYellow;
+module.exports = {
+  presets: ['@kbn/babel-preset/node_preset'],
+  ignore: ['**/*.test.js'],
+};

webpackShims/tinymath.js → packages/kbn-optimizer/index.d.ts

@@ -17,4 +17,4 @@
  * under the License.
  */
 
-module.exports = require('tinymath/lib/tinymath.es5.js');
+export * from './src/index';

packages/kbn-optimizer/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@kbn/optimizer",
+  "version": "1.0.0",
+  "private": true,
+  "license": "Apache-2.0",
+  "main": "./target/index.js",
+  "scripts": {
+    "build": "babel src --out-dir target --copy-files --delete-dir-on-start --extensions .ts --ignore *.test.ts --source-maps=inline",
+    "kbn:bootstrap": "yarn build",
+    "kbn:watch": "yarn build --watch"
+  },
+  "dependencies": {
+    "@babel/cli": "^7.5.5",
+    "@kbn/babel-preset": "1.0.0",
+    "@kbn/dev-utils": "1.0.0",
+    "@kbn/ui-shared-deps": "1.0.0",
+    "@types/loader-utils": "^1.1.3",
+    "@types/webpack": "^4.41.3",
+    "autoprefixer": "^9.7.4",
+    "babel-loader": "^8.0.6",
+    "clean-webpack-plugin": "^3.0.0",
+    "cpy": "^8.0.0",
+    "css-loader": "^3.4.2",
+    "del": "^5.1.0",
+    "file-loader": "^4.2.0",
+    "is-path-inside": "^3.0.2",
+    "istanbul-instrumenter-loader": "^3.0.1",
+    "jest-raw-loader": "^1.0.1",
+    "loader-utils": "^1.2.3",
+    "node-sass": "^4.13.0",
+    "postcss-loader": "^3.0.0",
+    "raw-loader": "^3.1.0",
+    "rxjs": "^6.5.3",
+    "sass-loader": "^8.0.2",
+    "script-loader": "^0.7.2",
+    "style-loader": "^1.1.3",
+    "terser-webpack-plugin": "^2.1.2",
+    "tinymath": "1.2.1",
+    "url-loader": "^2.2.0",
+    "val-loader": "^1.1.1",
+    "webpack": "^4.41.5",
+    "webpack-merge": "^4.2.2"
+  }
+}

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json

@@ -0,0 +1,4 @@
+{
+  "id": "bar",
+  "ui": true
+}

src/legacy/core_plugins/navigation/public/index.ts → packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts

@@ -17,7 +17,4 @@
  * under the License.
  */
 
-// TODO these are imports from the old plugin world.
-// Once the new platform is ready, they can get removed
-// and handled by the platform itself in the setup method
-// of the ExpressionExectorService
+export * from './lib';

src/legacy/core_plugins/inspector_views/index.js → packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/lib.ts

@@ -17,12 +17,6 @@
  * under the License.
  */
 
-import { resolve } from 'path';
-
-export default function(kibana) {
-  return new kibana.Plugin({
-    uiExports: {
-      styleSheetPaths: resolve(__dirname, 'public/index.scss'),
-    },
-  });
+export function barLibFn() {
+  return 'bar';
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/baz/kibana.json

@@ -0,0 +1,3 @@
+{
+  "id": "baz"
+}