Merge pull request #114 from mikrostew/parallel-babel

Parallel babel transpilation

README.md

@@ -99,3 +99,137 @@ In order to aid plugin developers in this process, broccoli-babel-transpiler wil
   `babel-plugin-htmlbars-inline-precompile` (uses `ember-template-compiler.js` to compile inlined templates).
 * `baseDir` - This method is expected to return the plugins base dir. The provided `baseDir` is used to ensure the cache is invalidated if any of the
   plugin's files change (including its deps). Each plugin should implement `baseDir` as: `Plugin.prototype.baseDir = function() { return \_\_dirname; };`.
+
+## Parallel Transpilation
+
+broccoli-babel-transpiler can run multiple babel transpiles in parallel using a pool of workers, to take advantage of multi-core systems.
+Because these workers are separate processes, the plugins and callback functions that are normally passed as options to babel must be specified in a serializable form.
+To enable this parallelization there is an API to tell the worker how to construct the plugin or callback in its process.
+
+Plugins are specified as an object with a `_parallelBabel` property:
+
+```js
+var plugin = {
+  _parallelBabel: {
+    requireFile: '/full/path/to/the/file',
+    useMethod: 'methodName',
+    buildUsing: 'buildFunction',
+    params: { ok: 'this object will be passed to buildFunction()' }
+  }
+};
+```
+
+Callbacks can be specified like plugins, or as functions with a `_parallelBabel` property:
+
+```js
+var callback = function() { /* do something */ };
+callback._parallelBabel = {
+  requireFile: '/full/path/to/the/file',
+  useMethod: 'methodName',
+  buildUsing: 'buildFunction',
+  params: { ok: 'this object will be passed to buildFunction()' }
+};
+```
+
+### requireFile (required)
+
+This property specifies the file to require in the worker process to create the plugin or callback.
+This must be given as an absolute path.
+
+```js
+var esTranspiler = require('broccoli-babel-transpiler');
+
+var somePlugin = {
+  _parallelBabel: {
+    requireFile: '/full/path/to/the/file'
+  }
+});
+
+var scriptTree = esTranspiler(inputTree, {
+  plugins: [
+    'transform-strict-mode', // plugins that are given as strings will automatically be parallelized
+    somePlugin
+  ]
+});
+```
+
+### useMethod (optional)
+
+This property specifies the method to use from the file that is required.
+
+If you have a plugin defined like this:
+
+```js
+// some_plugin.js
+
+module.exports = {
+  pluginFunction(babel) {
+    // do plugin things
+  }
+};
+```
+
+You can tell broccoli-babel-transpiler to use that function in the worker processes like so:
+
+```js
+var esTranspiler = require('broccoli-babel-transpiler');
+
+var somePlugin = {
+  _parallelBabel: {
+    requireFile: '/path/to/some_plugin',
+    useMethod: 'pluginFunction'
+  }
+});
+
+var scriptTree = esTranspiler(inputTree, {
+  plugins: [ somePlugin ]
+});
+```
+
+### buildUsing and params (optional)
+
+These properties specify a function to run to build the plugin (or callback), and any parameters
+to pass to that function.
+
+If the plugin needs to be built dynamically, you can do that like so:
+
+```js
+// some_plugin.js
+
+module.exports = {
+  buildPlugin(params) {
+    var pluginInstance = doSomethingWith(params.text);
+    return pluginInstance;
+  }
+};
+```
+
+This will tell the worker process to require the plugin and call the `buildPlugin` function with the `params` object as an argument:
+
+```js
+var esTranspiler = require('broccoli-babel-transpiler');
+
+var somePlugin = {
+  _parallelBabel: {
+    requireFile: '/path/to/some_plugin',
+    buildUsing: 'buildPlugin',
+    params: { text: 'some text' }
+  }
+});
+
+var scriptTree = esTranspiler(inputTree, {
+  plugins: [ somePlugin ]
+});
+```
+
+Note: If both `useMethod` and `buildUsing` are specified, `useMethod` takes precedence.
+
+### Number of jobs
+
+The number of parallel jobs defaults to the number of detected CPUs - 1.
+
+This can be changed with the `JOBS` environment variable:
+
+```
+JOBS=4 ember build
+```

expectations/comments.js

@@ -0,0 +1 @@
+//comment 1

expectations/imports-getModuleId.js

@@ -0,0 +1,19 @@
+define('testModule', ['exports', './fixtures-classes'], function (exports, _fixturesClasses) {
+  'use strict';
+
+  Object.defineProperty(exports, "__esModule", {
+    value: true
+  });
+  exports.FooTwo = undefined;
+
+  var _fixturesClasses2 = _interopRequireDefault(_fixturesClasses);
+
+  function _interopRequireDefault(obj) {
+    return obj && obj.__esModule ? obj : {
+      default: obj
+    };
+  }
+
+  class FooTwo extends _fixturesClasses2.default {}
+  exports.FooTwo = FooTwo;
+});

expectations/imports.js

@@ -0,0 +1,5 @@
+'use strict';
+
+import Foo from 'fixtures-classes';
+
+export class FooTwo extends Foo {}

fixtures/amd-name-resolver-parallel/index.js

@@ -0,0 +1,6 @@
+"use strict";
+
+module.exports = {
+  name: 'amd-name-resolver',
+  moduleResolve: require('amd-name-resolver').moduleResolve,
+};

fixtures/errors/fixtures.js

@@ -0,0 +1 @@
+const foo;

fixtures/file/fixtures.js

@@ -0,0 +1 @@
+const x = 0;

fixtures/files/fixtures-comments.js

@@ -0,0 +1,2 @@
+//comment 1
+//another comment

fixtures/files/fixtures-imports.js

@@ -0,0 +1,3 @@
+import Foo from './fixtures-classes';
+
+export class FooTwo extends Foo {}

fixtures/get-module-id-parallel/index.js

@@ -0,0 +1,7 @@
+"use strict";
+
+module.exports = {
+  build(options) {
+    return (moduleName) => options.name;
+  }
+};

fixtures/print-comment-parallel/index.js

@@ -0,0 +1,7 @@
+"use strict";
+
+module.exports = {
+  buildMe(options) {
+    return (comment) => comment === options.contents;
+  }
+};

fixtures/transform-es2015-block-scoping-parallel/index.js

@@ -0,0 +1,15 @@
+"use strict";
+
+module.exports = {
+  name: 'transform-es2015-block-scoping',
+
+  pluginFunction: require('babel-plugin-transform-es2015-block-scoping'),
+
+  build() {
+    return 'transform-es2015-block-scoping';
+  },
+
+  buildTwo(options) {
+    return 'for-testing' + options.text;
+  }
+};

fixtures/transform-strict-mode-parallel/index.js

@@ -0,0 +1,3 @@
+"use strict";
+
+module.exports = 'transform-strict-mode';

fixtures/transform-strict-mode-process-exit/index.js

@@ -0,0 +1,13 @@
+"use strict";
+
+var pluginFunction = require('babel-plugin-transform-strict-mode');
+
+module.exports = {
+  name: 'transform-strict-mode',
+
+  buildMeAFunction(options) {
+    return function() {
+      process.exit(1);
+    };
+  }
+};

index.js

@@ -1,15 +1,15 @@
 'use strict';
 
-var transpiler = require('babel-core');
 var Filter     = require('broccoli-persistent-filter');
 var clone      = require('clone');
-var path       = require('path');
 var fs         = require('fs');
 var stringify  = require('json-stable-stringify');
 var mergeTrees = require('broccoli-merge-trees');
 var funnel     = require('broccoli-funnel');
 var crypto     = require('crypto');
 var hashForDep = require('hash-for-dep');
+var ParallelApi = require('./lib/parallel-api');
+
 
 function getExtensionsRegex(extensions) {
   return extensions.map(function(extension) {
@@ -32,9 +32,11 @@ function Babel(inputTree, _options) {
 
   var options = _options || {};
   options.persist = 'persist' in options ? options.persist : true;
+  options.async = true;
   Filter.call(this, inputTree, options);
 
   delete options.persist;
+  delete options.async;
   delete options.annotation;
   delete options.description;
 
@@ -75,7 +77,7 @@ Babel.prototype.baseDir = function() {
 };
 
 Babel.prototype.transform = function(string, options) {
-  return transpiler.transform(string, options);
+  return ParallelApi.transformString(string, options);
 };
 
 /*
@@ -174,17 +176,20 @@ Babel.prototype.processString = function(string, relativePath) {
     options.moduleId = replaceExtensions(this.extensionsRegex, options.filename);
   }
 
-  var transpiled = this.transform(string, options);
+  var plugin = this;
+  return this.transform(string, options)
+  .then(function (transpiled) {
 
-  if (this.helperWhiteList) {
-    var invalidHelpers = transpiled.metadata.usedHelpers.filter(function(helper) {
-      return this.helperWhiteList.indexOf(helper) === -1;
-    }, this);
+    if (plugin.helperWhiteList) {
+      var invalidHelpers = transpiled.metadata.usedHelpers.filter(function(helper) {
+        return plugin.helperWhiteList.indexOf(helper) === -1;
+      }, plugin);
 
-    validateHelpers(invalidHelpers, relativePath);
-  }
+      validateHelpers(invalidHelpers, relativePath);
+    }
 
-  return transpiled.code;
+    return transpiled.code;
+  });
 };
 
 Babel.prototype.copyOptions = function() {
@@ -210,7 +215,7 @@ function validateHelpers(invalidHelpers, relativePath) {
         return item + '`, `';
       }).join('');
 
-      message = relativePath + ' was transformed and relies on `' + helpers + '`, which were not included in the helper whitelist. Either add these helpers to the whitelist or refactor to not be dependent on these runtime helper.';
+      message = relativePath + ' was transformed and relies on `' + helpers + '`, which were not included in the helper whitelist. Either add these helpers to the whitelist or refactor to not be dependent on these runtime helpers.';
     }
 
     throw new Error(message);