Merge pull request #681 from vicary/feat/concurrency-5.4.0

Add concurrency support for more than one thread

.gitignore

@@ -1,4 +1,5 @@
 node_modules
+node_modules.nosync
 dist
 .webpack
 .serverless

README.md

@@ -541,12 +541,13 @@ if you are trying to override the entry in webpack.config.js with other unsuppor
 The individual packaging needs more time at the packaging phase, but you'll
 get that paid back twice at runtime.
 
-#### Individual packaging serializedCompile
+#### Individual packaging concurrency
 ```yaml
 # serverless.yml
 custom:
   webpack:
-    serializedCompile: true
+    concurrency: 5          # desired concurrency, defaults to the number of available cores
+    serializedCompile: true # backward compatible, this translates to concurrency: 1
 ```
 Will run each webpack build one at a time which helps reduce memory usage and in some cases impoves overall build performance.
 

lib/Configuration.js

@@ -4,6 +4,7 @@
  */
 
 const _ = require('lodash');
+const os = require('os');
 
 /**
  * Plugin defaults
@@ -15,7 +16,7 @@ const DefaultConfig = {
   packagerOptions: {},
   keepOutputDirectory: false,
   config: null,
-  serializedCompile: false
+  concurrency: os.cpus().length
 };
 
 class Configuration {
@@ -38,6 +39,21 @@ class Configuration {
       }
     }
 
+    // Concurrency may be passed via CLI, e.g.
+    // custom:
+    //   webpack:
+    //     concurrency: ${opt:compile-concurrency, 7}
+    // In this case it is typed as a string and we have to validate it
+    if (this._config.concurrency !== undefined) {
+      this._config.concurrency = Number(this._config.concurrency);
+      if (isNaN(this._config.concurrency) || this._config.concurrency < 1) {
+        throw new Error('concurrency option must be a positive number');
+      }
+    } else if (this._config.serializedCompile === true) {
+      // Backwards compatibility with serializedCompile setting
+      this._config.concurrency = 1;
+    }
+
     // Set defaults for all missing properties
     _.defaults(this._config, DefaultConfig);
   }
@@ -53,7 +69,7 @@ class Configuration {
   get excludeFiles() {
     return this._config.excludeFiles;
   }
-  
+
   get excludeRegex() {
     return this._config.excludeRegex;
   }
@@ -78,8 +94,8 @@ class Configuration {
     return this._config.keepOutputDirectory;
   }
 
-  get serializedCompile() {
-    return this._config.serializedCompile;
+  get concurrency() {
+    return this._config.concurrency;
   }
 
   toJSON() {

lib/Configuration.test.js

@@ -3,6 +3,7 @@
  * Unit tests for Configuration.
  */
 
+const os = require('os');
 const chai = require('chai');
 const Configuration = require('./Configuration');
 
@@ -20,7 +21,7 @@ describe('Configuration', () => {
         packagerOptions: {},
         keepOutputDirectory: false,
         config: null,
-        serializedCompile: false
+        concurrency: os.cpus().length
       };
     });
 
@@ -70,7 +71,7 @@ describe('Configuration', () => {
         packagerOptions: {},
         keepOutputDirectory: false,
         config: null,
-        serializedCompile: false
+        concurrency: os.cpus().length
       });
     });
   });
@@ -91,7 +92,7 @@ describe('Configuration', () => {
         packagerOptions: {},
         keepOutputDirectory: false,
         config: null,
-        serializedCompile: false
+        concurrency: os.cpus().length
       });
     });
 
@@ -111,8 +112,52 @@ describe('Configuration', () => {
         packagerOptions: {},
         keepOutputDirectory: false,
         config: null,
-        serializedCompile: false
+        concurrency: os.cpus().length
       });
     });
+
+    it('should accept a numeric string as concurrency value', () => {
+      const testCustom = {
+        webpack: {
+          includeModules: { forceInclude: ['mod1'] },
+          webpackConfig: 'myWebpackFile.js',
+          concurrency: '3'
+        }
+      };
+      const config = new Configuration(testCustom);
+      expect(config.concurrency).to.equal(3);
+    });
+
+    it('should not accept an invalid string as concurrency value', () => {
+      const testCustom = {
+        webpack: {
+          includeModules: { forceInclude: ['mod1'] },
+          webpackConfig: 'myWebpackFile.js',
+          concurrency: '3abc'
+        }
+      };
+      expect(() => new Configuration(testCustom)).throws();
+    });
+
+    it('should not accept a non-positive number as concurrency value', () => {
+      const testCustom = {
+        webpack: {
+          includeModules: { forceInclude: ['mod1'] },
+          webpackConfig: 'myWebpackFile.js',
+          concurrency: 0
+        }
+      };
+      expect(() => new Configuration(testCustom)).throws();
+    });
+
+    it('should be backward compatible with serializedCompile', () => {
+      const testCustom = {
+        webpack: {
+          serializedCompile: true
+        }
+      };
+      const config = new Configuration(testCustom);
+      expect(config.concurrency).to.equal(1);
+    });
   });
 });

lib/compile.js

@@ -22,27 +22,25 @@ function getStatsLogger(statsConfig, consoleLog) {
 }
 
 function webpackCompile(config, logStats) {
-  return BbPromise
-    .fromCallback(cb => webpack(config).run(cb))
-    .then(stats => {
-      // ensure stats in any array in the case of multiCompile
-      stats = stats.stats ? stats.stats : [stats];
-
-      _.forEach(stats, compileStats => {
-        logStats(compileStats);
-        if (compileStats.hasErrors()) {
-          throw new Error('Webpack compilation error, see stats above');
-        }
-      });
-
-      return stats;
+  return BbPromise.fromCallback(cb => webpack(config).run(cb)).then(stats => {
+    // ensure stats in any array in the case of concurrent build.
+    stats = stats.stats ? stats.stats : [stats];
+
+    _.forEach(stats, compileStats => {
+      logStats(compileStats);
+      if (compileStats.hasErrors()) {
+        throw new Error('Webpack compilation error, see stats above');
+      }
     });
+
+    return stats;
+  });
 }
 
-function webpackCompileSerial(configs, logStats) {
-  return BbPromise
-    .mapSeries(configs, config => webpackCompile(config, logStats))
-    .then(stats => _.flatten(stats));
+function webpackConcurrentCompile(configs, logStats, concurrency) {
+  return BbPromise.map(configs, config => webpackCompile(config, logStats), { concurrency }).then(stats =>
+    _.flatten(stats)
+  );
 }
 
 module.exports = {
@@ -52,10 +50,14 @@ module.exports = {
     const configs = ensureArray(this.webpackConfig);
     const logStats = getStatsLogger(configs[0].stats, this.serverless.cli.consoleLog);
 
-    return (this.serializedCompile ? webpackCompileSerial : webpackCompile)(configs, logStats)
-      .then(stats => {
-        this.compileStats = { stats };
-        return BbPromise.resolve();
-      });
+    if (!this.configuration) {
+      return BbPromise.reject('Missing plugin configuration');
+    }
+    const concurrency = this.configuration.concurrency;
+
+    return webpackConcurrentCompile(configs, logStats, concurrency).then(stats => {
+      this.compileStats = { stats };
+      return BbPromise.resolve();
+    });
   }
 };

lib/validate.js

@@ -189,11 +189,9 @@ module.exports = {
 
       // In case of individual packaging we have to create a separate config for each function
       if (_.has(this.serverless, 'service.package') && this.serverless.service.package.individually) {
-        this.multiCompile = true;
-        this.serializedCompile = this.configuration.serializedCompile;
         this.options.verbose &&
           this.serverless.cli.log(
-            `Using ${this.serializedCompile ? 'serialized' : 'multi'}-compile (individual packaging)`
+            `Individually packaging with concurrency at ${this.configuration.concurrency} entries a time.`
           );
 
         if (this.webpackConfig.entry && !_.isEqual(this.webpackConfig.entry, entries)) {