Merge pull request #517 from takeshape/serialized-compile

Serialized compile to address #299

README.md

@@ -512,6 +512,15 @@ if you are trying to override the entry in webpack.config.js with other unsuppor
 The individual packaging needs more time at the packaging phase, but you'll
 get that paid back twice at runtime.
 
+#### Individual packaging serializedCompile
+```yaml
+# serverless.yml
+custom:
+  webpack:
+    serializedCompile: true
+```
+Will run each webpack build one at a time which helps reduce memory usage and in some cases impoves overall build performance.
+
 ## Usage
 
 ### Automatic bundling

lib/Configuration.js

@@ -14,7 +14,8 @@ const DefaultConfig = {
   packager: 'npm',
   packagerOptions: {},
   keepOutputDirectory: false,
-  config: null
+  config: null,
+  serializedCompile: false
 };
 
 class Configuration {
@@ -73,6 +74,10 @@ class Configuration {
     return this._config.keepOutputDirectory;
   }
 
+  get serializedCompile() {
+    return this._config.serializedCompile;
+  }
+
   toJSON() {
     return _.omitBy(this._config, _.isNil);
   }

lib/Configuration.test.js

@@ -19,7 +19,8 @@ describe('Configuration', () => {
         packager: 'npm',
         packagerOptions: {},
         keepOutputDirectory: false,
-        config: null
+        config: null,
+        serializedCompile: false
       };
     });
 
@@ -78,7 +79,8 @@ describe('Configuration', () => {
         packager: 'npm',
         packagerOptions: {},
         keepOutputDirectory: false,
-        config: null
+        config: null,
+        serializedCompile: false
       });
     });
   });
@@ -98,7 +100,8 @@ describe('Configuration', () => {
         packager: 'npm',
         packagerOptions: {},
         keepOutputDirectory: false,
-        config: null
+        config: null,
+        serializedCompile: false
       });
     });
 
@@ -117,7 +120,8 @@ describe('Configuration', () => {
         packager: 'npm',
         packagerOptions: {},
         keepOutputDirectory: false,
-        config: null
+        config: null,
+        serializedCompile: false
       });
     });
   });

lib/compile.js

@@ -5,44 +5,57 @@ const BbPromise = require('bluebird');
 const webpack = require('webpack');
 const tty = require('tty');
 
-module.exports = {
-  compile() {
-    this.serverless.cli.log('Bundling with Webpack...');
+const defaultStatsConfig = {
+  colors: tty.isatty(process.stdout.fd),
+  hash: false,
+  version: false,
+  chunks: false,
+  children: false
+};
 
-    const compiler = webpack(this.webpackConfig);
-
-    return BbPromise.fromCallback(cb => compiler.run(cb)).then(stats => {
-      if (!this.multiCompile) {
-        stats = { stats: [stats] };
-      }
-
-      const compileOutputPaths = [];
-      const consoleStats = this.webpackConfig.stats ||
-        _.get(this, 'webpackConfig[0].stats') || {
-        colors: tty.isatty(process.stdout.fd),
-        hash: false,
-        version: false,
-        chunks: false,
-        children: false
-      };
-
-      _.forEach(stats.stats, compileStats => {
-        const statsOutput = compileStats.toString(consoleStats);
-        if (statsOutput) {
-          this.serverless.cli.consoleLog(statsOutput);
+function ensureArray(obj) {
+  return _.isArray(obj) ? obj : [obj];
+}
+
+function getStatsLogger(statsConfig, consoleLog) {
+  return stats => consoleLog(stats.toString(statsConfig || defaultStatsConfig));
+}
+
+function webpackCompile(config, logStats) {
+  return BbPromise
+    .fromCallback(cb => webpack(config).run(cb))
+    .then(stats => {
+      // ensure stats in any array in the case of multiCompile
+      stats = stats.stats ? stats.stats : [stats];
+
+      _.forEach(stats, compileStats => {
+        logStats(compileStats);
+        if (compileStats.hasErrors()) {
+          throw new Error('Webpack compilation error, see stats above');
         }
+      });
 
-        if (compileStats.compilation.errors.length) {
-          throw new Error('Webpack compilation error, see above');
-        }
+      return stats;
+    });
+}
 
-        compileOutputPaths.push(compileStats.compilation.compiler.outputPath);
-      });
+function webpackCompileSerial(configs, logStats) {
+  return BbPromise
+    .mapSeries(configs, config => webpackCompile(config, logStats))
+    .then(stats => _.flatten(stats));
+}
 
-      this.compileOutputPaths = compileOutputPaths;
-      this.compileStats = stats;
+module.exports = {
+  compile() {
+    this.serverless.cli.log('Bundling with Webpack...');
 
-      return BbPromise.resolve();
-    });
+    const configs = ensureArray(this.webpackConfig);
+    const logStats = getStatsLogger(configs[0].stats, this.serverless.cli.consoleLog);
+
+    return (this.serializedCompile ? webpackCompileSerial : webpackCompile)(configs, logStats)
+      .then(stats => {
+        this.compileStats = { stats };
+        return BbPromise.resolve();
+      });
   }
 };

lib/validate.js

@@ -178,8 +178,9 @@ module.exports = {
 
       // In case of individual packaging we have to create a separate config for each function
       if (_.has(this.serverless, 'service.package') && this.serverless.service.package.individually) {
-        this.options.verbose && this.serverless.cli.log('Using multi-compile (individual packaging)');
         this.multiCompile = true;
+        this.serializedCompile = this.configuration.serializedCompile;
+        this.options.verbose && this.serverless.cli.log(`Using ${this.serializedCompile ? 'serialized' : 'multi'}-compile (individual packaging)`);
 
         if (this.webpackConfig.entry && !_.isEqual(this.webpackConfig.entry, entries)) {
           return BbPromise.reject(

tests/compile.test.js

@@ -44,13 +44,10 @@ describe('compile', () => {
       consoleLog: sandbox.stub()
     };
 
-    module = _.assign(
-      {
-        serverless,
-        options: {}
-      },
-      baseModule
-    );
+    module = _.assign({
+      serverless,
+      options: {},
+    }, baseModule);
   });
 
   afterEach(() => {
@@ -65,8 +62,9 @@ describe('compile', () => {
   it('should compile with webpack from a context configuration', () => {
     const testWebpackConfig = 'testconfig';
     module.webpackConfig = testWebpackConfig;
-    return expect(module.compile()).to.be.fulfilled.then(() => {
-      expect(webpackMock).to.have.been.calledWith(testWebpackConfig);
+    return expect(module.compile()).to.be.fulfilled
+    .then(() => {
+      expect(webpackMock).to.have.been.calledWith([testWebpackConfig]);
       expect(webpackMock.compilerMock.run).to.have.been.calledOnce;
       return null;
     });
@@ -80,29 +78,58 @@ describe('compile', () => {
   });
 
   it('should work with multi compile', () => {
-    const testWebpackConfig = 'testconfig';
-    const multiStats = [
-      {
+    const testWebpackConfig = ['testconfig'];
+    const multiStats = {
+      stats: [{
         compilation: {
           errors: [],
           compiler: {
-            outputPath: 'statsMock-outputPath'
-          }
+            outputPath: 'statsMock-outputPath',
+          },
         },
-        toString: sandbox.stub().returns('testStats')
-      }
-    ];
+        toString: sandbox.stub().returns('testStats'),
+        hasErrors: _.constant(false)
+      }]
+    };
     module.webpackConfig = testWebpackConfig;
     module.multiCompile = true;
     webpackMock.compilerMock.run.reset();
     webpackMock.compilerMock.run.yields(null, multiStats);
-    return expect(module.compile()).to.be.fulfilled.then(() => {
+    return expect(module.compile()).to.be.fulfilled
+    .then(() => {
       expect(webpackMock).to.have.been.calledWith(testWebpackConfig);
       expect(webpackMock.compilerMock.run).to.have.been.calledOnce;
       return null;
     });
   });
 
+  it('should work with serialized compile', () => {
+    const testWebpackConfig = ['testconfig'];
+    const multiStats = {
+      stats: [{
+        compilation: {
+          errors: [],
+          compiler: {
+            outputPath: 'statsMock-outputPath',
+          },
+        },
+        toString: sandbox.stub().returns('testStats'),
+        hasErrors: _.constant(false)
+      }]
+    };
+    module.webpackConfig = testWebpackConfig;
+    module.multiCompile = true;
+    module.serializedCompile = true;
+    webpackMock.compilerMock.run.reset();
+    webpackMock.compilerMock.run.yields(null, multiStats);
+    return expect(module.compile()).to.be.fulfilled
+      .then(() => {
+        expect(webpackMock).to.have.been.calledWith(testWebpackConfig);
+        expect(webpackMock.compilerMock.run).to.have.been.calledOnce;
+        return null;
+      });
+  });
+
   it('should use correct stats option', () => {
     const testWebpackConfig = {
       stats: 'minimal'
@@ -114,23 +141,24 @@ describe('compile', () => {
           outputPath: 'statsMock-outputPath'
         }
       },
-      toString: sandbox.stub().returns('testStats')
+      toString: sandbox.stub().returns('testStats'),
+      hasErrors: _.constant(false)
     };
 
     module.webpackConfig = testWebpackConfig;
     webpackMock.compilerMock.run.reset();
     webpackMock.compilerMock.run.yields(null, mockStats);
-    return expect(module.compile())
-      .to.be.fulfilled.then(() => {
-        expect(webpackMock).to.have.been.calledWith(testWebpackConfig);
-        expect(mockStats.toString.firstCall.args).to.eql([testWebpackConfig.stats]);
-        module.webpackConfig = [testWebpackConfig];
-        return expect(module.compile()).to.be.fulfilled;
-      })
-      .then(() => {
-        expect(webpackMock).to.have.been.calledWith([testWebpackConfig]);
-        expect(mockStats.toString.args).to.eql([ [testWebpackConfig.stats], [testWebpackConfig.stats] ]);
-        return null;
-      });
+    return (expect(module.compile()).to.be.fulfilled)
+    .then(() => {
+      expect(webpackMock).to.have.been.calledWith([testWebpackConfig]);
+      expect(mockStats.toString.firstCall.args).to.eql([testWebpackConfig.stats]);
+      module.webpackConfig = [testWebpackConfig];
+      return (expect(module.compile()).to.be.fulfilled);
+    })
+    .then(() => {
+      expect(webpackMock).to.have.been.calledWith([testWebpackConfig]);
+      expect(mockStats.toString.args).to.eql([ [testWebpackConfig.stats], [testWebpackConfig.stats] ]);
+      return null;
+    });
   });
 });

tests/webpack.mock.js

@@ -6,20 +6,24 @@ const StatsMock = () => ({
   compilation: {
     errors: [],
     compiler: {
-      outputPath: 'statsMock-outputPath'
-    }
+      outputPath: 'statsMock-outputPath',
+    },
+  },
+  toString: sinon.stub().returns('testStats'),
+  hasErrors() {
+    return Boolean(this.compilation.errors.length);
   },
-  toString: sinon.stub().returns('testStats')
 });
 
+
 const CompilerMock = (sandbox, statsMock) => ({
   run: sandbox.stub().yields(null, statsMock),
   watch: sandbox.stub().yields(null, statsMock),
   hooks: {
     beforeCompile: {
       tapPromise: sandbox.stub()
     }
-  }
+  },
 });
 
 const webpackMock = sandbox => {