Skip to content

Commit

Permalink
fix(packager): implement better way to bundle actions
Browse files Browse the repository at this point in the history
fixes #966, fixes #672, fixes #589
  • Loading branch information
tripodsan committed Jun 13, 2019
1 parent bc13466 commit c2adb80
Show file tree
Hide file tree
Showing 7 changed files with 314 additions and 237 deletions.
9 changes: 9 additions & 0 deletions src/deploy.cmd.js
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class DeployCommand extends StaticCommand {
this._dryRun = false;
this._createPackages = 'auto';
this._addStrain = null;
this._enableMinify = null;
}

get requireConfigFile() {
Expand Down Expand Up @@ -133,6 +134,11 @@ class DeployCommand extends StaticCommand {
return this;
}

withMinify(value) {
this._enableMinify = value;
return this;
}

actionName(script) {
if (script.main.indexOf(path.resolve(__dirname, 'openwhisk')) === 0) {
return `hlx--${script.name}`;
Expand Down Expand Up @@ -330,6 +336,9 @@ Alternatively you can auto-add one using the {grey --add <name>} option.`);
.withTarget(this._target)
.withDirectory(this.directory)
.withOnlyModified(this._createPackages === 'auto');
if (this._enableMinify !== null) {
pgkCommand.withMinify(this._enableMinify);
}
await pgkCommand.run();
}

Expand Down
151 changes: 67 additions & 84 deletions src/package.cmd.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,22 @@
* OF ANY KIND, either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/

'use strict';

const chalk = require('chalk');
const glob = require('glob');
const path = require('path');
const fs = require('fs-extra');
const ProgressBar = require('progress');
const archiver = require('archiver');
const StaticCommand = require('./static.cmd.js');
const packageCfg = require('./parcel/packager-config.js');
const ExternalsCollector = require('./parcel/ExternalsCollector.js');
const ActionBundler = require('./parcel/ActionBundler.js');
const { flattenDependencies } = require('./packager-utils.js');

class PackageCommand extends StaticCommand {
constructor(logger) {
super(logger);
this._target = null;
this._onlyModified = false;
this._enableMinify = true;
}

// eslint-disable-next-line class-methods-use-this
Expand All @@ -45,6 +42,11 @@ class PackageCommand extends StaticCommand {
return this;
}

withMinify(value) {
this._enableMinify = value;
return this;
}

async init() {
await super.init();
this._target = path.resolve(this.directory, this._target);
Expand Down Expand Up @@ -78,7 +80,6 @@ class PackageCommand extends StaticCommand {
};

return new Promise((resolve, reject) => {
const ticks = {};
const archiveName = path.basename(info.zipFile);
let hadErrors = false;

Expand All @@ -98,9 +99,7 @@ class PackageCommand extends StaticCommand {
});
archive.on('entry', (data) => {
log.debug(`${archiveName}: A ${data.name}`);
if (ticks[data.name]) {
tick('', data.name);
}
tick('', data.name);
});
archive.on('warning', (err) => {
log.error(`${chalk.redBright('[error] ')}Unable to create archive: ${err.message}`);
Expand All @@ -112,6 +111,7 @@ class PackageCommand extends StaticCommand {
hadErrors = true;
reject(err);
});
archive.pipe(output);

const packageJson = {
name: info.name,
Expand All @@ -121,44 +121,41 @@ class PackageCommand extends StaticCommand {
license: 'Apache-2.0',
};

archive.pipe(output);
archive.append(JSON.stringify(packageJson, null, ' '), { name: 'package.json' });
archive.file(info.bundlePath, { name: path.basename(info.main) });
archive.finalize();
});
}

info.files.forEach((file) => {
const name = path.basename(file);
archive.file(path.resolve(this._target, file), { name });
ticks[name] = true;
});

// add modules that cause problems when embeded in webpack
Object.keys(info.externals).forEach((mod) => {
// if the module was linked via `npm link`, then it is a checked-out module, and should
// not be included as-is.
const modPath = info.externals[mod];
if (modPath.indexOf('/node_modules/') < 0 && modPath.indexOf('\\node_modules\\') < 0) {
// todo: async
// todo: read .npmignore
const files = [
...glob.sync('!(.git|node_modules|logs|docs|coverage)/**', {
cwd: modPath,
matchBase: false,
}),
...glob.sync('*', {
cwd: modPath,
matchBase: false,
nodir: true,
})];
files.forEach((name) => {
archive.file(path.resolve(modPath, name), { name: `node_modules/${mod}/${name}` });
});
} else {
archive.directory(modPath, `node_modules/${mod}`);
ticks[`node_modules/${mod}/package.json`] = true;
}
});
/**
* Creates the action bundles
* @param {*[]} scripts the scripts information
* @param {ProgressBar} bar the progress bar
*/
async createBundles(scripts, bar) {
const progressHandler = (percent, msg, ...args) => {
const action = msg === 'building' ? `bundling ${args[0]}` : msg;
bar.update(percent * 0.8, { action });
};

archive.finalize();
// create the bundles
const bundler = new ActionBundler()
.withDirectory(this._target)
.withModulePaths(['node_modules', path.resolve(__dirname, '..', 'node_modules')])
.withLogger(this.log)
.withProgressHandler(progressHandler)
.withMinify(this._enableMinify);
const files = {};
scripts.forEach((script) => {
files[script.name] = path.resolve(this._target, script.main);
});
const stats = await bundler.run(files);
if (stats.errors) {
stats.errors.forEach(this.log.error);
}
if (stats.warnings) {
stats.warnings.forEach(this.log.warn);
}
}

async run() {
Expand Down Expand Up @@ -196,52 +193,38 @@ class PackageCommand extends StaticCommand {
scripts = scripts.filter(script => !script.zipFile);
}

// generate additional infos
scripts.forEach((script) => {
/* eslint-disable no-param-reassign */
script.name = path.basename(script.main, '.js');
script.dirname = script.isStatic ? '' : path.dirname(script.main);
script.archiveName = `${script.name}.zip`;
script.zipFile = path.resolve(this._target, script.dirname, script.archiveName);
script.infoFile = path.resolve(this._target, script.dirname, `${script.name}.info.json`);
/* eslint-enable no-param-reassign */
});

const bar = new ProgressBar('[:bar] :action :etas', {
total: scripts.length * 2,
width: 50,
renderThrottle: 1,
stream: process.stdout,
});

// collect all the external modules of the scripts
let steps = 0;
await Promise.all(scripts.map(async (script) => {
const collector = new ExternalsCollector()
.withDirectory(this._target)
.withExternals(Object.keys(packageCfg.externals));

// eslint-disable-next-line no-param-reassign
script.files = [script.main, ...script.requires].map(f => path.resolve(this._target, f));
bar.tick(1, {
action: `analyzing ${path.basename(script.main)}`,
if (scripts.length > 0) {
// generate additional infos
scripts.forEach((script) => {
/* eslint-disable no-param-reassign */
script.name = path.basename(script.main, '.js');
script.bundleName = `${script.name}.bundle.js`;
script.bundlePath = path.resolve(this._target, script.bundleName);
script.dirname = script.isStatic ? '' : path.dirname(script.main);
script.archiveName = `${script.name}.zip`;
script.zipFile = path.resolve(this._target, script.dirname, script.archiveName);
script.infoFile = path.resolve(this._target, script.dirname, `${script.name}.info.json`);
/* eslint-enable no-param-reassign */
});
// eslint-disable-next-line no-param-reassign
script.externals = await collector.collectModules(script.files);
steps += Object.keys(script.externals).length + script.files.length;
bar.tick(1, {
action: `analyzing ${path.basename(script.main)}`,

// we reserve 80% for bundling the scripts and 20% for creating the zip files.
const bar = new ProgressBar('[:bar] :action :elapseds', {
total: scripts.length * 2 * 5,
width: 50,
renderThrottle: 1,
stream: process.stdout,
});
}));

// trigger new progress bar
bar.total += steps;
// create bundles
await this.createBundles(scripts, bar);

// package actions
await Promise.all(scripts.map(script => this.createPackage(script, bar)));
// package actions
await Promise.all(scripts.map(script => this.createPackage(script, bar)));

// write back the updated infos
await Promise.all(scripts.map(script => fs.writeJson(script.infoFile, script, { spaces: 2 })));
// write back the updated infos
// eslint-disable-next-line max-len
await Promise.all(scripts.map(script => fs.writeJson(script.infoFile, script, { spaces: 2 })));
}

this.log.info('✅ packaging completed');
return this;
Expand Down
144 changes: 144 additions & 0 deletions src/parcel/ActionBundler.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
/*
* Copyright 2019 Adobe. All rights reserved.
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. You may obtain a copy
* of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
* OF ANY KIND, either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
const webpack = require('webpack');

/**
* Helper class that packs the script into 1 bundle using webpack.
*/
class ActionBundler {
constructor() {
this._cwd = process.cwd();
this._modulesPaths = ['node_modules'];
this._minify = true;
this._logger = console;
this._progressHandler = null;
}

withDirectory(d) {
this._cwd = d;
return this;
}

withLogger(value) {
this._logger = value;
return this;
}

withModulePaths(value) {
this._modulesPaths = value;
return this;
}

withMinify(value) {
this._minify = value;
return this;
}

withProgressHandler(value) {
this._progressHandler = value;
return this;
}

async run(files) {
const options = {
target: 'node',
mode: 'none',
entry: files,
cache: true,
output: {
path: this._cwd,
filename: '[name].bundle.js',
library: 'main',
libraryTarget: 'umd',
},
externals: [],
resolve: {
modules: this._modulesPaths,
},
devtool: false,
optimization: {
minimize: this._minify,
},
plugins: [],
};

if (this._progressHandler) {
options.plugins.push(new webpack.ProgressPlugin(this._progressHandler));
}

const compiler = webpack(options);

const ignoredWarnings = [{
message: /Critical dependency: the request of a dependency is an expression/,
resource: '/@babel/core/lib/config/files/configuration.js',
}, {
message: /Critical dependency: the request of a dependency is an expression/,
resource: '/@babel/core/lib/config/files/plugins.js',
}, {
message: /Critical dependency: the request of a dependency is an expression/,
resource: '/@babel/core/lib/config/files/plugins.js',
}, {
message: /Critical dependency: require function is used in a way in which dependencies cannot be statically extracted/,
resource: '/@adobe/htlengine/src/compiler/Compiler.js',
}, {
message: /Critical dependency: the request of a dependency is an expression/,
resource: '/@adobe/htlengine/src/runtime/Runtime.js',
}, {
message: /^Module not found: Error: Can't resolve 'bufferutil'/,
resource: '/ws/lib/buffer-util.js',
}, {
message: /^Module not found: Error: Can't resolve 'utf-8-validate'/,
resource: '/ws/lib/validation.js',
}];

const ignoredErrors = [{
message: /^Module not found: Error: Can't resolve 'canvas'/,
resource: '/jsdom/lib/jsdom/utils.js',
}];

const matchWarning = rules => (w) => {
const msg = w.message;
const res = w.module.resource;
return !rules.find((r) => {
if (!res.endsWith(r.resource)) {
return false;
}
return r.message.test(msg);
});
};

return new Promise((resolve, reject) => {
compiler.run((err, stats) => {
if (err) {
reject(err);
return;
}

// filter out the expected warnings and errors
// eslint-disable-next-line no-param-reassign,max-len
stats.compilation.warnings = stats.compilation.warnings.filter(matchWarning(ignoredWarnings));
// eslint-disable-next-line no-param-reassign
stats.compilation.errors = stats.compilation.errors.filter(matchWarning(ignoredErrors));

if (stats.hasErrors() || stats.hasWarnings()) {
resolve(stats.toJson({
errorDetails: false,
}));
return;
}
resolve({});
});
});
}
}

module.exports = ActionBundler;
Loading

0 comments on commit c2adb80

Please sign in to comment.