From b5bda04a04fdcf690048feab3de2e8848358d6cf Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Fri, 19 Mar 2021 13:58:10 -0700 Subject: [PATCH] refactor(bundle-source): Isolate transform (#2685) This isolates a transformSource function so that it can be applied for the Endo Base-64 Zip bundle format in a subsequent change. The change reveals that the transformSource function is necessarilly async--it uses the Mozilla SourceMapConsumer, which in turn relies on a pool of workers for performance. This in turn suggests the next phase of work will be adding support for asynchronous transforms to the Endo archiver in the Compartment Mapper. Refactoring revealed an opportunity to parallelize transforms, which might better leverage the source map worker pool, and also revealed a resource leak since we did not previously release source map workers. --- packages/bundle-source/src/index.js | 203 ++++++++++++++++------------ 1 file changed, 115 insertions(+), 88 deletions(-) diff --git a/packages/bundle-source/src/index.js b/packages/bundle-source/src/index.js index cbcfc542ea3..6c46fd3832e 100644 --- a/packages/bundle-source/src/index.js +++ b/packages/bundle-source/src/index.js @@ -36,6 +36,102 @@ export function tildotPlugin() { }, }; } + +function rewriteComment(node, unmapLoc) { + node.type = 'CommentBlock'; + // Within comments... + node.value = node.value + // ...strip extraneous comment whitespace + .replace(/^\s+/gm, ' ') + // ...replace HTML comments with a defanged version to pass SES restrictions. + .replace(HTML_COMMENT_START_RE, '') + // ...replace import expressions with a defanged version to pass SES restrictions. + .replace(IMPORT_RE, 'X$1$2') + // ...replace end-of-comment markers + .replace(/\*\//g, '*X/'); + if (unmapLoc) { + unmapLoc(node.loc); + } + // console.log(JSON.stringify(node, undefined, 2)); +} + +async function makeLocationUnmapper({ sourceMap, ast }) { + // We rearrange the rolled-up chunk according to its sourcemap to move + // its source lines back to the right place. + // eslint-disable-next-line no-await-in-loop + const consumer = await new SourceMapConsumer(sourceMap); + try { + const unmapped = new WeakSet(); + let lastPos = { ...ast.loc.start }; + return loc => { + if (!loc || unmapped.has(loc)) { + return; + } + // Make sure things start at least at the right place. + loc.end = { ...loc.start }; + for (const pos of ['start', 'end']) { + if (loc[pos]) { + const newPos = consumer.originalPositionFor(loc[pos]); + if (newPos.source !== null) { + lastPos = { + line: newPos.line, + column: newPos.column, + }; + } + loc[pos] = lastPos; + } + } + unmapped.add(loc); + }; + } finally { + consumer.destroy(); + } +} + +function transformAst(ast, unmapLoc) { + babelTraverse(ast, { + enter(p) { + const { loc, leadingComments, trailingComments } = p.node; + if (p.node.comments) { + p.node.comments = []; + } + // Rewrite all comments. + (leadingComments || []).forEach(node => rewriteComment(node, unmapLoc)); + if (p.node.type.startsWith('Comment')) { + rewriteComment(p.node, unmapLoc); + } + // If not a comment, and we are unmapping the source maps, + // then do it for this location. + if (unmapLoc) { + unmapLoc(loc); + } + (trailingComments || []).forEach(node => rewriteComment(node, unmapLoc)); + }, + }); +} + +async function transformSource(code, { sourceMap, useLocationUnmap }) { + // Parse the rolled-up chunk with Babel. + // We are prepared for different module systems. + const ast = (babelParser.parse || babelParser)(code, { + plugins: ['bigInt'], + }); + + let unmapLoc; + if (useLocationUnmap) { + unmapLoc = await makeLocationUnmapper({ + sourceMap, + ast, + }); + } + + transformAst(ast, unmapLoc); + + // Now generate the sources with the new positions. + return babelGenerate(ast, { retainLines: true }); +} + /** @type {BundleSource} */ export default async function bundleSource( startFilename, @@ -86,97 +182,28 @@ export default async function bundleSource( // Create a source bundle. const sourceBundle = {}; let entrypoint; - for (const chunk of output) { - if (chunk.isAsset) { - throw Error(`unprepared for assets: ${chunk.fileName}`); - } - const { code, fileName, isEntry } = chunk; - if (isEntry) { - entrypoint = fileName; - } - - // Parse the rolled-up chunk with Babel. - // We are prepared for different module systems. - const ast = (babelParser.parse || babelParser)(code, { - plugins: ['bigInt'], - }); - - let unmapLoc; - if ( - moduleFormat === 'nestedEvaluate' && - !fileName.startsWith('_virtual/') - ) { - // We rearrange the rolled-up chunk according to its sourcemap to move - // its source lines back to the right place. - // eslint-disable-next-line no-await-in-loop - const consumer = await new SourceMapConsumer(chunk.map); - const unmapped = new WeakSet(); - let lastPos = { ...ast.loc.start }; - unmapLoc = loc => { - if (!loc || unmapped.has(loc)) { - return; - } - // Make sure things start at least at the right place. - loc.end = { ...loc.start }; - for (const pos of ['start', 'end']) { - if (loc[pos]) { - const newPos = consumer.originalPositionFor(loc[pos]); - if (newPos.source !== null) { - lastPos = { - line: newPos.line, - column: newPos.column, - }; - } - loc[pos] = lastPos; - } - } - unmapped.add(loc); - }; - } - - const rewriteComment = node => { - node.type = 'CommentBlock'; - // Within comments... - node.value = node.value - // ...strip extraneous comment whitespace - .replace(/^\s+/gm, ' ') - // ...replace HTML comments with a defanged version to pass SES restrictions. - .replace(HTML_COMMENT_START_RE, '') - // ...replace import expressions with a defanged version to pass SES restrictions. - .replace(IMPORT_RE, 'X$1$2') - // ...replace end-of-comment markers - .replace(/\*\//g, '*X/'); - if (unmapLoc) { - unmapLoc(node.loc); + await Promise.all( + output.map(async chunk => { + if (chunk.isAsset) { + throw Error(`unprepared for assets: ${chunk.fileName}`); + } + const { code, fileName, isEntry } = chunk; + if (isEntry) { + entrypoint = fileName; } - // console.log(JSON.stringify(node, undefined, 2)); - }; - babelTraverse(ast, { - enter(p) { - const { loc, leadingComments, trailingComments } = p.node; - if (p.node.comments) { - p.node.comments = []; - } - // Rewrite all comments. - (leadingComments || []).forEach(rewriteComment); - if (p.node.type.startsWith('Comment')) { - rewriteComment(p.node); - } - // If not a comment, and we are unmapping the source maps, - // then do it for this location. - if (unmapLoc) { - unmapLoc(loc); - } - (trailingComments || []).forEach(rewriteComment); - }, - }); + const useLocationUnmap = + moduleFormat === 'nestedEvaluate' && !fileName.startsWith('_virtual/'); - // Now generate the sources with the new positions. - sourceBundle[fileName] = babelGenerate(ast, { retainLines: true }).code; - // console.log(`==== sourceBundle[${fileName}]\n${sourceBundle[fileName]}\n====`); - } + const { code: transformedCode } = await transformSource(code, { + sourceMap: chunk.map, + useLocationUnmap, + }); + sourceBundle[fileName] = transformedCode; + + // console.log(`==== sourceBundle[${fileName}]\n${sourceBundle[fileName]}\n====`); + }), + ); if (!entrypoint) { throw Error('No entrypoint found in output bundle');