Support fs static eval and encoding

.circleci/config.yml

@@ -0,0 +1,17 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/node:10
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - run:
+          name: Installing dependencies
+          command: yarn install
+      # - run:
+      #     name: Linting
+      #     command: yarn lint
+      - run:
+          name: Tests
+          command: yarn test

.gitignore

@@ -0,0 +1 @@
+node_modules

fs-inliner.js

@@ -3,6 +3,7 @@ const fs = require('fs');
 const { walk } = require('estree-walker');
 const MagicString = require('magic-string');
 const { attachScopes } = require('rollup-pluginutils');
+const evaluate = require('static-eval');
 
 // Very basic first-pass fs.readFileSync inlining
 function isReference(node, parent) {
@@ -20,23 +21,6 @@ function isReference(node, parent) {
 	return true;
 }
 
-/*
- * For now we only support one expression type:
- * __dirname + '/asset.name'
- * 
- */
-function extractStaticFileSource (expr, id, scope) {
-  if (expr.type === 'BinaryExpression' &&
-      expr.left.type === 'Identifier' &&
-      expr.left.name === '__dirname' &&
-      !scope.contains('__dirname') &&
-      expr.right.type === 'Literal') {
-    const assetPath = path.resolve(id, '..', expr.right.value.substr(1));
-    const assetSource = fs.readFileSync(assetPath).toString('base64');
-    return 'Buffer.from("' + assetSource + '", "base64")';
-  }
-}
-
 exports.transform = function (code, id) {
   if (code.indexOf('readFileSync') === -1)
     return;
@@ -46,45 +30,135 @@ exports.transform = function (code, id) {
   let scope = attachScopes(ast, 'scope');
 
   let fsId, readFileSyncId;
-  let fsShadowDepth = 0;
-  let readFileSyncShadowDepth = 0;
+  let pathId, pathImportIds = {};
+  const shadowDepths = Object.create(null);
+  shadowDepths.__filename = 0;
+  shadowDepths.__dirname = 0;
   for (const decl of ast.body) {
     // Detects:
     // import * as fs from 'fs';
     // import fs from 'fs';
     // import { readFileSync } from 'fs';
-    if (decl.type === 'ImportDeclaration' &&
-        (decl.source.value === 'fs' || decl.source.value === '\u0000commonjs-proxy:fs')) {
-      for (const impt of decl.specifiers) {
-        if (impt.type === 'ImportNamespaceSpecifier' || impt.type === 'ImportDefaultSpecifier') {
-          fsId = impt.local.name;
-        } else if (impt.type === 'ImportSpecifier' && impt.imported.name === 'readFileSync') {
-          readFileSyncId = impt.local.name;
+    // import * as path from 'path';
+    // import path from 'path';
+    // import { join } from 'path';
+    if (decl.type === 'ImportDeclaration') {
+      const source = decl.source.value.startsWith('\u0000commonjs-proxy:') ? decl.source.value.substr(16) : decl.source.value;
+      if (source === 'fs') {
+        for (const impt of decl.specifiers) {
+          if (impt.type === 'ImportNamespaceSpecifier' || impt.type === 'ImportDefaultSpecifier') {
+            fsId = impt.local.name;
+            shadowDepths[fsId] = 0;
+          } else if (impt.type === 'ImportSpecifier' && impt.imported.name === 'readFileSync') {
+            readFileSyncId = impt.local.name;
+            shadowDepths[readFileSyncId] = 0;
+          }
+        }
+      }
+      else if (source === 'path') {
+        for (const impt of decl.specifiers) {
+          if (impt.type === 'ImportNamespaceSpecifier' || impt.type === 'ImportDefaultSpecifier') {
+            pathId = impt.local.name;
+            shadowDepths[pathId] = 0;
+          } else if (impt.type === 'ImportSpecifier') {
+            pathImportIds[impt.local.name] = impt.imported.name;
+            shadowDepths[impt.local.name] = 0;
+          }
         }
       }
     }
   }
   let didInline = false;
 
+  function extractStaticFileSource (expr, id, enc) {
+    const vars = {};
+    if (shadowDepths.__filename === 0)
+      vars.__dirname = path.resolve(id, '..');
+    if (shadowDepths.__dirname === 0)
+      vars.__filename = id;
+    if (pathId) {
+      if (shadowDepths[pathId] === 0)
+        vars[pathId] = path;
+      for (const pathFn of Object.keys(pathImportIds)) {
+        if (shadowDepths[pathFn] === 0)
+          vars[pathFn] = path[pathImportIds[pathFn]];
+      }
+    }
+
+    // evaluate returns undefined for non-statically-analyzable
+    const assetPath = evaluate(expr, vars);
+    if (assetPath) {
+      const source = fs.readFileSync(assetPath);
+      if (enc) {
+        try {
+          return JSON.stringify(source.toString(enc));
+        }
+        catch (e) {
+          return;
+        }
+      }
+      else {
+        const assetSource = source.toString('base64');
+        return 'Buffer.from("' + assetSource + '", "base64")';
+      }
+    }
+  }
+
   walk(ast, {
     enter (node, parent) {
       if (node.scope) {
         scope = node.scope;
-        if (node.scope.declarations[fsId])
-          fsShadowDepth++
-        if (node.scope.declarations[readFileSyncId])
-          readFileSyncShadowDepth++;
+        for (const id in node.scope.declarations) {
+          if (id in shadowDepths)
+            shadowDepths[id]++;
+        }
       }
 
-      if (node.type === 'MemberExpression') {
+      // for now we only support top-level variable declarations
+      // so "var { join } = path" will only work in the top scope.
+      if (parent === ast && node.type === 'VariableDeclaration') {
+        for (const decl of node.declarations) {
+          // var { join } = path;
+          if (decl.id.type === 'ObjectPattern' &&
+              decl.init &&
+              decl.init.type === 'Identifier' &&
+              decl.init.name === pathId &&
+              shadowDepths[pathId] === 0) {
+            for (const prop of decl.id.properties) {
+              if (prop.type !== 'Property' ||
+                  prop.key.type !== 'Identifier' ||
+                  prop.value.type !== 'Identifier')
+                continue;
+              pathImportIds[prop.value.name] = prop.key.name;
+              shadowDepths[prop.key.name] = 0;
+            }
+          }
+          // var join = path.join;
+          if (decl.id.type === 'Identifier' &&
+              decl.init &&
+              decl.init.type === 'MemberExpression' &&
+              decl.init.object.type === 'Identifier' &&
+              decl.init.object.name === pathId &&
+              shadowDepths.pathId === 0 &&
+              decl.init.computed === false &&
+              decl.init.property.type === 'Identifier') {
+            pathImportIds[decl.init.property.name] = decl.id.name;
+            shadowDepths[decl.id.name] = 0;
+          }
+        }
+      }
+
+      else if (node.type === 'MemberExpression') {
         if (parent.type === 'CallExpression' &&
             parent.callee === node &&
-            parent.arguments.length === 1 &&
+            (parent.arguments.length === 1 || parent.arguments.length === 2) &&
             node.object.type === 'Identifier' &&
-            (node.object.name === 'fs' || (fsShadowDepth === 0 && node.object.name === fsId)) &&
+            (node.object.name === 'fs' || (node.object.name === fsId && shadowDepths[fsId] === 0)) &&
+            !scope.contains(fsId) &&
             node.property.type === 'Identifier' &&
             node.property.name === 'readFileSync') {
-          const inlined = extractStaticFileSource(parent.arguments[0], id, scope);
+          const inlined = extractStaticFileSource(parent.arguments[0], id,
+              parent.arguments[1] && parent.arguments[1].type === 'Literal' ? parent.arguments[1].value : null);
           if (inlined) {
             didInline = true;
             magicString.overwrite(parent.start, parent.end, inlined);
@@ -93,13 +167,13 @@ exports.transform = function (code, id) {
         }
       }
 
-      else if (node.type === 'Identifier' &&
+      else if (parent && node.type === 'Identifier' &&
                isReference(node, parent) &&
-               readFileSyncShadowDepth === 0 &&
                node.name === readFileSyncId &&
+               !scope.contains(readFileSyncId) &&
                parent.type === 'CallExpression' &&
                parent.callee === node) {
-        const inlined = extractStaticFileSource(parent.arguments[0], id, scope);
+        const inlined = extractStaticFileSource(parent.arguments[0], id);
         if (inlined) {
           didInline = true;
           magicString.overwrite(parent.start, parent.end, inlined);
@@ -110,10 +184,10 @@ exports.transform = function (code, id) {
     leave (node) {
       if (node.scope) {
         scope = scope.parent;
-        if (node.scope.declarations[fsId])
-          fsShadowDepth--;
-        if (node.scope.declarations[readFileSyncId])
-          readFileSyncShadowDepth--;
+        for (const id in node.scope.declarations) {
+          if (id in shadowDepths)
+            shadowDepths[id]--;
+        }
       }
     }
   });

index.js

@@ -6,7 +6,7 @@ const { terser } = require("rollup-plugin-terser");
 const builtins = require("builtins")();
 const fsInliner = require("./fs-inliner.js");
 
-module.exports = async (input, { minify = true } = {}) => {
+module.exports = async (input, { minify = true, sourcemap = false } = {}) => {
   const resolve = nodeResolve({
     module: false,
     jsnext: false,
@@ -15,6 +15,9 @@ module.exports = async (input, { minify = true } = {}) => {
   });
   const bundle = await rollup.rollup({
     input,
+    treeshake: {
+      pureExternalModules: [...builtins]
+    },
     plugins: [
       resolve,
       commonjs({
@@ -24,7 +27,7 @@ module.exports = async (input, { minify = true } = {}) => {
             if (builtins[id] || await resolve.resolveId(id, parentId))
               return false;
           }
-          catch {}
+          catch (e) {}
           return true;
         }
       }),
@@ -36,6 +39,7 @@ module.exports = async (input, { minify = true } = {}) => {
   });
 
   return await bundle.generate({
+    sourcemap,
     format: "cjs"
   });
 };

jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  testEnvironment: 'node',
+  rootDir: 'test',
+};

package.json

@@ -11,19 +11,22 @@
     "rollup-plugin-json": "^3.1.0",
     "rollup-plugin-node-resolve": "3.4.0",
     "rollup-plugin-terser": "^3.0.0",
-    "rollup-pluginutils": "^2.3.3"
+    "rollup-pluginutils": "^2.3.3",
+    "static-eval": "^2.0.0"
   },
   "license": "MIT",
   "main": "index.js",
   "bin": {
     "ncc": "./cli.js"
   },
   "scripts": {
-    "test": "node test",
+    "test": "jest",
     "prepublish": "npm test"
   },
   "devDependencies": {
     "axios": "^0.18.0",
-    "fontkit": "^1.7.7"
+    "fontkit": "^1.7.7",
+    "jest": "^23.6.0",
+    "source-map-support": "^0.5.9"
   }
 }

test/index.test.js

@@ -0,0 +1,44 @@
+const fs = require("fs");
+const sourceMapSupport = require('source-map-support');
+const ncc = require("../");
+
+const sourceMapSources = {};
+sourceMapSupport.install({
+  retrieveSourceMap (source) {
+    if (!sourceMapSources[source])
+      return null;
+
+    return {
+      url: source,
+      map: sourceMapSources[source]
+    };
+  }
+});
+
+for (const unitTest of fs.readdirSync(`${__dirname}/unit`)) {
+  it(`should generate correct output for ${unitTest}`, async () => {
+    const expected = fs.readFileSync(`${__dirname}/unit/${unitTest}/output.js`)
+        .toString().trim()
+        // Windows support
+        .replace(/\r/g, '');
+    await ncc(`${__dirname}/unit/${unitTest}/input.js`, { minify: false }).then(async t => {
+      const actual = t.code.trim();
+      expect(actual).toBe(expected);
+    });
+  });
+}
+
+jest.setTimeout(10000);
+
+for (const integrationTest of fs.readdirSync(__dirname + "/integration")) {
+  it(`should evaluate ${integrationTest} without errors`, async () => {
+    const { code, map } = await ncc(`${__dirname}/integration/${integrationTest}`, { minify: false, sourcemap: true });
+    sourceMapSources[integrationTest] = map;
+    module.exports = null;
+    eval(`${code}\n//# sourceURL=${integrationTest}`);
+    if ("function" !== typeof module.exports) {
+      throw new Error(`Integration test "${integrationTest}" evaluation failed. It does not export a function`)
+    }
+    await module.exports();
+  });
+}