New analysis updates

package.json

@@ -18,11 +18,12 @@
   "devDependencies": {
     "@azure/cosmos": "^2.0.5",
     "@bugsnag/js": "^5.0.1",
+    "@ffmpeg-installer/ffmpeg": "^1.0.17",
     "@google-cloud/bigquery": "^2.0.1",
     "@google-cloud/firestore": "^0.19.0",
     "@sentry/node": "^4.3.0",
     "@tensorflow/tfjs-node": "^0.3.0",
-    "@zeit/webpack-asset-relocator-loader": "0.4.5",
+    "@zeit/webpack-asset-relocator-loader": "0.5.0-beta.5",
     "analytics-node": "^3.3.0",
     "apollo-server-express": "^2.2.2",
     "arg": "^4.1.0",
@@ -60,6 +61,7 @@
     "jugglingdb": "2.0.1",
     "koa": "^2.6.2",
     "leveldown": "^4.0.1",
+    "lighthouse": "^5.0.0",
     "loopback": "^3.24.0",
     "mailgun": "^0.5.0",
     "mariadb": "^2.0.1-beta",
@@ -73,6 +75,7 @@
     "passport": "^0.4.0",
     "passport-google-oauth": "^1.0.0",
     "path-platform": "^0.11.15",
+    "pdf2json": "^1.1.8",
     "pdfkit": "^0.8.3",
     "pg": "^7.6.1",
     "pug": "^2.0.3",

scripts/build.js

@@ -62,13 +62,12 @@ async function main() {
   // detect unexpected asset emissions from core build
   const unknownAssets = [
     ...Object.keys(cliAssets),
-    ...Object.keys(indexAssets).filter(asset => !asset.startsWith('locales/')),
+    ...Object.keys(indexAssets).filter(asset => !asset.startsWith('locales/') && asset !== 'worker.js' && asset !== 'index1.js'),
     ...Object.keys(relocateLoaderAssets),
     ...Object.keys(shebangLoaderAssets),
-    ...Object.keys(tsLoaderAssets).filter(asset => !asset.startsWith('lib/')),
+    ...Object.keys(tsLoaderAssets).filter(asset => !asset.startsWith('lib/') && !asset.startsWith('typescript/lib')),
     ...Object.keys(sourcemapAssets)
   ].filter(asset => !asset.endsWith('.js.cache') && !asset.endsWith('.cache.js'));
-
   if (unknownAssets.length) {
     console.error("New assets are being emitted by the core build");
     console.log(unknownAssets);
@@ -111,6 +110,8 @@ module.exports = typescript;
   writeFileSync(__dirname + "/../dist/ncc/loaders/ts-loader.js", tsLoader);
   writeFileSync(__dirname + "/../dist/ncc/loaders/uncacheable.js", readFileSync(__dirname + "/../src/loaders/uncacheable.js"));
   writeFileSync(__dirname + "/../dist/ncc/loaders/empty-loader.js", readFileSync(__dirname + "/../src/loaders/empty-loader.js"));
+  writeFileSync(__dirname + "/../dist/ncc/loaders/notfound-loader.js", readFileSync(__dirname + "/../src/loaders/notfound-loader.js"));
+  writeFileSync(__dirname + "/../dist/ncc/@@notfound.js", readFileSync(__dirname + "/../src/@@notfound.js"));
 
   // copy typescript types
   await copy(

src/@@notfound.js

@@ -0,0 +1,9 @@
+const id = 'UNKNOWN';
+if (id.startsWith('./') || id.startsWith('../')) {
+  const e = new Error('Cannot find module "' + id + '".');
+  e.code = 'MODULE_NOT_FOUND';
+  throw e;
+}
+else {
+  __non_webpack_require__(id);
+}

src/index.js

@@ -66,7 +66,6 @@ module.exports = (
     existingAssetNames.push(`${filename}.cache.js`);
   }
   const resolvePlugins = [];
-  let tsconfigMatchPath;
   // add TsconfigPathsPlugin to support `paths` resolution in tsconfig
   // we need to catch here because the plugin will
   // error if there's no tsconfig in the working directory
@@ -79,6 +78,21 @@ module.exports = (
     }
   } catch (e) {}
 
+  resolvePlugins.push({
+    apply(resolver) {
+      const resolve = resolver.resolve;
+      resolver.resolve = function (context, path, request, resolveContext, callback) {
+        resolve.call(resolver, context, path, request, resolveContext, function (err, result) {
+          if (!err) return callback(null, result);
+          if (!err.missing || !err.missing.length)
+            return callback(err);
+          // make not found errors runtime errors
+          callback(null, __dirname + '/@@notfound.js' + '?' + request);
+        });
+      };
+    }
+  });
+
   const externalSet = new Set(externals);
 
   let watcher, watchHandler, rebuildHandler;
@@ -118,30 +132,16 @@ module.exports = (
     node: false,
     externals: async ({ context, request }, callback) => {
       if (externalSet.has(request)) return callback(null, `commonjs ${request}`);
-      if (request[0] === "." && (request[1] === "/" || request[1] === "." && request[2] === "/")) {
-        if (request.startsWith("./node_modules/")) request = request.substr(15);
-        else if (request.startsWith("../node_modules/")) request = request.substr(16);
-        else return callback();
-      }
-      if (request[0] === "/" || /^[a-z]:\\/i.test(request) || nodeBuiltins.has(request) ||
-          tsconfigMatchPath && tsconfigMatchPath(request, undefined, undefined, SUPPORTED_EXTENSIONS))
-        return callback();
-      const pkgNameMatch = request.match(pkgNameRegEx);
-      if (pkgNameMatch) request = pkgNameMatch[0];
-      let pkgPath = context + sep + 'node_modules' + sep + request;
-      do {
-        if (await new Promise((resolve, reject) =>
-          compiler.inputFileSystem.stat(pkgPath, (err, stats) =>
-            err && err.code !== 'ENOENT' ? reject(err) : resolve(stats ? stats.isDirectory() : false)
-          )
-        ))
-          return callback();
-      } while (pkgPath.length > (pkgPath = pkgPath.substr(0, pkgPath.lastIndexOf(sep, pkgPath.length - 15 - request.length)) + sep + 'node_modules' + sep + request).length);
-      console.error(`ncc: Module directory "${context}" attempted to require "${request}" but could not be resolved, assuming external.`);
-      return callback(null, `commonjs ${request}`);
+      return callback();
     },
     module: {
       rules: [
+        {
+          test: /@@notfound\.js$/,
+          use: [{
+            loader: eval('__dirname + "/loaders/notfound-loader.js"')
+          }]
+        },
         {
           test: /\.(js|mjs|tsx?|node)$/,
           use: [{
@@ -184,6 +184,36 @@ module.exports = (
     plugins: [
       {
         apply(compiler) {
+          // override "not found" context to try built require first
+          compiler.hooks.compilation.tap("ncc", compilation => {
+            compilation.moduleTemplates.javascript.hooks.render.tap(
+              "ncc",
+              (
+                moduleSourcePostModule,
+                module,
+                options,
+                dependencyTemplates
+              ) => {
+                if (
+                  module._contextDependencies &&
+                  moduleSourcePostModule._value.match(
+                    /webpackEmptyAsyncContext|webpackEmptyContext/
+                  )
+                ) {
+                  // ensure __webpack_require__ is added to wrapper
+                  module.type = 'custom';
+                  return moduleSourcePostModule._value.replace(
+                    "var e = new Error",
+                    `if (typeof req === 'number' && __webpack_require__.m[req])\n` +
+                    `  return __webpack_require__(req);\n` +
+                    `try { return require(req) }\n` + 
+                    `catch (e) { if (e.code !== 'MODULE_NOT_FOUND') throw e }\n` +
+                    `var e = new Error`
+                  );
+                }
+              }
+            );
+          });
           compiler.hooks.compilation.tap("relocate-loader", relocateLoader.initAssetPermissionsCache);
           compiler.hooks.watchRun.tap("ncc", () => {
             if (rebuildHandler)

src/loaders/notfound-loader.js

@@ -0,0 +1,7 @@
+module.exports = function (input, map) {
+  if (this.cacheable)
+    this.cacheable();
+  const id = this.resourceQuery.substr(1);
+  input = input.replace('\'UNKNOWN\'', JSON.stringify(id));
+  this.callback(null, input, map);
+};

test/integration/ffmpeg.js

@@ -0,0 +1,2 @@
+let {path} = require('@ffmpeg-installer/ffmpeg');
+console.log(path);

test/integration/lighthouse.js

@@ -0,0 +1 @@
+require('lighthouse');

test/integration/pdf2json.js

@@ -0,0 +1,8 @@
+const { parse } = require('url')
+const PDFParser = require('pdf2json')
+
+module.exports = (req, res) => {
+  const { query } = parse(req.url, true)
+  const { name = 'World' } = query
+  res.end(`Hello ${name}!`)
+}

test/integration/twilio.js

@@ -6,3 +6,4 @@ try {
     throw err;
   }
 }
+

test/unit/runtime-notfound/input.js

@@ -0,0 +1,2 @@
+require('./not-found.js');
+require('./not-foud2.js');

test/unit/runtime-notfound/output-coverage.js

@@ -0,0 +1,85 @@
+module.exports =
+/******/ (function(modules, runtime) { // webpackBootstrap
+/******/ 	"use strict";
+/******/ 	// The module cache
+/******/ 	var installedModules = {};
+/******/
+/******/ 	// The require function
+/******/ 	function __webpack_require__(moduleId) {
+/******/
+/******/ 		// Check if module is in cache
+/******/ 		if(installedModules[moduleId]) {
+/******/ 			return installedModules[moduleId].exports;
+/******/ 		}
+/******/ 		// Create a new module (and put it into the cache)
+/******/ 		var module = installedModules[moduleId] = {
+/******/ 			i: moduleId,
+/******/ 			l: false,
+/******/ 			exports: {}
+/******/ 		};
+/******/
+/******/ 		// Execute the module function
+/******/ 		modules[moduleId].call(module.exports, module, module.exports, __webpack_require__);
+/******/
+/******/ 		// Flag the module as loaded
+/******/ 		module.l = true;
+/******/
+/******/ 		// Return the exports of the module
+/******/ 		return module.exports;
+/******/ 	}
+/******/
+/******/
+/******/
+/******/ 	// the startup function
+/******/ 	function startup() {
+/******/ 		// Load entry module and return exports
+/******/ 		return __webpack_require__(226);
+/******/ 	};
+/******/
+/******/ 	// run startup
+/******/ 	return startup();
+/******/ })
+/************************************************************************/
+/******/ ({
+
+/***/ 226:
+/***/ (function(__unusedmodule, __unusedexports, __webpack_require__) {
+
+__webpack_require__(590);
+__webpack_require__(548);
+
+/***/ }),
+
+/***/ 548:
+/***/ (function() {
+
+const id = "./not-foud2.js";
+if (id.startsWith('./') || id.startsWith('../')) {
+  const e = new Error('Cannot find module "' + id + '".');
+  e.code = 'MODULE_NOT_FOUND';
+  throw e;
+}
+else {
+  eval("require")(id);
+}
+
+
+/***/ }),
+
+/***/ 590:
+/***/ (function() {
+
+const id = "./not-found.js";
+if (id.startsWith('./') || id.startsWith('../')) {
+  const e = new Error('Cannot find module "' + id + '".');
+  e.code = 'MODULE_NOT_FOUND';
+  throw e;
+}
+else {
+  eval("require")(id);
+}
+
+
+/***/ })
+
+/******/ });

test/unit/runtime-notfound/output.js

@@ -0,0 +1,85 @@
+module.exports =
+/******/ (function(modules, runtime) { // webpackBootstrap
+/******/ 	"use strict";
+/******/ 	// The module cache
+/******/ 	var installedModules = {};
+/******/
+/******/ 	// The require function
+/******/ 	function __webpack_require__(moduleId) {
+/******/
+/******/ 		// Check if module is in cache
+/******/ 		if(installedModules[moduleId]) {
+/******/ 			return installedModules[moduleId].exports;
+/******/ 		}
+/******/ 		// Create a new module (and put it into the cache)
+/******/ 		var module = installedModules[moduleId] = {
+/******/ 			i: moduleId,
+/******/ 			l: false,
+/******/ 			exports: {}
+/******/ 		};
+/******/
+/******/ 		// Execute the module function
+/******/ 		modules[moduleId].call(module.exports, module, module.exports, __webpack_require__);
+/******/
+/******/ 		// Flag the module as loaded
+/******/ 		module.l = true;
+/******/
+/******/ 		// Return the exports of the module
+/******/ 		return module.exports;
+/******/ 	}
+/******/
+/******/
+/******/
+/******/ 	// the startup function
+/******/ 	function startup() {
+/******/ 		// Load entry module and return exports
+/******/ 		return __webpack_require__(768);
+/******/ 	};
+/******/
+/******/ 	// run startup
+/******/ 	return startup();
+/******/ })
+/************************************************************************/
+/******/ ({
+
+/***/ 663:
+/***/ (function() {
+
+const id = "./not-foud2.js";
+if (id.startsWith('./') || id.startsWith('../')) {
+  const e = new Error('Cannot find module "' + id + '".');
+  e.code = 'MODULE_NOT_FOUND';
+  throw e;
+}
+else {
+  eval("require")(id);
+}
+
+
+/***/ }),
+
+/***/ 768:
+/***/ (function(__unusedmodule, __unusedexports, __webpack_require__) {
+
+__webpack_require__(925);
+__webpack_require__(663);
+
+/***/ }),
+
+/***/ 925:
+/***/ (function() {
+
+const id = "./not-found.js";
+if (id.startsWith('./') || id.startsWith('../')) {
+  const e = new Error('Cannot find module "' + id + '".');
+  e.code = 'MODULE_NOT_FOUND';
+  throw e;
+}
+else {
+  eval("require")(id);
+}
+
+
+/***/ })
+
+/******/ });

test/unit/shebang/output-coverage.js

@@ -34,7 +34,7 @@ module.exports =
 /******/ 	// the startup function
 /******/ 	function startup() {
 /******/ 		// Load entry module and return exports
-/******/ 		return __webpack_require__(817);
+/******/ 		return __webpack_require__(374);
 /******/ 	};
 /******/
 /******/ 	// run startup
@@ -43,7 +43,7 @@ module.exports =
 /************************************************************************/
 /******/ ({
 
-/***/ 817:
+/***/ 374:
 /***/ (function(module) {
 
 module.exports = 'asdf';