From 459c1c13d054b37442126889077b7056269eeb35 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Sun, 16 Sep 2018 16:06:02 +0200 Subject: [PATCH] Improvements to on-demand-entries (#5176) First wave of changes. Just landing this first so that there is a base to build on. --- .../webpack/plugins/pages-manifest-plugin.js | 3 +- package.json | 2 +- server/hot-reloader.js | 2 +- server/on-demand-entry-handler.js | 161 ++++++++++-------- 4 files changed, 93 insertions(+), 75 deletions(-) diff --git a/build/webpack/plugins/pages-manifest-plugin.js b/build/webpack/plugins/pages-manifest-plugin.js index 69eda8971920b..ff58063729fab 100644 --- a/build/webpack/plugins/pages-manifest-plugin.js +++ b/build/webpack/plugins/pages-manifest-plugin.js @@ -7,7 +7,7 @@ import {PAGES_MANIFEST, ROUTE_NAME_REGEX} from '../../../lib/constants' // It's also used by next export to provide defaultPathMap export default class PagesManifestPlugin { apply (compiler: any) { - compiler.hooks.emit.tapAsync('NextJsPagesManifest', (compilation, callback) => { + compiler.hooks.emit.tap('NextJsPagesManifest', (compilation) => { const {entries} = compilation const pages = {} @@ -32,7 +32,6 @@ export default class PagesManifestPlugin { } compilation.assets[PAGES_MANIFEST] = new RawSource(JSON.stringify(pages)) - callback() }) } } diff --git a/package.json b/package.json index 76abd706befe6..1be5ca7678a4b 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,7 @@ "terser-webpack-plugin": "1.0.2", "unfetch": "3.0.0", "url": "0.11.0", - "webpack": "4.17.1", + "webpack": "4.19.0", "webpack-dev-middleware": "3.2.0", "webpack-hot-middleware": "2.22.3", "webpack-sources": "1.2.0", diff --git a/server/hot-reloader.js b/server/hot-reloader.js index 20b9bced13c84..a5f04e43cf612 100644 --- a/server/hot-reloader.js +++ b/server/hot-reloader.js @@ -319,7 +319,7 @@ export default class HotReloader { heartbeat: 2500 }) - const onDemandEntries = onDemandEntryHandler(webpackDevMiddleware, multiCompiler.compilers, { + const onDemandEntries = onDemandEntryHandler(webpackDevMiddleware, multiCompiler, { dir: this.dir, buildId: this.buildId, dev: true, diff --git a/server/on-demand-entry-handler.js b/server/on-demand-entry-handler.js index c2052d52de012..adfb7b7ff7767 100644 --- a/server/on-demand-entry-handler.js +++ b/server/on-demand-entry-handler.js @@ -16,7 +16,18 @@ const BUILT = Symbol('built') const glob = promisify(globModule) const access = promisify(fs.access) -export default function onDemandEntryHandler (devMiddleware, compilers, { +// Based on https://github.com/webpack/webpack/blob/master/lib/DynamicEntryPlugin.js#L29-L37 +function addEntry (compilation, context, name, entry) { + return new Promise((resolve, reject) => { + const dep = DynamicEntryPlugin.createDependency(entry, name) + compilation.addEntry(context, dep, name, (err) => { + if (err) return reject(err) + resolve() + }) + }) +} + +export default function onDemandEntryHandler (devMiddleware, multiCompiler, { buildId, dir, dev, @@ -25,20 +36,18 @@ export default function onDemandEntryHandler (devMiddleware, compilers, { maxInactiveAge = 1000 * 60, pagesBufferLength = 2 }) { + const {compilers} = multiCompiler + const invalidator = new Invalidator(devMiddleware, multiCompiler) let entries = {} let lastAccessPages = [''] let doneCallbacks = new EventEmitter() - const invalidator = new Invalidator(devMiddleware) let reloading = false let stopped = false let reloadCallbacks = new EventEmitter() - // Keep the names of compilers which are building pages at a given moment. - const currentBuilders = new Set() - compilers.forEach(compiler => { - compiler.hooks.make.tapAsync('NextJsOnDemandEntries', function (compilation, done) { + for (const compiler of compilers) { + compiler.hooks.make.tapPromise('NextJsOnDemandEntries', (compilation) => { invalidator.startBuilding() - currentBuilders.add(compiler.name) const allEntries = Object.keys(entries).map(async (page) => { const { name, entry } = entries[page] @@ -57,65 +66,79 @@ export default function onDemandEntryHandler (devMiddleware, compilers, { return addEntry(compilation, compiler.context, name, entry) }) - Promise.all(allEntries) - .then(() => done()) - .catch(done) + return Promise.all(allEntries) }) + } - compiler.hooks.done.tap('NextJsOnDemandEntries', function (stats) { - // Wait until all the compilers mark the build as done. - currentBuilders.delete(compiler.name) - if (currentBuilders.size !== 0) return + multiCompiler.hooks.done.tap('NextJsOnDemandEntries', (multiStats) => { + const clientStats = multiStats.stats[0] + const { compilation } = clientStats + const hardFailedPages = compilation.errors + .filter(e => { + // Make sure to only pick errors which marked with missing modules + const hasNoModuleFoundError = /ENOENT/.test(e.message) || /Module not found/.test(e.message) + if (!hasNoModuleFoundError) return false + + // The page itself is missing. So this is a failed page. + if (IS_BUNDLED_PAGE_REGEX.test(e.module.name)) return true + + // No dependencies means this is a top level page. + // So this is a failed page. + return e.module.dependencies.length === 0 + }) + .map(e => e.module.chunks) + .reduce((a, b) => [...a, ...b], []) + .map(c => { + const pageName = ROUTE_NAME_REGEX.exec(c.name)[1] + return normalizePage(`/${pageName}`) + }) - const { compilation } = stats - const hardFailedPages = compilation.errors - .filter(e => { - // Make sure to only pick errors which marked with missing modules - const hasNoModuleFoundError = /ENOENT/.test(e.message) || /Module not found/.test(e.message) - if (!hasNoModuleFoundError) return false + // compilation.entrypoints is a Map object, so iterating over it 0 is the key and 1 is the value + for (const [, entrypoint] of compilation.entrypoints.entries()) { + const result = ROUTE_NAME_REGEX.exec(entrypoint.name) + if (!result) { + continue + } - // The page itself is missing. So this is a failed page. - if (IS_BUNDLED_PAGE_REGEX.test(e.module.name)) return true + const pagePath = result[1] - // No dependencies means this is a top level page. - // So this is a failed page. - return e.module.dependencies.length === 0 - }) - .map(e => e.module.chunks) - .reduce((a, b) => [...a, ...b], []) - .map(c => { - const pageName = ROUTE_NAME_REGEX.exec(c.name)[1] - return normalizePage(`/${pageName}`) - }) + if (!pagePath) { + continue + } - // Call all the doneCallbacks - Object.keys(entries).forEach((page) => { - const entryInfo = entries[page] - if (entryInfo.status !== BUILDING) return + const page = normalizePage('/' + pagePath) - entryInfo.status = BUILT - entries[page].lastActiveTime = Date.now() - doneCallbacks.emit(page) - }) + const entry = entries[page] + if (!entry) { + continue + } - invalidator.doneBuilding(compiler.name) - - if (hardFailedPages.length > 0 && !reloading) { - console.log(`> Reloading webpack due to inconsistant state of pages(s): ${hardFailedPages.join(', ')}`) - reloading = true - reload() - .then(() => { - console.log('> Webpack reloaded.') - reloadCallbacks.emit('done') - stop() - }) - .catch(err => { - console.error(`> Webpack reloading failed: ${err.message}`) - console.error(err.stack) - process.exit(1) - }) + if (entry.status !== BUILDING) { + continue } - }) + + entry.status = BUILT + entry.lastActiveTime = Date.now() + doneCallbacks.emit(page) + } + + invalidator.doneBuilding() + + if (hardFailedPages.length > 0 && !reloading) { + console.log(`> Reloading webpack due to inconsistant state of pages(s): ${hardFailedPages.join(', ')}`) + reloading = true + reload() + .then(() => { + console.log('> Webpack reloaded.') + reloadCallbacks.emit('done') + stop() + }) + .catch(err => { + console.error(`> Webpack reloading failed: ${err.message}`) + console.error(err.stack) + process.exit(1) + }) + } }) const disposeHandler = setInterval(function () { @@ -248,17 +271,6 @@ export default function onDemandEntryHandler (devMiddleware, compilers, { } } -// Based on https://github.com/webpack/webpack/blob/master/lib/DynamicEntryPlugin.js#L29-L37 -function addEntry (compilation, context, name, entry) { - return new Promise((resolve, reject) => { - const dep = DynamicEntryPlugin.createDependency(entry, name) - compilation.addEntry(context, dep, name, (err) => { - if (err) return reject(err) - resolve() - }) - }) -} - function disposeInactiveEntries (devMiddleware, entries, lastAccessPages, maxInactiveAge) { const disposingPages = [] @@ -291,10 +303,11 @@ function disposeInactiveEntries (devMiddleware, entries, lastAccessPages, maxIna // /index and / is the same. So, we need to identify both pages as the same. // This also applies to sub pages as well. export function normalizePage (page) { - if (page === '/index' || page === '/') { + const unixPagePath = page.replace(/\\/g, '/') + if (unixPagePath === '/index' || unixPagePath === '/') { return '/' } - return page.replace(/\/index$/, '') + return unixPagePath.replace(/\/index$/, '') } function sendJson (res, payload) { @@ -306,7 +319,8 @@ function sendJson (res, payload) { // Make sure only one invalidation happens at a time // Otherwise, webpack hash gets changed and it'll force the client to reload. class Invalidator { - constructor (devMiddleware) { + constructor (devMiddleware, multiCompiler) { + this.multiCompiler = multiCompiler this.devMiddleware = devMiddleware // contains an array of types of compilers currently building this.building = false @@ -324,6 +338,11 @@ class Invalidator { } this.building = true + // Work around a bug in webpack, calling `invalidate` on Watching.js + // doesn't trigger the invalid call used to keep track of the `.done` hook on multiCompiler + for (const compiler of this.multiCompiler.compilers) { + compiler.hooks.invalid.call() + } this.devMiddleware.invalidate() }