From bdf82000a8996a24e6e6672ff751035373f2134f Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 17 Jul 2024 14:10:02 -0700 Subject: [PATCH] Add additional handling for experimental tracing (#67785) This adds additional trace handling under the experimental trace mode. Outside of the feature flag the bloom filter for app router paths used in pages to hard navigate was updated to no longer be inlined by default with webpack to avoid invalidating the main bundle for pages too often. Instead the bloom filter data is stored in the `_buildManifest` file which is only used in the pages router and is easier to update as it's per-build already. This tracing feature is still very experimental and should not be leveraged in production or outside of testing. --------- Co-authored-by: Zack Tanner <1939140+ztanner@users.noreply.github.com> --- .../flying-shuttle/detect-changed-entries.ts | 221 ++++++++ .../src/build/flying-shuttle/stitch-builds.ts | 489 ++++++++++++++++++ .../src/build/flying-shuttle/store-shuttle.ts | 73 +++ packages/next/src/build/index.ts | 175 +++++-- packages/next/src/build/webpack-config.ts | 47 +- .../webpack/plugins/build-manifest-plugin.ts | 34 +- .../webpack/plugins/define-env-plugin.ts | 2 + .../next/src/server/app-render/app-render.tsx | 6 +- .../server/app-render/required-scripts.tsx | 7 +- packages/next/src/server/get-page-files.ts | 4 + packages/next/src/shared/lib/router/router.ts | 120 +++-- .../dynamic-client/[category]/[id]/page.js | 6 + .../app-dir/app/components/button/button.js | 15 + .../app/components/button/button.module.css | 4 + test/e2e/app-dir/app/flying-shuttle.test.ts | 90 ++++ test/e2e/app-dir/app/next.config.js | 8 + test/e2e/app-dir/app/pages/index.js | 6 + test/e2e/app-dir/app/provide-paths.test.ts | 55 -- 18 files changed, 1194 insertions(+), 168 deletions(-) create mode 100644 packages/next/src/build/flying-shuttle/detect-changed-entries.ts create mode 100644 packages/next/src/build/flying-shuttle/stitch-builds.ts create mode 100644 packages/next/src/build/flying-shuttle/store-shuttle.ts create mode 100644 test/e2e/app-dir/app/components/button/button.js create mode 100644 test/e2e/app-dir/app/components/button/button.module.css delete mode 100644 test/e2e/app-dir/app/provide-paths.test.ts diff --git a/packages/next/src/build/flying-shuttle/detect-changed-entries.ts b/packages/next/src/build/flying-shuttle/detect-changed-entries.ts new file mode 100644 index 0000000000000..61c1b68e43ddc --- /dev/null +++ b/packages/next/src/build/flying-shuttle/detect-changed-entries.ts @@ -0,0 +1,221 @@ +import fs from 'fs' +import path from 'path' +import crypto from 'crypto' +import { getPageFromPath } from '../entries' +import { Sema } from 'next/dist/compiled/async-sema' + +export interface DetectedEntriesResult { + app: string[] + pages: string[] +} + +let _hasShuttle: undefined | boolean = undefined +export async function hasShuttle(shuttleDir: string) { + if (typeof _hasShuttle === 'boolean') { + return _hasShuttle + } + _hasShuttle = await fs.promises + .access(path.join(shuttleDir, 'server')) + .then(() => true) + .catch(() => false) + + return _hasShuttle +} + +export async function detectChangedEntries({ + appPaths, + pagesPaths, + pageExtensions, + distDir, + shuttleDir, +}: { + appPaths?: string[] + pagesPaths?: string[] + pageExtensions: string[] + distDir: string + shuttleDir: string +}): Promise<{ + changed: DetectedEntriesResult + unchanged: DetectedEntriesResult +}> { + const changedEntries: { + app: string[] + pages: string[] + } = { + app: [], + pages: [], + } + const unchangedEntries: typeof changedEntries = { + app: [], + pages: [], + } + + if (!(await hasShuttle(shuttleDir))) { + // no shuttle so consider everything changed + console.log(`no shuttle. can't detect changes`) + return { + changed: { + pages: pagesPaths || [], + app: appPaths || [], + }, + unchanged: { + pages: [], + app: [], + }, + } + } + + const hashCache = new Map() + + async function computeHash(p: string): Promise { + let hash = hashCache.get(p) + if (hash) { + return hash + } + return new Promise((resolve, reject) => { + const hashInst = crypto.createHash('sha1') + const stream = fs.createReadStream(p) + stream.on('error', (err) => reject(err)) + stream.on('data', (chunk) => hashInst.update(chunk)) + stream.on('end', () => { + const digest = hashInst.digest('hex') + resolve(digest) + hashCache.set(p, digest) + }) + }) + } + + const hashSema = new Sema(16) + let globalEntryChanged = false + + async function detectChange({ + normalizedEntry, + entry, + type, + }: { + entry: string + normalizedEntry: string + type: keyof typeof changedEntries + }) { + const traceFile = path.join( + shuttleDir, + 'server', + type, + `${normalizedEntry}.js.nft.json` + ) + let changed = true + + // we don't need to check any further entry's dependencies if + // a global entry changed since that invalidates everything + if (!globalEntryChanged) { + try { + const traceData: { + fileHashes: Record + } = JSON.parse(await fs.promises.readFile(traceFile, 'utf8')) + + if (traceData) { + let changedDependency = false + await Promise.all( + Object.keys(traceData.fileHashes).map(async (file) => { + try { + if (changedDependency) return + await hashSema.acquire() + const relativeTraceFile = path.relative( + path.join(shuttleDir, 'server', type), + traceFile + ) + const originalTraceFile = path.join( + distDir, + 'server', + type, + relativeTraceFile + ) + const absoluteFile = path.join( + path.dirname(originalTraceFile), + file + ) + + if (absoluteFile.startsWith(distDir)) { + return + } + + const prevHash = traceData.fileHashes[file] + const curHash = await computeHash(absoluteFile) + + if (prevHash !== curHash) { + console.log('detected change on', { + prevHash, + curHash, + file, + entry: normalizedEntry, + }) + changedDependency = true + } + } finally { + hashSema.release() + } + }) + ) + + if (!changedDependency) { + changed = false + } + } else { + console.error('missing trace data', traceFile, normalizedEntry) + } + } catch (err) { + console.error(`Failed to detect change for ${entry}`, err) + } + } + + // we always rebuild global entries so we have a version + // that matches the newest build/runtime + const isGlobalEntry = /(_app|_document|_error)/.test(entry) + + if (changed || isGlobalEntry) { + // if a global entry changed all entries are changed + if (!globalEntryChanged && isGlobalEntry) { + console.log(`global entry ${entry} changed invalidating all entries`) + globalEntryChanged = true + // move unchanged to changed + changedEntries[type].push(...unchangedEntries[type]) + } + changedEntries[type].push(entry) + } else { + unchangedEntries[type].push(entry) + } + } + + // loop over entries and their dependency's hashes + // to detect which changed + for (const entry of pagesPaths || []) { + let normalizedEntry = getPageFromPath(entry, pageExtensions) + + if (normalizedEntry === '/') { + normalizedEntry = '/index' + } + await detectChange({ entry, normalizedEntry, type: 'pages' }) + } + + for (const entry of appPaths || []) { + const normalizedEntry = getPageFromPath(entry, pageExtensions) + await detectChange({ entry, normalizedEntry, type: 'app' }) + } + + console.log( + 'changed entries', + JSON.stringify( + { + changedEntries, + unchangedEntries, + }, + null, + 2 + ) + ) + + return { + changed: changedEntries, + unchanged: unchangedEntries, + } +} diff --git a/packages/next/src/build/flying-shuttle/stitch-builds.ts b/packages/next/src/build/flying-shuttle/stitch-builds.ts new file mode 100644 index 0000000000000..c4d596c63971f --- /dev/null +++ b/packages/next/src/build/flying-shuttle/stitch-builds.ts @@ -0,0 +1,489 @@ +import type { Rewrite, Redirect } from '../../lib/load-custom-routes' +import type { PagesManifest } from '../webpack/plugins/pages-manifest-plugin' + +import fs from 'fs' +import path from 'path' +import { getPageFromPath } from '../entries' +import { Sema } from 'next/dist/compiled/async-sema' +import { recursiveCopy } from '../../lib/recursive-copy' +import { getSortedRoutes } from '../../shared/lib/router/utils' +import { generateClientManifest } from '../webpack/plugins/build-manifest-plugin' +import { createClientRouterFilter } from '../../lib/create-client-router-filter' +import { + hasShuttle, + type DetectedEntriesResult, +} from './detect-changed-entries' +import { + APP_BUILD_MANIFEST, + APP_PATH_ROUTES_MANIFEST, + APP_PATHS_MANIFEST, + AUTOMATIC_FONT_OPTIMIZATION_MANIFEST, + BUILD_MANIFEST, + CLIENT_REFERENCE_MANIFEST, + FUNCTIONS_CONFIG_MANIFEST, + MIDDLEWARE_BUILD_MANIFEST, + MIDDLEWARE_MANIFEST, + MIDDLEWARE_REACT_LOADABLE_MANIFEST, + NEXT_FONT_MANIFEST, + PAGES_MANIFEST, + REACT_LOADABLE_MANIFEST, + SERVER_REFERENCE_MANIFEST, + ROUTES_MANIFEST, +} from '../../shared/lib/constants' +import { normalizeAppPath } from '../../shared/lib/router/utils/app-paths' + +export async function stitchBuilds( + { + distDir, + shuttleDir, + buildId, + rewrites, + redirects, + allowedErrorRate, + encryptionKey, + edgePreviewProps, + }: { + buildId: string + distDir: string + shuttleDir: string + rewrites: { + beforeFiles: Rewrite[] + afterFiles: Rewrite[] + fallback: Rewrite[] + } + redirects: Redirect[] + allowedErrorRate?: number + encryptionKey: string + edgePreviewProps: Record + }, + entries: { + changed: DetectedEntriesResult + unchanged: DetectedEntriesResult + pageExtensions: string[] + } +): Promise<{ + pagesManifest?: PagesManifest +}> { + if (!(await hasShuttle(shuttleDir))) { + // no shuttle directory nothing to stitch + return {} + } + // if a manifest is needed in the rest of the build + // we return it from here so it can be used without + // re-reading from disk after changing + const updatedManifests: { + pagesManifest?: PagesManifest + } = {} + + // we need to copy the chunks from the shuttle folder + // to the distDir (we copy all server split chunks currently) + await recursiveCopy( + path.join(shuttleDir, 'server'), + path.join(distDir, 'server'), + { + filter(item) { + // we copy page chunks separately to not copy stale entries + return !item.match(/^[/\\](pages|app)[/\\]/) + }, + overwrite: true, + } + ) + // copy static chunks (this includes stale chunks but won't be loaded) + // unless referenced + await recursiveCopy( + path.join(shuttleDir, 'static'), + path.join(distDir, 'static'), + { overwrite: true } + ) + + async function copyPageChunk(entry: string, type: 'app' | 'pages') { + // copy entry chunk and flight manifest stuff + // TODO: copy .map files? + const entryFile = path.join('server', type, `${entry}.js`) + + await fs.promises.mkdir(path.join(distDir, path.dirname(entryFile)), { + recursive: true, + }) + await fs.promises.copyFile( + path.join(shuttleDir, entryFile + '.nft.json'), + path.join(distDir, entryFile + '.nft.json') + ) + + if (type === 'app' && !entry.endsWith('/route')) { + const clientRefManifestFile = path.join( + 'server', + type, + `${entry}_${CLIENT_REFERENCE_MANIFEST}.js` + ) + await fs.promises.copyFile( + path.join(shuttleDir, clientRefManifestFile), + path.join(distDir, clientRefManifestFile) + ) + } + await fs.promises.copyFile( + path.join(shuttleDir, entryFile), + path.join(distDir, entryFile) + ) + } + const copySema = new Sema(8) + + // restore unchanged entries avoiding copying stale + // entries from the shuttle/previous build + for (const { type, curEntries } of [ + { type: 'app', curEntries: entries.unchanged.app }, + { type: 'pages', curEntries: entries.unchanged.pages }, + ] as Array<{ type: 'app' | 'pages'; curEntries: string[] }>) { + await Promise.all( + curEntries.map(async (entry) => { + try { + await copySema.acquire() + let normalizedEntry = getPageFromPath(entry, entries.pageExtensions) + if (normalizedEntry === '/') { + normalizedEntry = '/index' + } + await copyPageChunk(normalizedEntry, type) + } finally { + copySema.release() + } + }) + ) + } + // always attempt copying not-found chunk + await copyPageChunk('/_not-found/page', 'app').catch(() => {}) + + // merge dynamic/static routes in routes-manifest + const [restoreRoutesManifest, currentRoutesManifest] = await Promise.all( + [ + path.join(shuttleDir, 'manifests', ROUTES_MANIFEST), + path.join(distDir, ROUTES_MANIFEST), + ].map(async (f) => JSON.parse(await fs.promises.readFile(f, 'utf8'))) + ) + const dynamicRouteMap: Record = {} + const combinedDynamicRoutes: Record[] = [ + ...currentRoutesManifest.dynamicRoutes, + ...restoreRoutesManifest.dynamicRoutes, + ] + for (const route of combinedDynamicRoutes) { + dynamicRouteMap[route.page] = route + } + + const mergedRoutesManifest = { + ...currentRoutesManifest, + dynamicRoutes: getSortedRoutes( + combinedDynamicRoutes.map((item) => item.page) + ).map((page) => dynamicRouteMap[page]), + staticRoutes: [ + ...currentRoutesManifest.staticRoutes, + ...restoreRoutesManifest.staticRoutes, + ], + } + await fs.promises.writeFile( + path.join(distDir, ROUTES_MANIFEST), + JSON.stringify(mergedRoutesManifest, null, 2) + ) + + // for build-manifest we use latest runtime files + // and only merge previous page chunk entries + // middleware-build-manifest.js (needs to be regenerated) + const [restoreBuildManifest, currentBuildManifest] = await Promise.all( + [ + path.join(shuttleDir, 'manifests', BUILD_MANIFEST), + path.join(distDir, BUILD_MANIFEST), + ].map(async (file) => JSON.parse(await fs.promises.readFile(file, 'utf8'))) + ) + const mergedBuildManifest = { + // we want to re-use original runtime + // chunks so we favor restored version + // over new + ...currentBuildManifest, + pages: { + ...restoreBuildManifest.pages, + ...currentBuildManifest.pages, + }, + } + + // _app and _error is unique per runtime + // so nest under each specific entry in build-manifest + const internalEntries = ['/_error', '/_app'] + + for (const entry of Object.keys(restoreBuildManifest.pages)) { + if (currentBuildManifest.pages[entry]) { + continue + } + for (const internalEntry of internalEntries) { + for (const chunk of restoreBuildManifest.pages[internalEntry]) { + if (!restoreBuildManifest.pages[entry].includes(chunk)) { + mergedBuildManifest.pages[entry].unshift(chunk) + } + } + } + } + + for (const entry of Object.keys(currentBuildManifest.pages)) { + for (const internalEntry of internalEntries) { + for (const chunk of currentBuildManifest.pages[internalEntry]) { + if (!currentBuildManifest.pages[entry].includes(chunk)) { + mergedBuildManifest.pages[entry].unshift(chunk) + } + } + } + } + + for (const key of internalEntries) { + mergedBuildManifest.pages[key] = [] + } + + for (const entry of entries.unchanged.app || []) { + const normalizedEntry = getPageFromPath(entry, entries.pageExtensions) + mergedBuildManifest.rootMainFilesTree[normalizedEntry] = + restoreBuildManifest.rootMainFilesTree[normalizedEntry] || + restoreBuildManifest.rootMainFiles + } + + await fs.promises.writeFile( + path.join(distDir, BUILD_MANIFEST), + JSON.stringify(mergedBuildManifest, null, 2) + ) + await fs.promises.writeFile( + path.join(distDir, 'server', `${MIDDLEWARE_BUILD_MANIFEST}.js`), + `self.__BUILD_MANIFEST=${JSON.stringify(mergedBuildManifest)}` + ) + await fs.promises.writeFile( + path.join(distDir, 'static', buildId, `_buildManifest.js`), + `self.__BUILD_MANIFEST = ${generateClientManifest( + mergedBuildManifest, + rewrites, + createClientRouterFilter( + [ + ...[ + // client filter always has all app paths + ...(entries.unchanged?.app || []), + ...(entries.changed?.app || []), + ].map((entry) => + normalizeAppPath(getPageFromPath(entry, entries.pageExtensions)) + ), + ...(entries.unchanged.pages.length + ? entries.changed?.pages || [] + : [] + ).map((item) => getPageFromPath(item, entries.pageExtensions)), + ], + redirects, + allowedErrorRate + ) + )};self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB()` + ) + + // for react-loadable-manifest we just merge directly + // prioritizing current manifest over previous, + // middleware-react-loadable-manifest (needs to be regenerated) + const [restoreLoadableManifest, currentLoadableManifest] = await Promise.all( + [ + path.join(shuttleDir, 'manifests', REACT_LOADABLE_MANIFEST), + path.join(distDir, REACT_LOADABLE_MANIFEST), + ].map(async (file) => JSON.parse(await fs.promises.readFile(file, 'utf8'))) + ) + const mergedLoadableManifest = { + ...restoreLoadableManifest, + ...currentLoadableManifest, + } + + await fs.promises.writeFile( + path.join(distDir, REACT_LOADABLE_MANIFEST), + JSON.stringify(mergedLoadableManifest, null, 2) + ) + await fs.promises.writeFile( + path.join(distDir, 'server', `${MIDDLEWARE_REACT_LOADABLE_MANIFEST}.js`), + `self.__REACT_LOADABLE_MANIFEST=${JSON.stringify( + JSON.stringify(mergedLoadableManifest) + )}` + ) + + // for server/middleware-manifest we just merge the functions + // and middleware fields + const [restoreMiddlewareManifest, currentMiddlewareManifest] = + await Promise.all( + [ + path.join(shuttleDir, 'server', MIDDLEWARE_MANIFEST), + path.join(distDir, 'server', MIDDLEWARE_MANIFEST), + ].map(async (file) => + JSON.parse(await fs.promises.readFile(file, 'utf8')) + ) + ) + const mergedMiddlewareManifest = { + ...currentMiddlewareManifest, + functions: { + ...restoreMiddlewareManifest.functions, + ...currentMiddlewareManifest.functions, + }, + } + // update edge function env + const updatedEdgeEnv: Record = { + __NEXT_BUILD_ID: buildId, + NEXT_SERVER_ACTIONS_ENCRYPTION_KEY: encryptionKey, + ...edgePreviewProps, + } + if (mergedMiddlewareManifest.middleware['/']) { + Object.assign(mergedMiddlewareManifest.middleware['/'].env, updatedEdgeEnv) + } + for (const key of Object.keys(mergedMiddlewareManifest.functions)) { + Object.assign(mergedMiddlewareManifest.functions[key].env, updatedEdgeEnv) + } + + await fs.promises.writeFile( + path.join(distDir, 'server', MIDDLEWARE_MANIFEST), + JSON.stringify(mergedMiddlewareManifest, null, 2) + ) + + // for server/next-font-manifest we just merge nested + // page/app fields and regenerate server/next-font-manifest.js + const [restoreNextFontManifest, currentNextFontManifest] = await Promise.all( + [ + path.join(shuttleDir, 'server', `${NEXT_FONT_MANIFEST}.json`), + path.join(distDir, 'server', `${NEXT_FONT_MANIFEST}.json`), + ].map(async (file) => JSON.parse(await fs.promises.readFile(file, 'utf8'))) + ) + const mergedNextFontManifest = { + ...currentNextFontManifest, + pages: { + ...restoreNextFontManifest.pages, + ...currentNextFontManifest.pages, + }, + app: { + ...restoreNextFontManifest.app, + ...currentNextFontManifest.app, + }, + } + + await fs.promises.writeFile( + path.join(distDir, 'server', `${NEXT_FONT_MANIFEST}.json`), + JSON.stringify(mergedNextFontManifest, null, 2) + ) + await fs.promises.writeFile( + path.join(distDir, 'server', `${NEXT_FONT_MANIFEST}.js`), + `self.__NEXT_FONT_MANIFEST=${JSON.stringify( + JSON.stringify(mergedNextFontManifest) + )}` + ) + + // for server/font-manifest.json we just merge the arrays + for (const file of [AUTOMATIC_FONT_OPTIMIZATION_MANIFEST]) { + const [restoreFontManifest, currentFontManifest] = await Promise.all( + [ + path.join(shuttleDir, 'server', file), + path.join(distDir, 'server', file), + ].map(async (f) => JSON.parse(await fs.promises.readFile(f, 'utf8'))) + ) + const mergedFontManifest = [...restoreFontManifest, ...currentFontManifest] + + await fs.promises.writeFile( + path.join(distDir, 'server', file), + JSON.stringify(mergedFontManifest, null, 2) + ) + } + + // for server/functions-config-manifest.json we just merge + // the functions field + const [restoreFunctionsConfigManifest, currentFunctionsConfigManifest] = + await Promise.all( + [ + path.join(shuttleDir, 'server', FUNCTIONS_CONFIG_MANIFEST), + path.join(distDir, 'server', FUNCTIONS_CONFIG_MANIFEST), + ].map(async (file) => + JSON.parse(await fs.promises.readFile(file, 'utf8')) + ) + ) + const mergedFunctionsConfigManifest = { + ...currentFunctionsConfigManifest, + functions: { + ...restoreFunctionsConfigManifest.functions, + ...currentFunctionsConfigManifest.functions, + }, + } + await fs.promises.writeFile( + path.join(distDir, 'server', FUNCTIONS_CONFIG_MANIFEST), + JSON.stringify(mergedFunctionsConfigManifest, null, 2) + ) + + for (const file of [APP_BUILD_MANIFEST, APP_PATH_ROUTES_MANIFEST]) { + const [restorePagesManifest, currentPagesManifest] = await Promise.all( + [path.join(shuttleDir, 'manifests', file), path.join(distDir, file)].map( + async (f) => JSON.parse(await fs.promises.readFile(f, 'utf8')) + ) + ) + const mergedPagesManifest = { + ...restorePagesManifest, + ...currentPagesManifest, + + ...(file === APP_BUILD_MANIFEST + ? { + pages: { + ...restorePagesManifest.pages, + ...currentPagesManifest.pages, + }, + } + : {}), + } + await fs.promises.writeFile( + path.join(distDir, file), + JSON.stringify(mergedPagesManifest, null, 2) + ) + } + + for (const file of [PAGES_MANIFEST, APP_PATHS_MANIFEST]) { + const [restoreAppManifest, currentAppManifest] = await Promise.all( + [ + path.join(shuttleDir, 'server', file), + path.join(distDir, 'server', file), + ].map(async (f) => JSON.parse(await fs.promises.readFile(f, 'utf8'))) + ) + const mergedManifest = { + ...restoreAppManifest, + ...currentAppManifest, + } + await fs.promises.writeFile( + path.join(distDir, 'server', file), + JSON.stringify(mergedManifest, null, 2) + ) + if (file === PAGES_MANIFEST) { + updatedManifests.pagesManifest = mergedManifest + } + } + + // for server/server-reference-manifest.json we merge + // and regenerate server/server-reference-manifest.js + const [restoreServerRefManifest, currentServerRefManifest] = + await Promise.all( + [ + path.join(shuttleDir, 'server', `${SERVER_REFERENCE_MANIFEST}.json`), + path.join(distDir, 'server', `${SERVER_REFERENCE_MANIFEST}.json`), + ].map(async (file) => + JSON.parse(await fs.promises.readFile(file, 'utf8')) + ) + ) + const mergedServerRefManifest = { + ...currentServerRefManifest, + node: { + ...restoreServerRefManifest.node, + ...currentServerRefManifest.node, + }, + edge: { + ...restoreServerRefManifest.edge, + ...currentServerRefManifest.edge, + }, + } + await fs.promises.writeFile( + path.join(distDir, 'server', `${SERVER_REFERENCE_MANIFEST}.json`), + JSON.stringify(mergedServerRefManifest, null, 2) + ) + await fs.promises.writeFile( + path.join(distDir, 'server', `${SERVER_REFERENCE_MANIFEST}.js`), + `self.__RSC_SERVER_MANIFEST=${JSON.stringify( + JSON.stringify(mergedServerRefManifest) + )}` + ) + + // TODO: inline env variables post build by find/replace + // in all the chunks for NEXT_PUBLIC_? + + return updatedManifests +} diff --git a/packages/next/src/build/flying-shuttle/store-shuttle.ts b/packages/next/src/build/flying-shuttle/store-shuttle.ts new file mode 100644 index 0000000000000..f53050bd0b9c6 --- /dev/null +++ b/packages/next/src/build/flying-shuttle/store-shuttle.ts @@ -0,0 +1,73 @@ +import fs from 'fs' +import path from 'path' +import { + BUILD_MANIFEST, + APP_BUILD_MANIFEST, + REACT_LOADABLE_MANIFEST, + APP_PATH_ROUTES_MANIFEST, + PAGES_MANIFEST, + ROUTES_MANIFEST, +} from '../../shared/lib/constants' +import { recursiveCopy } from '../../lib/recursive-copy' + +// we can create a new shuttle with the outputs before env values have +// been inlined, can be done after stitching takes place +export async function storeShuttle({ + distDir, + shuttleDir, +}: { + distDir: string + shuttleDir: string +}) { + await fs.promises.rm(shuttleDir, { force: true, recursive: true }) + await fs.promises.mkdir(shuttleDir, { recursive: true }) + + // copy all server entries + await recursiveCopy( + path.join(distDir, 'server'), + path.join(shuttleDir, 'server'), + { + filter(item) { + return !item.match(/\.(rsc|meta|html)$/) + }, + } + ) + + const pagesManifest = JSON.parse( + await fs.promises.readFile( + path.join(shuttleDir, 'server', PAGES_MANIFEST), + 'utf8' + ) + ) + // ensure manifest isn't modified to .html as it's before static gen + for (const key of Object.keys(pagesManifest)) { + pagesManifest[key] = pagesManifest[key].replace(/\.html$/, '.js') + } + await fs.promises.writeFile( + path.join(shuttleDir, 'server', PAGES_MANIFEST), + JSON.stringify(pagesManifest) + ) + + // copy static assets + await recursiveCopy( + path.join(distDir, 'static'), + path.join(shuttleDir, 'static') + ) + + // copy manifests not nested in {distDir}/server/ + await fs.promises.mkdir(path.join(shuttleDir, 'manifests'), { + recursive: true, + }) + + for (const item of [ + BUILD_MANIFEST, + ROUTES_MANIFEST, + APP_BUILD_MANIFEST, + REACT_LOADABLE_MANIFEST, + APP_PATH_ROUTES_MANIFEST, + ]) { + const outputPath = path.join(shuttleDir, 'manifests', item) + await fs.promises.mkdir(path.dirname(outputPath), { recursive: true }) + await fs.promises.copyFile(path.join(distDir, item), outputPath) + } +} diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index a9ebd536e6e76..fe356258db50b 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -101,7 +101,7 @@ import { import type { EventBuildFeatureUsage } from '../telemetry/events' import { Telemetry } from '../telemetry/storage' import { getPageStaticInfo } from './analysis/get-page-static-info' -import { createPagesMapping, sortByPageExts } from './entries' +import { createPagesMapping, getPageFromPath, sortByPageExts } from './entries' import { PAGE_TYPES } from '../lib/page-types' import { generateBuildId } from './generate-build-id' import { isWriteable } from './is-writeable' @@ -187,6 +187,12 @@ import { checkIsAppPPREnabled, checkIsRoutePPREnabled, } from '../server/lib/experimental/ppr' +import { + detectChangedEntries, + type DetectedEntriesResult, +} from './flying-shuttle/detect-changed-entries' +import { storeShuttle } from './flying-shuttle/store-shuttle' +import { stitchBuilds } from './flying-shuttle/stitch-builds' interface ExperimentalBypassForInfo { experimentalBypassFor?: RouteHas[] @@ -717,8 +723,10 @@ export default async function build( ) NextBuildContext.buildId = buildId + const shuttleDir = path.join(distDir, 'cache', 'shuttle') + if (config.experimental.flyingShuttle) { - await fs.mkdir(path.join(distDir, 'cache', 'shuttle'), { + await fs.mkdir(shuttleDir, { recursive: true, }) } @@ -838,20 +846,32 @@ export default async function build( appDir ) - const providedPagePaths: string[] = JSON.parse( - process.env.NEXT_PROVIDED_PAGE_PATHS || '[]' - ) - let pagesPaths = - providedPagePaths.length > 0 - ? providedPagePaths - : !appDirOnly && pagesDir - ? await nextBuildSpan.traceChild('collect-pages').traceAsyncFn(() => - recursiveReadDir(pagesDir, { - pathnameFilter: validFileMatcher.isPageFile, - }) - ) - : [] + !appDirOnly && pagesDir + ? await nextBuildSpan.traceChild('collect-pages').traceAsyncFn(() => + recursiveReadDir(pagesDir, { + pathnameFilter: validFileMatcher.isPageFile, + }) + ) + : [] + + let changedPagePathsResult: + | undefined + | { + changed: DetectedEntriesResult + unchanged: DetectedEntriesResult + } + + if (pagesPaths && config.experimental.flyingShuttle) { + changedPagePathsResult = await detectChangedEntries({ + pagesPaths, + pageExtensions: config.pageExtensions, + distDir, + shuttleDir, + }) + console.log({ changedPagePathsResult }) + pagesPaths = changedPagePathsResult.changed.pages + } const middlewareDetectionRegExp = new RegExp( `^${MIDDLEWARE_FILENAME}\\.(?:${config.pageExtensions.join('|')})$` @@ -912,27 +932,37 @@ export default async function build( let mappedAppPages: MappedPages | undefined let denormalizedAppPages: string[] | undefined + let changedAppPathsResult: + | undefined + | { + changed: DetectedEntriesResult + unchanged: DetectedEntriesResult + } if (appDir) { - const providedAppPaths: string[] = JSON.parse( - process.env.NEXT_PROVIDED_APP_PATHS || '[]' - ) + let appPaths = await nextBuildSpan + .traceChild('collect-app-paths') + .traceAsyncFn(() => + recursiveReadDir(appDir, { + pathnameFilter: (absolutePath) => + validFileMatcher.isAppRouterPage(absolutePath) || + // For now we only collect the root /not-found page in the app + // directory as the 404 fallback + validFileMatcher.isRootNotFound(absolutePath), + ignorePartFilter: (part) => part.startsWith('_'), + }) + ) - let appPaths = - providedAppPaths.length > 0 - ? providedAppPaths - : await nextBuildSpan - .traceChild('collect-app-paths') - .traceAsyncFn(() => - recursiveReadDir(appDir, { - pathnameFilter: (absolutePath) => - validFileMatcher.isAppRouterPage(absolutePath) || - // For now we only collect the root /not-found page in the app - // directory as the 404 fallback - validFileMatcher.isRootNotFound(absolutePath), - ignorePartFilter: (part) => part.startsWith('_'), - }) - ) + if (appPaths && config.experimental.flyingShuttle) { + changedAppPathsResult = await detectChangedEntries({ + appPaths, + pageExtensions: config.pageExtensions, + distDir, + shuttleDir, + }) + console.log({ changedAppPathsResult }) + appPaths = changedAppPathsResult.changed.app + } mappedAppPages = await nextBuildSpan .traceChild('create-app-mapping') @@ -1137,19 +1167,41 @@ export default async function build( ), } } + let clientRouterFilters: + | undefined + | ReturnType if (config.experimental.clientRouterFilter) { const nonInternalRedirects = (config._originalRedirects || []).filter( (r: any) => !r.internal ) - const clientRouterFilters = createClientRouterFilter( - appPaths, + const filterPaths: string[] = [] + + if (config.experimental.flyingShuttle) { + filterPaths.push( + ...[ + // client filter always has all app paths + ...(changedAppPathsResult?.unchanged?.app || []), + ...(changedAppPathsResult?.changed?.app || []), + ].map((entry) => + normalizeAppPath(getPageFromPath(entry, config.pageExtensions)) + ), + ...(changedPagePathsResult?.unchanged.pages.length + ? changedPagePathsResult.changed?.pages || [] + : [] + ).map((item) => getPageFromPath(item, config.pageExtensions)) + ) + } else { + filterPaths.push(...appPaths) + } + + clientRouterFilters = createClientRouterFilter( + filterPaths, config.experimental.clientRouterFilterRedirects ? nonInternalRedirects : [], config.experimental.clientRouterFilterAllowedRate ) - NextBuildContext.clientRouterFilters = clientRouterFilters } @@ -1323,7 +1375,7 @@ export default async function build( env: process.env as Record, defineEnv: createDefineEnv({ isTurbopack: true, - clientRouterFilters: NextBuildContext.clientRouterFilters, + clientRouterFilters, config, dev, distDir, @@ -1736,7 +1788,7 @@ export default async function build( const appDynamicParamPaths = new Set() const appDefaultConfigs = new Map() const pageInfos: PageInfos = new Map() - const pagesManifest = await readManifest(pagesManifestPath) + let pagesManifest = await readManifest(pagesManifestPath) const buildManifest = await readManifest(buildManifestPath) const appBuildManifest = appDir ? await readManifest(appBuildManifestPath) @@ -2424,6 +2476,53 @@ export default async function build( path.join(distDir, SERVER_DIRECTORY, MIDDLEWARE_MANIFEST) ) + if (!isGenerateMode) { + if (config.experimental.flyingShuttle) { + console.log('stitching builds...') + const stitchResult = await stitchBuilds( + { + buildId, + distDir, + shuttleDir, + rewrites, + redirects, + edgePreviewProps: { + __NEXT_PREVIEW_MODE_ID: + NextBuildContext.previewProps!.previewModeId, + __NEXT_PREVIEW_MODE_ENCRYPTION_KEY: + NextBuildContext.previewProps!.previewModeEncryptionKey, + __NEXT_PREVIEW_MODE_SIGNING_KEY: + NextBuildContext.previewProps!.previewModeSigningKey, + }, + encryptionKey, + allowedErrorRate: + config.experimental.clientRouterFilterAllowedRate, + }, + { + changed: { + pages: changedPagePathsResult?.changed.pages || [], + app: changedAppPathsResult?.changed.app || [], + }, + unchanged: { + pages: changedPagePathsResult?.unchanged.pages || [], + app: changedAppPathsResult?.unchanged.app || [], + }, + pageExtensions: config.pageExtensions, + } + ) + // reload pagesManifest since it's been updated on disk + if (stitchResult.pagesManifest) { + pagesManifest = stitchResult.pagesManifest + } + + console.log('storing shuttle') + await storeShuttle({ + distDir, + shuttleDir, + }) + } + } + const finalPrerenderRoutes: { [route: string]: SsgRoute } = {} const finalDynamicRoutes: PrerenderManifest['dynamicRoutes'] = {} const tbdPrerenderRoutes: string[] = [] diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index 169162a2a1d67..b04ea5b3089c1 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -945,24 +945,11 @@ export default async function getBaseWebpackConfig( ), ], - ...(config.experimental.flyingShuttle - ? { - recordsPath: path.join(distDir, 'cache', 'shuttle', 'records.json'), - } - : {}), - optimization: { emitOnErrors: !dev, checkWasmTypes: false, nodeEnv: false, - ...(config.experimental.flyingShuttle - ? { - moduleIds: 'deterministic', - portableRecords: true, - } - : {}), - splitChunks: ((): | Required['optimization']['splitChunks'] | false => { @@ -1019,7 +1006,7 @@ export default async function getBaseWebpackConfig( if (isNodeServer || isEdgeServer) { return { - filename: `${isEdgeServer ? 'edge-chunks/' : ''}[name].js`, + filename: `${isEdgeServer ? `edge-chunks${config.experimental.flyingShuttle ? `-${buildId}` : ''}/` : ''}[name].js`, chunks: 'all', minChunks: 2, } @@ -1107,6 +1094,7 @@ export default async function getBaseWebpackConfig( runtimeChunk: isClient ? { name: CLIENT_STATIC_FILES_RUNTIME_WEBPACK } : undefined, + minimize: !dev && (isClient || @@ -1199,13 +1187,28 @@ export default async function getBaseWebpackConfig( ...(config.experimental.flyingShuttle ? { // ensure we only use contenthash as it's more deterministic - filename: isNodeOrEdgeCompilation - ? dev || isEdgeServer - ? `[name].js` - : `../[name].js` - : `static/chunks/${isDevFallback ? 'fallback/' : ''}[name]${ - dev ? '' : '-[contenthash]' - }.js`, + filename: (p) => { + if (isNodeOrEdgeCompilation) { + // runtime chunk needs hash so it can be isolated + // across builds + const isRuntimeChunk = p.chunk?.name?.match( + /webpack-(api-runtime|runtime)/ + ) + return `${isEdgeServer ? '' : '../'}[name]${isRuntimeChunk ? `-${buildId}` : ''}.js` + } + // client filename + return `static/chunks/[name]-[contenthash].js` + }, + + path: isNodeServer + ? path.join(outputPath, `chunks-${buildId}`) + : outputPath, + + chunkFilename: isNodeOrEdgeCompilation + ? `[name].js` + : `static/chunks/[contenthash].js`, + + webassemblyModuleFilename: 'static/wasm/[contenthash].wasm', } : {}), }, @@ -1796,7 +1799,6 @@ export default async function getBaseWebpackConfig( }), getDefineEnvPlugin({ isTurbopack: false, - clientRouterFilters, config, dev, distDir, @@ -1894,6 +1896,7 @@ export default async function getBaseWebpackConfig( rewrites, isDevFallback, appDirEnabled: hasAppDir, + clientRouterFilters, }), new ProfilingPlugin({ runWebpackSpan, rootDir: dir }), config.optimizeFonts && diff --git a/packages/next/src/build/webpack/plugins/build-manifest-plugin.ts b/packages/next/src/build/webpack/plugins/build-manifest-plugin.ts index 5bdc80e627c74..1355c59905e1d 100644 --- a/packages/next/src/build/webpack/plugins/build-manifest-plugin.ts +++ b/packages/next/src/build/webpack/plugins/build-manifest-plugin.ts @@ -1,3 +1,4 @@ +import type { BloomFilter } from '../../../shared/lib/bloom-filter' import type { Rewrite, CustomRoutes } from '../../../lib/load-custom-routes' import devalue from 'next/dist/compiled/devalue' import { webpack, sources } from 'next/dist/compiled/webpack/webpack' @@ -17,6 +18,7 @@ import getRouteFromEntrypoint from '../../../server/get-route-from-entrypoint' import { ampFirstEntryNamesMap } from './next-drop-client-page-plugin' import { getSortedRoutes } from '../../../shared/lib/router/utils' import { spans } from './profiling-plugin' +import { Span } from '../../../trace' type DeepMutable = { -readonly [P in keyof T]: DeepMutable } @@ -91,13 +93,22 @@ export function normalizeRewritesForBuildManifest( // This function takes the asset map generated in BuildManifestPlugin and creates a // reduced version to send to the client. -function generateClientManifest( - compiler: any, - compilation: any, +export function generateClientManifest( assetMap: BuildManifest, - rewrites: CustomRoutes['rewrites'] + rewrites: CustomRoutes['rewrites'], + clientRouterFilters?: { + staticFilter: ReturnType + dynamicFilter: ReturnType + }, + compiler?: any, + compilation?: any ): string | undefined { - const compilationSpan = spans.get(compilation) || spans.get(compiler) + const compilationSpan = compilation + ? spans.get(compilation) + : compiler + ? spans.get(compiler) + : new Span({ name: 'client-manifest' }) + const genClientManifestSpan = compilationSpan?.traceChild( 'NextJsBuildManifest-generateClientManifest' ) @@ -105,6 +116,8 @@ function generateClientManifest( return genClientManifestSpan?.traceFn(() => { const clientManifest: ClientBuildManifest = { __rewrites: normalizeRewritesForBuildManifest(rewrites) as any, + __routerFilterStatic: clientRouterFilters?.staticFilter as any, + __routerFilterDynamic: clientRouterFilters?.dynamicFilter as any, } const appDependencies = new Set(assetMap.pages['/_app']) const sortedPageKeys = getSortedRoutes(Object.keys(assetMap.pages)) @@ -162,12 +175,14 @@ export default class BuildManifestPlugin { private rewrites: CustomRoutes['rewrites'] private isDevFallback: boolean private appDirEnabled: boolean + private clientRouterFilters?: Parameters[2] constructor(options: { buildId: string rewrites: CustomRoutes['rewrites'] isDevFallback?: boolean appDirEnabled: boolean + clientRouterFilters?: Parameters[2] }) { this.buildId = options.buildId this.isDevFallback = !!options.isDevFallback @@ -177,6 +192,7 @@ export default class BuildManifestPlugin { fallback: [], } this.appDirEnabled = options.appDirEnabled + this.clientRouterFilters = options.clientRouterFilters this.rewrites.beforeFiles = options.rewrites.beforeFiles.map(processRoute) this.rewrites.afterFiles = options.rewrites.afterFiles.map(processRoute) this.rewrites.fallback = options.rewrites.fallback.map(processRoute) @@ -195,6 +211,7 @@ export default class BuildManifestPlugin { ampDevFiles: [], lowPriorityFiles: [], rootMainFiles: [], + rootMainFilesTree: {}, pages: { '/_app': [] }, ampFirstPages: [], } @@ -308,10 +325,11 @@ export default class BuildManifestPlugin { assets[clientManifestPath] = new sources.RawSource( `self.__BUILD_MANIFEST = ${generateClientManifest( - compiler, - compilation, assetMap, - this.rewrites + this.rewrites, + this.clientRouterFilters, + compiler, + compilation )};self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB()` ) } diff --git a/packages/next/src/build/webpack/plugins/define-env-plugin.ts b/packages/next/src/build/webpack/plugins/define-env-plugin.ts index edd19f6105440..ea1b3452ddde6 100644 --- a/packages/next/src/build/webpack/plugins/define-env-plugin.ts +++ b/packages/next/src/build/webpack/plugins/define-env-plugin.ts @@ -197,6 +197,8 @@ export function getDefineEnv({ ? 5 * 60 // 5 minutes : config.experimental.staleTimes?.static ), + 'process.env.__NEXT_FLYING_SHUTTLE': + config.experimental.flyingShuttle ?? false, 'process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED': config.experimental.clientRouterFilter ?? true, 'process.env.__NEXT_CLIENT_ROUTER_S_FILTER': diff --git a/packages/next/src/server/app-render/app-render.tsx b/packages/next/src/server/app-render/app-render.tsx index c48f874a74866..452c48210c984 100644 --- a/packages/next/src/server/app-render/app-render.tsx +++ b/packages/next/src/server/app-render/app-render.tsx @@ -1017,7 +1017,8 @@ async function renderToHTMLOrFlightImpl( renderOpts.crossOrigin, subresourceIntegrityManifest, getAssetQueryString(ctx, true), - nonce + nonce, + renderOpts.page ) const rscPayload = await getRSCPayload(tree, ctx, asNotFound) @@ -1385,7 +1386,8 @@ async function renderToHTMLOrFlightImpl( renderOpts.crossOrigin, subresourceIntegrityManifest, getAssetQueryString(ctx, false), - nonce + nonce, + '/_not-found/page' ) const errorRSCPayload = await getErrorRSCPayload(tree, ctx, errorType) diff --git a/packages/next/src/server/app-render/required-scripts.tsx b/packages/next/src/server/app-render/required-scripts.tsx index 113bd0ac183db..0bdc268afe3a5 100644 --- a/packages/next/src/server/app-render/required-scripts.tsx +++ b/packages/next/src/server/app-render/required-scripts.tsx @@ -9,7 +9,8 @@ export function getRequiredScripts( crossOrigin: undefined | '' | 'anonymous' | 'use-credentials', SRIManifest: undefined | Record, qs: string, - nonce: string | undefined + nonce: string | undefined, + pagePath: string ): [ () => void, { src: string; integrity?: string; crossOrigin?: string | undefined }, @@ -25,7 +26,9 @@ export function getRequiredScripts( crossOrigin, } - const files = buildManifest.rootMainFiles.map(encodeURIPath) + const files = ( + buildManifest.rootMainFilesTree?.[pagePath] || buildManifest.rootMainFiles + ).map(encodeURIPath) if (files.length === 0) { throw new Error( 'Invariant: missing bootstrap script. This is a bug in Next.js' diff --git a/packages/next/src/server/get-page-files.ts b/packages/next/src/server/get-page-files.ts index 229729b3e3974..9890b9f36e724 100644 --- a/packages/next/src/server/get-page-files.ts +++ b/packages/next/src/server/get-page-files.ts @@ -7,6 +7,10 @@ export type BuildManifest = { polyfillFiles: readonly string[] lowPriorityFiles: readonly string[] rootMainFiles: readonly string[] + // this is a separate field for flying shuttle to allow + // different root main files per entries/build (ideally temporary) + // until we can stitch the runtime chunks together safely + rootMainFilesTree: { [appRoute: string]: readonly string[] } pages: { '/_app': readonly string[] [page: string]: readonly string[] diff --git a/packages/next/src/shared/lib/router/router.ts b/packages/next/src/shared/lib/router/router.ts index 9265c94a74faf..fee787615db4d 100644 --- a/packages/next/src/shared/lib/router/router.ts +++ b/packages/next/src/shared/lib/router/router.ts @@ -765,45 +765,6 @@ export default class Router implements BaseRouter { ], } - if (process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED) { - const { BloomFilter } = - require('../../lib/bloom-filter') as typeof import('../../lib/bloom-filter') - - type Filter = ReturnType< - import('../../lib/bloom-filter').BloomFilter['export'] - > - - const routerFilterSValue: Filter | false = process.env - .__NEXT_CLIENT_ROUTER_S_FILTER as any - - const staticFilterData: Filter | undefined = routerFilterSValue - ? routerFilterSValue - : undefined - - const routerFilterDValue: Filter | false = process.env - .__NEXT_CLIENT_ROUTER_D_FILTER as any - - const dynamicFilterData: Filter | undefined = routerFilterDValue - ? routerFilterDValue - : undefined - - if (staticFilterData?.numHashes) { - this._bfl_s = new BloomFilter( - staticFilterData.numItems, - staticFilterData.errorRate - ) - this._bfl_s.import(staticFilterData) - } - - if (dynamicFilterData?.numHashes) { - this._bfl_d = new BloomFilter( - dynamicFilterData.numItems, - dynamicFilterData.errorRate - ) - this._bfl_d.import(dynamicFilterData) - } - } - // Backwards compat for Router.router.events // TODO: Should be remove the following major version as it was never documented this.events = Router.events @@ -1060,10 +1021,86 @@ export default class Router implements BaseRouter { skipNavigate?: boolean ) { if (process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED) { + if (!this._bfl_s && !this._bfl_d) { + const { BloomFilter } = + require('../../lib/bloom-filter') as typeof import('../../lib/bloom-filter') + + type Filter = ReturnType< + import('../../lib/bloom-filter').BloomFilter['export'] + > + let staticFilterData: Filter | undefined + let dynamicFilterData: Filter | undefined + + try { + ;({ + __routerFilterStatic: staticFilterData, + __routerFilterDynamic: dynamicFilterData, + } = (await getClientBuildManifest()) as any as { + __routerFilterStatic?: Filter + __routerFilterDynamic?: Filter + }) + } catch (err) { + // failed to load build manifest hard navigate + // to be safe + console.error(err) + if (skipNavigate) { + return true + } + handleHardNavigation({ + url: addBasePath( + addLocale(as, locale || this.locale, this.defaultLocale) + ), + router: this, + }) + return new Promise(() => {}) + } + + const routerFilterSValue: Filter | false = process.env + .__NEXT_CLIENT_ROUTER_S_FILTER as any + + if (!staticFilterData && routerFilterSValue) { + staticFilterData = routerFilterSValue ? routerFilterSValue : undefined + } + + const routerFilterDValue: Filter | false = process.env + .__NEXT_CLIENT_ROUTER_D_FILTER as any + + if (!dynamicFilterData && routerFilterDValue) { + dynamicFilterData = routerFilterDValue + ? routerFilterDValue + : undefined + } + + if (staticFilterData?.numHashes) { + this._bfl_s = new BloomFilter( + staticFilterData.numItems, + staticFilterData.errorRate + ) + this._bfl_s.import(staticFilterData) + } + + if (dynamicFilterData?.numHashes) { + this._bfl_d = new BloomFilter( + dynamicFilterData.numItems, + dynamicFilterData.errorRate + ) + this._bfl_d.import(dynamicFilterData) + } + } + let matchesBflStatic = false let matchesBflDynamic = false + const pathsToCheck: Array<{ as?: string; allowMatchCurrent?: boolean }> = + [{ as }, { as: resolvedAs }] + + if (process.env.__NEXT_FLYING_SHUTTLE) { + // if existing page changed we hard navigate to + // avoid runtime conflict with new page + // TODO: check buildManifest files instead? + pathsToCheck.push({ as: this.asPath, allowMatchCurrent: true }) + } - for (const curAs of [as, resolvedAs]) { + for (const { as: curAs, allowMatchCurrent } of pathsToCheck) { if (curAs) { const asNoSlash = removeTrailingSlash( new URL(curAs, 'http://n').pathname @@ -1073,8 +1110,9 @@ export default class Router implements BaseRouter { ) if ( + allowMatchCurrent || asNoSlash !== - removeTrailingSlash(new URL(this.asPath, 'http://n').pathname) + removeTrailingSlash(new URL(this.asPath, 'http://n').pathname) ) { matchesBflStatic = matchesBflStatic || diff --git a/test/e2e/app-dir/app/app/dynamic-client/[category]/[id]/page.js b/test/e2e/app-dir/app/app/dynamic-client/[category]/[id]/page.js index ca5cd77f0f571..d807ae8910cd7 100644 --- a/test/e2e/app-dir/app/app/dynamic-client/[category]/[id]/page.js +++ b/test/e2e/app-dir/app/app/dynamic-client/[category]/[id]/page.js @@ -1,6 +1,11 @@ 'use client' import { useSearchParams } from 'next/navigation' +import dynamic from 'next/dynamic' + +const Button = dynamic(() => + import('../../../../components/button/button').then((mod) => mod.Button) +) export default function IdPage({ children, params }) { return ( @@ -14,6 +19,7 @@ export default function IdPage({ children, params }) {

{JSON.stringify(Object.fromEntries(useSearchParams()))}

+ ) } diff --git a/test/e2e/app-dir/app/components/button/button.js b/test/e2e/app-dir/app/components/button/button.js new file mode 100644 index 0000000000000..b8836ebcc7274 --- /dev/null +++ b/test/e2e/app-dir/app/components/button/button.js @@ -0,0 +1,15 @@ +'use client' +import * as buttonStyle from './button.module.css' + +export function Button({ children }) { + return ( + + ) +} diff --git a/test/e2e/app-dir/app/components/button/button.module.css b/test/e2e/app-dir/app/components/button/button.module.css new file mode 100644 index 0000000000000..be1d0b06f1cd1 --- /dev/null +++ b/test/e2e/app-dir/app/components/button/button.module.css @@ -0,0 +1,4 @@ +.button { + background: #000; + color: #fff; +} diff --git a/test/e2e/app-dir/app/flying-shuttle.test.ts b/test/e2e/app-dir/app/flying-shuttle.test.ts index f07c1daecbd43..5711475f46c31 100644 --- a/test/e2e/app-dir/app/flying-shuttle.test.ts +++ b/test/e2e/app-dir/app/flying-shuttle.test.ts @@ -1,6 +1,7 @@ import fs from 'fs' import path from 'path' import { nextTestSetup, isNextStart } from 'e2e-utils' +import { retry } from 'next-test-utils' // This feature is only relevant to Webpack. ;(process.env.TURBOPACK ? describe.skip : describe)( @@ -70,5 +71,94 @@ import { nextTestSetup, isNextStart } from 'e2e-utils' } } }) + + async function checkAppPagesNavigation() { + for (const path of [ + '/', + '/blog/123', + '/dynamic-client/first/second', + '/dashboard', + '/dashboard/deployments/123', + ]) { + require('console').error('checking', path) + const res = await next.fetch(path) + expect(res.status).toBe(200) + + const browser = await next.browser(path) + // TODO: check for hydration success properly + await retry(async () => { + expect(await browser.eval('!!window.next.router')).toBe(true) + }) + const browserLogs = await browser.log() + expect( + browserLogs.some(({ message }) => message.includes('error')) + ).toBeFalse() + } + // TODO: check we hard navigate boundaries properly + } + + it('should only rebuild just a changed app route correctly', async () => { + await next.stop() + + const dataPath = 'app/dashboard/deployments/[id]/data.json' + const originalContent = await next.readFile(dataPath) + + try { + await next.patchFile(dataPath, JSON.stringify({ hello: 'again' })) + await next.start() + + await checkAppPagesNavigation() + } finally { + await next.patchFile(dataPath, originalContent) + } + }) + + it('should only rebuild just a changed pages route correctly', async () => { + await next.stop() + + const pagePath = 'pages/index.js' + const originalContent = await next.readFile(pagePath) + + try { + await next.patchFile( + pagePath, + originalContent.replace( + 'hello from pages/index', + 'hello from pages/index!!' + ) + ) + await next.start() + + await checkAppPagesNavigation() + } finally { + await next.patchFile(pagePath, originalContent) + } + }) + + it('should only rebuild a changed app and pages route correctly', async () => { + await next.stop() + + const pagePath = 'pages/index.js' + const originalPageContent = await next.readFile(pagePath) + const dataPath = 'app/dashboard/deployments/[id]/data.json' + const originalDataContent = await next.readFile(dataPath) + + try { + await next.patchFile( + pagePath, + originalPageContent.replace( + 'hello from pages/index', + 'hello from pages/index!!' + ) + ) + await next.patchFile(dataPath, JSON.stringify({ hello: 'again' })) + await next.start() + + await checkAppPagesNavigation() + } finally { + await next.patchFile(pagePath, originalPageContent) + await next.patchFile(dataPath, originalDataContent) + } + }) } ) diff --git a/test/e2e/app-dir/app/next.config.js b/test/e2e/app-dir/app/next.config.js index 927e60e0ea5c6..6cb266cb54a28 100644 --- a/test/e2e/app-dir/app/next.config.js +++ b/test/e2e/app-dir/app/next.config.js @@ -8,6 +8,14 @@ module.exports = { parallelServerBuildTraces: true, webpackBuildWorker: true, }, + webpack(cfg) { + if (process.env.NEXT_PRIVATE_FLYING_SHUTTLE) { + // disable the webpack cache to make sure we're + // deterministic without + cfg.cache = false + } + return cfg + }, // output: 'standalone', rewrites: async () => { return { diff --git a/test/e2e/app-dir/app/pages/index.js b/test/e2e/app-dir/app/pages/index.js index b1037a470b719..7055a6eb21af3 100644 --- a/test/e2e/app-dir/app/pages/index.js +++ b/test/e2e/app-dir/app/pages/index.js @@ -1,7 +1,12 @@ import React from 'react' import Link from 'next/link' +import dynamic from 'next/dynamic' import styles from '../styles/shared.module.css' +const Button = dynamic(() => + import('../components/button/button').then((mod) => mod.Button) +) + export default function Page() { return ( <> @@ -10,6 +15,7 @@ export default function Page() {

Dashboard

{React.version}

+ ) } diff --git a/test/e2e/app-dir/app/provide-paths.test.ts b/test/e2e/app-dir/app/provide-paths.test.ts deleted file mode 100644 index ed5d38ad91b09..0000000000000 --- a/test/e2e/app-dir/app/provide-paths.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { nextTestSetup } from 'e2e-utils' -import glob from 'glob' -import path from 'path' - -describe('Provided page/app paths', () => { - const { next, isNextDev } = nextTestSetup({ - files: __dirname, - // Deployments are unable to inspect the `.next` directory. - skipDeployment: true, - dependencies: { - nanoid: '4.0.1', - }, - env: { - NEXT_PROVIDED_PAGE_PATHS: JSON.stringify(['/index.js', '/ssg.js']), - NEXT_PROVIDED_APP_PATHS: JSON.stringify([ - '/dashboard/page.js', - '/(newroot)/dashboard/another/page.js', - ]), - }, - }) - - if (isNextDev) { - it('should skip dev', () => {}) - return - } - - it('should only build the provided paths', async () => { - const appPaths = await glob.sync('**/*.js', { - cwd: path.join(next.testDir, '.next/server/app'), - }) - const pagePaths = await glob.sync('**/*.js', { - cwd: path.join(next.testDir, '.next/server/pages'), - }) - - expect(appPaths).toEqual([ - '_not-found/page_client-reference-manifest.js', - '_not-found/page.js', - '(newroot)/dashboard/another/page_client-reference-manifest.js', - '(newroot)/dashboard/another/page.js', - 'dashboard/page_client-reference-manifest.js', - 'dashboard/page.js', - ]) - expect(pagePaths).toEqual([ - '_app.js', - '_document.js', - '_error.js', - 'ssg.js', - ]) - - for (const pathname of ['/', '/ssg', '/dashboard', '/dashboard/another']) { - const res = await next.fetch(pathname) - expect(res.status).toBe(200) - } - }) -})