From 3a832f1e0a8c582e7bacf6db4ef232f7178dc005 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 26 Sep 2024 13:54:00 -0700 Subject: [PATCH] Add cache scope handling for dynamic IO for dev/build (#70408) As discussed this adds an in memory cache scope which is leveraged for seeding during prefetch and then leveraged during non-prefetch requests in development and during build it shares a cache scope across one build worker. During production server mode the cache scopes are specific per-request with no prefetch cache seeding. --- packages/next/src/build/index.ts | 2 + packages/next/src/build/utils.ts | 6 ++ ...tatic-generation-async-storage.external.ts | 1 + .../helpers/create-incremental-cache.ts | 3 + packages/next/src/export/worker.ts | 27 +++++--- .../next/src/server/app-render/app-render.tsx | 1 + .../src/server/async-storage/cache-scope.ts | 27 ++++++++ .../with-static-generation-store.ts | 3 + packages/next/src/server/base-server.ts | 63 ++++++++++++++++++- .../next/src/server/dev/next-dev-server.ts | 1 + .../src/server/dev/static-paths-worker.ts | 2 + .../src/server/lib/incremental-cache/index.ts | 35 ++++++++++- .../src/server/lib/prefetch-cache-scopes.ts | 40 ++++++++++++ packages/next/src/server/next-server.ts | 1 + packages/next/src/server/request/cookies.ts | 5 +- .../next/src/server/request/draft-mode.ts | 7 ++- packages/next/src/server/request/headers.ts | 5 +- packages/next/src/server/request/params.ts | 5 +- .../next/src/server/request/search-params.ts | 5 +- packages/next/src/server/web-server.ts | 1 + 20 files changed, 221 insertions(+), 19 deletions(-) create mode 100644 packages/next/src/server/async-storage/cache-scope.ts create mode 100644 packages/next/src/server/lib/prefetch-cache-scopes.ts diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 3d5cb5002d982..f954d5eb9d23f 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1897,6 +1897,7 @@ export default async function build( distDir, configFileName, runtimeEnvConfig, + dynamicIO: Boolean(config.experimental.dynamicIO), httpAgentOptions: config.httpAgentOptions, locales: config.i18n?.locales, defaultLocale: config.i18n?.defaultLocale, @@ -2112,6 +2113,7 @@ export default async function build( pageRuntime, edgeInfo, pageType, + dynamicIO: Boolean(config.experimental.dynamicIO), cacheHandler: config.cacheHandler, isrFlushToDisk: ciEnvironment.hasNextSupport ? false diff --git a/packages/next/src/build/utils.ts b/packages/next/src/build/utils.ts index e0abdf17ff31d..c772a0bcbc3bd 100644 --- a/packages/next/src/build/utils.ts +++ b/packages/next/src/build/utils.ts @@ -1348,6 +1348,7 @@ export async function buildAppStaticPaths({ dir, page, distDir, + dynamicIO, configFileName, generateParams, isrFlushToDisk, @@ -1362,6 +1363,7 @@ export async function buildAppStaticPaths({ }: { dir: string page: string + dynamicIO: boolean configFileName: string generateParams: GenerateParamsResults distDir: string @@ -1390,6 +1392,7 @@ export async function buildAppStaticPaths({ const incrementalCache = new IncrementalCache({ fs: nodeFs, dev: true, + dynamicIO, flushToDisk: isrFlushToDisk, serverDistDir: path.join(distDir, 'server'), fetchCacheKeyPrefix, @@ -1581,6 +1584,7 @@ export async function isPageStatic({ pageRuntime, edgeInfo, pageType, + dynamicIO, originalAppPath, isrFlushToDisk, maxMemoryCacheSize, @@ -1592,6 +1596,7 @@ export async function isPageStatic({ dir: string page: string distDir: string + dynamicIO: boolean configFileName: string runtimeEnvConfig: any httpAgentOptions: NextConfigComplete['httpAgentOptions'] @@ -1722,6 +1727,7 @@ export async function isPageStatic({ await buildAppStaticPaths({ dir, page, + dynamicIO, configFileName, generateParams, distDir, diff --git a/packages/next/src/client/components/static-generation-async-storage.external.ts b/packages/next/src/client/components/static-generation-async-storage.external.ts index 6df289fcd8c84..ac5a8569a060d 100644 --- a/packages/next/src/client/components/static-generation-async-storage.external.ts +++ b/packages/next/src/client/components/static-generation-async-storage.external.ts @@ -63,6 +63,7 @@ export interface StaticGenerationStore { isDraftMode?: boolean isUnstableNoStore?: boolean + isPrefetchRequest?: boolean requestEndedState?: { ended?: boolean } } diff --git a/packages/next/src/export/helpers/create-incremental-cache.ts b/packages/next/src/export/helpers/create-incremental-cache.ts index 00dd82f9166dd..fa209b19f6e3b 100644 --- a/packages/next/src/export/helpers/create-incremental-cache.ts +++ b/packages/next/src/export/helpers/create-incremental-cache.ts @@ -7,12 +7,14 @@ import { formatDynamicImportPath } from '../../lib/format-dynamic-import-path' export async function createIncrementalCache({ cacheHandler, + dynamicIO, cacheMaxMemorySize, fetchCacheKeyPrefix, distDir, dir, flushToDisk, }: { + dynamicIO: boolean cacheHandler?: string cacheMaxMemorySize?: number fetchCacheKeyPrefix?: string @@ -34,6 +36,7 @@ export async function createIncrementalCache({ dev: false, requestHeaders: {}, flushToDisk, + dynamicIO, fetchCache: true, maxMemoryCacheSize: cacheMaxMemorySize, fetchCacheKeyPrefix, diff --git a/packages/next/src/export/worker.ts b/packages/next/src/export/worker.ts index 1c0b8c988ba96..f1d061df08e1a 100644 --- a/packages/next/src/export/worker.ts +++ b/packages/next/src/export/worker.ts @@ -46,6 +46,7 @@ import { type FallbackRouteParams, } from '../server/request/fallback-params' import { needsExperimentalReact } from '../lib/needs-experimental-react' +import { runWithCacheScope } from '../server/async-storage/cache-scope' const envConfig = require('../shared/lib/runtime-config.external') @@ -352,6 +353,7 @@ export async function exportPages( fetchCacheKeyPrefix, distDir, dir, + dynamicIO: Boolean(nextConfig.experimental.dynamicIO), // skip writing to disk in minimal mode for now, pending some // changes to better support it flushToDisk: !hasNextSupport, @@ -459,21 +461,26 @@ export async function exportPages( return { result, path, pageKey } } + // for each build worker we share one dynamic IO cache scope + // this is only leveraged if the flag is enabled + const dynamicIOCacheScope = new Map() - for (let i = 0; i < paths.length; i += maxConcurrency) { - const subset = paths.slice(i, i + maxConcurrency) + await runWithCacheScope({ cache: dynamicIOCacheScope }, async () => { + for (let i = 0; i < paths.length; i += maxConcurrency) { + const subset = paths.slice(i, i + maxConcurrency) - const subsetResults = await Promise.all( - subset.map((path) => - exportPageWithRetry( - path, - nextConfig.experimental.staticGenerationRetryCount ?? 1 + const subsetResults = await Promise.all( + subset.map((path) => + exportPageWithRetry( + path, + nextConfig.experimental.staticGenerationRetryCount ?? 1 + ) ) ) - ) - results.push(...subsetResults) - } + results.push(...subsetResults) + } + }) return results } diff --git a/packages/next/src/server/app-render/app-render.tsx b/packages/next/src/server/app-render/app-render.tsx index d4c3d77286f0d..71e9ed8b32706 100644 --- a/packages/next/src/server/app-render/app-render.tsx +++ b/packages/next/src/server/app-render/app-render.tsx @@ -1284,6 +1284,7 @@ export const renderToHTMLOrFlight: AppPageRender = ( fallbackRouteParams, renderOpts, requestEndedState, + isPrefetchRequest: Boolean(req.headers[NEXT_ROUTER_PREFETCH_HEADER]), }, (staticGenerationStore) => renderToHTMLOrFlightImpl( diff --git a/packages/next/src/server/async-storage/cache-scope.ts b/packages/next/src/server/async-storage/cache-scope.ts new file mode 100644 index 0000000000000..f8d0970dc94ad --- /dev/null +++ b/packages/next/src/server/async-storage/cache-scope.ts @@ -0,0 +1,27 @@ +import { AsyncLocalStorage } from 'async_hooks' + +export interface CacheScopeStore { + cache?: Map +} + +export const cacheScopeAsyncLocalStorage = + new AsyncLocalStorage() + +/** + * For dynamic IO handling we want to have a scoped memory + * cache which can live either the lifetime of a build worker, + * the lifetime of a specific request, or from a prefetch request + * to the request for non-prefetch version of a page (with + * drop-off after so long to prevent memory inflating) + */ +export function runWithCacheScope( + store: Partial, + fn: (...args: any[]) => Promise +) { + return cacheScopeAsyncLocalStorage.run( + { + cache: store.cache || new Map(), + }, + fn + ) +} diff --git a/packages/next/src/server/async-storage/with-static-generation-store.ts b/packages/next/src/server/async-storage/with-static-generation-store.ts index 2b289471df8f7..71d702fe110e4 100644 --- a/packages/next/src/server/async-storage/with-static-generation-store.ts +++ b/packages/next/src/server/async-storage/with-static-generation-store.ts @@ -21,6 +21,7 @@ export type StaticGenerationContext = { fallbackRouteParams: FallbackRouteParams | null requestEndedState?: { ended?: boolean } + isPrefetchRequest?: boolean renderOpts: { incrementalCache?: IncrementalCache isOnDemandRevalidate?: boolean @@ -69,6 +70,7 @@ export const withStaticGenerationStore: WithStore< fallbackRouteParams, renderOpts, requestEndedState, + isPrefetchRequest, }: StaticGenerationContext, callback: (store: StaticGenerationStore) => Result ): Result => { @@ -111,6 +113,7 @@ export const withStaticGenerationStore: WithStore< isDraftMode: renderOpts.isDraftMode, requestEndedState, + isPrefetchRequest, } // TODO: remove this when we resolve accessing the store outside the execution context diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 22935ef81c58e..26985461dc6f4 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -173,6 +173,11 @@ import type { RouteModule } from './route-modules/route-module' import { FallbackMode, parseFallbackField } from '../lib/fallback' import { toResponseCacheEntry } from './response-cache/utils' import { scheduleOnNextTick } from '../lib/scheduler' +import { PrefetchCacheScopes } from './lib/prefetch-cache-scopes' +import { + runWithCacheScope, + type CacheScopeStore, +} from './async-storage/cache-scope' export type FindComponentsResult = { components: LoadComponentsReturnType @@ -454,6 +459,14 @@ export default abstract class Server< private readonly isAppPPREnabled: boolean + private readonly prefetchCacheScopesDev = new PrefetchCacheScopes() + + /** + * This is used to persist cache scopes across + * prefetch -> full route requests for dynamic IO + * it's only fully used in dev + */ + public constructor(options: ServerOptions) { const { dir = '.', @@ -2748,7 +2761,7 @@ export default abstract class Server< } } - const responseGenerator: ResponseGenerator = async ({ + let responseGenerator: ResponseGenerator = async ({ hasResolved, previousCacheEntry, isRevalidating, @@ -2999,6 +3012,54 @@ export default abstract class Server< } } + if (this.nextConfig.experimental.dynamicIO) { + const originalResponseGenerator = responseGenerator + + responseGenerator = async ( + ...args: Parameters + ): ReturnType => { + let cache: CacheScopeStore['cache'] | undefined + + if (this.renderOpts.dev) { + cache = this.prefetchCacheScopesDev.get(urlPathname) + + // we need to seed the prefetch cache scope in dev + // since we did not have a prefetch cache available + // and this is not a prefetch request + if ( + !cache && + !isPrefetchRSCRequest && + routeModule?.definition.kind === RouteKind.APP_PAGE + ) { + req.headers[RSC_HEADER] = '1' + req.headers[NEXT_ROUTER_PREFETCH_HEADER] = '1' + + cache = new Map() + + await runWithCacheScope({ cache }, () => + originalResponseGenerator(...args) + ) + this.prefetchCacheScopesDev.set(urlPathname, cache) + + delete req.headers[RSC_HEADER] + delete req.headers[NEXT_ROUTER_PREFETCH_HEADER] + } + } + + return runWithCacheScope({ cache }, () => + originalResponseGenerator(...args) + ).finally(() => { + if (this.renderOpts.dev) { + if (isPrefetchRSCRequest) { + this.prefetchCacheScopesDev.set(urlPathname, cache) + } else { + this.prefetchCacheScopesDev.del(urlPathname) + } + } + }) + } + } + const cacheEntry = await this.responseCache.get( ssgCacheKey, responseGenerator, diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index 40b2ea7346402..cc19c32853d4d 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -762,6 +762,7 @@ export default class DevServer extends Server { configFileName, publicRuntimeConfig, serverRuntimeConfig, + dynamicIO: Boolean(this.nextConfig.experimental.dynamicIO), }, httpAgentOptions, locales, diff --git a/packages/next/src/server/dev/static-paths-worker.ts b/packages/next/src/server/dev/static-paths-worker.ts index dad3333f19dc3..5f80c98dbcf02 100644 --- a/packages/next/src/server/dev/static-paths-worker.ts +++ b/packages/next/src/server/dev/static-paths-worker.ts @@ -30,6 +30,7 @@ type RuntimeConfig = { configFileName: string publicRuntimeConfig: { [key: string]: any } serverRuntimeConfig: { [key: string]: any } + dynamicIO: boolean } // we call getStaticPaths in a separate process to ensure @@ -115,6 +116,7 @@ export async function loadStaticPaths({ return await buildAppStaticPaths({ dir, page: pathname, + dynamicIO: config.dynamicIO, generateParams, configFileName: config.configFileName, distDir, diff --git a/packages/next/src/server/lib/incremental-cache/index.ts b/packages/next/src/server/lib/incremental-cache/index.ts index 10c5d899d2626..6a44cee910440 100644 --- a/packages/next/src/server/lib/incremental-cache/index.ts +++ b/packages/next/src/server/lib/incremental-cache/index.ts @@ -10,6 +10,7 @@ import { import type { Revalidate } from '../revalidate' import type { DeepReadonly } from '../../../shared/lib/deep-readonly' +import { cacheScopeAsyncLocalStorage } from '../../async-storage/cache-scope' import FetchCache from './fetch-cache' import FileSystemCache from './file-system-cache' import { normalizePagePath } from '../../../shared/lib/page-path/normalize-page-path' @@ -76,6 +77,7 @@ export class IncrementalCache implements IncrementalCacheType { readonly fetchCacheKeyPrefix?: string readonly revalidatedTags?: string[] readonly isOnDemandRevalidate?: boolean + readonly hasDynamicIO?: boolean private readonly locks = new Map>() private readonly unlocks = new Map Promise>() @@ -89,6 +91,7 @@ export class IncrementalCache implements IncrementalCacheType { constructor({ fs, dev, + dynamicIO, flushToDisk, fetchCache, minimalMode, @@ -103,6 +106,7 @@ export class IncrementalCache implements IncrementalCacheType { }: { fs?: CacheFs dev: boolean + dynamicIO: boolean fetchCache?: boolean minimalMode?: boolean serverDistDir?: string @@ -143,6 +147,7 @@ export class IncrementalCache implements IncrementalCacheType { maxMemoryCacheSize = parseInt(process.env.__NEXT_TEST_MAX_ISR_CACHE, 10) } this.dev = dev + this.hasDynamicIO = dynamicIO this.disableForTestmode = process.env.NEXT_PRIVATE_TEST_PROXY === 'true' // this is a hack to avoid Webpack knowing this is equal to this.minimalMode // because we replace this.minimalMode to true in production bundles. @@ -404,6 +409,23 @@ export class IncrementalCache implements IncrementalCacheType { let entry: IncrementalCacheEntry | null = null let revalidate = ctx.revalidate + if (this.hasDynamicIO && ctx.kind === IncrementalCacheKind.FETCH) { + const cacheScope = cacheScopeAsyncLocalStorage.getStore() + + if (cacheScope?.cache) { + const memoryCacheData = cacheScope.cache.get(cacheKey) + + if (memoryCacheData?.kind === CachedRouteKind.FETCH) { + return { + isStale: false, + value: memoryCacheData, + revalidateAfter: false, + isFallback: false, + } + } + } + } + const cacheData = await this.cacheHandler?.get(cacheKey, ctx) if (cacheData?.value?.kind === CachedRouteKind.FETCH) { @@ -502,6 +524,17 @@ export class IncrementalCache implements IncrementalCacheType { } ) { if (this.disableForTestmode || (this.dev && !ctx.fetchCache)) return + + pathname = this._getPathname(pathname, ctx.fetchCache) + + if (this.hasDynamicIO && data?.kind === CachedRouteKind.FETCH) { + const cacheScope = cacheScopeAsyncLocalStorage.getStore() + + if (cacheScope?.cache) { + cacheScope.cache.set(pathname, data) + } + } + // FetchCache has upper limit of 2MB per-entry currently const itemSize = JSON.stringify(data).length if ( @@ -519,8 +552,6 @@ export class IncrementalCache implements IncrementalCacheType { return } - pathname = this._getPathname(pathname, ctx.fetchCache) - try { // Set the value for the revalidate seconds so if it changes we can // update the cache with the new value. diff --git a/packages/next/src/server/lib/prefetch-cache-scopes.ts b/packages/next/src/server/lib/prefetch-cache-scopes.ts new file mode 100644 index 0000000000000..72e55a2810115 --- /dev/null +++ b/packages/next/src/server/lib/prefetch-cache-scopes.ts @@ -0,0 +1,40 @@ +import type { CacheScopeStore } from '../async-storage/cache-scope' + +export class PrefetchCacheScopes { + private cacheScopes = new Map< + string, + { + cache: CacheScopeStore['cache'] + // we track timestamp as we evict after 30s + // if a prefetch cache scope isn't used by then + timestamp: number + } + >() + + private evict() { + for (const [key, value] of this.cacheScopes) { + if (value.timestamp < Date.now() - 30_000) { + this.cacheScopes.delete(key) + } + } + } + + // TODO: should this key include query params if so we need to + // filter _rsc query + get(url: string) { + setImmediate(() => this.evict()) + return this.cacheScopes.get(url)?.cache + } + + set(url: string, cache: CacheScopeStore['cache']) { + setImmediate(() => this.evict()) + return this.cacheScopes.set(url, { + cache, + timestamp: Date.now(), + }) + } + + del(url: string) { + this.cacheScopes.delete(url) + } +} diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index e4db0e68b71d2..646d79eda100b 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -371,6 +371,7 @@ export default class NextNodeServer extends BaseServer< dev, requestHeaders, requestProtocol, + dynamicIO: Boolean(this.nextConfig.experimental.dynamicIO), allowedRevalidateHeaderKeys: this.nextConfig.experimental.allowedRevalidateHeaderKeys, minimalMode: this.minimalMode, diff --git a/packages/next/src/server/request/cookies.ts b/packages/next/src/server/request/cookies.ts index fe1f77ecd33ca..769e218780cd1 100644 --- a/packages/next/src/server/request/cookies.ts +++ b/packages/next/src/server/request/cookies.ts @@ -126,7 +126,10 @@ export function cookies(): Promise { underlyingCookies = requestStore.cookies } - if (process.env.NODE_ENV === 'development') { + if ( + process.env.NODE_ENV === 'development' && + !staticGenerationStore?.isPrefetchRequest + ) { return makeUntrackedExoticCookiesWithDevWarnings( underlyingCookies, staticGenerationStore?.route diff --git a/packages/next/src/server/request/draft-mode.ts b/packages/next/src/server/request/draft-mode.ts index 2ecdb55f06188..1fc18ab41370c 100644 --- a/packages/next/src/server/request/draft-mode.ts +++ b/packages/next/src/server/request/draft-mode.ts @@ -34,9 +34,12 @@ export type UnsafeUnwrappedDraftMode = DraftMode export function draftMode(): Promise { const callingExpression = 'draftMode' const requestStore = getExpectedRequestStore(callingExpression) + const staticGenerationStore = staticGenerationAsyncStorage.getStore() - if (process.env.NODE_ENV === 'development') { - const staticGenerationStore = staticGenerationAsyncStorage.getStore() + if ( + process.env.NODE_ENV === 'development' && + !staticGenerationStore?.isPrefetchRequest + ) { const route = staticGenerationStore?.route return createExoticDraftModeWithDevWarnings(requestStore.draftMode, route) } else { diff --git a/packages/next/src/server/request/headers.ts b/packages/next/src/server/request/headers.ts index 9d8dfbf9908e0..55398133c3bfb 100644 --- a/packages/next/src/server/request/headers.ts +++ b/packages/next/src/server/request/headers.ts @@ -112,7 +112,10 @@ export function headers(): Promise { trackDynamicDataInDynamicRender(staticGenerationStore) } - if (process.env.NODE_ENV === 'development') { + if ( + process.env.NODE_ENV === 'development' && + !staticGenerationStore?.isPrefetchRequest + ) { return makeUntrackedExoticHeadersWithDevWarnings( requestStore.headers, staticGenerationStore?.route diff --git a/packages/next/src/server/request/params.ts b/packages/next/src/server/request/params.ts index 1cdf6be2a101f..b105cff268c45 100644 --- a/packages/next/src/server/request/params.ts +++ b/packages/next/src/server/request/params.ts @@ -161,7 +161,10 @@ function createRenderParams( underlyingParams: Params, staticGenerationStore: StaticGenerationStore ): Promise { - if (process.env.NODE_ENV === 'development') { + if ( + process.env.NODE_ENV === 'development' && + !staticGenerationStore.isPrefetchRequest + ) { return makeDynamicallyTrackedExoticParamsWithDevWarnings( underlyingParams, staticGenerationStore diff --git a/packages/next/src/server/request/search-params.ts b/packages/next/src/server/request/search-params.ts index b0171480e26a8..a1dde6cf8e3e2 100644 --- a/packages/next/src/server/request/search-params.ts +++ b/packages/next/src/server/request/search-params.ts @@ -140,7 +140,10 @@ function createRenderSearchParams( // dictionary object. return Promise.resolve({}) } else { - if (process.env.NODE_ENV === 'development') { + if ( + process.env.NODE_ENV === 'development' && + !staticGenerationStore.isPrefetchRequest + ) { return makeDynamicallyTrackedExoticSearchParamsWithDevWarnings( underlyingSearchParams, staticGenerationStore diff --git a/packages/next/src/server/web-server.ts b/packages/next/src/server/web-server.ts index 12d2c12d9ae7d..459fc1409073f 100644 --- a/packages/next/src/server/web-server.ts +++ b/packages/next/src/server/web-server.ts @@ -80,6 +80,7 @@ export default class NextWebServer extends BaseServer< return new IncrementalCache({ dev, requestHeaders, + dynamicIO: Boolean(this.nextConfig.experimental.dynamicIO), requestProtocol: 'https', allowedRevalidateHeaderKeys: this.nextConfig.experimental.allowedRevalidateHeaderKeys,