-
Notifications
You must be signed in to change notification settings - Fork 27.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add additional handling for experimental tracing (#67785)
This adds additional trace handling under the experimental trace mode. Outside of the feature flag the bloom filter for app router paths used in pages to hard navigate was updated to no longer be inlined by default with webpack to avoid invalidating the main bundle for pages too often. Instead the bloom filter data is stored in the `_buildManifest` file which is only used in the pages router and is easier to update as it's per-build already. This tracing feature is still very experimental and should not be leveraged in production or outside of testing. --------- Co-authored-by: Zack Tanner <[email protected]>
- Loading branch information
Showing
18 changed files
with
1,194 additions
and
168 deletions.
There are no files selected for viewing
221 changes: 221 additions & 0 deletions
221
packages/next/src/build/flying-shuttle/detect-changed-entries.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,221 @@ | ||
import fs from 'fs' | ||
import path from 'path' | ||
import crypto from 'crypto' | ||
import { getPageFromPath } from '../entries' | ||
import { Sema } from 'next/dist/compiled/async-sema' | ||
|
||
export interface DetectedEntriesResult { | ||
app: string[] | ||
pages: string[] | ||
} | ||
|
||
let _hasShuttle: undefined | boolean = undefined | ||
export async function hasShuttle(shuttleDir: string) { | ||
if (typeof _hasShuttle === 'boolean') { | ||
return _hasShuttle | ||
} | ||
_hasShuttle = await fs.promises | ||
.access(path.join(shuttleDir, 'server')) | ||
.then(() => true) | ||
.catch(() => false) | ||
|
||
return _hasShuttle | ||
} | ||
|
||
export async function detectChangedEntries({ | ||
appPaths, | ||
pagesPaths, | ||
pageExtensions, | ||
distDir, | ||
shuttleDir, | ||
}: { | ||
appPaths?: string[] | ||
pagesPaths?: string[] | ||
pageExtensions: string[] | ||
distDir: string | ||
shuttleDir: string | ||
}): Promise<{ | ||
changed: DetectedEntriesResult | ||
unchanged: DetectedEntriesResult | ||
}> { | ||
const changedEntries: { | ||
app: string[] | ||
pages: string[] | ||
} = { | ||
app: [], | ||
pages: [], | ||
} | ||
const unchangedEntries: typeof changedEntries = { | ||
app: [], | ||
pages: [], | ||
} | ||
|
||
if (!(await hasShuttle(shuttleDir))) { | ||
// no shuttle so consider everything changed | ||
console.log(`no shuttle. can't detect changes`) | ||
return { | ||
changed: { | ||
pages: pagesPaths || [], | ||
app: appPaths || [], | ||
}, | ||
unchanged: { | ||
pages: [], | ||
app: [], | ||
}, | ||
} | ||
} | ||
|
||
const hashCache = new Map<string, string>() | ||
|
||
async function computeHash(p: string): Promise<string> { | ||
let hash = hashCache.get(p) | ||
if (hash) { | ||
return hash | ||
} | ||
return new Promise((resolve, reject) => { | ||
const hashInst = crypto.createHash('sha1') | ||
const stream = fs.createReadStream(p) | ||
stream.on('error', (err) => reject(err)) | ||
stream.on('data', (chunk) => hashInst.update(chunk)) | ||
stream.on('end', () => { | ||
const digest = hashInst.digest('hex') | ||
resolve(digest) | ||
hashCache.set(p, digest) | ||
}) | ||
}) | ||
} | ||
|
||
const hashSema = new Sema(16) | ||
let globalEntryChanged = false | ||
|
||
async function detectChange({ | ||
normalizedEntry, | ||
entry, | ||
type, | ||
}: { | ||
entry: string | ||
normalizedEntry: string | ||
type: keyof typeof changedEntries | ||
}) { | ||
const traceFile = path.join( | ||
shuttleDir, | ||
'server', | ||
type, | ||
`${normalizedEntry}.js.nft.json` | ||
) | ||
let changed = true | ||
|
||
// we don't need to check any further entry's dependencies if | ||
// a global entry changed since that invalidates everything | ||
if (!globalEntryChanged) { | ||
try { | ||
const traceData: { | ||
fileHashes: Record<string, string> | ||
} = JSON.parse(await fs.promises.readFile(traceFile, 'utf8')) | ||
|
||
if (traceData) { | ||
let changedDependency = false | ||
await Promise.all( | ||
Object.keys(traceData.fileHashes).map(async (file) => { | ||
try { | ||
if (changedDependency) return | ||
await hashSema.acquire() | ||
const relativeTraceFile = path.relative( | ||
path.join(shuttleDir, 'server', type), | ||
traceFile | ||
) | ||
const originalTraceFile = path.join( | ||
distDir, | ||
'server', | ||
type, | ||
relativeTraceFile | ||
) | ||
const absoluteFile = path.join( | ||
path.dirname(originalTraceFile), | ||
file | ||
) | ||
|
||
if (absoluteFile.startsWith(distDir)) { | ||
return | ||
} | ||
|
||
const prevHash = traceData.fileHashes[file] | ||
const curHash = await computeHash(absoluteFile) | ||
|
||
if (prevHash !== curHash) { | ||
console.log('detected change on', { | ||
prevHash, | ||
curHash, | ||
file, | ||
entry: normalizedEntry, | ||
}) | ||
changedDependency = true | ||
} | ||
} finally { | ||
hashSema.release() | ||
} | ||
}) | ||
) | ||
|
||
if (!changedDependency) { | ||
changed = false | ||
} | ||
} else { | ||
console.error('missing trace data', traceFile, normalizedEntry) | ||
} | ||
} catch (err) { | ||
console.error(`Failed to detect change for ${entry}`, err) | ||
} | ||
} | ||
|
||
// we always rebuild global entries so we have a version | ||
// that matches the newest build/runtime | ||
const isGlobalEntry = /(_app|_document|_error)/.test(entry) | ||
|
||
if (changed || isGlobalEntry) { | ||
// if a global entry changed all entries are changed | ||
if (!globalEntryChanged && isGlobalEntry) { | ||
console.log(`global entry ${entry} changed invalidating all entries`) | ||
globalEntryChanged = true | ||
// move unchanged to changed | ||
changedEntries[type].push(...unchangedEntries[type]) | ||
} | ||
changedEntries[type].push(entry) | ||
} else { | ||
unchangedEntries[type].push(entry) | ||
} | ||
} | ||
|
||
// loop over entries and their dependency's hashes | ||
// to detect which changed | ||
for (const entry of pagesPaths || []) { | ||
let normalizedEntry = getPageFromPath(entry, pageExtensions) | ||
|
||
if (normalizedEntry === '/') { | ||
normalizedEntry = '/index' | ||
} | ||
await detectChange({ entry, normalizedEntry, type: 'pages' }) | ||
} | ||
|
||
for (const entry of appPaths || []) { | ||
const normalizedEntry = getPageFromPath(entry, pageExtensions) | ||
await detectChange({ entry, normalizedEntry, type: 'app' }) | ||
} | ||
|
||
console.log( | ||
'changed entries', | ||
JSON.stringify( | ||
{ | ||
changedEntries, | ||
unchangedEntries, | ||
}, | ||
null, | ||
2 | ||
) | ||
) | ||
|
||
return { | ||
changed: changedEntries, | ||
unchanged: unchangedEntries, | ||
} | ||
} |
Oops, something went wrong.