Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Invalidate CC cache manifest when lockfile or config changes #10763

Merged
merged 18 commits into from
Apr 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/metal-terms-push.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"astro": patch
---

Invalidate CC cache manifest when lockfile or config changes
1 change: 1 addition & 0 deletions packages/astro/src/@types/astro.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2780,6 +2780,7 @@ export interface AstroIntegration {
dir: URL;
routes: RouteData[];
logger: AstroIntegrationLogger;
cacheManifest: boolean;
}) => void | Promise<void>;
};
}
Expand Down
1 change: 1 addition & 0 deletions packages/astro/src/core/build/consts.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
export const CHUNKS_PATH = 'chunks/';
1 change: 1 addition & 0 deletions packages/astro/src/core/build/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ class AstroBuilder {
.flat()
.map((pageData) => pageData.route),
logging: this.logger,
cacheManifest: internals.cacheManifestUsed,
});

if (this.logger.level && levels[this.logger.level()] <= levels['info']) {
Expand Down
2 changes: 2 additions & 0 deletions packages/astro/src/core/build/internal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ export interface BuildInternals {
discoveredScripts: Set<string>;

cachedClientEntries: string[];
cacheManifestUsed: boolean;

propagatedStylesMap: Map<string, Set<StylesheetAsset>>;
propagatedScriptsMap: Map<string, Set<string>>;
Expand Down Expand Up @@ -140,6 +141,7 @@ export function createBuildInternals(): BuildInternals {
componentMetadata: new Map(),
ssrSplitEntryChunks: new Map(),
entryPoints: new Map(),
cacheManifestUsed: false,
};
}

Expand Down
191 changes: 158 additions & 33 deletions packages/astro/src/core/build/plugins/plugin-content.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,30 @@ import fsMod from 'node:fs';
import { fileURLToPath } from 'node:url';
import pLimit from 'p-limit';
import { type Plugin as VitePlugin, normalizePath } from 'vite';
import { configPaths } from '../../config/index.js';
import { CONTENT_RENDER_FLAG, PROPAGATED_ASSET_FLAG } from '../../../content/consts.js';
import { type ContentLookupMap, hasContentFlag } from '../../../content/utils.js';
import {
generateContentEntryFile,
generateLookupMap,
} from '../../../content/vite-plugin-content-virtual-mod.js';
import { isServerLikeOutput } from '../../../prerender/utils.js';
import { joinPaths, removeFileExtension, removeLeadingForwardSlash } from '../../path.js';
import { joinPaths, removeFileExtension, removeLeadingForwardSlash, appendForwardSlash } from '../../path.js';
import { addRollupInput } from '../add-rollup-input.js';
import { type BuildInternals } from '../internal.js';
import type { AstroBuildPlugin } from '../plugin.js';
import { copyFiles } from '../static-build.js';
import type { StaticBuildOptions } from '../types.js';
import { encodeName } from '../util.js';
import { extendManualChunks } from './util.js';
import { emptyDir } from '../../fs/index.js';
import { CHUNKS_PATH } from '../consts.js';

const CONTENT_CACHE_DIR = './content/';
const CONTENT_MANIFEST_FILE = './manifest.json';
// IMPORTANT: Update this version when making significant changes to the manifest format.
// Only manifests generated with the same version number can be compared.
const CONTENT_MANIFEST_VERSION = 0;
const CONTENT_MANIFEST_VERSION = 1;

interface ContentManifestKey {
collection: string;
Expand All @@ -39,40 +42,44 @@ interface ContentManifest {
// Tracks components that should be passed to the client build
// When the cache is restored, these might no longer be referenced
clientEntries: string[];
// Hash of the lockfiles, pnpm-lock.yaml, package-lock.json, etc.
// Kept so that installing new packages results in a full rebuild.
lockfiles: string;
ematipico marked this conversation as resolved.
Show resolved Hide resolved
// Hash of the Astro config. Changing options results in invalidating the cache.
configs: string;
ematipico marked this conversation as resolved.
Show resolved Hide resolved
}

const virtualEmptyModuleId = `virtual:empty-content`;
const resolvedVirtualEmptyModuleId = `\0${virtualEmptyModuleId}`;
const NO_MANIFEST_VERSION = -1 as const;

function createContentManifest(): ContentManifest {
return { version: -1, entries: [], serverEntries: [], clientEntries: [] };
return { version: NO_MANIFEST_VERSION, entries: [], serverEntries: [], clientEntries: [], lockfiles: "", configs: "" };
}

function vitePluginContent(
opts: StaticBuildOptions,
lookupMap: ContentLookupMap,
internals: BuildInternals
internals: BuildInternals,
cachedBuildOutput: Array<{ cached: URL; dist: URL; }>
): VitePlugin {
const { config } = opts.settings;
const { cacheDir } = config;
const distRoot = config.outDir;
const distContentRoot = new URL('./content/', distRoot);
const cachedChunks = new URL('./chunks/', opts.settings.config.cacheDir);
const distChunks = new URL('./chunks/', opts.settings.config.outDir);
const contentCacheDir = new URL(CONTENT_CACHE_DIR, cacheDir);
const contentManifestFile = new URL(CONTENT_MANIFEST_FILE, contentCacheDir);
const cache = contentCacheDir;
const cacheTmp = new URL('./.tmp/', cache);
const cacheTmp = new URL('./.tmp/', contentCacheDir);
let oldManifest = createContentManifest();
let newManifest = createContentManifest();
let entries: ContentEntries;
let injectedEmptyFile = false;
let currentManifestState: ReturnType<typeof manifestState> = 'valid';

if (fsMod.existsSync(contentManifestFile)) {
try {
const data = fsMod.readFileSync(contentManifestFile, { encoding: 'utf8' });
oldManifest = JSON.parse(data);
internals.cachedClientEntries = oldManifest.clientEntries;
} catch {}
}

Expand All @@ -84,6 +91,32 @@ function vitePluginContent(
newManifest = await generateContentManifest(opts, lookupMap);
entries = getEntriesFromManifests(oldManifest, newManifest);

// If the manifest is valid, use the cached client entries as nothing has changed
currentManifestState = manifestState(oldManifest, newManifest);
if(currentManifestState === 'valid') {
internals.cachedClientEntries = oldManifest.clientEntries;
} else {
let logReason = '';
switch(currentManifestState) {
case 'config-mismatch':
logReason = 'Astro config has changed';
break;
case 'lockfile-mismatch':
logReason = 'Lockfiles have changed';
break;
case 'no-entries':
logReason = 'No content collections entries cached';
break;
case 'version-mismatch':
logReason = 'The cache manifest version has changed';
break;
case 'no-manifest':
logReason = 'No content manifest was found in the cache';
break;
}
opts.logger.info('build', `Cache invalid, rebuilding from source. Reason: ${logReason}.`);
}

// Of the cached entries, these ones need to be rebuilt
for (const { type, entry } of entries.buildFromSource) {
const fileURL = encodeURI(joinPaths(opts.settings.config.root.toString(), entry));
Expand All @@ -96,10 +129,18 @@ function vitePluginContent(
}
newOptions = addRollupInput(newOptions, inputs);
}
// Restores cached chunks from the previous build
if (fsMod.existsSync(cachedChunks)) {
await copyFiles(cachedChunks, distChunks, true);

// Restores cached chunks and assets from the previous build
// If the manifest state is not valid then it needs to rebuild everything
// so don't do that in this case.
if(currentManifestState === 'valid') {
for(const { cached, dist } of cachedBuildOutput) {
if (fsMod.existsSync(cached)) {
await copyFiles(cached, dist, true);
}
}
}

// If nothing needs to be rebuilt, we inject a fake entrypoint to appease Rollup
if (entries.buildFromSource.length === 0) {
newOptions = addRollupInput(newOptions, [virtualEmptyModuleId]);
Expand Down Expand Up @@ -199,16 +240,20 @@ function vitePluginContent(
]);
newManifest.serverEntries = Array.from(serverComponents);
newManifest.clientEntries = Array.from(clientComponents);

const cacheExists = fsMod.existsSync(contentCacheDir);
// If the manifest is invalid, empty the cache so that we can create a new one.
if(cacheExists && currentManifestState !== 'valid') {
emptyDir(contentCacheDir);
}

await fsMod.promises.mkdir(contentCacheDir, { recursive: true });
await fsMod.promises.writeFile(contentManifestFile, JSON.stringify(newManifest), {
encoding: 'utf8',
});

const cacheExists = fsMod.existsSync(cache);
fsMod.mkdirSync(cache, { recursive: true });
await fsMod.promises.mkdir(cacheTmp, { recursive: true });
await copyFiles(distContentRoot, cacheTmp, true);
if (cacheExists) {
if (cacheExists && currentManifestState === 'valid') {
await copyFiles(contentCacheDir, distContentRoot, false);
}
await copyFiles(cacheTmp, contentCacheDir);
Expand Down Expand Up @@ -242,12 +287,12 @@ function getEntriesFromManifests(
oldManifest: ContentManifest,
newManifest: ContentManifest
): ContentEntries {
const { version: oldVersion, entries: oldEntries } = oldManifest;
const { version: newVersion, entries: newEntries } = newManifest;
const { entries: oldEntries } = oldManifest;
const { entries: newEntries } = newManifest;
let entries: ContentEntries = { restoreFromCache: [], buildFromSource: [] };

const newEntryMap = new Map<ContentManifestKey, string>(newEntries);
if (oldVersion !== newVersion || oldEntries.length === 0) {
if (manifestState(oldManifest, newManifest) !== 'valid') {
entries.buildFromSource = Array.from(newEntryMap.keys());
return entries;
}
Expand All @@ -265,16 +310,37 @@ function getEntriesFromManifests(
return entries;
}

type ManifestState = 'valid' | 'no-manifest' | 'version-mismatch' | 'no-entries' | 'lockfile-mismatch' | 'config-mismatch';

function manifestState(oldManifest: ContentManifest, newManifest: ContentManifest): ManifestState {
// There isn't an existing manifest.
if(oldManifest.version === NO_MANIFEST_VERSION) {
return 'no-manifest';
}
// Version mismatch, always invalid
if (oldManifest.version !== newManifest.version) {
return 'version-mismatch';
}
if(oldManifest.entries.length === 0) {
return 'no-entries';
}
// Lockfiles have changed or there is no lockfile at all.
if((oldManifest.lockfiles !== newManifest.lockfiles) || newManifest.lockfiles === '') {
return 'lockfile-mismatch';
}
// Config has changed.
if(oldManifest.configs !== newManifest.configs) {
return 'config-mismatch';
}
return 'valid';
}

async function generateContentManifest(
opts: StaticBuildOptions,
lookupMap: ContentLookupMap
): Promise<ContentManifest> {
let manifest: ContentManifest = {
version: CONTENT_MANIFEST_VERSION,
entries: [],
serverEntries: [],
clientEntries: [],
};
let manifest = createContentManifest();
manifest.version = CONTENT_MANIFEST_VERSION;
const limit = pLimit(10);
const promises: Promise<void>[] = [];

Expand All @@ -290,13 +356,63 @@ async function generateContentManifest(
);
}
}

const [lockfiles, configs] = await Promise.all([
lockfilesHash(opts.settings.config.root),
configHash(opts.settings.config.root)
]);

manifest.lockfiles = lockfiles;
manifest.configs = configs;

await Promise.all(promises);
return manifest;
}

function checksum(data: string): string {
return createHash('sha1').update(data).digest('base64');
async function pushBufferInto(fileURL: URL, buffers: Uint8Array[]) {
try {
const handle = await fsMod.promises.open(fileURL, 'r');
ematipico marked this conversation as resolved.
Show resolved Hide resolved
const data = await handle.readFile();
buffers.push(data);
await handle.close();
} catch {
// File doesn't exist, ignore
}
}

async function lockfilesHash(root: URL) {
// Order is important so don't change this.
const lockfiles = ['package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lockb'];
ematipico marked this conversation as resolved.
Show resolved Hide resolved
const datas: Uint8Array[] = [];
const promises: Promise<void>[] = [];
for(const lockfileName of lockfiles) {
const fileURL = new URL(`./${lockfileName}`, root);
promises.push(pushBufferInto(fileURL, datas));
}
await Promise.all(promises);
return checksum(...datas);
}

async function configHash(root: URL) {
const configFileNames = configPaths;
for(const configPath of configFileNames) {
try {
const fileURL = new URL(`./${configPath}`, root);
const data = await fsMod.promises.readFile(fileURL);
const hash = checksum(data);
return hash;
} catch {
// File doesn't exist
}
}
// No config file, still create a hash since we can compare nothing against nothing.
return checksum(`export default {}`);
}

function checksum(...datas: string[] | Uint8Array[]): string {
const hash = createHash('sha1');
datas.forEach(data => hash.update(data));
return hash.digest('base64');
}

function collectionTypeToFlag(type: 'content' | 'data') {
Expand All @@ -308,8 +424,15 @@ export function pluginContent(
opts: StaticBuildOptions,
internals: BuildInternals
): AstroBuildPlugin {
const cachedChunks = new URL('./chunks/', opts.settings.config.cacheDir);
const distChunks = new URL('./chunks/', opts.settings.config.outDir);
const { cacheDir, outDir } = opts.settings.config;

const chunksFolder = './' + CHUNKS_PATH;
const assetsFolder = './' + appendForwardSlash(opts.settings.config.build.assets);
// These are build output that is kept in the cache.
const cachedBuildOutput = [
{ cached: new URL(chunksFolder, cacheDir), dist: new URL(chunksFolder, outDir) },
{ cached: new URL(assetsFolder, cacheDir), dist: new URL(assetsFolder, outDir) },
];

return {
targets: ['server'],
Expand All @@ -321,10 +444,9 @@ export function pluginContent(
if (isServerLikeOutput(opts.settings.config)) {
return { vitePlugin: undefined };
}

const lookupMap = await generateLookupMap({ settings: opts.settings, fs: fsMod });
return {
vitePlugin: vitePluginContent(opts, lookupMap, internals),
vitePlugin: vitePluginContent(opts, lookupMap, internals, cachedBuildOutput),
};
},

Expand All @@ -335,8 +457,11 @@ export function pluginContent(
if (isServerLikeOutput(opts.settings.config)) {
return;
}
if (fsMod.existsSync(distChunks)) {
await copyFiles(distChunks, cachedChunks, true);
// Cache build output of chunks and assets
for(const { cached, dist } of cachedBuildOutput) {
if (fsMod.existsSync(dist)) {
await copyFiles(dist, cached, true);
}
}
},
},
Expand Down
Loading
Loading