Content Collection cache (experimental) (#8854)

Co-authored-by: Sarah Rainsberger <[email protected]> Co-authored-by: Matthew Phillips <[email protected]>
withastro · Nov 9, 2023 · 3e1239e · 3e1239e
1 parent 5b16619
commit 3e1239e
Show file tree

Hide file tree

Showing 45 changed files with 1,962 additions and 209 deletions.
diff --git a/.changeset/lovely-pianos-build.md b/.changeset/lovely-pianos-build.md
@@ -0,0 +1,28 @@
+---
+'astro': minor
+---
+
+Provides a new, experimental build cache for [Content Collections](https://docs.astro.build/en/guides/content-collections/) as part of the [Incremental Build RFC](https://github.com/withastro/roadmap/pull/763). This includes multiple refactors to Astro's build process to optimize how Content Collections are handled, which should provide significant performance improvements for users with many collections. 
+
+Users building a `static` site can opt-in to preview the new build cache by adding the following flag to your Astro config:
+
+```js
+// astro.config.mjs
+export default {
+  experimental: {
+    contentCollectionCache: true,
+  },
+};
+```
+
+When this experimental feature is enabled, the files generated from your content collections will be stored in the [`cacheDir`](https://docs.astro.build/en/reference/configuration-reference/#cachedir) (by default, `node_modules/.astro`) and reused between builds. Most CI environments automatically restore files in `node_modules/` by default.
+
+In our internal testing on the real world [Astro Docs](https://github.com/withastro/docs) project, this feature reduces the bundling step of `astro build` from **133.20s** to **10.46s**, about 92% faster. The end-to-end `astro build` process used to take **4min 58s** and now takes just over `1min` for a total reduction of 80%.
+
+If you run into any issues with this experimental feature, please let us know! 
+
+You can always bypass the cache for a single build by passing the `--force` flag to `astro build`.
+
+```
+astro build --force
+```
diff --git a/packages/astro/content-module.template.mjs b/packages/astro/content-module.template.mjs
@@ -9,21 +9,18 @@ import {
 	createReference,
 } from 'astro/content/runtime';
 
+export { defineCollection } from 'astro/content/runtime';
 export { z } from 'astro/zod';
 
 const contentDir = '@@CONTENT_DIR@@';
 
-const contentEntryGlob = import.meta.glob('@@CONTENT_ENTRY_GLOB_PATH@@', {
-	query: { astroContentCollectionEntry: true },
-});
+const contentEntryGlob = '@@CONTENT_ENTRY_GLOB_PATH@@';
 const contentCollectionToEntryMap = createCollectionToGlobResultMap({
 	globResult: contentEntryGlob,
 	contentDir,
 });
 
-const dataEntryGlob = import.meta.glob('@@DATA_ENTRY_GLOB_PATH@@', {
-	query: { astroDataCollectionEntry: true },
-});
+const dataEntryGlob = '@@DATA_ENTRY_GLOB_PATH@@';
 const dataCollectionToEntryMap = createCollectionToGlobResultMap({
 	globResult: dataEntryGlob,
 	contentDir,
@@ -45,19 +42,12 @@ function createGlobLookup(glob) {
 	};
 }
 
-const renderEntryGlob = import.meta.glob('@@RENDER_ENTRY_GLOB_PATH@@', {
-	query: { astroRenderContent: true },
-});
+const renderEntryGlob = '@@RENDER_ENTRY_GLOB_PATH@@'
 const collectionToRenderEntryMap = createCollectionToGlobResultMap({
 	globResult: renderEntryGlob,
 	contentDir,
 });
 
-export function defineCollection(config) {
-	if (!config.type) config.type = 'content';
-	return config;
-}
-
 export const getCollection = createGetCollection({
 	contentCollectionToEntryMap,
 	dataCollectionToEntryMap,

diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts
@@ -1527,6 +1527,24 @@ export interface AstroUserConfig {
 			 */
 			routingStrategy?: 'prefix-always' | 'prefix-other-locales';
 		};
+		/**
+		 * @docs
+		 * @name experimental.contentCollectionCache
+		 * @type {boolean}
+		 * @default `false`
+		 * @version 3.5.0
+		 * @description
+		 * Enables a persistent cache for content collections when building in static mode.
+		 *
+		 * ```js
+		 * {
+		 * 	experimental: {
+		 * 		contentCollectionCache: true,
+		 * 	},
+		 * }
+		 * ```
+		 */
+		contentCollectionCache?: boolean;
 	};
 }
 

diff --git a/packages/astro/src/assets/vite-plugin-assets.ts b/packages/astro/src/assets/vite-plugin-assets.ts
@@ -42,7 +42,8 @@ export default function assets({
 				extendManualChunks(outputOptions, {
 					after(id) {
 						if (id.includes('astro/dist/assets/services/')) {
-							return `astro-assets-services`;
+							// By convention, library code is emitted to the `chunks/astro/*` directory
+							return `astro/assets-service`;
 						}
 					},
 				});

diff --git a/packages/astro/src/cli/build/index.ts b/packages/astro/src/cli/build/index.ts
@@ -26,5 +26,5 @@ export async function build({ flags }: BuildOptions) {
 
 	const inlineConfig = flagsToAstroInlineConfig(flags);
 
-	await _build(inlineConfig);
+	await _build(inlineConfig, { force: flags.force ?? false });
 }
diff --git a/packages/astro/src/content/consts.ts b/packages/astro/src/content/consts.ts
@@ -4,6 +4,7 @@ export const CONTENT_FLAG = 'astroContentCollectionEntry';
 export const DATA_FLAG = 'astroDataCollectionEntry';
 
 export const VIRTUAL_MODULE_ID = 'astro:content';
+export const RESOLVED_VIRTUAL_MODULE_ID = '\0' + VIRTUAL_MODULE_ID;
 export const LINKS_PLACEHOLDER = '@@ASTRO-LINKS@@';
 export const STYLES_PLACEHOLDER = '@@ASTRO-STYLES@@';
 export const SCRIPTS_PLACEHOLDER = '@@ASTRO-SCRIPTS@@';

diff --git a/packages/astro/src/content/runtime.ts b/packages/astro/src/content/runtime.ts
@@ -19,6 +19,11 @@ type GlobResult = Record<string, LazyImport>;
 type CollectionToEntryMap = Record<string, GlobResult>;
 type GetEntryImport = (collection: string, lookupId: string) => Promise<LazyImport>;
 
+export function defineCollection(config: any) {
+	if (!config.type) config.type = 'content';
+	return config;
+}
+
 export function createCollectionToGlobResultMap({
 	globResult,
 	contentDir,
@@ -69,7 +74,7 @@ export function createGetCollection({
 		let entries: any[] = [];
 		// Cache `getCollection()` calls in production only
 		// prevents stale cache in development
-		if (import.meta.env.PROD && cacheEntriesByCollection.has(collection)) {
+		if (!import.meta.env?.DEV && cacheEntriesByCollection.has(collection)) {
 			// Always return a new instance so consumers can safely mutate it
 			entries = [...cacheEntriesByCollection.get(collection)!];
 		} else {

diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts
@@ -326,6 +326,15 @@ export function parseFrontmatter(fileContents: string) {
  */
 export const globalContentConfigObserver = contentObservable({ status: 'init' });
 
+export function hasAnyContentFlag(viteId: string): boolean {
+	const flags = new URLSearchParams(viteId.split('?')[1] ?? '');
+	const flag = Array.from(flags.keys()).at(0);
+	if (typeof flag !== 'string') {
+		return false;
+	}
+	return CONTENT_FLAGS.includes(flag as any);
+}
+
 export function hasContentFlag(viteId: string, flag: (typeof CONTENT_FLAGS)[number]): boolean {
 	const flags = new URLSearchParams(viteId.split('?')[1] ?? '');
 	return flags.has(flag);

diff --git a/packages/astro/src/content/vite-plugin-content-assets.ts b/packages/astro/src/content/vite-plugin-content-assets.ts
@@ -1,6 +1,6 @@
 import { extname } from 'node:path';
 import { pathToFileURL } from 'node:url';
-import type { Plugin } from 'vite';
+import type { Plugin, Rollup } from 'vite';
 import type { AstroSettings } from '../@types/astro.js';
 import { moduleIsTopLevelPage, walkParentInfos } from '../core/build/graph.js';
 import { getPageDataByViteID, type BuildInternals } from '../core/build/internal.js';
@@ -110,16 +110,16 @@ export function astroConfigBuildPlugin(
 	options: StaticBuildOptions,
 	internals: BuildInternals
 ): AstroBuildPlugin {
-	let ssrPluginContext: any = undefined;
+	let ssrPluginContext: Rollup.PluginContext | undefined = undefined;
 	return {
-		build: 'ssr',
+		targets: ['server'],
 		hooks: {
-			'build:before': ({ build }) => {
+			'build:before': ({ target }) => {
 				return {
 					vitePlugin: {
 						name: 'astro:content-build-plugin',
 						generateBundle() {
-							if (build === 'ssr') {
+							if (target === 'server') {
 								ssrPluginContext = this;
 							}
 						},
@@ -144,24 +144,43 @@ export function astroConfigBuildPlugin(
 						let entryLinks = new Set<string>();
 						let entryScripts = new Set<string>();
 
-						for (const id of Object.keys(chunk.modules)) {
-							for (const [pageInfo] of walkParentInfos(id, ssrPluginContext)) {
-								if (moduleIsTopLevelPage(pageInfo)) {
-									const pageViteID = pageInfo.id;
-									const pageData = getPageDataByViteID(internals, pageViteID);
-									if (!pageData) continue;
+						if (options.settings.config.experimental.contentCollectionCache) {
+							// TODO: hoisted scripts are still handled on the pageData rather than the asset propagation point
+							for (const id of chunk.moduleIds) {
+								const _entryCss = internals.propagatedStylesMap.get(id);
+								const _entryScripts = internals.propagatedScriptsMap.get(id);
+								if (_entryCss) {
+									for (const value of _entryCss) {
+										if (value.type === 'inline') entryStyles.add(value.content);
+										if (value.type === 'external') entryLinks.add(value.src);
+									}
+								}
+								if (_entryScripts) {
+									for (const value of _entryScripts) {
+										entryScripts.add(value);
+									}
+								}
+							}
+						} else {
+							for (const id of Object.keys(chunk.modules)) {
+								for (const [pageInfo] of walkParentInfos(id, ssrPluginContext!)) {
+									if (moduleIsTopLevelPage(pageInfo)) {
+										const pageViteID = pageInfo.id;
+										const pageData = getPageDataByViteID(internals, pageViteID);
+										if (!pageData) continue;
 
-									const _entryCss = pageData.propagatedStyles?.get(id);
-									const _entryScripts = pageData.propagatedScripts?.get(id);
-									if (_entryCss) {
-										for (const value of _entryCss) {
-											if (value.type === 'inline') entryStyles.add(value.content);
-											if (value.type === 'external') entryLinks.add(value.src);
+										const _entryCss = internals.propagatedStylesMap?.get(id);
+										const _entryScripts = pageData.propagatedScripts?.get(id);
+										if (_entryCss) {
+											for (const value of _entryCss) {
+												if (value.type === 'inline') entryStyles.add(value.content);
+												if (value.type === 'external') entryLinks.add(value.src);
+											}
 										}
-									}
-									if (_entryScripts) {
-										for (const value of _entryScripts) {
-											entryScripts.add(value);
+										if (_entryScripts) {
+											for (const value of _entryScripts) {
+												entryScripts.add(value);
+											}
 										}
 									}
 								}
@@ -174,12 +193,22 @@ export function astroConfigBuildPlugin(
 								JSON.stringify(STYLES_PLACEHOLDER),
 								JSON.stringify(Array.from(entryStyles))
 							);
+						} else {
+							newCode = newCode.replace(
+								JSON.stringify(STYLES_PLACEHOLDER),
+								"[]"
+							);
 						}
 						if (entryLinks.size) {
 							newCode = newCode.replace(
 								JSON.stringify(LINKS_PLACEHOLDER),
 								JSON.stringify(Array.from(entryLinks).map(prependBase))
 							);
+						} else {
+							newCode = newCode.replace(
+								JSON.stringify(LINKS_PLACEHOLDER),
+								"[]"
+							);
 						}
 						if (entryScripts.size) {
 							const entryFileNames = new Set<string>();
@@ -205,8 +234,13 @@ export function astroConfigBuildPlugin(
 									}))
 								)
 							);
+						} else {
+							newCode = newCode.replace(
+								JSON.stringify(SCRIPTS_PLACEHOLDER),
+								"[]"
+							);
 						}
-						mutate(chunk, 'server', newCode);
+						mutate(chunk, ['server'], newCode);
 					}
 				}
 			},