Preserve content modules properly in cache (#10889)

* Wait until after build to preserve content modules * Properly build hoisted scripts * Add changeset * Fix tests hitting each other * Global state, the shame * Move the file copying over to earlier in the process
withastro · May 6, 2024 · 4d905cc · 4d905cc
1 parent 082abb8
commit 4d905cc
Show file tree

Hide file tree

Showing 33 changed files with 385 additions and 345 deletions.
diff --git a/.changeset/spicy-keys-own.md b/.changeset/spicy-keys-own.md
@@ -0,0 +1,5 @@
+---
+"astro": patch
+---
+
+Preserve content modules properly in cache
diff --git a/packages/astro/content-module.template.mjs b/packages/astro/content-module.template.mjs
@@ -48,10 +48,12 @@ const collectionToRenderEntryMap = createCollectionToGlobResultMap({
 	contentDir,
 });
 
+const cacheEntriesByCollection = new Map();
 export const getCollection = createGetCollection({
 	contentCollectionToEntryMap,
 	dataCollectionToEntryMap,
 	getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap),
+	cacheEntriesByCollection,
 });
 
 export const getEntryBySlug = createGetEntryBySlug({

diff --git a/packages/astro/src/assets/build/generate.ts b/packages/astro/src/assets/build/generate.ts
@@ -10,7 +10,7 @@ import { AstroError } from '../../core/errors/errors.js';
 import { AstroErrorData } from '../../core/errors/index.js';
 import type { Logger } from '../../core/logger/core.js';
 import { isRemotePath, removeLeadingForwardSlash } from '../../core/path.js';
-import { isServerLikeOutput } from '../../prerender/utils.js';
+import { isServerLikeOutput } from '../../core/util.js';
 import type { MapValue } from '../../type-utils.js';
 import { getConfiguredImageService } from '../internal.js';
 import type { LocalImageService } from '../services/service.js';

diff --git a/packages/astro/src/assets/vite-plugin-assets.ts b/packages/astro/src/assets/vite-plugin-assets.ts
@@ -12,7 +12,7 @@ import {
 	removeBase,
 	removeQueryString,
 } from '../core/path.js';
-import { isServerLikeOutput } from '../prerender/utils.js';
+import { isServerLikeOutput } from '../core/util.js';
 import { VALID_INPUT_FORMATS, VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js';
 import { emitESMImage } from './utils/emitAsset.js';
 import { getAssetsPrefix } from './utils/getAssetsPrefix.js';

diff --git a/packages/astro/src/content/runtime.ts b/packages/astro/src/content/runtime.ts
@@ -44,15 +44,16 @@ export function createCollectionToGlobResultMap({
 	return collectionToGlobResultMap;
 }
 
-const cacheEntriesByCollection = new Map<string, any[]>();
 export function createGetCollection({
 	contentCollectionToEntryMap,
 	dataCollectionToEntryMap,
 	getRenderEntryImport,
+	cacheEntriesByCollection,
 }: {
 	contentCollectionToEntryMap: CollectionToEntryMap;
 	dataCollectionToEntryMap: CollectionToEntryMap;
 	getRenderEntryImport: GetEntryImport;
+	cacheEntriesByCollection: Map<string, any[]>;
 }) {
 	return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
 		let type: 'content' | 'data';

diff --git a/packages/astro/src/content/vite-plugin-content-imports.ts b/packages/astro/src/content/vite-plugin-content-imports.ts
@@ -15,7 +15,7 @@ import type {
 import { getProxyCode } from '../assets/utils/proxy.js';
 import { AstroError } from '../core/errors/errors.js';
 import { AstroErrorData } from '../core/errors/index.js';
-import { isServerLikeOutput } from '../prerender/utils.js';
+import { isServerLikeOutput } from '../core/util.js';
 import { CONTENT_FLAG, DATA_FLAG } from './consts.js';
 import {
 	type ContentConfig,

diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts
@@ -8,8 +8,7 @@ import type { AstroSettings } from '../@types/astro.js';
 import { encodeName } from '../core/build/util.js';
 import { AstroError, AstroErrorData } from '../core/errors/index.js';
 import { appendForwardSlash, removeFileExtension } from '../core/path.js';
-import { rootRelativePath } from '../core/util.js';
-import { isServerLikeOutput } from '../prerender/utils.js';
+import { rootRelativePath, isServerLikeOutput } from '../core/util.js';
 import type { AstroPluginMetadata } from '../vite-plugin-astro/index.js';
 import {
 	CONTENT_FLAG,

diff --git a/packages/astro/src/core/build/generate.ts b/packages/astro/src/core/build/generate.ts
@@ -31,7 +31,7 @@ import {
 } from '../../core/path.js';
 import { toRoutingStrategy } from '../../i18n/utils.js';
 import { runHookBuildGenerated } from '../../integrations/hooks.js';
-import { getOutputDirectory, isServerLikeOutput } from '../../prerender/utils.js';
+import { getOutputDirectory } from '../../prerender/utils.js';
 import type { SSRManifestI18n } from '../app/types.js';
 import { NoPrerenderedRoutesWithDomains } from '../errors/errors-data.js';
 import { AstroError, AstroErrorData } from '../errors/index.js';
@@ -45,7 +45,7 @@ import { RenderContext } from '../render-context.js';
 import { callGetStaticPaths } from '../render/route-cache.js';
 import { createRequest } from '../request.js';
 import { matchRoute } from '../routing/match.js';
-import { getOutputFilename } from '../util.js';
+import { getOutputFilename, isServerLikeOutput } from '../util.js';
 import { getOutDirWithinCwd, getOutFile, getOutFolder } from './common.js';
 import {
 	cssOrder,

diff --git a/packages/astro/src/core/build/index.ts b/packages/astro/src/core/build/index.ts
@@ -19,7 +19,6 @@ import {
 	runHookConfigDone,
 	runHookConfigSetup,
 } from '../../integrations/hooks.js';
-import { isServerLikeOutput } from '../../prerender/utils.js';
 import { resolveConfig } from '../config/config.js';
 import { createNodeLogger } from '../config/logging.js';
 import { createSettings } from '../config/settings.js';
@@ -28,7 +27,7 @@ import type { Logger } from '../logger/core.js';
 import { levels, timerMessage } from '../logger/core.js';
 import { apply as applyPolyfill } from '../polyfill.js';
 import { createRouteManifest } from '../routing/index.js';
-import { ensureProcessNodeEnv } from '../util.js';
+import { ensureProcessNodeEnv, isServerLikeOutput } from '../util.js';
 import { collectPagesData } from './page-data.js';
 import { staticBuild, viteBuild } from './static-build.js';
 import type { StaticBuildOptions } from './types.js';

diff --git a/packages/astro/src/core/build/internal.ts b/packages/astro/src/core/build/internal.ts
@@ -25,6 +25,8 @@ export interface BuildInternals {
 	hoistedScriptIdToHoistedMap: Map<string, Set<string>>;
 	// A mapping of hoisted script ids back to the pages which reference it
 	hoistedScriptIdToPagesMap: Map<string, Set<string>>;
+	// A mapping of hoisted script ids back to the content which reference it
+	hoistedScriptIdToContentMap: Map<string, Set<string>>;
 
 	/**
 	 * Used by the `directRenderScript` option. If script is inlined, its id and
@@ -123,6 +125,7 @@ export function createBuildInternals(): BuildInternals {
 		cssModuleToChunkIdMap: new Map(),
 		hoistedScriptIdToHoistedMap,
 		hoistedScriptIdToPagesMap,
+		hoistedScriptIdToContentMap: new Map(),
 		inlinedScripts: new Map(),
 		entrySpecifierToBundleMap: new Map<string, string>(),
 		pageToBundleMap: new Map<string, string>(),

diff --git a/packages/astro/src/core/build/pipeline.ts b/packages/astro/src/core/build/pipeline.ts
@@ -1,5 +1,5 @@
 import type { RouteData, SSRLoadedRenderer, SSRResult } from '../../@types/astro.js';
-import { getOutputDirectory, isServerLikeOutput } from '../../prerender/utils.js';
+import { getOutputDirectory } from '../../prerender/utils.js';
 import { BEFORE_HYDRATION_SCRIPT_ID, PAGE_SCRIPT_ID } from '../../vite-plugin-scripts/index.js';
 import type { SSRManifest } from '../app/types.js';
 import { routeIsFallback, routeIsRedirect } from '../redirects/helpers.js';
@@ -21,6 +21,7 @@ import { getVirtualModulePageNameFromPath } from './plugins/util.js';
 import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js';
 import type { PageBuildData, StaticBuildOptions } from './types.js';
 import { i18nHasFallback } from './util.js';
+import { isServerLikeOutput } from '../util.js';
 
 /**
  * The build pipeline is responsible to gather the files emitted by the SSR build and generate the pages by executing these files.

diff --git a/packages/astro/src/core/build/plugins/plugin-analyzer.ts b/packages/astro/src/core/build/plugins/plugin-analyzer.ts
@@ -6,6 +6,7 @@ import type { AstroBuildPlugin } from '../plugin.js';
 
 import { PROPAGATED_ASSET_FLAG } from '../../../content/consts.js';
 import { prependForwardSlash } from '../../../core/path.js';
+import { isContentCollectionsCacheEnabled } from '../../../core/util.js';
 import {
 	getParentModuleInfos,
 	getTopLevelPageModuleInfos,
@@ -31,6 +32,7 @@ export function vitePluginAnalyzer(
 		const pageScripts = new Map<
 			string,
 			{
+				type: 'page' | 'content';
 				hoistedSet: Set<string>;
 				propagatedMapByImporter: Map<string, Set<string>>;
 			}
@@ -51,27 +53,46 @@ export function vitePluginAnalyzer(
 				if (hoistedScripts.size) {
 					for (const parentInfo of getParentModuleInfos(from, this, isPropagatedAsset)) {
 						if (isPropagatedAsset(parentInfo.id)) {
-							for (const nestedParentInfo of getParentModuleInfos(from, this)) {
-								if (moduleIsTopLevelPage(nestedParentInfo)) {
-									for (const hid of hoistedScripts) {
-										if (!pageScripts.has(nestedParentInfo.id)) {
-											pageScripts.set(nestedParentInfo.id, {
-												hoistedSet: new Set(),
-												propagatedMapByImporter: new Map(),
-											});
-										}
-										const entry = pageScripts.get(nestedParentInfo.id)!;
-										if (!entry.propagatedMapByImporter.has(parentInfo.id)) {
-											entry.propagatedMapByImporter.set(parentInfo.id, new Set());
+							if(isContentCollectionsCacheEnabled(options.settings.config)) {
+								if (!pageScripts.has(parentInfo.id)) {
+									pageScripts.set(parentInfo.id, {
+										type: 'content',
+										hoistedSet: new Set(),
+										propagatedMapByImporter: new Map(),
+									});
+								}
+								const propagaters = pageScripts.get(parentInfo.id)!.propagatedMapByImporter;
+								for(const hid of hoistedScripts) {
+									if(!propagaters.has(parentInfo.id)) {
+										propagaters.set(parentInfo.id, new Set());
+									}
+									propagaters.get(parentInfo.id)!.add(hid);
+								}
+							} else {
+								for (const nestedParentInfo of getParentModuleInfos(from, this)) {
+									if (moduleIsTopLevelPage(nestedParentInfo)) {
+										for (const hid of hoistedScripts) {
+											if (!pageScripts.has(nestedParentInfo.id)) {
+												pageScripts.set(nestedParentInfo.id, {
+													type: 'page',
+													hoistedSet: new Set(),
+													propagatedMapByImporter: new Map(),
+												});
+											}
+											const entry = pageScripts.get(nestedParentInfo.id)!;
+											if (!entry.propagatedMapByImporter.has(parentInfo.id)) {
+												entry.propagatedMapByImporter.set(parentInfo.id, new Set());
+											}
+											entry.propagatedMapByImporter.get(parentInfo.id)!.add(hid);
 										}
-										entry.propagatedMapByImporter.get(parentInfo.id)!.add(hid);
 									}
 								}
 							}
 						} else if (moduleIsTopLevelPage(parentInfo)) {
 							for (const hid of hoistedScripts) {
 								if (!pageScripts.has(parentInfo.id)) {
 									pageScripts.set(parentInfo.id, {
+										type: 'page',
 										hoistedSet: new Set(),
 										propagatedMapByImporter: new Map(),
 									});
@@ -84,12 +105,21 @@ export function vitePluginAnalyzer(
 			},
 
 			finalize() {
-				for (const [pageId, { hoistedSet, propagatedMapByImporter }] of pageScripts) {
-					const pageData = getPageDataByViteID(internals, pageId);
-					if (!pageData) continue;
+				for (const [pageId, { hoistedSet, propagatedMapByImporter, type }] of pageScripts) {
+					let astroModuleId: string;
+					if(type === 'page') {
+						const pageData = getPageDataByViteID(internals, pageId);
+						if (!pageData) {
+							continue;
+						};
+						const { component } = pageData;
+						astroModuleId = prependForwardSlash(component);
 
-					const { component } = pageData;
-					const astroModuleId = prependForwardSlash(component);
+						// Keep track of the importers
+						pageData.propagatedScripts = propagatedMapByImporter;
+					} else {
+						astroModuleId = pageId;
+					}
 
 					const uniqueHoistedId = JSON.stringify(Array.from(hoistedSet).sort());
 					let moduleId: string;
@@ -104,8 +134,6 @@ export function vitePluginAnalyzer(
 					}
 					internals.discoveredScripts.add(moduleId);
 
-					pageData.propagatedScripts = propagatedMapByImporter;
-
 					// Add propagated scripts to client build,
 					// but DON'T add to pages -> hoisted script map.
 					for (const propagatedScripts of propagatedMapByImporter.values()) {
@@ -114,13 +142,24 @@ export function vitePluginAnalyzer(
 						}
 					}
 
-					// Make sure to track that this page uses this set of hoisted scripts
-					if (internals.hoistedScriptIdToPagesMap.has(moduleId)) {
-						const pages = internals.hoistedScriptIdToPagesMap.get(moduleId);
-						pages!.add(astroModuleId);
+					if(type === 'page') {
+						// Make sure to track that this page uses this set of hoisted scripts
+						if (internals.hoistedScriptIdToPagesMap.has(moduleId)) {
+							const pages = internals.hoistedScriptIdToPagesMap.get(moduleId);
+							pages!.add(astroModuleId);
+						} else {
+							internals.hoistedScriptIdToPagesMap.set(moduleId, new Set([astroModuleId]));
+							internals.hoistedScriptIdToHoistedMap.set(moduleId, hoistedSet);
+						}
 					} else {
-						internals.hoistedScriptIdToPagesMap.set(moduleId, new Set([astroModuleId]));
-						internals.hoistedScriptIdToHoistedMap.set(moduleId, hoistedSet);
+						// For content collections save to hoistedScriptIdToContentMap instead
+						if (internals.hoistedScriptIdToContentMap.has(moduleId)) {
+							const contentModules = internals.hoistedScriptIdToContentMap.get(moduleId);
+							contentModules!.add(astroModuleId);
+						} else {
+							internals.hoistedScriptIdToContentMap.set(moduleId, new Set([astroModuleId]));
+							internals.hoistedScriptIdToHoistedMap.set(moduleId, hoistedSet);
+						}
 					}
 				}
 			},