feat: allow integrations to refresh content layer data (#11878)

* Allow integrations to refresh content layer data This reverts commit 90a862f. * Add test * Add changeset * Add wait in test * Dispose of queue * Skip if no content layer collections * Use spaces in markdown * fix: don't keep data store in node_modules during dev * Lint * Fix test * Apply suggestions from code review Co-authored-by: Sarah Rainsberger <[email protected]> * Re-order tests * Wait for data store * Lint * Handle case where Vite already knows about save --------- Co-authored-by: Sarah Rainsberger <[email protected]>
withastro · Sep 3, 2024 · 334948c · 334948c
1 parent 4563ffc
commit 334948c
Show file tree

Hide file tree

Showing 13 changed files with 209 additions and 39 deletions.
diff --git a/.changeset/curvy-walls-kneel.md b/.changeset/curvy-walls-kneel.md
@@ -0,0 +1,45 @@
+---
+'astro': patch
+---
+
+Adds a new function `refreshContent` to the `astro:server:setup` hook that allows integrations to refresh the content layer. This can be used, for example, to register a webhook endpoint during dev, or to open a socket to a CMS to listen for changes.
+
+By default, `refreshContent` will refresh all collections. You can optionally pass a `loaders` property, which is an array of loader names. If provided, only collections that use those loaders will be refreshed. For example, A CMS integration could use this property to only refresh its own collections.
+
+You can also pass a `context` object to the loaders. This can be used to pass arbitrary data, such as the webhook body, or an event from the websocket.
+
+```ts
+ {
+    name: 'my-integration',
+    hooks: {
+        'astro:server:setup': async ({ server, refreshContent }) => {
+            server.middlewares.use('/_refresh', async (req, res) => {
+                if(req.method !== 'POST') {
+                  res.statusCode = 405
+                  res.end('Method Not Allowed');
+                  return
+                }
+                let body = '';
+                req.on('data', chunk => {
+                    body += chunk.toString();
+                });
+                req.on('end', async () => {
+                    try {
+                        const webhookBody = JSON.parse(body);
+                        await refreshContent({
+                          context: { webhookBody },
+                          loaders: ['my-loader']
+                        });
+                        res.writeHead(200, { 'Content-Type': 'application/json' });
+                        res.end(JSON.stringify({ message: 'Content refreshed successfully' }));
+                    } catch (error) {
+                        res.writeHead(500, { 'Content-Type': 'application/json' });
+                        res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message }));
+                    }
+                });
+            });
+        }
+    }
+}
+```
+
diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts
@@ -73,6 +73,11 @@ export class ContentLayer {
 		this.#unsubscribe?.();
 	}
 
+	dispose() {
+		this.#queue.kill();
+		this.#unsubscribe?.();
+	}
+
 	async #getGenerateDigest() {
 		if (this.#generateDigest) {
 			return this.#generateDigest;
@@ -224,7 +229,7 @@ export class ContentLayer {
 		if (!existsSync(this.#settings.config.cacheDir)) {
 			await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
 		}
-		const cacheFile = new URL(DATA_STORE_FILE, this.#settings.config.cacheDir);
+		const cacheFile = getDataStoreFile(this.#settings);
 		await this.#store.writeToDisk(cacheFile);
 		if (!existsSync(this.#settings.dotAstroDir)) {
 			await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
@@ -285,17 +290,24 @@ export async function simpleLoader<TData extends { id: string }>(
 	}
 }
 
+export function getDataStoreFile(settings: AstroSettings) {
+	return new URL(
+		DATA_STORE_FILE,
+		process?.env.NODE_ENV === 'development' ? settings.dotAstroDir : settings.config.cacheDir,
+	);
+}
+
 function contentLayerSingleton() {
 	let instance: ContentLayer | null = null;
 	return {
 		init: (options: ContentLayerOptions) => {
-			instance?.unwatchContentConfig();
+			instance?.dispose();
 			instance = new ContentLayer(options);
 			return instance;
 		},
 		get: () => instance,
 		dispose: () => {
-			instance?.unwatchContentConfig();
+			instance?.dispose();
 			instance = null;
 		},
 	};

diff --git a/packages/astro/src/content/data-store.ts b/packages/astro/src/content/data-store.ts
@@ -91,6 +91,9 @@ export class DataStore {
 		try {
 			// @ts-expect-error - this is a virtual module
 			const data = await import('astro:data-layer-content');
+			if (data.default instanceof Map) {
+				return DataStore.fromMap(data.default);
+			}
 			const map = devalue.unflatten(data.default);
 			return DataStore.fromMap(map);
 		} catch {}

diff --git a/packages/astro/src/content/loaders/types.ts b/packages/astro/src/content/loaders/types.ts
@@ -33,6 +33,7 @@ export interface LoaderContext {
 	/** When running in dev, this is a filesystem watcher that can be used to trigger updates */
 	watcher?: FSWatcher;
 
+	/** If the loader has been triggered by an integration, this may optionally contain extra data set by that integration */
 	refreshContextData?: Record<string, unknown>;
 	entryTypes: Map<string, ContentEntryType>;
 }

diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts
@@ -54,12 +54,16 @@ export function astroContentVirtualModPlugin({
 }: AstroContentVirtualModPluginParams): Plugin {
 	let IS_DEV = false;
 	const IS_SERVER = isServerLikeOutput(settings.config);
-	const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
+	let dataStoreFile: URL;
 	return {
 		name: 'astro-content-virtual-mod-plugin',
 		enforce: 'pre',
 		configResolved(config) {
 			IS_DEV = config.mode === 'development';
+			dataStoreFile = new URL(
+				DATA_STORE_FILE,
+				IS_DEV ? settings.dotAstroDir : settings.config.cacheDir,
+			);
 		},
 		async resolveId(id) {
 			if (id === VIRTUAL_MODULE_ID) {
@@ -180,25 +184,31 @@ export function astroContentVirtualModPlugin({
 
 		configureServer(server) {
 			const dataStorePath = fileURLToPath(dataStoreFile);
-			// Watch for changes to the data store file
-			if (Array.isArray(server.watcher.options.ignored)) {
-				// The data store file is in node_modules, so is ignored by default,
-				// so we need to un-ignore it.
-				server.watcher.options.ignored.push(`!${dataStorePath}`);
-			}
+
 			server.watcher.add(dataStorePath);
 
+			function invalidateDataStore() {
+				const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID);
+				if (module) {
+					server.moduleGraph.invalidateModule(module);
+				}
+				server.ws.send({
+					type: 'full-reload',
+					path: '*',
+				});
+			}
+
+			// If the datastore file changes, invalidate the virtual module
+
+			server.watcher.on('add', (addedPath) => {
+				if (addedPath === dataStorePath) {
+					invalidateDataStore();
+				}
+			});
+
 			server.watcher.on('change', (changedPath) => {
-				// If the datastore file changes, invalidate the virtual module
 				if (changedPath === dataStorePath) {
-					const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID);
-					if (module) {
-						server.moduleGraph.invalidateModule(module);
-					}
-					server.ws.send({
-						type: 'full-reload',
-						path: '*',
-					});
+					invalidateDataStore();
 				}
 			});
 		},

diff --git a/packages/astro/src/core/build/index.ts b/packages/astro/src/core/build/index.ts
@@ -56,6 +56,9 @@ export default async function build(
 	const logger = createNodeLogger(inlineConfig);
 	const { userConfig, astroConfig } = await resolveConfig(inlineConfig, 'build');
 	telemetry.record(eventCliSession('build', userConfig));
+
+	const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root));
+
 	if (inlineConfig.force) {
 		if (astroConfig.experimental.contentCollectionCache) {
 			const contentCacheDir = new URL('./content/', astroConfig.cacheDir);
@@ -65,11 +68,9 @@ export default async function build(
 				logger.warn('content', 'content cache cleared (force)');
 			}
 		}
-		await clearContentLayerCache({ astroConfig, logger, fs });
+		await clearContentLayerCache({ settings, logger, fs });
 	}
 
-	const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root));
-
 	const builder = new AstroBuilder(settings, {
 		...options,
 		logger,

diff --git a/packages/astro/src/core/dev/dev.ts b/packages/astro/src/core/dev/dev.ts
@@ -5,8 +5,7 @@ import { performance } from 'node:perf_hooks';
 import { green } from 'kleur/colors';
 import { gt, major, minor, patch } from 'semver';
 import type * as vite from 'vite';
-import { DATA_STORE_FILE } from '../../content/consts.js';
-import { globalContentLayer } from '../../content/content-layer.js';
+import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js';
 import { attachContentServerListeners } from '../../content/index.js';
 import { MutableDataStore } from '../../content/mutable-data-store.js';
 import { globalContentConfigObserver } from '../../content/utils.js';
@@ -108,7 +107,7 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise<DevS
 
 	let store: MutableDataStore | undefined;
 	try {
-		const dataStoreFile = new URL(DATA_STORE_FILE, restart.container.settings.config.cacheDir);
+		const dataStoreFile = getDataStoreFile(restart.container.settings);
 		if (existsSync(dataStoreFile)) {
 			store = await MutableDataStore.fromFile(dataStoreFile);
 		}

diff --git a/packages/astro/src/core/sync/index.ts b/packages/astro/src/core/sync/index.ts
@@ -3,8 +3,8 @@ import { performance } from 'node:perf_hooks';
 import { fileURLToPath } from 'node:url';
 import { dim } from 'kleur/colors';
 import { type HMRPayload, createServer } from 'vite';
-import { CONTENT_TYPES_FILE, DATA_STORE_FILE } from '../../content/consts.js';
-import { globalContentLayer } from '../../content/content-layer.js';
+import { CONTENT_TYPES_FILE } from '../../content/consts.js';
+import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js';
 import { createContentTypesGenerator } from '../../content/index.js';
 import { MutableDataStore } from '../../content/mutable-data-store.js';
 import { getContentPaths, globalContentConfigObserver } from '../../content/utils.js';
@@ -13,7 +13,7 @@ import { telemetry } from '../../events/index.js';
 import { eventCliSession } from '../../events/session.js';
 import { runHookConfigDone, runHookConfigSetup } from '../../integrations/hooks.js';
 import type { AstroSettings } from '../../types/astro.js';
-import type { AstroConfig, AstroInlineConfig } from '../../types/public/config.js';
+import type { AstroInlineConfig } from '../../types/public/config.js';
 import { getTimeStat } from '../build/util.js';
 import { resolveConfig } from '../config/config.js';
 import { createNodeLogger } from '../config/logging.js';
@@ -70,11 +70,11 @@ export default async function sync(
  * Clears the content layer and content collection cache, forcing a full rebuild.
  */
 export async function clearContentLayerCache({
-	astroConfig,
+	settings,
 	logger,
 	fs = fsMod,
-}: { astroConfig: AstroConfig; logger: Logger; fs?: typeof fsMod }) {
-	const dataStore = new URL(DATA_STORE_FILE, astroConfig.cacheDir);
+}: { settings: AstroSettings; logger: Logger; fs?: typeof fsMod }) {
+	const dataStore = getDataStoreFile(settings);
 	if (fs.existsSync(dataStore)) {
 		logger.debug('content', 'clearing data store');
 		await fs.promises.rm(dataStore, { force: true });
@@ -96,7 +96,7 @@ export async function syncInternal({
 	force,
 }: SyncOptions): Promise<void> {
 	if (force) {
-		await clearContentLayerCache({ astroConfig: settings.config, logger, fs });
+		await clearContentLayerCache({ settings, logger, fs });
 	}
 
 	const timerStart = performance.now();
@@ -107,7 +107,7 @@ export async function syncInternal({
 			settings.timer.start('Sync content layer');
 			let store: MutableDataStore | undefined;
 			try {
-				const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
+				const dataStoreFile = getDataStoreFile(settings);
 				if (existsSync(dataStoreFile)) {
 					store = await MutableDataStore.fromFile(dataStoreFile);
 				}

diff --git a/packages/astro/src/integrations/hooks.ts b/packages/astro/src/integrations/hooks.ts
@@ -5,6 +5,9 @@ import { bold } from 'kleur/colors';
 import type { InlineConfig, ViteDevServer } from 'vite';
 import astroIntegrationActionsRouteHandler from '../actions/integration.js';
 import { isActionsFilePresent } from '../actions/utils.js';
+import { CONTENT_LAYER_TYPE } from '../content/consts.js';
+import { globalContentLayer } from '../content/content-layer.js';
+import { globalContentConfigObserver } from '../content/utils.js';
 import type { SerializedSSRManifest } from '../core/app/types.js';
 import type { PageBuildData } from '../core/build/types.js';
 import { buildClientDirectiveEntrypoint } from '../core/client-directive/index.js';
@@ -13,7 +16,11 @@ import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js';
 import { isServerLikeOutput } from '../core/util.js';
 import type { AstroSettings } from '../types/astro.js';
 import type { AstroConfig } from '../types/public/config.js';
-import type { ContentEntryType, DataEntryType } from '../types/public/content.js';
+import type {
+	ContentEntryType,
+	DataEntryType,
+	RefreshContentOptions,
+} from '../types/public/content.js';
 import type {
 	AstroIntegration,
 	AstroRenderer,
@@ -367,6 +374,24 @@ export async function runHookServerSetup({
 	server: ViteDevServer;
 	logger: Logger;
 }) {
+	let refreshContent: undefined | ((options: RefreshContentOptions) => Promise<void>);
+	if (config.experimental?.contentLayer) {
+		refreshContent = async (options: RefreshContentOptions) => {
+			const contentConfig = globalContentConfigObserver.get();
+			if (
+				contentConfig.status !== 'loaded' ||
+				!Object.values(contentConfig.config.collections).some(
+					(collection) => collection.type === CONTENT_LAYER_TYPE,
+				)
+			) {
+				return;
+			}
+
+			const contentLayer = await globalContentLayer.get();
+			await contentLayer?.sync(options);
+		};
+	}
+
 	for (const integration of config.integrations) {
 		if (integration?.hooks?.['astro:server:setup']) {
 			await withTakingALongTimeMsg({
@@ -376,6 +401,7 @@ export async function runHookServerSetup({
 					server,
 					logger: getLogger(integration, logger),
 					toolbar: getToolbarServerCommunicationHelpers(server),
+					refreshContent,
 				}),
 				logger,
 			});

diff --git a/packages/astro/src/types/public/integrations.ts b/packages/astro/src/types/public/integrations.ts
@@ -6,6 +6,7 @@ import type { AstroIntegrationLogger } from '../../core/logger/core.js';
 import type { getToolbarServerCommunicationHelpers } from '../../integrations/hooks.js';
 import type { DeepPartial } from '../../type-utils.js';
 import type { AstroConfig } from './config.js';
+import type { RefreshContentOptions } from './content.js';
 import type { RouteData } from './internal.js';
 import type { DevToolbarAppEntry } from './toolbar.js';
 
@@ -187,6 +188,7 @@ export interface BaseIntegrationHooks {
 		server: ViteDevServer;
 		logger: AstroIntegrationLogger;
 		toolbar: ReturnType<typeof getToolbarServerCommunicationHelpers>;
+		refreshContent?: (options: RefreshContentOptions) => Promise<void>;
 	}) => void | Promise<void>;
 	'astro:server:start': (options: {
 		address: AddressInfo;

diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js
@@ -196,7 +196,11 @@ describe('Content Layer', () => {
 		let devServer;
 		let json;
 		before(async () => {
-			devServer = await fixture.startDevServer();
+			devServer = await fixture.startDevServer({ force: true });
+			// Vite may not have noticed the saved data store yet. Wait a little just in case.
+			await fixture.onNextDataStoreChange(1000).catch(() => {
+				// Ignore timeout, because it may have saved before we get here.
+			})
 			const rawJsonResponse = await fixture.fetch('/collections.json');
 			const rawJson = await rawJsonResponse.text();
 			json = devalue.parse(rawJson);
@@ -275,6 +279,22 @@ describe('Content Layer', () => {
 			});
 		});
 
+		it('reloads data when an integration triggers a content refresh', async () => {
+			const rawJsonResponse = await fixture.fetch('/collections.json');
+			const initialJson = devalue.parse(await rawJsonResponse.text());
+			assert.equal(initialJson.increment.data.lastValue, 1);
+
+			const refreshResponse = await fixture.fetch('/_refresh', {
+				method: 'POST',
+				body: JSON.stringify({}),
+			});
+			const refreshData = await refreshResponse.json();
+			assert.equal(refreshData.message, 'Content refreshed successfully');
+			const updatedJsonResponse = await fixture.fetch('/collections.json');
+			const updated = devalue.parse(await updatedJsonResponse.text());
+			assert.equal(updated.increment.data.lastValue, 2);
+		});
+
 		it('updates collection when data file is changed', async () => {
 			const rawJsonResponse = await fixture.fetch('/collections.json');
 			const initialJson = devalue.parse(await rawJsonResponse.text());
@@ -286,9 +306,7 @@ describe('Content Layer', () => {
 				return JSON.stringify(data, null, 2);
 			});
 
-			// Writes are debounced to 500ms
-			await new Promise((r) => setTimeout(r, 700));
-
+			await fixture.onNextDataStoreChange();
 			const updatedJsonResponse = await fixture.fetch('/collections.json');
 			const updated = devalue.parse(await updatedJsonResponse.text());
 			assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy'));