diff --git a/packages/astro/package.json b/packages/astro/package.json index f2b232eccb60..2ebfec0537fc 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -152,6 +152,7 @@ "esbuild": "^0.21.5", "estree-walker": "^3.0.3", "fast-glob": "^3.3.2", + "fastq": "^1.17.1", "flattie": "^1.1.1", "github-slugger": "^2.0.0", "gray-matter": "^4.0.3", diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index 5c7f14355112..606754eac85b 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -1,6 +1,7 @@ import { promises as fs, existsSync } from 'node:fs'; import { isAbsolute } from 'node:path'; import { fileURLToPath } from 'node:url'; +import * as fastq from 'fastq'; import type { FSWatcher } from 'vite'; import xxhash from 'xxhash-wasm'; import type { AstroSettings, ContentEntryType, RefreshContentOptions } from '../@types/astro.js'; @@ -38,7 +39,8 @@ export class ContentLayer { #generateDigest?: (data: Record | string) => string; - #loading = false; + #queue: fastq.queueAsPromised; + constructor({ settings, logger, store, watcher }: ContentLayerOptions) { // The default max listeners is 10, which can be exceeded when using a lot of loaders watcher?.setMaxListeners(50); @@ -47,8 +49,15 @@ export class ContentLayer { this.#store = store; this.#settings = settings; this.#watcher = watcher; + this.#queue = fastq.promise(this.#doSync.bind(this), 1); } + /** + * Whether the content layer is currently loading content + */ + get loading() { + return !this.#queue.idle(); + } /** * Watch for changes to the content config and trigger a sync when it changes. @@ -56,11 +65,7 @@ export class ContentLayer { watchContentConfig() { this.#unsubscribe?.(); this.#unsubscribe = globalContentConfigObserver.subscribe(async (ctx) => { - if ( - !this.#loading && - ctx.status === 'loaded' && - ctx.config.digest !== this.#lastConfigDigest - ) { + if (ctx.status === 'loaded' && ctx.config.digest !== this.#lastConfigDigest) { this.sync(); } }); @@ -70,7 +75,6 @@ export class ContentLayer { this.#unsubscribe?.(); } - async #getGenerateDigest() { if (this.#generateDigest) { return this.#generateDigest; @@ -116,12 +120,17 @@ export class ContentLayer { } /** - * Run the `load()` method of each collection's loader, which will load the data and save it in the data store. + * Enqueues a sync job that runs the `load()` method of each collection's loader, which will load the data and save it in the data store. * The loader itself is responsible for deciding whether this will clear and reload the full collection, or - * perform an incremental update. After the data is loaded, the data store is written to disk. + * perform an incremental update. After the data is loaded, the data store is written to disk. Jobs are queued, + * so that only one sync can run at a time. The function returns a promise that resolves when this sync job is complete. */ - async sync(options?: RefreshContentOptions) { - + + sync(options: RefreshContentOptions = {}): Promise { + return this.#queue.push(options); + } + + async #doSync(options: RefreshContentOptions) { const contentConfig = globalContentConfigObserver.get(); const logger = this.#logger.forkIntegrationLogger('content'); if (contentConfig?.status !== 'loaded') { @@ -170,7 +179,8 @@ export class ContentLayer { // If loaders are specified, only sync the specified loaders if ( options?.loaders && - (typeof collection.loader !== 'object' || !options.loaders.includes(collection.loader.name)) + (typeof collection.loader !== 'object' || + !options.loaders.includes(collection.loader.name)) ) { return; } @@ -208,7 +218,7 @@ export class ContentLayer { collectionName: name, parseData, loaderName: collection.loader.name, - refreshContextData: options?.context + refreshContextData: options?.context, }); if (typeof collection.loader === 'function') { @@ -289,18 +299,12 @@ export async function simpleLoader( function contentLayerSingleton() { let instance: ContentLayer | null = null; return { - initialized: () => Boolean(instance), init: (options: ContentLayerOptions) => { instance?.unwatchContentConfig(); instance = new ContentLayer(options); return instance; }, - get: () => { - if (!instance) { - throw new Error('Content layer not initialized'); - } - return instance; - }, + get: () => instance, dispose: () => { instance?.unwatchContentConfig(); instance = null; diff --git a/packages/astro/src/core/dev/restart.ts b/packages/astro/src/core/dev/restart.ts index ee0ba995c446..30821362c744 100644 --- a/packages/astro/src/core/dev/restart.ts +++ b/packages/astro/src/core/dev/restart.ts @@ -185,9 +185,7 @@ export async function createContainerWithAutomaticRestart({ key: 's', description: 'sync content layer', action: () => { - if (globalContentLayer.initialized()) { - globalContentLayer.get().sync(); - } + globalContentLayer.get()?.sync(); }, }); } diff --git a/packages/astro/src/integrations/hooks.ts b/packages/astro/src/integrations/hooks.ts index 3e7b6e48b0bd..22523b716589 100644 --- a/packages/astro/src/integrations/hooks.ts +++ b/packages/astro/src/integrations/hooks.ts @@ -16,6 +16,7 @@ import type { RouteData, RouteOptions, } from '../@types/astro.js'; +import { globalContentLayer } from '../content/content-layer.js'; import type { SerializedSSRManifest } from '../core/app/types.js'; import type { PageBuildData } from '../core/build/types.js'; import { buildClientDirectiveEntrypoint } from '../core/client-directive/index.js'; @@ -23,7 +24,6 @@ import { mergeConfig } from '../core/config/index.js'; import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js'; import { isServerLikeOutput } from '../core/util.js'; import { validateSupportedFeatures } from './features-validation.js'; -import { globalContentLayer } from '../content/content-layer.js'; async function withTakingALongTimeMsg({ name, @@ -378,7 +378,7 @@ export async function runHookServerSetup({ if (config.experimental?.contentLayer) { refreshContent = async (options: RefreshContentOptions) => { const contentLayer = await globalContentLayer.get(); - await contentLayer.sync(options); + await contentLayer?.sync(options); }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ac8d67daa5c5..4ddb0c30f341 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -645,6 +645,9 @@ importers: fast-glob: specifier: ^3.3.2 version: 3.3.2 + fastq: + specifier: ^1.17.1 + version: 1.17.1 flattie: specifier: ^1.1.1 version: 1.1.1 @@ -9375,10 +9378,12 @@ packages: libsql@0.3.19: resolution: {integrity: sha512-Aj5cQ5uk/6fHdmeW0TiXK42FqUlwx7ytmMLPSaUQPin5HKKKuUPD62MAbN4OEweGBBI7q1BekoEN4gPUEL6MZA==} + cpu: [x64, arm64, wasm32] os: [darwin, linux, win32] libsql@0.4.1: resolution: {integrity: sha512-qZlR9Yu1zMBeLChzkE/cKfoKV3Esp9cn9Vx5Zirn4AVhDWPcjYhKwbtJcMuHehgk3mH+fJr9qW+3vesBWbQpBg==} + cpu: [x64, arm64, wasm32] os: [darwin, linux, win32] lilconfig@2.1.0: @@ -11367,6 +11372,9 @@ packages: resolution: {integrity: sha512-M/wqwtOEjgb956/+m5ZrYT/Iq6Hax0OakWbokj8+9PXOnB7b/4AxESHieEtnNEy7ZpjsjYW1/5nK8fATQMmRxw==} peerDependencies: vue: '>=3.2.13' + peerDependenciesMeta: + vue: + optional: true vite@5.4.2: resolution: {integrity: sha512-dDrQTRHp5C1fTFzcSaMxjk6vdpKvT+2/mIdE07Gw2ykehT49O0z/VHS3zZ8iV/Gh8BJJKHWOe5RjaNrW5xf/GA==} @@ -17927,6 +17935,7 @@ snapshots: vite-svg-loader@5.1.0(vue@3.4.38(typescript@5.5.4)): dependencies: svgo: 3.2.0 + optionalDependencies: vue: 3.4.38(typescript@5.5.4) vite@5.4.2(@types/node@18.19.31)(sass@1.77.8):