From c14bdedb00f1e9bc8ca723eb8a681a9e17db8a56 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 30 Aug 2024 14:02:02 +0100 Subject: [PATCH 1/2] fix: separate image extraction from schema parsing in content layer --- .changeset/odd-tips-jam.md | 5 ++++ packages/astro/src/content/content-layer.ts | 11 +------ packages/astro/src/content/data-store.ts | 1 + packages/astro/src/content/loaders/glob.ts | 15 ++++------ .../astro/src/content/mutable-data-store.ts | 29 +++++++++++++++++-- packages/astro/test/content-layer.test.js | 4 +++ .../content-outside-src/columbia-copy.md | 1 + .../content-layer/src/content/config.ts | 3 +- .../src/pages/collections.json.js | 1 - 9 files changed, 45 insertions(+), 25 deletions(-) create mode 100644 .changeset/odd-tips-jam.md diff --git a/.changeset/odd-tips-jam.md b/.changeset/odd-tips-jam.md new file mode 100644 index 000000000000..4332a4eba114 --- /dev/null +++ b/.changeset/odd-tips-jam.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Correctly handles content layer data where the transformed value does not match the input schema diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index 606754eac85b..be182bd6f36f 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -188,7 +188,7 @@ export class ContentLayer { const collectionWithResolvedSchema = { ...collection, schema }; const parseData: LoaderContext['parseData'] = async ({ id, data, filePath = '' }) => { - const { imageImports, data: parsedData } = await getEntryDataAndImages( + const { data: parsedData } = await getEntryDataAndImages( { id, collection: name, @@ -201,15 +201,6 @@ export class ContentLayer { collectionWithResolvedSchema, false, ); - if (imageImports?.length) { - this.#store.addAssetImports( - imageImports, - // This path may already be relative, if we're re-parsing an existing entry - isAbsolute(filePath) - ? posixRelative(fileURLToPath(this.#settings.config.root), filePath) - : filePath, - ); - } return parsedData; }; diff --git a/packages/astro/src/content/data-store.ts b/packages/astro/src/content/data-store.ts index 76cefc411a76..fbf31d0f1372 100644 --- a/packages/astro/src/content/data-store.ts +++ b/packages/astro/src/content/data-store.ts @@ -33,6 +33,7 @@ export interface DataEntry = Record; } /** diff --git a/packages/astro/src/content/loaders/glob.ts b/packages/astro/src/content/loaders/glob.ts index 260d32c62029..e0b1d79c1b94 100644 --- a/packages/astro/src/content/loaders/glob.ts +++ b/packages/astro/src/content/loaders/glob.ts @@ -107,15 +107,11 @@ export function glob(globOptions: GlobOptions): Loader { store.addModuleImport(existingEntry.filePath); } - if (existingEntry.rendered?.metadata?.imagePaths?.length) { + if (existingEntry.assetImports?.length) { // Add asset imports for existing entries - store.addAssetImports( - existingEntry.rendered.metadata.imagePaths, - existingEntry.filePath, - ); + store.addAssetImports(existingEntry.assetImports, existingEntry.filePath); } - // Re-parsing to resolve images and other effects - await parseData(existingEntry); + return; } @@ -156,10 +152,9 @@ export function glob(globOptions: GlobOptions): Loader { filePath: relativePath, digest, rendered, + assetImports: rendered?.metadata?.imagePaths, }); - if (rendered?.metadata?.imagePaths?.length) { - store.addAssetImports(rendered.metadata.imagePaths, relativePath); - } + // todo: add an explicit way to opt in to deferred rendering } else if ('contentModuleTypes' in entryType) { store.set({ diff --git a/packages/astro/src/content/mutable-data-store.ts b/packages/astro/src/content/mutable-data-store.ts index 200951848046..29acf4506bf7 100644 --- a/packages/astro/src/content/mutable-data-store.ts +++ b/packages/astro/src/content/mutable-data-store.ts @@ -1,7 +1,9 @@ import { promises as fs, type PathLike, existsSync } from 'node:fs'; import * as devalue from 'devalue'; +import { Traverse } from 'neotraverse/modern'; import { imageSrcToImportId, importIdToSymbolName } from '../assets/utils/resolveImports.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; +import { IMAGE_IMPORT_PREFIX } from './consts.js'; import { type DataEntry, DataStore, type RenderedContent } from './data-store.js'; import { contentModuleToId } from './utils.js'; @@ -53,7 +55,7 @@ export class MutableDataStore extends DataStore { this.#saveToDiskDebounced(); } - addAssetImport(assetImport: string, filePath: string) { + addAssetImport(assetImport: string, filePath?: string) { const id = imageSrcToImportId(assetImport, filePath); if (id) { this.#assetImports.add(id); @@ -64,7 +66,7 @@ export class MutableDataStore extends DataStore { } } - addAssetImports(assets: Array, filePath: string) { + addAssetImports(assets: Array, filePath?: string) { assets.forEach((asset) => this.addAssetImport(asset, filePath)); } @@ -195,7 +197,7 @@ export default new Map([\n${lines.join(',\n')}]); entries: () => this.entries(collectionName), values: () => this.values(collectionName), keys: () => this.keys(collectionName), - set: ({ id: key, data, body, filePath, deferredRender, digest, rendered }) => { + set: ({ id: key, data, body, filePath, deferredRender, digest, rendered, assetImports }) => { if (!key) { throw new Error(`ID must be a non-empty string`); } @@ -206,6 +208,15 @@ export default new Map([\n${lines.join(',\n')}]); return false; } } + const foundAssets = new Set(assetImports); + // Check for image imports in the data. These will have been prefixed during schema parsing + new Traverse(data).forEach((_, val) => { + if (typeof val === 'string' && val.startsWith(IMAGE_IMPORT_PREFIX)) { + const src = val.replace(IMAGE_IMPORT_PREFIX, ''); + foundAssets.add(src); + } + }); + const entry: DataEntry = { id, data, @@ -221,6 +232,12 @@ export default new Map([\n${lines.join(',\n')}]); } entry.filePath = filePath; } + + if (foundAssets.size) { + entry.assetImports = Array.from(foundAssets); + this.addAssetImports(entry.assetImports, filePath); + } + if (digest) { entry.digest = digest; } @@ -334,6 +351,12 @@ export interface ScopedDataStore { * If an entry is a deferred, its rendering phase is delegated to a virtual module during the runtime phase. */ deferredRender?: boolean; + /** + * Assets such as images to process during the build. These should be files on disk, with a path relative to filePath. + * Any values that use image() in the schema will already be added automatically. + * @internal + */ + assetImports?: Array; }) => boolean; values: () => Array; keys: () => Array; diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index d8d6a3b4512f..0590e7e59020 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -162,17 +162,21 @@ describe('Content Layer', () => { it('updates the store on new builds', async () => { assert.equal(json.increment.data.lastValue, 1); + assert.equal(json.entryWithReference.data.something?.content, 'transform me'); await fixture.build(); const newJson = devalue.parse(await fixture.readFile('/collections.json')); assert.equal(newJson.increment.data.lastValue, 2); + assert.equal(newJson.entryWithReference.data.something?.content, 'transform me'); }); it('clears the store on new build with force flag', async () => { let newJson = devalue.parse(await fixture.readFile('/collections.json')); assert.equal(newJson.increment.data.lastValue, 2); + assert.equal(newJson.entryWithReference.data.something?.content, 'transform me'); await fixture.build({ force: true }, {}); newJson = devalue.parse(await fixture.readFile('/collections.json')); assert.equal(newJson.increment.data.lastValue, 1); + assert.equal(newJson.entryWithReference.data.something?.content, 'transform me'); }); it('clears the store on new build if the config has changed', async () => { diff --git a/packages/astro/test/fixtures/content-layer/content-outside-src/columbia-copy.md b/packages/astro/test/fixtures/content-layer/content-outside-src/columbia-copy.md index e5ca2b3a5358..fba81378ee29 100644 --- a/packages/astro/test/fixtures/content-layer/content-outside-src/columbia-copy.md +++ b/packages/astro/test/fixtures/content-layer/content-outside-src/columbia-copy.md @@ -5,6 +5,7 @@ publishedDate: 'Sat May 21 2022 00:00:00 GMT-0400 (Eastern Daylight Time)' tags: [space, 90s] cat: tabby heroImage: "./shuttle.jpg" +something: "transform me" --- **Source:** [Wikipedia](https://en.wikipedia.org/wiki/Space_Shuttle_Endeavour) diff --git a/packages/astro/test/fixtures/content-layer/src/content/config.ts b/packages/astro/test/fixtures/content-layer/src/content/config.ts index 5e25d83b95de..8f06b4362909 100644 --- a/packages/astro/test/fixtures/content-layer/src/content/config.ts +++ b/packages/astro/test/fixtures/content-layer/src/content/config.ts @@ -78,6 +78,7 @@ const spacecraft = defineCollection({ tags: z.array(z.string()), heroImage: image().optional(), cat: reference('cats').optional(), + something: z.string().optional().transform(str => ({ type: 'test', content: str })) }), }); @@ -120,9 +121,9 @@ const increment = defineCollection({ schema: async () => z.object({ lastValue: z.number(), lastUpdated: z.date(), + }), }, - }); export const collections = { blog, dogs, cats, numbers, spacecraft, increment, images }; diff --git a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js index 7db3156ca2e0..87c8cc052680 100644 --- a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js +++ b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js @@ -17,7 +17,6 @@ export async function GET() { const increment = await getEntry('increment', 'value'); const images = await getCollection('images'); - return new Response( devalue.stringify({ customLoader, From aa43a77cd324ca5748ed60f1fbcb191926c1061b Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Sun, 1 Sep 2024 10:03:17 +0100 Subject: [PATCH 2/2] rm unused imports --- packages/astro/src/content/content-layer.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index be182bd6f36f..e14eed43b711 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -1,6 +1,4 @@ import { promises as fs, existsSync } from 'node:fs'; -import { isAbsolute } from 'node:path'; -import { fileURLToPath } from 'node:url'; import * as fastq from 'fastq'; import type { FSWatcher } from 'vite'; import xxhash from 'xxhash-wasm'; @@ -19,7 +17,6 @@ import { getEntryConfigByExtMap, getEntryDataAndImages, globalContentConfigObserver, - posixRelative, } from './utils.js'; export interface ContentLayerOptions {