From 325a57c543d88eab5e3ab32ee1bbfb534aed9c7c Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 20 Sep 2024 10:32:52 +0100 Subject: [PATCH] fix: correctly parse inline loader values (#12035) --- .changeset/cold-bananas-hear.md | 5 ++ packages/astro/src/content/content-layer.ts | 48 +++++++++++-- packages/astro/src/content/utils.ts | 51 ++++++++++---- packages/astro/src/core/errors/errors-data.ts | 70 +++++++++++++++++++ packages/astro/test/content-layer.test.js | 21 +++++- .../content-layer/src/content/config.ts | 67 +++++++++++++++++- .../src/pages/collections.json.js | 3 + packages/astro/types/content.d.ts | 8 ++- 8 files changed, 252 insertions(+), 21 deletions(-) create mode 100644 .changeset/cold-bananas-hear.md diff --git a/.changeset/cold-bananas-hear.md b/.changeset/cold-bananas-hear.md new file mode 100644 index 000000000000..dfa9ade4af39 --- /dev/null +++ b/.changeset/cold-bananas-hear.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Correctly parse values returned from inline loader diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index bf3213901517..2e092ae6150d 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -2,6 +2,7 @@ import { promises as fs, existsSync } from 'node:fs'; import * as fastq from 'fastq'; import type { FSWatcher } from 'vite'; import xxhash from 'xxhash-wasm'; +import { AstroError, AstroErrorData } from '../core/errors/index.js'; import type { Logger } from '../core/logger/core.js'; import type { AstroSettings } from '../types/astro.js'; import type { ContentEntryType, RefreshContentOptions } from '../types/public/content.js'; @@ -266,15 +267,54 @@ export class ContentLayer { } export async function simpleLoader( - handler: () => Array | Promise>, + handler: () => + | Array + | Promise> + | Record> + | Promise>>, context: LoaderContext, ) { const data = await handler(); context.store.clear(); - for (const raw of data) { - const item = await context.parseData({ id: raw.id, data: raw }); - context.store.set({ id: raw.id, data: item }); + if (Array.isArray(data)) { + for (const raw of data) { + if (!raw.id) { + throw new AstroError({ + ...AstroErrorData.ContentLoaderInvalidDataError, + message: AstroErrorData.ContentLoaderInvalidDataError.message( + context.collection, + `Entry missing ID:\n${JSON.stringify({ ...raw, id: undefined }, null, 2)}`, + ), + }); + } + const item = await context.parseData({ id: raw.id, data: raw }); + context.store.set({ id: raw.id, data: item }); + } + return; + } + if (typeof data === 'object') { + for (const [id, raw] of Object.entries(data)) { + if (raw.id && raw.id !== id) { + throw new AstroError({ + ...AstroErrorData.ContentLoaderInvalidDataError, + message: AstroErrorData.ContentLoaderInvalidDataError.message( + context.collection, + `Object key ${JSON.stringify(id)} does not match ID ${JSON.stringify(raw.id)}`, + ), + }); + } + const item = await context.parseData({ id, data: raw }); + context.store.set({ id, data: item }); + } + return; } + throw new AstroError({ + ...AstroErrorData.ExpectedImageOptions, + message: AstroErrorData.ContentLoaderInvalidDataError.message( + context.collection, + `Invalid data type: ${typeof data}`, + ), + }); } /** * Get the path to the data store file. diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index 1dd1a457fdd7..0fa7bed06ef0 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -32,6 +32,17 @@ export type ContentLookupMap = { [collectionName: string]: { type: 'content' | 'data'; entries: { [lookupId: string]: string } }; }; +const entryTypeSchema = z + .object({ + id: z + .string({ + invalid_type_error: 'Content entry `id` must be a string', + // Default to empty string so we can validate properly in the loader + }) + .catch(''), + }) + .catchall(z.unknown()); + const collectionConfigParser = z.union([ z.object({ type: z.literal('content').optional().default('content'), @@ -47,18 +58,31 @@ const collectionConfigParser = z.union([ loader: z.union([ z.function().returns( z.union([ - z.array( + z.array(entryTypeSchema), + z.promise(z.array(entryTypeSchema)), + z.record( + z.string(), z .object({ - id: z.string(), + id: z + .string({ + invalid_type_error: 'Content entry `id` must be a string', + }) + .optional(), }) .catchall(z.unknown()), ), + z.promise( - z.array( + z.record( + z.string(), z .object({ - id: z.string(), + id: z + .string({ + invalid_type_error: 'Content entry `id` must be a string', + }) + .optional(), }) .catchall(z.unknown()), ), @@ -194,16 +218,19 @@ export async function getEntryDataAndImages< data = parsed.data as TOutputData; } else { if (!formattedError) { + const errorType = + collectionConfig.type === 'content' + ? AstroErrorData.InvalidContentEntryFrontmatterError + : AstroErrorData.InvalidContentEntryDataError; formattedError = new AstroError({ - ...AstroErrorData.InvalidContentEntryFrontmatterError, - message: AstroErrorData.InvalidContentEntryFrontmatterError.message( - entry.collection, - entry.id, - parsed.error, - ), + ...errorType, + message: errorType.message(entry.collection, entry.id, parsed.error), location: { - file: entry._internal.filePath, - line: getYAMLErrorLine(entry._internal.rawData, String(parsed.error.errors[0].path[0])), + file: entry._internal?.filePath, + line: getYAMLErrorLine( + entry._internal?.rawData, + String(parsed.error.errors[0].path[0]), + ), column: 0, }, }); diff --git a/packages/astro/src/core/errors/errors-data.ts b/packages/astro/src/core/errors/errors-data.ts index 89bdfb55a02e..fd930cf33000 100644 --- a/packages/astro/src/core/errors/errors-data.ts +++ b/packages/astro/src/core/errors/errors-data.ts @@ -1490,6 +1490,76 @@ export const InvalidContentEntryFrontmatterError = { }, hint: 'See https://docs.astro.build/en/guides/content-collections/ for more information on content schemas.', } satisfies ErrorData; + +/** + * @docs + * @message + * **Example error message:**
+ * **blog** → **post** frontmatter does not match collection schema.
+ * "title" is required.
+ * "date" must be a valid date. + * @description + * A content entry does not match its collection schema. + * Make sure that all required fields are present, and that all fields are of the correct type. + * You can check against the collection schema in your `src/content/config.*` file. + * See the [Content collections documentation](https://docs.astro.build/en/guides/content-collections/) for more information. + */ +export const InvalidContentEntryDataError = { + name: 'InvalidContentEntryDataError', + title: 'Content entry data does not match schema.', + message(collection: string, entryId: string, error: ZodError) { + return [ + `**${String(collection)} → ${String(entryId)}** data does not match collection schema.`, + ...error.errors.map((zodError) => zodError.message), + ].join('\n'); + }, + hint: 'See https://docs.astro.build/en/guides/content-collections/ for more information on content schemas.', +} satisfies ErrorData; + +/** + * @docs + * @message + * **Example error message:**
+ * **blog** → **post** data does not match collection schema.
+ * "title" is required.
+ * "date" must be a valid date. + * @description + * A content entry does not match its collection schema. + * Make sure that all required fields are present, and that all fields are of the correct type. + * You can check against the collection schema in your `src/content/config.*` file. + * See the [Content collections documentation](https://docs.astro.build/en/guides/content-collections/) for more information. + */ +export const ContentEntryDataError = { + name: 'ContentEntryDataError', + title: 'Content entry data does not match schema.', + message(collection: string, entryId: string, error: ZodError) { + return [ + `**${String(collection)} → ${String(entryId)}** data does not match collection schema.`, + ...error.errors.map((zodError) => zodError.message), + ].join('\n'); + }, + hint: 'See https://docs.astro.build/en/guides/content-collections/ for more information on content schemas.', +} satisfies ErrorData; + +/** + * @docs + * @message + * **Example error message:**
+ * The loader for **blog** returned invalid data.
+ * Object is missing required property "id". + * @description + * The loader for a content collection returned invalid data. + * Inline loaders must return an array of objects with unique ID fields or a plain object with IDs as keys and entries as values. + */ +export const ContentLoaderInvalidDataError = { + name: 'ContentLoaderInvalidDataError', + title: 'Content entry is missing an ID', + message(collection: string, extra: string) { + return `**${String(collection)}** entry is missing an ID.\n${extra}`; + }, + hint: 'See https://docs.astro.build/en/guides/content-collections/ for more information on content loaders.', +} satisfies ErrorData; + /** * @docs * @message `COLLECTION_NAME` → `ENTRY_ID` has an invalid slug. `slug` must be a string. diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 6fceaec44f46..5be3953867ed 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -3,8 +3,8 @@ import { promises as fs } from 'node:fs'; import { sep } from 'node:path'; import { sep as posixSep } from 'node:path/posix'; import { after, before, describe, it } from 'node:test'; -import * as devalue from 'devalue'; import * as cheerio from 'cheerio'; +import * as devalue from 'devalue'; import { loadFixture } from './test-utils.js'; describe('Content Layer', () => { @@ -134,6 +134,23 @@ describe('Content Layer', () => { }); }); + it('returns a collection from a simple loader that uses an object', async () => { + assert.ok(json.hasOwnProperty('simpleLoaderObject')); + assert.ok(Array.isArray(json.simpleLoaderObject)); + assert.deepEqual(json.simpleLoaderObject[0], { + id: 'capybara', + collection: 'rodents', + data: { + name: 'Capybara', + scientificName: 'Hydrochoerus hydrochaeris', + lifespan: 10, + weight: 50000, + diet: ['grass', 'aquatic plants', 'bark', 'fruits'], + nocturnal: false, + }, + }); + }); + it('transforms a reference id to a reference object', async () => { assert.ok(json.hasOwnProperty('entryWithReference')); assert.deepEqual(json.entryWithReference.data.cat, { collection: 'cats', id: 'tabby' }); @@ -168,7 +185,7 @@ describe('Content Layer', () => { }); it('displays public images unchanged', async () => { - assert.equal($('img[alt="buzz"]').attr('src'), "/buzz.jpg"); + assert.equal($('img[alt="buzz"]').attr('src'), '/buzz.jpg'); }); it('renders local images', async () => { diff --git a/packages/astro/test/fixtures/content-layer/src/content/config.ts b/packages/astro/test/fixtures/content-layer/src/content/config.ts index 79412da6606f..402bad7fc5f5 100644 --- a/packages/astro/test/fixtures/content-layer/src/content/config.ts +++ b/packages/astro/test/fixtures/content-layer/src/content/config.ts @@ -18,6 +18,59 @@ const dogs = defineCollection({ }), }); +const rodents = defineCollection({ + loader: () => ({ + capybara: { + name: 'Capybara', + scientificName: 'Hydrochoerus hydrochaeris', + lifespan: 10, + weight: 50000, + diet: ['grass', 'aquatic plants', 'bark', 'fruits'], + nocturnal: false, + }, + hamster: { + name: 'Golden Hamster', + scientificName: 'Mesocricetus auratus', + lifespan: 2, + weight: 120, + diet: ['seeds', 'nuts', 'insects'], + nocturnal: true, + }, + rat: { + name: 'Brown Rat', + scientificName: 'Rattus norvegicus', + lifespan: 2, + weight: 350, + diet: ['grains', 'fruits', 'vegetables', 'meat'], + nocturnal: true, + }, + mouse: { + name: 'House Mouse', + scientificName: 'Mus musculus', + lifespan: 1, + weight: 20, + diet: ['seeds', 'grains', 'fruits'], + nocturnal: true, + }, + guineaPig: { + name: 'Guinea Pig', + scientificName: 'Cavia porcellus', + lifespan: 5, + weight: 1000, + diet: ['hay', 'vegetables', 'fruits'], + nocturnal: false, + }, + }), + schema: z.object({ + name: z.string(), + scientificName: z.string(), + lifespan: z.number().int().positive(), + weight: z.number().positive(), + diet: z.array(z.string()), + nocturnal: z.boolean(), + }), +}); + const cats = defineCollection({ loader: async function () { return [ @@ -131,7 +184,7 @@ const increment = defineCollection({ data: { lastValue: lastValue + 1, lastUpdated: new Date(), - refreshContextData + refreshContextData, }, }); }, @@ -145,4 +198,14 @@ const increment = defineCollection({ }, }); -export const collections = { blog, dogs, cats, numbers, spacecraft, increment, images, probes }; +export const collections = { + blog, + dogs, + cats, + numbers, + spacecraft, + increment, + images, + probes, + rodents, +}; diff --git a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js index aea8bfc9ad8b..761ff7dba6fa 100644 --- a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js +++ b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js @@ -20,6 +20,8 @@ export async function GET() { const images = await getCollection('images'); + const simpleLoaderObject = await getCollection('rodents') + const probes = await getCollection('probes'); return new Response( devalue.stringify({ @@ -27,6 +29,7 @@ export async function GET() { fileLoader, dataEntry, simpleLoader, + simpleLoaderObject, entryWithReference, entryWithImagePath, referencedEntry, diff --git a/packages/astro/types/content.d.ts b/packages/astro/types/content.d.ts index 1715a30a42e3..8bcf23d8c819 100644 --- a/packages/astro/types/content.d.ts +++ b/packages/astro/types/content.d.ts @@ -60,7 +60,13 @@ declare module 'astro:content' { type ContentLayerConfig = { type?: 'content_layer'; schema?: S | ((context: SchemaContext) => S); - loader: import('astro/loaders').Loader | (() => Array | Promise>); + loader: + | import('astro/loaders').Loader + | (() => + | Array + | Promise> + | Record & { id?: string }> + | Promise & { id?: string }>>); }; type DataCollectionConfig = {