From ab6d42e13ef8a2f52ad5a737216416a223b3adc1 Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Fri, 20 Sep 2024 14:49:41 -0400 Subject: [PATCH 01/12] add custom file format support --- packages/astro/src/content/loaders/file.ts | 50 +++++++++++++++++----- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/packages/astro/src/content/loaders/file.ts b/packages/astro/src/content/loaders/file.ts index 22d498b12e39..2044e36ac018 100644 --- a/packages/astro/src/content/loaders/file.ts +++ b/packages/astro/src/content/loaders/file.ts @@ -2,24 +2,54 @@ import { promises as fs, existsSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { posixRelative } from '../utils.js'; import type { Loader, LoaderContext } from './types.js'; +import yaml from 'js-yaml'; + +export interface FileOptions { + /** + * the parsing function to use for this data + * @default JSON.parse or yaml.load, depending on the extension of the file + * */ + parser?: (text: string) => any; +} /** * Loads entries from a JSON file. The file must contain an array of objects that contain unique `id` fields, or an object with string keys. * @todo Add support for other file types, such as YAML, CSV etc. * @param fileName The path to the JSON file to load, relative to the content directory. + * @param options Additional options for */ -export function file(fileName: string): Loader { +export function file(fileName: string, options?: FileOptions): Loader { if (fileName.includes('*')) { // TODO: AstroError throw new Error('Glob patterns are not supported in `file` loader. Use `glob` loader instead.'); } + let parse: ((text: string) => any) | null = null; + + const ext = fileName.split('.').at(-1); + if (ext === 'json') { + parse = JSON.parse; + } else if (ext === 'yml' || ext === 'yaml') { + parse = (text) => + yaml.load(text, { + filename: fileName, + }); + } + if (options?.parser) parse = options.parser; + + if (parse === null) { + // TODO: AstroError + throw new Error( + `No parser found for file '${fileName}'. Try passing a parser to the \`file\` loader.`, + ); + } + async function syncData(filePath: string, { logger, parseData, store, config }: LoaderContext) { - let json: Array>; + let data: Array>; try { - const data = await fs.readFile(filePath, 'utf-8'); - json = JSON.parse(data); + const contents = await fs.readFile(filePath, 'utf-8'); + data = parse!(contents); } catch (error: any) { logger.error(`Error reading data from ${fileName}`); logger.debug(error.message); @@ -28,13 +58,13 @@ export function file(fileName: string): Loader { const normalizedFilePath = posixRelative(fileURLToPath(config.root), filePath); - if (Array.isArray(json)) { - if (json.length === 0) { + if (Array.isArray(data)) { + if (data.length === 0) { logger.warn(`No items found in ${fileName}`); } - logger.debug(`Found ${json.length} item array in ${fileName}`); + logger.debug(`Found ${data.length} item array in ${fileName}`); store.clear(); - for (const rawItem of json) { + for (const rawItem of data) { const id = (rawItem.id ?? rawItem.slug)?.toString(); if (!id) { logger.error(`Item in ${fileName} is missing an id or slug field.`); @@ -43,8 +73,8 @@ export function file(fileName: string): Loader { const data = await parseData({ id, data: rawItem, filePath }); store.set({ id, data, filePath: normalizedFilePath }); } - } else if (typeof json === 'object') { - const entries = Object.entries>(json); + } else if (typeof data === 'object') { + const entries = Object.entries>(data); logger.debug(`Found object with ${entries.length} entries in ${fileName}`); store.clear(); for (const [id, rawItem] of entries) { From 366eba18c31f865e9995e5378183f9e1120003a3 Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Fri, 20 Sep 2024 15:30:43 -0400 Subject: [PATCH 02/12] add tests --- .../test/fixtures/content-layer/package.json | 3 +- .../content-layer/src/content/config.ts | 13 ++- .../fixtures/content-layer/src/data/fish.yaml | 41 +++++++++ .../content-layer/src/data/music.toml | 89 +++++++++++++++++++ .../src/pages/artists/[slug].astro | 32 +++++++ .../content-layer/src/pages/fish/[slug].astro | 34 +++++++ .../src/pages/songs/[slug].astro | 47 ++++++++++ pnpm-lock.yaml | 8 ++ 8 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 packages/astro/test/fixtures/content-layer/src/data/fish.yaml create mode 100644 packages/astro/test/fixtures/content-layer/src/data/music.toml create mode 100644 packages/astro/test/fixtures/content-layer/src/pages/artists/[slug].astro create mode 100644 packages/astro/test/fixtures/content-layer/src/pages/fish/[slug].astro create mode 100644 packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro diff --git a/packages/astro/test/fixtures/content-layer/package.json b/packages/astro/test/fixtures/content-layer/package.json index fc73ce6f7ac7..166155da13a8 100644 --- a/packages/astro/test/fixtures/content-layer/package.json +++ b/packages/astro/test/fixtures/content-layer/package.json @@ -3,7 +3,8 @@ "version": "0.0.0", "private": true, "dependencies": { + "@astrojs/mdx": "workspace:*", "astro": "workspace:*", - "@astrojs/mdx": "workspace:*" + "toml": "^3.0.0" } } diff --git a/packages/astro/test/fixtures/content-layer/src/content/config.ts b/packages/astro/test/fixtures/content-layer/src/content/config.ts index 402bad7fc5f5..34c831f238ce 100644 --- a/packages/astro/test/fixtures/content-layer/src/content/config.ts +++ b/packages/astro/test/fixtures/content-layer/src/content/config.ts @@ -1,6 +1,7 @@ import { defineCollection, z, reference } from 'astro:content'; import { file, glob } from 'astro/loaders'; import { loader } from '../loaders/post-loader.js'; +import { parse as parseToml } from 'toml'; const blog = defineCollection({ loader: loader({ url: 'https://jsonplaceholder.typicode.com/posts' }), @@ -118,6 +119,16 @@ const cats = defineCollection({ }), }); +const fish = defineCollection({ + loader: file('src/data/fish.yaml'), + schema: z.object({ + id: z.string(), + name: z.string(), + breed: z.string(), + age: z.number(), + }), +}); + // Absolute paths should also work const absoluteRoot = new URL('../../content/space', import.meta.url); @@ -176,7 +187,7 @@ const images = defineCollection({ const increment = defineCollection({ loader: { name: 'increment-loader', - load: async ({ store, refreshContextData }) => { + load: async ({ store }) => { const entry = store.get<{ lastValue: number }>('value'); const lastValue = entry?.data.lastValue ?? 0; store.set({ diff --git a/packages/astro/test/fixtures/content-layer/src/data/fish.yaml b/packages/astro/test/fixtures/content-layer/src/data/fish.yaml new file mode 100644 index 000000000000..5a49d9983c03 --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/data/fish.yaml @@ -0,0 +1,41 @@ +- id: "bubbles" + name: "Bubbles" + breed: "Goldfish" + age: 2 +- id: "finn" + name: "Finn" + breed: "Betta" + age: 1 +- id: "shadow" + name: "Shadow" + breed: "Catfish" + age: 3 +- id: "spark" + name: "Spark" + breed: "Tetra" + age: 1 +- id: "splash" + name: "Splash" + breed: "Guppy" + age: 2 +- id: "nemo" + name: "Nemo" + breed: "Clownfish" + age: 3 +- id: "angel-fish" + name: "Angel Fish" + breed: "Angelfish" + age: 4 +- id: "gold-stripe" + name: "Gold Stripe" + breed: "Molly" + age: 1 +- id: "blue-tail" + name: "Blue Tail" + breed: "Swordtail" + age: 2 +- id: "bubble-buddy" + name: "Bubble Buddy" + breed: "Betta" + age: 3 + diff --git a/packages/astro/test/fixtures/content-layer/src/data/music.toml b/packages/astro/test/fixtures/content-layer/src/data/music.toml new file mode 100644 index 000000000000..89e15c9bbbf5 --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/data/music.toml @@ -0,0 +1,89 @@ +[[artists]] +id = "kendrick-lamar" +name = "Kendrick Lamar" +genre = ["Hip-Hop", "Rap"] + +[[artists]] +id = "mac-miller" +name = "Mac Miller" +genre = ["Hip-Hop", "Rap"] + +[[artists]] +id = "jid" +name = "JID" +genre = ["Hip-Hop", "Rap"] + +[[artists]] +id = "yasiin-bey" +name = "Yasiin Bey" +genre = ["Hip-Hop", "Rap"] + +[[artists]] +id = "kanye-west" +name = "Kanye West" +genre = ["Hip-Hop", "Rap"] + +[[artists]] +id = "jay-z" +name = "JAY-Z" +genre = ["Hip-Hop", "Rap"] + +[[artists]] +id = "j-ivy" +name = "J. Ivy" +genre = ["Spoken Word", "Rap"] + +[[artists]] +id = "frank-ocean" +name = "Frank Ocean" +genre = ["R&B", "Hip-Hop"] + +[[artists]] +id = "the-dream" +name = "The-Dream" +genre = ["R&B", "Hip-Hop"] + +[[artists]] +id = "baby-keem" +name = "Baby Keem" +genre = ["Hip-Hop", "Rap"] + +[[songs]] +id = "crown" +name = "Crown" +artists = ["kendrick-lamar"] + +[[songs]] +id = "nikes-on-my-feet" +name = "Nikes on My Feet" +artists = ["mac-miller"] + +[[songs]] +id = "stars" +name = "Stars" +artists = ["jid", "yasiin-bey"] + +[[songs]] +id = "never-let-me-down" +name = "Never Let Me Down" +artists = ["kanye-west", "jay-z", "j-ivy"] + +[[songs]] +id = "no-church-in-the-wild" +name = "No Church In The Wild" +artists = ["jay-z", "kanye-west", "frank-ocean", "the-dream"] + +[[songs]] +id = "family-ties" +name = "family ties" +artists = ["kendrick-lamar", "baby-keem"] + +[[songs]] +id = "somebody" +name = "Somebody" +artists = ["jid"] + +[[songs]] +id = "honest" +name = "HONEST" +artists = ["baby-keem"] diff --git a/packages/astro/test/fixtures/content-layer/src/pages/artists/[slug].astro b/packages/astro/test/fixtures/content-layer/src/pages/artists/[slug].astro new file mode 100644 index 000000000000..560dc873a2fe --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/pages/artists/[slug].astro @@ -0,0 +1,32 @@ +--- +import type { GetStaticPaths } from 'astro'; +import { getCollection } from 'astro:content'; + +export const getStaticPaths = (async () => { + const collection = await getCollection('artists'); + if (!collection) return []; + return collection.map((artist) => ({ + params: { + slug: artist.id, + }, + props: { + artist: artist.data, + }, + })); +}) satisfies GetStaticPaths; + +interface Props { + artist: { + id: string; + name: string; + genre: string[]; + }; +} + +const { artist } = Astro.props; +--- + +

{artist.name}

+
    +
  • Genres: {artist.genre.join(', ')}
    • +
diff --git a/packages/astro/test/fixtures/content-layer/src/pages/fish/[slug].astro b/packages/astro/test/fixtures/content-layer/src/pages/fish/[slug].astro new file mode 100644 index 000000000000..4a2759622081 --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/pages/fish/[slug].astro @@ -0,0 +1,34 @@ +--- +import type { GetStaticPaths } from 'astro'; +import { getCollection } from 'astro:content'; + +export const getStaticPaths = (async () => { + const collection = await getCollection('fish'); + if (!collection) return []; + return collection.map((fish) => ({ + params: { + slug: fish.id, + }, + props: { + fish: fish.data, + }, + })); +}) satisfies GetStaticPaths; + +interface Props { + fish: { + id: string; + name: string; + breed: string; + age: number; + }; +} + +const { fish } = Astro.props; +--- + +

{fish.name}

+
    +
  • Breed: {fish.breed}
    • +
  • Age: {fish.age}
    • +
diff --git a/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro b/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro new file mode 100644 index 000000000000..c91581c6556d --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro @@ -0,0 +1,47 @@ +--- +import type { GetStaticPaths } from 'astro'; +import { getCollection, getEntry, type CollectionEntry } from 'astro:content'; + +export const getStaticPaths = (async () => { + const collection = await getCollection('songs'); + if (!collection) return []; + return Promise.all( + collection.map(async (song) => ({ + params: { + slug: song.id, + }, + props: { + song: song.data, + artists: await Promise.all( + song.data.artists.map(async ({ id }) => (await getEntry('artists', id)).data), + ), + }, + })), + ); +}) satisfies GetStaticPaths; + +interface Props { + song: { + id: string; + name: string; + }; + artists: CollectionEntry<'artists'>[]; +} + +const { song, artists } = Astro.props; + +console.log(artists); +--- + +

{song.name}

+

Authors:

+
    + { + artists.map((artist) => ( +
  • + {artist.name} +

    Genres: {artist.genre.join(', ')}

    +
    • + )) + } +
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 39a5ebb0b5d5..fc0a04a966c9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2722,6 +2722,9 @@ importers: astro: specifier: workspace:* version: link:../../.. + toml: + specifier: ^3.0.0 + version: 3.0.0 packages/astro/test/fixtures/content-layer-markdoc: dependencies: @@ -10411,6 +10414,9 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + toml@3.0.0: + resolution: {integrity: sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==} + totalist@3.0.1: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} @@ -16656,6 +16662,8 @@ snapshots: toidentifier@1.0.1: {} + toml@3.0.0: {} + totalist@3.0.1: {} tough-cookie@4.1.3: From 8c9cbb30a5e18ecb3547e4498ca33dbf470d0c55 Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Fri, 20 Sep 2024 15:35:10 -0400 Subject: [PATCH 03/12] lint/format --- packages/astro/src/content/loaders/file.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/astro/src/content/loaders/file.ts b/packages/astro/src/content/loaders/file.ts index 2044e36ac018..d6dee1d33991 100644 --- a/packages/astro/src/content/loaders/file.ts +++ b/packages/astro/src/content/loaders/file.ts @@ -1,8 +1,8 @@ import { promises as fs, existsSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; +import yaml from 'js-yaml'; import { posixRelative } from '../utils.js'; import type { Loader, LoaderContext } from './types.js'; -import yaml from 'js-yaml'; export interface FileOptions { /** @@ -70,16 +70,16 @@ export function file(fileName: string, options?: FileOptions): Loader { logger.error(`Item in ${fileName} is missing an id or slug field.`); continue; } - const data = await parseData({ id, data: rawItem, filePath }); - store.set({ id, data, filePath: normalizedFilePath }); + const parsedData = await parseData({ id, data: rawItem, filePath }); + store.set({ id, data: parsedData, filePath: normalizedFilePath }); } } else if (typeof data === 'object') { const entries = Object.entries>(data); logger.debug(`Found object with ${entries.length} entries in ${fileName}`); store.clear(); for (const [id, rawItem] of entries) { - const data = await parseData({ id, data: rawItem, filePath }); - store.set({ id, data, filePath: normalizedFilePath }); + const parsedData = await parseData({ id, data: rawItem, filePath }); + store.set({ id, data: parsedData, filePath: normalizedFilePath }); } } else { logger.error(`Invalid data in ${fileName}. Must be an array or object.`); From 5dbc75492218712cad35bcc9b020d164c1a8259f Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Fri, 20 Sep 2024 15:36:27 -0400 Subject: [PATCH 04/12] changeset --- .changeset/lovely-pianos-breathe.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lovely-pianos-breathe.md diff --git a/.changeset/lovely-pianos-breathe.md b/.changeset/lovely-pianos-breathe.md new file mode 100644 index 000000000000..0a81675ef999 --- /dev/null +++ b/.changeset/lovely-pianos-breathe.md @@ -0,0 +1,5 @@ +--- +'astro': minor +--- + +feat: custom file formats in astro/loader/file.ts From daa98ff50bf5d6bf2d5773e125089072a5df80e3 Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Fri, 20 Sep 2024 15:55:36 -0400 Subject: [PATCH 05/12] nits --- packages/astro/src/content/loaders/file.ts | 3 +-- packages/astro/test/fixtures/content-layer/package.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/astro/src/content/loaders/file.ts b/packages/astro/src/content/loaders/file.ts index d6dee1d33991..2b387246e3f2 100644 --- a/packages/astro/src/content/loaders/file.ts +++ b/packages/astro/src/content/loaders/file.ts @@ -14,9 +14,8 @@ export interface FileOptions { /** * Loads entries from a JSON file. The file must contain an array of objects that contain unique `id` fields, or an object with string keys. - * @todo Add support for other file types, such as YAML, CSV etc. * @param fileName The path to the JSON file to load, relative to the content directory. - * @param options Additional options for + * @param options Additional options for the file loader */ export function file(fileName: string, options?: FileOptions): Loader { if (fileName.includes('*')) { diff --git a/packages/astro/test/fixtures/content-layer/package.json b/packages/astro/test/fixtures/content-layer/package.json index 166155da13a8..4057b1c35a64 100644 --- a/packages/astro/test/fixtures/content-layer/package.json +++ b/packages/astro/test/fixtures/content-layer/package.json @@ -3,8 +3,8 @@ "version": "0.0.0", "private": true, "dependencies": { - "@astrojs/mdx": "workspace:*", "astro": "workspace:*", + "@astrojs/mdx": "workspace:*", "toml": "^3.0.0" } } From b6a1b15d53fdfc07f6f7a5686f6fcd08423504dc Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Fri, 20 Sep 2024 16:19:33 -0400 Subject: [PATCH 06/12] finish tests --- packages/astro/test/content-layer.test.js | 18 +++++++++--------- .../src/pages/collections.json.js | 15 +++++++++++---- .../content-layer/src/pages/songs/[slug].astro | 2 -- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 5be3953867ed..b37c65507ed8 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -53,11 +53,11 @@ describe('Content Layer', () => { assert.equal(json.customLoader.length, 5); }); - it('Returns `file()` loader collection', async () => { - assert.ok(json.hasOwnProperty('fileLoader')); - assert.ok(Array.isArray(json.fileLoader)); + it('Returns json `file()` loader collection', async () => { + assert.ok(json.hasOwnProperty('jsonLoader')); + assert.ok(Array.isArray(json.jsonLoader)); - const ids = json.fileLoader.map((item) => item.data.id); + const ids = json.jsonLoader.map((item) => item.data.id); assert.deepEqual(ids, [ 'labrador-retriever', 'german-shepherd', @@ -276,10 +276,10 @@ describe('Content Layer', () => { }); it('Returns `file()` loader collection', async () => { - assert.ok(json.hasOwnProperty('fileLoader')); - assert.ok(Array.isArray(json.fileLoader)); + assert.ok(json.hasOwnProperty('jsonLoader')); + assert.ok(Array.isArray(json.jsonLoader)); - const ids = json.fileLoader.map((item) => item.data.id); + const ids = json.jsonLoader.map((item) => item.data.id); assert.deepEqual(ids, [ 'labrador-retriever', 'german-shepherd', @@ -348,7 +348,7 @@ describe('Content Layer', () => { it('updates collection when data file is changed', async () => { const rawJsonResponse = await fixture.fetch('/collections.json'); const initialJson = devalue.parse(await rawJsonResponse.text()); - assert.equal(initialJson.fileLoader[0].data.temperament.includes('Bouncy'), false); + assert.equal(initialJson.jsonLoader[0].data.temperament.includes('Bouncy'), false); await fixture.editFile('/src/data/dogs.json', (prev) => { const data = JSON.parse(prev); @@ -359,7 +359,7 @@ describe('Content Layer', () => { await fixture.onNextDataStoreChange(); const updatedJsonResponse = await fixture.fetch('/collections.json'); const updated = devalue.parse(await updatedJsonResponse.text()); - assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy')); + assert.ok(updated.jsonLoader[0].data.temperament.includes('Bouncy')); await fixture.resetAllFiles(); }); }); diff --git a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js index 761ff7dba6fa..fdb2f6172dc2 100644 --- a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js +++ b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js @@ -5,7 +5,7 @@ export async function GET() { const customLoader = await getCollection('blog', (entry) => { return entry.data.id < 6; }); - const fileLoader = await getCollection('dogs'); + const jsonLoader = await getCollection('dogs'); const dataEntry = await getEntry('dogs', 'beagle'); @@ -23,10 +23,15 @@ export async function GET() { const simpleLoaderObject = await getCollection('rodents') const probes = await getCollection('probes'); + + const yamlLoader = await getCollection('fish'); + + const tomlLoader = await getCollection('songs'); + return new Response( devalue.stringify({ customLoader, - fileLoader, + jsonLoader, dataEntry, simpleLoader, simpleLoaderObject, @@ -35,7 +40,9 @@ export async function GET() { referencedEntry, increment, images, - probes - }) + probes, + yamlLoader, + tomlLoader, + }), ); } diff --git a/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro b/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro index c91581c6556d..eea2985186fd 100644 --- a/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro +++ b/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro @@ -29,8 +29,6 @@ interface Props { } const { song, artists } = Astro.props; - -console.log(artists); ---

{song.name}

From a31abad5fae53b9b3e1296d749befa6d34157e4c Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Fri, 20 Sep 2024 16:44:02 -0400 Subject: [PATCH 07/12] add nested json test --- .../content-layer/src/content/config.ts | 14 ++++++++ .../content-layer/src/data/birds.json | 34 +++++++++++++++++++ .../src/pages/collections.json.js | 3 ++ 3 files changed, 51 insertions(+) create mode 100644 packages/astro/test/fixtures/content-layer/src/data/birds.json diff --git a/packages/astro/test/fixtures/content-layer/src/content/config.ts b/packages/astro/test/fixtures/content-layer/src/content/config.ts index 34c831f238ce..4b05f4e4dce8 100644 --- a/packages/astro/test/fixtures/content-layer/src/content/config.ts +++ b/packages/astro/test/fixtures/content-layer/src/content/config.ts @@ -129,6 +129,18 @@ const fish = defineCollection({ }), }); +const birds = defineCollection({ + loader: file('src/data/birds.json', { + parser: (text) => JSON.parse(text).birds, + }), + schema: z.object({ + id: z.string(), + name: z.string(), + breed: z.string(), + age: z.number(), + }), +}); + // Absolute paths should also work const absoluteRoot = new URL('../../content/space', import.meta.url); @@ -213,6 +225,8 @@ export const collections = { blog, dogs, cats, + fish, + birds, numbers, spacecraft, increment, diff --git a/packages/astro/test/fixtures/content-layer/src/data/birds.json b/packages/astro/test/fixtures/content-layer/src/data/birds.json new file mode 100644 index 000000000000..3e7d83795a12 --- /dev/null +++ b/packages/astro/test/fixtures/content-layer/src/data/birds.json @@ -0,0 +1,34 @@ +{ + "birds": [ + { + "id": "bluejay", + "name": "Blue Jay", + "breed": "Cyanocitta cristata", + "age": 3 + }, + { + "id": "robin", + "name": "Robin", + "breed": "Turdus migratorius", + "age": 2 + }, + { + "id": "sparrow", + "name": "Sparrow", + "breed": "Passer domesticus", + "age": 1 + }, + { + "id": "cardinal", + "name": "Cardinal", + "breed": "Cardinalis cardinalis", + "age": 4 + }, + { + "id": "goldfinch", + "name": "Goldfinch", + "breed": "Spinus tristis", + "age": 2 + } + ] +} diff --git a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js index fdb2f6172dc2..6bced27e45e3 100644 --- a/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js +++ b/packages/astro/test/fixtures/content-layer/src/pages/collections.json.js @@ -28,6 +28,8 @@ export async function GET() { const tomlLoader = await getCollection('songs'); + const nestedJsonLoader = await getCollection('birds'); + return new Response( devalue.stringify({ customLoader, @@ -43,6 +45,7 @@ export async function GET() { probes, yamlLoader, tomlLoader, + nestedJsonLoader, }), ); } From 40ae1eabc4352f1b5bfd86fa5ba8943c8abaea85 Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Mon, 23 Sep 2024 12:44:40 -0400 Subject: [PATCH 08/12] requested changes --- .changeset/lovely-pianos-breathe.md | 37 +++++++++++++++++++++- packages/astro/src/content/loaders/file.ts | 4 ++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.changeset/lovely-pianos-breathe.md b/.changeset/lovely-pianos-breathe.md index 0a81675ef999..93d5f56fc54d 100644 --- a/.changeset/lovely-pianos-breathe.md +++ b/.changeset/lovely-pianos-breathe.md @@ -2,4 +2,39 @@ 'astro': minor --- -feat: custom file formats in astro/loader/file.ts +Adds support for custom parsers to file loader + +For example, with a toml file of this format: +```toml +[[dogs]] +id = "..." +age = "..." + +[[dogs]] +id = "..." +age = "..." +``` +a content collection using this file could look like this +```typescript +import { defineCollection } from "astro:content" +import { file } from "astro/loaders" +import { parse as parseToml } from "toml" +const dogs = defineCollection({ + loader: file("src/data/dogs.toml", { parser: (text) => parseToml(text).dogs }), + schema: /* ... */ +}) +``` + +This also adds support for nested json documents. For example: +```json +{"dogs": [{}], "cats": [{}]} +``` +can be consumed using +```typescript +const dogs = defineCollection({ + loader: file("src/data/pets.json", { parser: (text) => JSON.parse(text).dogs }) +}) +const cats = defineCollection({ + loader: file("src/data/pets.json", { parser: (text) => JSON.parse(text).cats }) +}) +``` diff --git a/packages/astro/src/content/loaders/file.ts b/packages/astro/src/content/loaders/file.ts index 2b387246e3f2..ed1701e11108 100644 --- a/packages/astro/src/content/loaders/file.ts +++ b/packages/astro/src/content/loaders/file.ts @@ -9,7 +9,9 @@ export interface FileOptions { * the parsing function to use for this data * @default JSON.parse or yaml.load, depending on the extension of the file * */ - parser?: (text: string) => any; + parser?: ( + text: string, + ) => Record | Array<{ id: string } & Record>; } /** From c545e87ada53f056334d1ec8fdcef51ed703e536 Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Tue, 24 Sep 2024 14:41:53 -0400 Subject: [PATCH 09/12] update changeset with @sarah11918 suggestions --- .changeset/lovely-pianos-breathe.md | 42 ++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/.changeset/lovely-pianos-breathe.md b/.changeset/lovely-pianos-breathe.md index 93d5f56fc54d..39a70127dcad 100644 --- a/.changeset/lovely-pianos-breathe.md +++ b/.changeset/lovely-pianos-breathe.md @@ -2,9 +2,11 @@ 'astro': minor --- -Adds support for custom parsers to file loader +Adds a new optional `parser` property to the built-in `file()` loader for content collections to support additional file types such as `toml` and `csv`. -For example, with a toml file of this format: +The `file()` loader now accepts a second argument that defines a `parser` function. This allows you to specify a custom parser (e.g. `toml.parse` or `csv-parse`) to create a collection from a file's contents. The `file()` loader will automatically detect and parse JSON and YAML files (based on their file extension) with no need for a `parser`. + +This works with any type of custom file formats including `csv` and `toml`. The following example defines a content collection `dogs` using a `.toml` file. ```toml [[dogs]] id = "..." @@ -14,27 +16,53 @@ age = "..." id = "..." age = "..." ``` -a content collection using this file could look like this +After importing TOML's parser, you can load the `dogs` collection into your project by passing both a file path and `parser` to the `file()` loader. ```typescript import { defineCollection } from "astro:content" import { file } from "astro/loaders" import { parse as parseToml } from "toml" + const dogs = defineCollection({ loader: file("src/data/dogs.toml", { parser: (text) => parseToml(text).dogs }), schema: /* ... */ }) + +// it also works with CSVs! +import { parse as parseCsv } from "csv-parse/sync"; + +const cats = defineCollection({ + loader: file("src/data/cats.csv", { parser: (text) => parseCsv(text, { columns: true, skipEmptyLines: true })}) +}); ``` -This also adds support for nested json documents. For example: +The `parser` argument also allows you to load a single collection from a nested JSON document. For example, this JSON file contains multiple collections: ```json {"dogs": [{}], "cats": [{}]} ``` -can be consumed using + +You can seperate these collections by passing a custom `parser` to the `file()` loader like so: ```typescript const dogs = defineCollection({ loader: file("src/data/pets.json", { parser: (text) => JSON.parse(text).dogs }) -}) +}); const cats = defineCollection({ loader: file("src/data/pets.json", { parser: (text) => JSON.parse(text).cats }) -}) +}); +``` + +```typescript +``` + +And it continues to work with maps of `id` to `data` +```yaml +bubbles: + breed: "Goldfish" + age: 2 +finn: + breed: "Betta" + age: 1 +``` + +```typescript +const fish = defineCollection({ loader: file("src/data/fish.yaml"), schema: z.object({ breed: z.string(), age: z.number() }) }); ``` From 6052036e004cd54ac5bf84c45d035ce74f9cdec3 Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Tue, 24 Sep 2024 14:52:18 -0400 Subject: [PATCH 10/12] typos/formatting --- .changeset/lovely-pianos-breathe.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.changeset/lovely-pianos-breathe.md b/.changeset/lovely-pianos-breathe.md index 39a70127dcad..d0d2df7923a6 100644 --- a/.changeset/lovely-pianos-breathe.md +++ b/.changeset/lovely-pianos-breathe.md @@ -50,9 +50,6 @@ const cats = defineCollection({ }); ``` -```typescript -``` - And it continues to work with maps of `id` to `data` ```yaml bubbles: @@ -64,5 +61,8 @@ finn: ``` ```typescript -const fish = defineCollection({ loader: file("src/data/fish.yaml"), schema: z.object({ breed: z.string(), age: z.number() }) }); +const fish = defineCollection({ + loader: file("src/data/fish.yaml"), + schema: z.object({ breed: z.string(), age: z.number() }) +}); ``` From 5f82af12c62e500eddd7a847737a01962621d9c0 Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Tue, 24 Sep 2024 14:52:59 -0400 Subject: [PATCH 11/12] add map yaml test --- packages/astro/src/content/loaders/file.ts | 4 +- packages/astro/test/content-layer.test.js | 52 +++++++++++++++++++ .../content-layer/src/content/config.ts | 1 - .../fixtures/content-layer/src/data/fish.yaml | 21 ++++---- 4 files changed, 65 insertions(+), 13 deletions(-) diff --git a/packages/astro/src/content/loaders/file.ts b/packages/astro/src/content/loaders/file.ts index ed1701e11108..d109f95b6994 100644 --- a/packages/astro/src/content/loaders/file.ts +++ b/packages/astro/src/content/loaders/file.ts @@ -11,7 +11,7 @@ export interface FileOptions { * */ parser?: ( text: string, - ) => Record | Array<{ id: string } & Record>; + ) => Record> | Array>; } /** @@ -46,7 +46,7 @@ export function file(fileName: string, options?: FileOptions): Loader { } async function syncData(filePath: string, { logger, parseData, store, config }: LoaderContext) { - let data: Array>; + let data: Array> | Record>; try { const contents = await fs.readFile(filePath, 'utf-8'); diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index b37c65507ed8..77ebcf2f0da1 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -97,6 +97,58 @@ describe('Content Layer', () => { ); }); + it('Returns nested json `file()` loader collection', async () => { + assert.ok(json.hasOwnProperty('nestedJsonLoader')); + assert.ok(Array.isArray(json.nestedJsonLoader)); + + const ids = json.nestedJsonLoader.map((item) => item.data.id); + assert.deepEqual(ids, ['bluejay', 'robin', 'sparrow', 'cardinal', 'goldfinch']); + }); + + it('Returns yaml `file()` loader collection', async () => { + assert.ok(json.hasOwnProperty('yamlLoader')); + assert.ok(Array.isArray(json.yamlLoader)); + + const ids = json.yamlLoader.map((item) => item.data.id); + assert.deepEqual(ids, [ + 'bubbles', + 'finn', + 'shadow', + 'spark', + 'splash', + 'nemo', + 'angel-fish', + 'gold-stripe', + 'blue-tail', + 'bubble-buddy', + ]); + }); + + it('Returns toml `file()` loader collection', async () => { + assert.ok(json.hasOwnProperty('tomlLoader')); + assert.ok(Array.isArray(json.tomlLoader)); + + const ids = json.tomlLoader.map((item) => item.data.id); + assert.deepEqual(ids, [ + 'crown', + 'nikes-on-my-feet', + 'stars', + 'never-let-me-down', + 'no-church-in-the-wild', + 'family-ties', + 'somebody', + 'honest', + ]); + }); + + it('Returns nested json `file()` loader collection', async () => { + assert.ok(json.hasOwnProperty('nestedJsonLoader')); + assert.ok(Array.isArray(json.nestedJsonLoader)); + + const ids = json.nestedJsonLoader.map((item) => item.data.id); + assert.deepEqual(ids, ['bluejay', 'robin', 'sparrow', 'cardinal', 'goldfinch']); + }); + it('Returns data entry by id', async () => { assert.ok(json.hasOwnProperty('dataEntry')); assert.equal(json.dataEntry.filePath?.split(sep).join(posixSep), 'src/data/dogs.json'); diff --git a/packages/astro/test/fixtures/content-layer/src/content/config.ts b/packages/astro/test/fixtures/content-layer/src/content/config.ts index 4b05f4e4dce8..58f368349300 100644 --- a/packages/astro/test/fixtures/content-layer/src/content/config.ts +++ b/packages/astro/test/fixtures/content-layer/src/content/config.ts @@ -122,7 +122,6 @@ const cats = defineCollection({ const fish = defineCollection({ loader: file('src/data/fish.yaml'), schema: z.object({ - id: z.string(), name: z.string(), breed: z.string(), age: z.number(), diff --git a/packages/astro/test/fixtures/content-layer/src/data/fish.yaml b/packages/astro/test/fixtures/content-layer/src/data/fish.yaml index 5a49d9983c03..a9ac4e4352b4 100644 --- a/packages/astro/test/fixtures/content-layer/src/data/fish.yaml +++ b/packages/astro/test/fixtures/content-layer/src/data/fish.yaml @@ -1,40 +1,41 @@ -- id: "bubbles" +# map of ids to data +bubbles: name: "Bubbles" breed: "Goldfish" age: 2 -- id: "finn" +finn: name: "Finn" breed: "Betta" age: 1 -- id: "shadow" +shadow: name: "Shadow" breed: "Catfish" age: 3 -- id: "spark" +spark: name: "Spark" breed: "Tetra" age: 1 -- id: "splash" +splash: name: "Splash" breed: "Guppy" age: 2 -- id: "nemo" +nemo: name: "Nemo" breed: "Clownfish" age: 3 -- id: "angel-fish" +angel-fish: name: "Angel Fish" breed: "Angelfish" age: 4 -- id: "gold-stripe" +gold-stripe: name: "Gold Stripe" breed: "Molly" age: 1 -- id: "blue-tail" +blue-tail: name: "Blue Tail" breed: "Swordtail" age: 2 -- id: "bubble-buddy" +bubble-buddy: name: "Bubble Buddy" breed: "Betta" age: 3 From 1fb4e2d103886c6b7f3d65694ee839d9ceab3948 Mon Sep 17 00:00:00 2001 From: Rohan Godha Date: Wed, 25 Sep 2024 17:39:11 -0400 Subject: [PATCH 12/12] fix tests and rebase --- packages/astro/test/content-layer.test.js | 2 +- .../content-layer/src/content/config.ts | 22 ++++++++- .../src/pages/artists/[slug].astro | 32 ------------- .../content-layer/src/pages/fish/[slug].astro | 34 -------------- .../src/pages/songs/[slug].astro | 45 ------------------- 5 files changed, 22 insertions(+), 113 deletions(-) delete mode 100644 packages/astro/test/fixtures/content-layer/src/pages/artists/[slug].astro delete mode 100644 packages/astro/test/fixtures/content-layer/src/pages/fish/[slug].astro delete mode 100644 packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro diff --git a/packages/astro/test/content-layer.test.js b/packages/astro/test/content-layer.test.js index 77ebcf2f0da1..abf91f36345f 100644 --- a/packages/astro/test/content-layer.test.js +++ b/packages/astro/test/content-layer.test.js @@ -109,7 +109,7 @@ describe('Content Layer', () => { assert.ok(json.hasOwnProperty('yamlLoader')); assert.ok(Array.isArray(json.yamlLoader)); - const ids = json.yamlLoader.map((item) => item.data.id); + const ids = json.yamlLoader.map((item) => item.id); assert.deepEqual(ids, [ 'bubbles', 'finn', diff --git a/packages/astro/test/fixtures/content-layer/src/content/config.ts b/packages/astro/test/fixtures/content-layer/src/content/config.ts index 58f368349300..776c44f6811e 100644 --- a/packages/astro/test/fixtures/content-layer/src/content/config.ts +++ b/packages/astro/test/fixtures/content-layer/src/content/config.ts @@ -198,7 +198,7 @@ const images = defineCollection({ const increment = defineCollection({ loader: { name: 'increment-loader', - load: async ({ store }) => { + load: async ({ store, refreshContextData }) => { const entry = store.get<{ lastValue: number }>('value'); const lastValue = entry?.data.lastValue ?? 0; store.set({ @@ -220,6 +220,24 @@ const increment = defineCollection({ }, }); +const artists = defineCollection({ + loader: file('src/data/music.toml', { parser: (text) => parseToml(text).artists }), + schema: z.object({ + id: z.string(), + name: z.string(), + genre: z.string().array(), + }), +}); + +const songs = defineCollection({ + loader: file('src/data/music.toml', { parser: (text) => parseToml(text).songs }), + schema: z.object({ + id: z.string(), + name: z.string(), + artists: z.array(reference('artists')), + }), +}); + export const collections = { blog, dogs, @@ -230,6 +248,8 @@ export const collections = { spacecraft, increment, images, + artists, + songs, probes, rodents, }; diff --git a/packages/astro/test/fixtures/content-layer/src/pages/artists/[slug].astro b/packages/astro/test/fixtures/content-layer/src/pages/artists/[slug].astro deleted file mode 100644 index 560dc873a2fe..000000000000 --- a/packages/astro/test/fixtures/content-layer/src/pages/artists/[slug].astro +++ /dev/null @@ -1,32 +0,0 @@ ---- -import type { GetStaticPaths } from 'astro'; -import { getCollection } from 'astro:content'; - -export const getStaticPaths = (async () => { - const collection = await getCollection('artists'); - if (!collection) return []; - return collection.map((artist) => ({ - params: { - slug: artist.id, - }, - props: { - artist: artist.data, - }, - })); -}) satisfies GetStaticPaths; - -interface Props { - artist: { - id: string; - name: string; - genre: string[]; - }; -} - -const { artist } = Astro.props; ---- - -

{artist.name}

-
    -
  • Genres: {artist.genre.join(', ')}
    • -
diff --git a/packages/astro/test/fixtures/content-layer/src/pages/fish/[slug].astro b/packages/astro/test/fixtures/content-layer/src/pages/fish/[slug].astro deleted file mode 100644 index 4a2759622081..000000000000 --- a/packages/astro/test/fixtures/content-layer/src/pages/fish/[slug].astro +++ /dev/null @@ -1,34 +0,0 @@ ---- -import type { GetStaticPaths } from 'astro'; -import { getCollection } from 'astro:content'; - -export const getStaticPaths = (async () => { - const collection = await getCollection('fish'); - if (!collection) return []; - return collection.map((fish) => ({ - params: { - slug: fish.id, - }, - props: { - fish: fish.data, - }, - })); -}) satisfies GetStaticPaths; - -interface Props { - fish: { - id: string; - name: string; - breed: string; - age: number; - }; -} - -const { fish } = Astro.props; ---- - -

{fish.name}

-
    -
  • Breed: {fish.breed}
    • -
  • Age: {fish.age}
    • -
diff --git a/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro b/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro deleted file mode 100644 index eea2985186fd..000000000000 --- a/packages/astro/test/fixtures/content-layer/src/pages/songs/[slug].astro +++ /dev/null @@ -1,45 +0,0 @@ ---- -import type { GetStaticPaths } from 'astro'; -import { getCollection, getEntry, type CollectionEntry } from 'astro:content'; - -export const getStaticPaths = (async () => { - const collection = await getCollection('songs'); - if (!collection) return []; - return Promise.all( - collection.map(async (song) => ({ - params: { - slug: song.id, - }, - props: { - song: song.data, - artists: await Promise.all( - song.data.artists.map(async ({ id }) => (await getEntry('artists', id)).data), - ), - }, - })), - ); -}) satisfies GetStaticPaths; - -interface Props { - song: { - id: string; - name: string; - }; - artists: CollectionEntry<'artists'>[]; -} - -const { song, artists } = Astro.props; ---- - -

{song.name}

-

Authors:

-
    - { - artists.map((artist) => ( -
  • - {artist.name} -

    Genres: {artist.genre.join(', ')}

    -
    • - )) - } -