diff --git a/src/module.ts b/src/module.ts index e1bb82031..370f442ec 100644 --- a/src/module.ts +++ b/src/module.ts @@ -225,14 +225,7 @@ export default defineNuxtModule({ const { resolve } = createResolver(import.meta.url) const resolveRuntimeModule = (path: string) => resolveModule(path, { paths: resolve('./runtime') }) const contentContext: ContentContext = { - transformers: [ - // Register internal content plugins - resolveRuntimeModule('./server/transformers/markdown'), - resolveRuntimeModule('./server/transformers/yaml'), - resolveRuntimeModule('./server/transformers/json'), - resolveRuntimeModule('./server/transformers/csv'), - resolveRuntimeModule('./server/transformers/path-meta') - ], + transformers: [], ...options } @@ -313,7 +306,7 @@ export default defineNuxtModule({ nitroConfig.virtual['#content/virtual/transformers'] = [ // TODO: remove kit usage templateUtils.importSources(contentContext.transformers), - `const transformers = [${contentContext.transformers.map(templateUtils.importName).join(', ')}]`, + `export const transformers = [${contentContext.transformers.map(templateUtils.importName).join(', ')}]`, 'export const getParser = (ext) => transformers.find(p => ext.match(new RegExp(p.extensions.join("|"), "i")) && p.parse)', 'export const getTransformers = (ext) => transformers.filter(p => ext.match(new RegExp(p.extensions.join("|"), "i")) && p.transform)', 'export default () => {}' @@ -398,7 +391,9 @@ export default defineNuxtModule({ // Register highlighter if (options.highlight) { - contentContext.transformers.push(resolveRuntimeModule('./server/transformers/shiki')) + contentContext.transformers.push(resolveRuntimeModule('./transformers/shiki')) + // @ts-ignore + contentContext.highlight.apiURL = `/api/${options.base}/highlight` nuxt.hook('nitro:config', (nitroConfig) => { nitroConfig.handlers = nitroConfig.handlers || [] diff --git a/src/runtime/server/index.ts b/src/runtime/server/index.ts index bfc3d50b6..a5e0ad631 100644 --- a/src/runtime/server/index.ts +++ b/src/runtime/server/index.ts @@ -1,2 +1 @@ -export { serverQueryContent } from './storage' -export { parseContent } from './transformers' +export { serverQueryContent, parseContent } from './storage' diff --git a/src/runtime/server/navigation.ts b/src/runtime/server/navigation.ts index 31c897428..ee94da9aa 100644 --- a/src/runtime/server/navigation.ts +++ b/src/runtime/server/navigation.ts @@ -1,5 +1,5 @@ import { NavItem, ParsedContentMeta } from '../types' -import { generateTitle } from './transformers/path-meta' +import { generateTitle } from '../transformers/path-meta' import { useRuntimeConfig } from '#imports' type PrivateNavItem = NavItem & { path?: string } diff --git a/src/runtime/server/storage.ts b/src/runtime/server/storage.ts index 0d4363e44..d1b3d56bb 100644 --- a/src/runtime/server/storage.ts +++ b/src/runtime/server/storage.ts @@ -2,13 +2,30 @@ import { prefixStorage } from 'unstorage' import { joinURL, withLeadingSlash, withoutTrailingSlash } from 'ufo' import { hash as ohash } from 'ohash' import type { CompatibilityEvent } from 'h3' -import type { QueryBuilderParams, ParsedContent, QueryBuilder } from '../types' +import defu from 'defu' +import type { QueryBuilderParams, ParsedContent, QueryBuilder, ContentTransformer } from '../types' import { createQuery } from '../query/query' import { createPipelineFetcher } from '../query/match/pipeline' -import { parseContent } from './transformers' +import { transformContent } from '../transformers' +import type { ModuleOptions } from '../../module' import { getPreview, isPreview } from './preview' // eslint-disable-next-line import/named -import { useRuntimeConfig, useStorage } from '#imports' +import { useNitroApp, useRuntimeConfig, useStorage } from '#imports' +import { transformers as customTransformers } from '#content/virtual/transformers' + +interface ParseContentOptions { + csv?: ModuleOptions['csv'] + yaml?: ModuleOptions['yaml'] + highlight?: ModuleOptions['highlight'] + markdown?: ModuleOptions['markdown'] + transformers?: ContentTransformer[] + pathMeta?: { + locales?: ModuleOptions['locales'] + defaultLocale?: ModuleOptions['defaultLocale'] + } + // Allow passing options for custom transformers + [key: string]: any +} export const sourceStorage = prefixStorage(useStorage(), 'content:source') export const cacheStorage = prefixStorage(useStorage(), 'cache:content') @@ -127,13 +144,45 @@ export const getContent = async (event: CompatibilityEvent, id: string): Promise return { _id: contentId, body: null } } - const parsed = await parseContent(contentId, body as string) + const parsed = await parseContent(contentId, body as string) as ParsedContent await cacheParsedStorage.setItem(id, { parsed, hash }).catch(() => {}) return parsed } +/** + * Parse content file using registered plugins + */ +export async function parseContent (id: string, content: string, opts: ParseContentOptions = {}) { + const nitroApp = useNitroApp() + const options = defu( + opts, + { + markdown: contentConfig.markdown, + csv: contentConfig.csv, + yaml: contentConfig.yaml, + highlight: contentConfig.highlight, + transformers: customTransformers, + pathMeta: { + defaultLocale: contentConfig.defaultLocale, + locales: contentConfig.locales + } + } + ) + + // Call hook before parsing the file + const file = { _id: id, body: content } + await nitroApp.hooks.callHook('content:file:beforeParse', file) + + const result = await transformContent(id, file.body, options) + + // Call hook after parsing the file + await nitroApp.hooks.callHook('content:file:afterParse', result) + + return result +} + export const createServerQueryFetch = (event: CompatibilityEvent, path?: string) => (query: QueryBuilder) => { if (path) { if (query.params().first) { diff --git a/src/runtime/server/transformers/csv.ts b/src/runtime/server/transformers/csv.ts deleted file mode 100644 index 949d887b2..000000000 --- a/src/runtime/server/transformers/csv.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { useRuntimeConfig } from '#imports' - -export default { - name: 'csv', - extensions: ['.csv'], - parse: async (_id, content) => { - const config = { ...useRuntimeConfig().content?.csv || {} } - - const csvToJson: any = await import('csvtojson').then(m => m.default || m) - - const parsed = await csvToJson({ output: 'json', ...config }) - .fromString(content) - - return { - _id, - _type: 'csv', - body: parsed - } - } -} diff --git a/src/runtime/server/transformers/index.ts b/src/runtime/server/transformers/index.ts deleted file mode 100644 index c4fbbd2fc..000000000 --- a/src/runtime/server/transformers/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { extname } from 'pathe' -import type { ParsedContent, ContentTransformer } from '../../types' -import { getParser, getTransformers } from '#content/virtual/transformers' -// eslint-disable-next-line import/named -import { useNitroApp } from '#imports' -/** - * Parse content file using registered plugins - */ -export async function parseContent (id: string, content: string) { - const nitroApp = useNitroApp() - - // Call hook before parsing the file - const file = { _id: id, body: content } - await nitroApp.hooks.callHook('content:file:beforeParse', file) - - const ext = extname(id) - const plugin: ContentTransformer = getParser(ext) - if (!plugin) { - // eslint-disable-next-line no-console - console.warn(`${ext} files are not supported, "${id}" falling back to raw content`) - return file - } - - const parsed: ParsedContent = await plugin.parse!(file._id, file.body) - - const transformers = getTransformers(ext) - const result = await transformers.reduce(async (prev, cur) => { - const next = (await prev) || parsed - - return cur.transform!(next) - }, Promise.resolve(parsed)) - - // Call hook after parsing the file - await nitroApp.hooks.callHook('content:file:afterParse', result) - - return result -} diff --git a/src/runtime/transformers/csv.ts b/src/runtime/transformers/csv.ts new file mode 100644 index 000000000..3ddd3d45c --- /dev/null +++ b/src/runtime/transformers/csv.ts @@ -0,0 +1,19 @@ +import { ParsedContent } from '../types' +import { defineTransformer } from './utils' + +export default defineTransformer({ + name: 'csv', + extensions: ['.csv'], + parse: async (_id, content, options = {}) => { + const csvToJson: any = await import('csvtojson').then(m => m.default || m) + + const parsed = await csvToJson({ output: 'json', ...options }) + .fromString(content) + + return { + _id, + _type: 'csv', + body: parsed + } + } +}) diff --git a/src/runtime/transformers/index.ts b/src/runtime/transformers/index.ts new file mode 100644 index 000000000..0991d2f4c --- /dev/null +++ b/src/runtime/transformers/index.ts @@ -0,0 +1,64 @@ +import { extname } from 'pathe' +import { camelCase } from 'scule' +import type { ContentTransformer, TransformContentOptions } from '../types' +import csv from './csv' +import markdown from './markdown' +import yaml from './yaml' +import pathMeta from './path-meta' +import json from './json' +import highlight from './shiki' + +const TRANSFORMERS = [ + csv, + markdown, + json, + yaml, + pathMeta, + highlight +] + +function getParser (ext, additionalTransformers: ContentTransformer[] = []): ContentTransformer { + let parser = additionalTransformers.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse) + if (!parser) { + parser = TRANSFORMERS.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse) + } + + return parser +} + +function getTransformers (ext, additionalTransformers: ContentTransformer[] = []) { + return [ + ...additionalTransformers.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform), + ...TRANSFORMERS.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform) + ] +} + +/** + * Parse content file using registered plugins + */ +export async function transformContent (id, content, options: TransformContentOptions = {}) { + const { transformers = [] } = options + // Call hook before parsing the file + const file = { _id: id, body: content } + + const ext = extname(id) + const parser: ContentTransformer = getParser(ext, transformers) + if (!parser) { + // eslint-disable-next-line no-console + console.warn(`${ext} files are not supported, "${id}" falling back to raw content`) + return file + } + + const parserOptions = options[camelCase(parser.name)] || {} + const parsed = await parser.parse!(file._id, file.body, parserOptions) + + const matchedTransformers = getTransformers(ext, transformers) + const result = await matchedTransformers.reduce(async (prev, cur) => { + const next = (await prev) || parsed + + const transformOptions = options[camelCase(cur.name)] || {} + return cur.transform!(next, transformOptions) + }, Promise.resolve(parsed)) + + return result +} diff --git a/src/runtime/server/transformers/json.ts b/src/runtime/transformers/json.ts similarity index 80% rename from src/runtime/server/transformers/json.ts rename to src/runtime/transformers/json.ts index fd5139630..677eed2eb 100644 --- a/src/runtime/server/transformers/json.ts +++ b/src/runtime/transformers/json.ts @@ -1,10 +1,12 @@ import destr from 'destr' +import { ParsedContent } from '../types' +import { defineTransformer } from './utils' -export default { +export default defineTransformer({ name: 'Json', extensions: ['.json', '.json5'], parse: async (_id, content) => { - let parsed = content + let parsed if (typeof content === 'string') { if (_id.endsWith('json5')) { @@ -24,10 +26,10 @@ export default { } } - return { + return { ...parsed, _id, _type: 'json' } } -} +}) diff --git a/src/runtime/server/transformers/markdown.ts b/src/runtime/transformers/markdown.ts similarity index 68% rename from src/runtime/server/transformers/markdown.ts rename to src/runtime/transformers/markdown.ts index a9103025b..f3342832e 100644 --- a/src/runtime/server/transformers/markdown.ts +++ b/src/runtime/transformers/markdown.ts @@ -1,13 +1,13 @@ -import { parse } from '../../markdown-parser' -import type { MarkdownOptions, MarkdownPlugin } from '../../types' -import { MarkdownParsedContent } from '../../types' -import { useRuntimeConfig } from '#imports' +import { parse } from '../markdown-parser' +import type { MarkdownOptions, MarkdownPlugin } from '../types' +import { MarkdownParsedContent } from '../types' +import { defineTransformer } from './utils' -export default { +export default defineTransformer({ name: 'markdown', extensions: ['.md'], - parse: async (_id, content) => { - const config: MarkdownOptions = { ...useRuntimeConfig().content?.markdown || {} } + parse: async (_id, content, options = {}) => { + const config = { ...options } as MarkdownOptions config.rehypePlugins = await importPlugins(config.rehypePlugins) config.remarkPlugins = await importPlugins(config.remarkPlugins) @@ -20,7 +20,7 @@ export default { _id } } -} +}) async function importPlugins (plugins: Record = {}) { const resolvedPlugins = {} diff --git a/src/runtime/server/transformers/path-meta.ts b/src/runtime/transformers/path-meta.ts similarity index 88% rename from src/runtime/server/transformers/path-meta.ts rename to src/runtime/transformers/path-meta.ts index ea73f6d74..a01baa0dd 100644 --- a/src/runtime/server/transformers/path-meta.ts +++ b/src/runtime/transformers/path-meta.ts @@ -1,8 +1,8 @@ import { pascalCase } from 'scule' import slugify from 'slugify' import { withoutTrailingSlash, withLeadingSlash } from 'ufo' -import { ParsedContentMeta } from '../../types' -import { useRuntimeConfig } from '#imports' +import { ParsedContent } from '../types' +import { defineTransformer } from './utils' const SEMVER_REGEX = /^(\d+)(\.\d+)*(\.x)?$/ @@ -13,7 +13,7 @@ const describeId = (_id: string) => { parts[parts.length - 1] = filename const _path = parts.join('/') - return > { + return { _source, _path, _extension, @@ -21,11 +21,11 @@ const describeId = (_id: string) => { } } -export default { +export default defineTransformer({ name: 'path-meta', extensions: ['.*'], - transform (content) { - const { locales, defaultLocale } = useRuntimeConfig().content || {} + transform (content, options: any = {}) { + const { locales = [], defaultLocale = 'en' } = options const { _source, _file, _path, _extension } = describeId(content._id) const parts = _path.split('/') @@ -34,7 +34,7 @@ export default { const filePath = parts.join('/') - return { + return { _path: generatePath(filePath), _draft: isDraft(filePath), _partial: isPartial(filePath), @@ -47,7 +47,7 @@ export default { _extension } } -} +}) /** * When file name ends with `.draft` then it will mark as draft. diff --git a/src/runtime/server/transformers/shiki.ts b/src/runtime/transformers/shiki.ts similarity index 85% rename from src/runtime/server/transformers/shiki.ts rename to src/runtime/transformers/shiki.ts index 0fb2417d1..cbb246679 100644 --- a/src/runtime/server/transformers/shiki.ts +++ b/src/runtime/transformers/shiki.ts @@ -1,17 +1,10 @@ import { visit } from 'unist-util-visit' -import { withBase } from 'ufo' -import { useRuntimeConfig } from '#imports' +import { defineTransformer } from './utils' -const highlightConfig = useRuntimeConfig().content.highlight - -const withContentBase = (url: string) => { - return withBase(url, `/api/${useRuntimeConfig().public.content.base}`) -} - -export default { - name: 'markdown', +export default defineTransformer({ + name: 'highlight', extensions: ['.md'], - transform: async (content) => { + transform: async (content, options = {}) => { const tokenColors: Record = {} const codeBlocks: any[] = [] const inlineCodes: any = [] @@ -59,12 +52,12 @@ export default { const code = node.children[0].value // Fetch highlighted tokens - const lines = await $fetch(withContentBase('highlight'), { + const lines = await $fetch(options.apiURL, { method: 'POST', body: { code, lang: node.props.lang || node.props.language, - theme: highlightConfig.theme + theme: options.theme } }) @@ -84,12 +77,12 @@ export default { const { code, language: lang, highlights = [] } = node.props // Fetch highlighted tokens - const lines = await $fetch(withContentBase('highlight'), { + const lines = await $fetch(options.apiURL, { method: 'POST', body: { code, lang, - theme: highlightConfig.theme + theme: options.theme } }) @@ -130,4 +123,4 @@ export default { } } } -} +}) diff --git a/src/runtime/transformers/utils.ts b/src/runtime/transformers/utils.ts new file mode 100644 index 000000000..169348a70 --- /dev/null +++ b/src/runtime/transformers/utils.ts @@ -0,0 +1,5 @@ +import { ContentTransformer } from '../types' + +export const defineTransformer = (transformer: ContentTransformer) => { + return transformer +} diff --git a/src/runtime/server/transformers/yaml.ts b/src/runtime/transformers/yaml.ts similarity index 77% rename from src/runtime/server/transformers/yaml.ts rename to src/runtime/transformers/yaml.ts index b7c875316..70af82bec 100644 --- a/src/runtime/server/transformers/yaml.ts +++ b/src/runtime/transformers/yaml.ts @@ -1,6 +1,8 @@ import { parseFrontMatter } from 'remark-mdc' +import { ParsedContent } from '../types' +import { defineTransformer } from './utils' -export default { +export default defineTransformer({ name: 'Yaml', extensions: ['.yml', '.yaml'], parse: async (_id, content) => { @@ -14,10 +16,10 @@ export default { parsed = { body: data } } - return { + return { ...parsed, _id, _type: 'yaml' } } -} +}) diff --git a/src/runtime/types.d.ts b/src/runtime/types.d.ts index f04b5a5a5..a8df4b855 100644 --- a/src/runtime/types.d.ts +++ b/src/runtime/types.d.ts @@ -140,8 +140,14 @@ export interface MarkdownParsedContent extends ParsedContent { export interface ContentTransformer { name: string extensions: string[] - parse?(id: string, content: string): Promise | ParsedContent - transform?: ((content: ParsedContent) => Promise) | ((content: ParsedContent) => ParsedContent) + parse?(id: string, content: string, options: any): Promise | ParsedContent + transform?(content: ParsedContent, options: any): Promise | ParsedContent +} + +export interface TransformContentOptions { + transformers?: ContentTransformer[] + + [key: string]: any } /** diff --git a/test/basic.test.ts b/test/basic.test.ts index 02a40cc8c..3e2c8b907 100644 --- a/test/basic.test.ts +++ b/test/basic.test.ts @@ -16,6 +16,7 @@ import { testModuleOptions } from './features/module-options' import { testContentQuery } from './features/content-query' import { testHighlighter } from './features/highlighter' import { testMarkdownRenderer } from './features/renderer-markdown' +import { testParserOptions } from './features/parser-options' const spyConsoleWarn = vi.spyOn(global.console, 'warn') @@ -141,4 +142,6 @@ describe('Basic usage', async () => { testModuleOptions() testHighlighter() + + testParserOptions() }) diff --git a/test/features/parser-options.ts b/test/features/parser-options.ts new file mode 100644 index 000000000..acf297433 --- /dev/null +++ b/test/features/parser-options.ts @@ -0,0 +1,42 @@ +import { describe, test, expect, assert } from 'vitest' +import { $fetch } from '@nuxt/test-utils' + +export const testParserOptions = () => { + describe('Parser Options', () => { + test('disable MDC syntax', async () => { + const parsed = await $fetch('/api/parse', { + method: 'POST', + body: { + id: 'content:index.md', + content: ':component', + options: { + markdown: { + mdc: false + } + } + } + }) + expect(parsed).toHaveProperty('_id') + assert(parsed.body.children[0].tag === 'p') + assert(parsed.body.children[0].children[0].type === 'text') + assert(parsed.body.children[0].children[0].value === ':component') + }) + + test('custom locale', async () => { + const parsed = await $fetch('/api/parse', { + method: 'POST', + body: { + id: 'content:index.md', + content: ':component', + options: { + pathMeta: { + defaultLocale: 'jp' + } + } + } + }) + expect(parsed).toHaveProperty('_id') + expect(parsed._locale).toBe('jp') + }) + }) +} diff --git a/test/fixtures/basic/server/api/parse.ts b/test/fixtures/basic/server/api/parse.ts index 73d9c96de..4665378cb 100644 --- a/test/fixtures/basic/server/api/parse.ts +++ b/test/fixtures/basic/server/api/parse.ts @@ -2,10 +2,10 @@ import { defineEventHandler, useBody } from 'h3' import { parseContent } from '#content/server' export default defineEventHandler(async (event) => { - const { id, content } = await useBody(event) + const { id, content, options } = await useBody(event) // @ts-ignore - const parsedContent = await parseContent(id, content) + const parsedContent = await parseContent(id, content, options) return parsedContent })