feat(module): integrate with unifiedjs VFile

src/module.ts

@@ -24,7 +24,7 @@ import type { ResolvedCollection } from './types/collection'
 import type { ModuleOptions, SqliteDatabaseConfig } from './types/module'
 import { getContentChecksum, localDatabase, logger, watchContents, chunks, watchComponents, watchConfig, startSocketServer } from './utils/dev'
 import { loadLayersConfig } from './utils/config'
-import { parseContent } from './utils/content'
+import { createParser } from './utils/content'
 import { installMDCModule } from './utils/mdc'
 import { findPreset } from './presets'
 import type { Manifest } from './types/manifest'
@@ -268,6 +268,8 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio
     if (!collection.source) {
       continue
     }
+    const parse = await createParser(collection, nuxt)
+
     for await (const source of collection.source) {
       if (source.prepare) {
         await source.prepare(nuxt)
@@ -285,8 +287,9 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio
         await Promise.all(chunk.map(async (key) => {
           key = key.substring(fixed.length)
           const keyInCollection = join(collection.name, source?.prefix || '', key)
+          const fullPath = join(cwd, fixed, key)
 
-          const content = await readFile(join(cwd, fixed, key), 'utf8')
+          const content = await readFile(fullPath, 'utf8')
           const checksum = getContentChecksum(configHash + collectionHash + content)
           const cache = databaseContents[keyInCollection]
 
@@ -297,7 +300,11 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio
           }
           else {
             parsedFilesCount += 1
-            parsedContent = await parseContent(keyInCollection, content, collection, nuxt)
+            parsedContent = await parse({
+              id: keyInCollection,
+              body: content,
+              path: fullPath,
+            })
             db.insertDevelopmentCache(keyInCollection, checksum, JSON.stringify(parsedContent))
           }
 

src/types/content.ts

@@ -2,6 +2,14 @@ import type { Highlighter, MDCRoot, Toc } from '@nuxtjs/mdc'
 
 export type { Toc, TocLink } from '@nuxtjs/mdc'
 
+export interface ContentFile extends Record<string, unknown> {
+  id: string
+  body: string
+  path: string
+  dirname?: string
+  extension?: string
+}
+
 export interface TransformedContent {
   id: string
   [key: string]: unknown
@@ -15,12 +23,12 @@ export interface TransformContentOptions {
 export type ContentTransformer = {
   name: string
   extensions: string[]
-  parse(id: string, content: string, options: Record<string, unknown>): Promise<TransformedContent> | TransformedContent
+  parse(file: ContentFile, options: Record<string, unknown>): Promise<TransformedContent> | TransformedContent
   transform?(content: TransformedContent, options: Record<string, unknown>): Promise<TransformedContent> | TransformedContent
 } | {
   name: string
   extensions: string[]
-  parse?(id: string, content: string, options: Record<string, unknown>): Promise<TransformedContent> | TransformedContent
+  parse?(file: ContentFile, options: Record<string, unknown>): Promise<TransformedContent> | TransformedContent
   transform(content: TransformedContent, options: Record<string, unknown>): Promise<TransformedContent> | TransformedContent
 }
 

src/utils/content/index.ts

@@ -1,3 +1,4 @@
+import { dirname } from 'node:path'
 import { createShikiHighlighter, rehypeHighlight } from '@nuxtjs/mdc/runtime'
 import { hash } from 'ohash'
 import type { Highlighter, MdcConfig, ModuleOptions as MDCModuleOptions } from '@nuxtjs/mdc'
@@ -8,6 +9,7 @@ import { visit } from 'unist-util-visit'
 import type { ResolvedCollection } from '../../types/collection'
 import type { ModuleOptions } from '../../types/module'
 import { transformContent } from './transformers'
+import type { ContentFile } from '~/src/types'
 
 let parserOptions = {
   mdcConfigs: [] as MdcConfig[],
@@ -97,15 +99,15 @@ async function _getHighlightPlugin(options: HighlighterOptions) {
   return highlightPlugin
 }
 
-export async function parseContent(key: string, content: string, collection: ResolvedCollection, nuxt?: Nuxt) {
+export async function createParser(collection: ResolvedCollection, nuxt?: Nuxt) {
   const mdcOptions = (nuxt?.options as unknown as { mdc: MDCModuleOptions })?.mdc || {}
   const { pathMeta = {}, markdown = {} } = (nuxt?.options as unknown as { content: ModuleOptions })?.content?.build || {}
 
   const rehypeHighlightPlugin = markdown.highlight !== false
     ? await getHighlightPluginInstance(defu(markdown.highlight as HighlighterOptions, mdcOptions.highlight, { compress: true }))
     : undefined
 
-  const parsedContent = await transformContent(key, content, {
+  const parserOptions = {
     pathMeta: pathMeta,
     markdown: {
       compress: true,
@@ -121,36 +123,43 @@ export async function parseContent(key: string, content: string, collection: Res
         ...mdcOptions?.remarkPlugins,
         ...markdown?.remarkPlugins,
       },
-      highlight: undefined,
+      highlight: undefined as HighlighterOptions,
     },
-  })
-
-  const { id: id, ...parsedContentFields } = parsedContent
-  const result = { id } as typeof collection.extendedSchema._type
-  const meta = {} as Record<string, unknown>
+  }
 
-  const collectionKeys = Object.keys(collection.extendedSchema.shape)
-  for (const key of Object.keys(parsedContentFields)) {
-    if (collectionKeys.includes(key)) {
-      result[key] = parsedContent[key]
+  return async function parse(file: ContentFile) {
+    if (file.path && !file.dirname) {
+      file.dirname = dirname(file.path)
     }
-    else {
-      meta[key] = parsedContent[key]
+
+    const parsedContent = await transformContent(file, parserOptions)
+    const { id: id, ...parsedContentFields } = parsedContent
+    const result = { id } as typeof collection.extendedSchema._type
+    const meta = {} as Record<string, unknown>
+
+    const collectionKeys = Object.keys(collection.extendedSchema.shape)
+    for (const key of Object.keys(parsedContentFields)) {
+      if (collectionKeys.includes(key)) {
+        result[key] = parsedContent[key]
+      }
+      else {
+        meta[key] = parsedContent[key]
+      }
     }
-  }
 
-  result.meta = meta
+    result.meta = meta
 
-  // Storing `content` into `rawbody` field
-  if (collectionKeys.includes('rawbody')) {
-    result.rawbody = result.rawbody ?? content
-  }
+    // Storing `content` into `rawbody` field
+    if (collectionKeys.includes('rawbody')) {
+      result.rawbody = result.rawbody ?? file.body
+    }
 
-  if (collectionKeys.includes('seo')) {
-    result.seo = result.seo || {}
-    result.seo.title = result.seo.title || result.title
-    result.seo.description = result.seo.description || result.description
-  }
+    if (collectionKeys.includes('seo')) {
+      result.seo = result.seo || {}
+      result.seo.title = result.seo.title || result.title
+      result.seo.description = result.seo.description || result.description
+    }
 
-  return result
+    return result
+  }
 }

src/utils/content/transformers/csv/index.ts

@@ -45,16 +45,16 @@ function csvParse(options) {
 export default defineTransformer({
   name: 'csv',
   extensions: ['.csv'],
-  parse: async (id, content, options = {}) => {
+  parse: async (file, options = {}) => {
     const stream = unified().use(csvParse, {
       delimiter: ',',
       json: true,
       ...options,
     })
-    const { result } = await stream.process(content)
+    const { result } = await stream.process(file.body)
 
     return {
-      id,
+      id: file.id,
       body: result,
     }
   },

src/utils/content/transformers/index.ts

@@ -1,6 +1,6 @@
 import { extname } from 'pathe'
 import { camelCase } from 'scule'
-import type { ContentTransformer, TransformedContent, TransformContentOptions } from '../../../types/content'
+import type { ContentTransformer, TransformContentOptions, ContentFile } from '../../../types/content'
 import csv from './csv'
 import markdown from './markdown'
 import yaml from './yaml'
@@ -34,20 +34,19 @@ function getTransformers(ext: string, additionalTransformers: ContentTransformer
 /**
  * Parse content file using registered plugins
  */
-export async function transformContent(id: string, content: string, options: TransformContentOptions = {}) {
+export async function transformContent(file: ContentFile, options: TransformContentOptions = {}) {
   const { transformers = [] } = options
   // Call hook before parsing the file
-  const file = { id: id, body: content } as TransformedContent
 
-  const ext = extname(id)
+  const ext = file.extension || extname(file.id)
   const parser = getParser(ext, transformers)
   if (!parser) {
-    console.warn(`${ext} files are not supported, "${id}" falling back to raw content`)
+    console.warn(`${ext} files are not supported, "${file.id}" falling back to raw content`)
     return file
   }
 
   const parserOptions = (options[camelCase(parser.name)] || {}) as Record<string, unknown>
-  const parsed = await parser.parse!(file.id, file.body as string, parserOptions)
+  const parsed = await parser.parse!(file, parserOptions)
 
   const matchedTransformers = getTransformers(ext, transformers)
   const result = await matchedTransformers.reduce(async (prev, cur) => {

src/utils/content/transformers/json.ts

@@ -4,14 +4,15 @@ import { defineTransformer } from './utils'
 export default defineTransformer({
   name: 'Json',
   extensions: ['.json'],
-  parse: async (id, content) => {
+  parse: async (file) => {
+    const { id, body } = file
     let parsed: Record<string, unknown>
 
-    if (typeof content === 'string') {
-      parsed = destr(content)
+    if (typeof body === 'string') {
+      parsed = destr(body)
     }
     else {
-      parsed = content
+      parsed = body
     }
 
     // Keep array contents under `body` key

src/utils/content/transformers/markdown.ts

@@ -12,7 +12,7 @@ import { generatePath } from './path-meta'
 export default defineTransformer({
   name: 'markdown',
   extensions: ['.md'],
-  parse: async (id, content, options: Partial<MarkdownOptions> = {}) => {
+  parse: async (file, options: Partial<MarkdownOptions> = {}) => {
     const config = { ...options } as MarkdownOptions
     config.rehypePlugins = await importPlugins(config.rehypePlugins)
     config.remarkPlugins = await importPlugins(config.remarkPlugins)
@@ -27,7 +27,7 @@ export default defineTransformer({
         }
       : undefined
 
-    const parsed = await parseMarkdown(content as string, {
+    const parsed = await parseMarkdown(file.body as string, {
       ...config,
       highlight,
       toc: config.toc,
@@ -36,6 +36,8 @@ export default defineTransformer({
         plugins: config.rehypePlugins,
         options: { handlers: { link } },
       },
+    }, {
+      fileOptions: file,
     })
 
     if ((options as { compress: boolean }).compress) {
@@ -46,7 +48,7 @@ export default defineTransformer({
           ...compressTree(parsed.body),
           toc: parsed.toc,
         },
-        id,
+        id: file.id,
       }
     }
 
@@ -57,7 +59,7 @@ export default defineTransformer({
         ...parsed.body,
         toc: parsed.toc,
       },
-      id,
+      id: file.id,
     }
   },
 })

src/utils/content/transformers/yaml.ts

@@ -4,8 +4,9 @@ import { defineTransformer } from './utils'
 export default defineTransformer({
   name: 'Yaml',
   extensions: ['.yml', '.yaml'],
-  parse: (id, content) => {
-    const { data } = parseFrontMatter(`---\n${content}\n---`)
+  parse: (file) => {
+    const { id, body } = file
+    const { data } = parseFrontMatter(`---\n${body}\n---`)
 
     // Keep array contents under `body` key
     let parsed = data

src/utils/dev.ts

@@ -15,7 +15,7 @@ import { withTrailingSlash } from 'ufo'
 import type { ModuleOptions, ResolvedCollection } from '../types'
 import type { Manifest } from '../types/manifest'
 import { generateCollectionInsert } from './collection'
-import { parseContent } from './content'
+import { createParser } from './content'
 import { moduleTemplates } from './templates'
 import { parseSourceBase } from './source'
 
@@ -86,6 +86,8 @@ export async function startSocketServer(nuxt: Nuxt, options: ModuleOptions, mani
 }
 
 export async function watchContents(nuxt: Nuxt, options: ModuleOptions, manifest: Manifest, socket: Awaited<ReturnType<typeof startSocketServer>>) {
+  const collectionParsers = {} as Record<string, Awaited<ReturnType<typeof createParser>>>
+
   const db = localDatabase(options._localDatabase!.filename)
   const collections = manifest.collections
 
@@ -113,8 +115,9 @@ export async function watchContents(nuxt: Nuxt, options: ModuleOptions, manifest
 
       const filePath = path.substring(fixed.length)
       const keyInCollection = join(collection.name, source?.prefix || '', filePath)
+      const fullPath = join(cwd, path)
 
-      const content = await readFile(join(cwd, path), 'utf8')
+      const content = await readFile(fullPath, 'utf8')
       const checksum = getContentChecksum(content)
       const localCache = db.fetchDevelopmentCacheForKey(keyInCollection)
 
@@ -124,7 +127,15 @@ export async function watchContents(nuxt: Nuxt, options: ModuleOptions, manifest
         return
       }
 
-      const parsedContent = await parseContent(keyInCollection, content, collection, nuxt)
+      if (!collectionParsers[collection.name]) {
+        collectionParsers[collection.name] = await createParser(collection, nuxt)
+      }
+      const parser = collectionParsers[collection.name]!
+      const parsedContent = await parser({
+        id: keyInCollection,
+        body: content,
+        path: fullPath,
+      })
 
       db.insertDevelopmentCache(keyInCollection, checksum, JSON.stringify(parsedContent))
 

test/unit/parseContent.csv.test.ts

@@ -1,7 +1,7 @@
 import { describe, test, expect, assert } from 'vitest'
 import csvToJson from 'csvtojson'
 import { z } from 'zod'
-import { parseContent } from '../../src/utils/content'
+import { parseContent } from '../utils/content'
 import { defineCollection } from '../../src/utils'
 import { resolveCollection } from '../../src/utils/collection'
 

test/unit/parseContent.json.test.ts

@@ -1,7 +1,7 @@
 import { describe, test, expect, assert } from 'vitest'
 
 import { z } from 'zod'
-import { parseContent } from '../../src/utils/content'
+import { parseContent } from '../utils/content'
 import { defineCollection } from '../../src/utils'
 import { resolveCollection } from '../../src/utils/collection'
 

test/unit/parseContent.md-highlighter.test.ts

@@ -3,7 +3,7 @@ import { z } from 'zod'
 import GithubLight from 'shiki/themes/github-light.mjs'
 import type { MDCElement } from '@nuxtjs/mdc'
 import type { Nuxt } from '@nuxt/schema'
-import { parseContent } from '../../src/utils/content'
+import { parseContent } from '../utils/content'
 import { defineCollection } from '../../src/utils'
 import { resolveCollection } from '../../src/utils/collection'
 import type { MarkdownRoot } from '../../src/types/content'

test/unit/parseContent.md.test.ts

@@ -2,7 +2,7 @@ import { describe, test, expect, assert } from 'vitest'
 import { z } from 'zod'
 import { visit } from 'unist-util-visit'
 import type { Nuxt } from '@nuxt/schema'
-import { parseContent } from '../../src/utils/content'
+import { parseContent } from '../utils/content'
 import { defineCollection } from '../../src/utils'
 import { resolveCollection } from '../../src/utils/collection'
 

test/unit/parseContent.path-meta.test.ts

@@ -1,6 +1,6 @@
 import { describe, test, expect, assert } from 'vitest'
 import { z } from 'zod'
-import { parseContent } from '../../src/utils/content'
+import { parseContent } from '../utils/content'
 import { defineCollection } from '../../src/utils'
 import { resolveCollection } from '../../src/utils/collection'
 

test/unit/parseContent.yaml.test.ts

@@ -1,6 +1,6 @@
 import { describe, test, expect, assert } from 'vitest'
 import { z } from 'zod'
-import { parseContent } from '../../src/utils/content'
+import { parseContent } from '../utils/content'
 import { defineCollection } from '../../src/utils'
 import { resolveCollection } from '../../src/utils/collection'