feat(gatsby-plugin-mdx): add ToC and excerpt support

packages/gatsby-plugin-mdx/package.json

@@ -53,18 +53,21 @@
     "change-case": "^4.1.2",
     "deepmerge": "^4.2.2",
     "fs-extra": "^10.1.0",
-    "gatsby-plugin-utils": "^3.10.0-next.1",
+    "gatsby-plugin-utils": "^3.12.0-next.1",
     "gray-matter": "^4.0.3",
     "mdast-util-mdx": "^2.0.0",
+    "mdast-util-toc": "^6.1.0",
     "mdast-util-to-markdown": "^1.3.0",
+    "rehype-infer-description-meta": "^1.0.1",
     "unified": "^10.1.2",
     "unist-util-visit": "^4.1.0",
     "vfile": "^5.3.2"
   },
   "devDependencies": {
     "@babel/cli": "^7.17.10",
     "@babel/core": "^7.18.2",
-    "babel-preset-gatsby-package": "^2.16.0-next.0",
+    "@types/mdast": "^3.0.10",
+    "babel-preset-gatsby-package": "^2.18.0-next.0",
     "cross-env": "^7.0.3",
     "typescript": "^4.7.2"
   },

packages/gatsby-plugin-mdx/src/compile-mdx.ts

@@ -1,27 +1,115 @@
+import deepmerge from "deepmerge"
+import type { NodePluginArgs } from "gatsby"
 import type { ProcessorOptions } from "@mdx-js/mdx"
-import type { IFileNode, IMdxNode } from "./types"
+import type { IFileNode, IMdxMetadata, IMdxNode } from "./types"
+
+import { enhanceMdxOptions, IMdxPluginOptions } from "./plugin-options"
+import { ERROR_CODES } from "./error-utils"
 
 // Compiles MDX into JS
-// This could be replaced by @mdx-js/mdx if MDX compile would
-// accept custom data passed to the unified pipeline via processor.data()
-export default async function compileMDX(
-  source: string,
+// Differences to original @mdx-js/loader:
+// * We pass the MDX node and a metadata object to the processor
+// * We inject the path to the original mdx file into the VFile which is used by the processor
+export async function compileMDX(
   mdxNode: IMdxNode,
   fileNode: IFileNode,
-  options: ProcessorOptions
-): Promise<string> {
-  const { createProcessor } = await import(`@mdx-js/mdx`)
-  const { VFile } = await import(`vfile`)
+  options: ProcessorOptions,
+  reporter: NodePluginArgs["reporter"]
+): Promise<{ processedMDX: string; metadata: IMdxMetadata } | null> {
+  try {
+    const { createProcessor } = await import(`@mdx-js/mdx`)
+    const { VFile } = await import(`vfile`)
+
+    const processor = createProcessor(options)
+
+    // Pass required custom data into the processor
+    const metadata: IMdxMetadata = {}
+    processor.data(`mdxNode`, mdxNode)
+    processor.data(`mdxMetadata`, metadata)
+
+    const result = await processor.process(
+      // Inject path to original file for remark plugins. See: https://github.com/gatsbyjs/gatsby/issues/26914
+      new VFile({ value: mdxNode.body, path: fileNode.absolutePath })
+    )
 
-  const processor = createProcessor(options)
+    // Clone metadata so ensure it won't be overridden by later processings
+    const clonedMetadata = Object.assign(
+      {},
+      processor.data(`mdxMetadata`) as IMdxMetadata
+    )
+    const processedMDX = result.toString()
 
-  // If we could pass this via MDX loader config, this whole custom loader might be obsolete.
-  processor.data(`mdxNode`, mdxNode)
+    return { processedMDX, metadata: clonedMetadata }
+  } catch (err) {
+    const errorMeta = [
+      mdxNode.title && `Title: ${mdxNode.title}`,
+      mdxNode.slug && `Slug: ${mdxNode.slug}`,
+      fileNode.relativePath && `Path: ${fileNode.relativePath}`,
+    ]
+      .filter(Boolean)
+      .join(`\n`)
 
-  const result = await processor.process(
-    // Inject path to original file for remark plugins. See: https://github.com/gatsbyjs/gatsby/issues/26914
-    new VFile({ value: source, path: fileNode.absolutePath })
+    reporter.panicOnBuild(
+      {
+        id: ERROR_CODES.MdxCompilation,
+        context: {
+          errorMeta,
+        },
+      },
+      err
+    )
+    return null
+  }
+}
+
+/**
+ * This helper function allows you to inject additional plugins and configuration into the MDX
+ * compilation pipeline. Very useful to create your own resolvers that return custom metadata.
+ * Internally used to generate the tables of contents and the excerpts.
+ */
+export const compileMDXWithCustomOptions = async ({
+  pluginOptions,
+  customOptions,
+  getNode,
+  getNodesByType,
+  pathPrefix,
+  reporter,
+  cache,
+  mdxNode,
+}: {
+  pluginOptions: IMdxPluginOptions
+  customOptions: Partial<IMdxPluginOptions>
+  getNode: NodePluginArgs["getNode"]
+  getNodesByType: NodePluginArgs["getNodesByType"]
+  pathPrefix: string
+  reporter: NodePluginArgs["reporter"]
+  cache: NodePluginArgs["cache"]
+  mdxNode: IMdxNode
+}): Promise<{
+  processedMDX: string
+  metadata: IMdxMetadata
+} | null> => {
+  const customPluginOptions = deepmerge(
+    Object.assign({}, pluginOptions),
+    customOptions
   )
 
-  return result.toString()
+  // Prepare MDX compile
+  const mdxOptions = await enhanceMdxOptions(customPluginOptions, {
+    getNode,
+    getNodesByType,
+    pathPrefix,
+    reporter,
+    cache,
+  })
+  if (!mdxNode.parent) {
+    return null
+  }
+  const fileNode = getNode(mdxNode.parent)
+  if (!fileNode) {
+    return null
+  }
+
+  // Compile MDX and extract metadata
+  return compileMDX(mdxNode, fileNode, mdxOptions, reporter)
 }

packages/gatsby-plugin-mdx/src/error-utils.ts

@@ -0,0 +1,13 @@
+export const ERROR_CODES = {
+  MdxCompilation: `10001`,
+}
+
+export const ERROR_MAP = {
+  [ERROR_CODES.MdxCompilation]: {
+    text: (context: { errorMeta: string }): string =>
+      `Failed to compile MDX. Information about the file:\n${context.errorMeta}`,
+    level: `ERROR`,
+    type: `PLUGIN`,
+    category: `USER`,
+  },
+}

packages/gatsby-plugin-mdx/src/gatsby-layout-loader.ts

@@ -4,7 +4,6 @@ import type { Options } from "mdast-util-to-markdown"
 import type { NodeMap } from "./types"
 import type { IMdxPluginOptions } from "./plugin-options"
 
-import grayMatter from "gray-matter"
 import { getOptions } from "loader-utils"
 
 export interface IGatsbyLayoutLoaderOptions {
@@ -13,7 +12,7 @@ export interface IGatsbyLayoutLoaderOptions {
 }
 
 // Wrap MDX content with Gatsby Layout component
-const gatsbyLayoutLoader: LoaderDefinition = async function (source) {
+const gatsbyLayoutLoader: LoaderDefinition = async function () {
   const { options, nodeMap }: IGatsbyLayoutLoaderOptions = getOptions(this)
 
   const res = nodeMap.get(this.resourcePath)
@@ -24,6 +23,12 @@ const gatsbyLayoutLoader: LoaderDefinition = async function (source) {
     )
   }
 
+  if (!res.mdxNode.body) {
+    throw new Error(
+      `MDX node is empty: ${JSON.stringify(res.mdxNode, null, 2)}`
+    )
+  }
+
   const { sourceInstanceName } = res.fileNode
 
   // Get the default layout for the file source instance group name,
@@ -34,20 +39,17 @@ const gatsbyLayoutLoader: LoaderDefinition = async function (source) {
 
   // No default layout set? Nothing to do here!
   if (!layoutPath) {
-    return source
+    return res.mdxNode.body
   }
 
-  // Remove frontmatter
-  const { content } = grayMatter(source)
-
   const { fromMarkdown } = await import(`mdast-util-from-markdown`)
   const { toMarkdown } = await import(`mdast-util-to-markdown`)
 
   const { mdxjs } = await import(`micromark-extension-mdxjs`)
   const { mdxFromMarkdown, mdxToMarkdown } = await import(`mdast-util-mdx`)
 
   // Parse MDX to AST
-  const tree = fromMarkdown(content, {
+  const tree = fromMarkdown(res.mdxNode.body, {
     extensions: [mdxjs()],
     mdastExtensions: [mdxFromMarkdown()],
   })
@@ -59,14 +61,13 @@ const gatsbyLayoutLoader: LoaderDefinition = async function (source) {
   )
 
   if (hasDefaultExport) {
-    return content
+    return res.mdxNode.body
   }
 
   tree.children.unshift({
     type: `mdxjsEsm` as `text`,
     value: `import GatsbyMDXLayout from "${layoutPath}"`,
   })
-
   tree.children.push({
     type: `mdxjsEsm` as `text`,
     value: `export default GatsbyMDXLayout`,

packages/gatsby-plugin-mdx/src/gatsby-mdx-loader.ts

@@ -1,22 +1,25 @@
 /* eslint-disable @babel/no-invalid-this */
-import type { LoaderDefinition } from "webpack"
 import type { ProcessorOptions } from "@mdx-js/mdx"
+import type { NodePluginArgs } from "gatsby"
+import type { LoaderDefinition } from "webpack"
 import type { NodeMap } from "./types"
 
 import { getOptions } from "loader-utils"
 
-import compileMDX from "./compile-mdx"
+import { compileMDX } from "./compile-mdx"
 
 export interface IGatsbyMDXLoaderOptions {
   options: ProcessorOptions
   nodeMap: NodeMap
+  reporter: NodePluginArgs["reporter"]
 }
 
 // Custom MDX Loader that injects the GraphQL MDX node into plugin data
 // This whole loaded could be replaced by @mdx-js/loader if MDX would
 // accept custom data passed to the unified pipeline via processor.data()
 const gatsbyMDXLoader: LoaderDefinition = async function (source) {
-  const { options, nodeMap }: IGatsbyMDXLoaderOptions = getOptions(this)
+  const { options, nodeMap, reporter }: IGatsbyMDXLoaderOptions =
+    getOptions(this)
   const res = nodeMap.get(this.resourcePath)
 
   if (!res) {
@@ -25,7 +28,19 @@ const gatsbyMDXLoader: LoaderDefinition = async function (source) {
 
   const { mdxNode, fileNode } = res
 
-  return compileMDX(source, mdxNode, fileNode, options)
+  const compileRes = await compileMDX(
+    // We want to work with the transformed source from our layout plugin
+    { ...mdxNode, body: source },
+    fileNode,
+    options,
+    reporter
+  )
+
+  if (compileRes && compileRes.processedMDX) {
+    return compileRes.processedMDX
+  }
+
+  return source
 }
 
 export default gatsbyMDXLoader