feat(gatsby-plugin-mdx): add ToC and excerpt support

packages/gatsby-plugin-mdx/package.json

@@ -56,14 +56,17 @@
     "gatsby-plugin-utils": "^3.10.0-next.1",
     "gray-matter": "^4.0.3",
     "mdast-util-mdx": "^2.0.0",
+    "mdast-util-toc": "^6.1.0",
     "mdast-util-to-markdown": "^1.3.0",
+    "rehype-infer-description-meta": "^1.0.1",
     "unified": "^10.1.2",
     "unist-util-visit": "^4.1.0",
     "vfile": "^5.3.2"
   },
   "devDependencies": {
     "@babel/cli": "^7.17.10",
     "@babel/core": "^7.18.2",
+    "@types/mdast": "^3.0.10",
     "babel-preset-gatsby-package": "^2.16.0-next.0",
     "cross-env": "^7.0.3",
     "typescript": "^4.7.2"

packages/gatsby-plugin-mdx/src/compile-mdx.ts

@@ -1,27 +1,108 @@
+import type { NodePluginArgs } from "gatsby"
 import type { ProcessorOptions } from "@mdx-js/mdx"
-import type { IFileNode, IMdxNode } from "./types"
+import type { IFileNode, IMdxMetadata, IMdxNode } from "./types"
+
+import deepmerge from "deepmerge"
+
+import { enhanceMdxOptions, IMdxPluginOptions } from "./plugin-options"
 
 // Compiles MDX into JS
 // This could be replaced by @mdx-js/mdx if MDX compile would
 // accept custom data passed to the unified pipeline via processor.data()
-export default async function compileMDX(
-  source: string,
+export async function compileMDX(
   mdxNode: IMdxNode,
   fileNode: IFileNode,
   options: ProcessorOptions
-): Promise<string> {
-  const { createProcessor } = await import(`@mdx-js/mdx`)
-  const { VFile } = await import(`vfile`)
+): Promise<{ processedMDX: string; metadata: IMdxMetadata }> {
+  try {
+    const { createProcessor } = await import(`@mdx-js/mdx`)
+    const { VFile } = await import(`vfile`)
+
+    const processor = createProcessor(options)
+
+    // If we could pass this via MDX loader config, this whole custom loader might be obsolete.
+    const metadata: IMdxMetadata = {}
+    processor.data(`mdxNode`, mdxNode)
+    processor.data(`mdxMetadata`, metadata)
 
-  const processor = createProcessor(options)
+    const result = await processor.process(
+      // Inject path to original file for remark plugins. See: https://github.com/gatsbyjs/gatsby/issues/26914
+      new VFile({ value: mdxNode.body, path: fileNode.absolutePath })
+    )
 
-  // If we could pass this via MDX loader config, this whole custom loader might be obsolete.
-  processor.data(`mdxNode`, mdxNode)
+    // Clone metadata so ensure it won't be overridden by later processings
+    const clonedMetadata = Object.assign(
+      {},
+      processor.data(`mdxMetadata`) as IMdxMetadata
+    )
+    const processedMDX = result.toString()
 
-  const result = await processor.process(
-    // Inject path to original file for remark plugins. See: https://github.com/gatsbyjs/gatsby/issues/26914
-    new VFile({ value: source, path: fileNode.absolutePath })
+    return { processedMDX, metadata: clonedMetadata }
+  } catch (err) {
+    const errorMeta = [
+      mdxNode.title && `Title: ${mdxNode.title}`,
+      mdxNode.slug && `Slug: ${mdxNode.slug}`,
+      mdxNode.frontmatter &&
+        `Frontmatter:\n${JSON.stringify(mdxNode.frontmatter, null, 2)}`,
+      fileNode.absolutePath && `Path: ${fileNode.absolutePath}`,
+      mdxNode.body && `Content:\n ${mdxNode.body}`,
+    ]
+      .filter(Boolean)
+      .join(`\n`)
+
+    err.message = `Unable to compile MDX:\n${errorMeta}\n\n---\nOriginal error:\n\n${err.message}`
+    throw err
+  }
+}
+
+/**
+ * This helper function allows you to inject additional plugins and configuration into the MDX
+ * compilation pipeline. Very useful to create your own resolvers that return custom metadata.
+ * Internally used to generate the tables of contents and the excerpts.
+ */
+export const compileMDXWithCustomOptions = async ({
+  pluginOptions,
+  customOptions,
+  getNode,
+  getNodesByType,
+  pathPrefix,
+  reporter,
+  cache,
+  mdxNode,
+}: {
+  pluginOptions: IMdxPluginOptions
+  customOptions: Partial<IMdxPluginOptions>
+  getNode: NodePluginArgs["getNode"]
+  getNodesByType: NodePluginArgs["getNodesByType"]
+  pathPrefix: string
+  reporter: NodePluginArgs["reporter"]
+  cache: NodePluginArgs["cache"]
+  mdxNode: IMdxNode
+}): Promise<{
+  processedMDX: string
+  metadata: IMdxMetadata
+} | null> => {
+  const customPluginOptions = deepmerge(
+    Object.assign({}, pluginOptions),
+    customOptions
   )
 
-  return result.toString()
+  // Prepare MDX compile
+  const mdxOptions = await enhanceMdxOptions(customPluginOptions, {
+    getNode,
+    getNodesByType,
+    pathPrefix,
+    reporter,
+    cache,
+  })
+  if (!mdxNode.parent) {
+    return null
+  }
+  const fileNode = getNode(mdxNode.parent)
+  if (!fileNode) {
+    return null
+  }
+
+  // Compile MDX and extract metadata
+  return compileMDX(mdxNode, fileNode, mdxOptions)
 }

packages/gatsby-plugin-mdx/src/gatsby-layout-loader.ts

@@ -4,7 +4,6 @@ import type { Options } from "mdast-util-to-markdown"
 import type { NodeMap } from "./types"
 import type { IMdxPluginOptions } from "./plugin-options"
 
-import grayMatter from "gray-matter"
 import { getOptions } from "loader-utils"
 
 export interface IGatsbyLayoutLoaderOptions {
@@ -13,7 +12,7 @@ export interface IGatsbyLayoutLoaderOptions {
 }
 
 // Wrap MDX content with Gatsby Layout component
-const gatsbyLayoutLoader: LoaderDefinition = async function (source) {
+const gatsbyLayoutLoader: LoaderDefinition = async function () {
   const { options, nodeMap }: IGatsbyLayoutLoaderOptions = getOptions(this)
 
   const res = nodeMap.get(this.resourcePath)
@@ -24,6 +23,12 @@ const gatsbyLayoutLoader: LoaderDefinition = async function (source) {
     )
   }
 
+  if (!res.mdxNode.body) {
+    throw new Error(
+      `MDX node is empty: ${JSON.stringify(res.mdxNode, null, 2)}`
+    )
+  }
+
   const { sourceInstanceName } = res.fileNode
 
   // Get the default layout for the file source instance group name,
@@ -34,20 +39,17 @@ const gatsbyLayoutLoader: LoaderDefinition = async function (source) {
 
   // No default layout set? Nothing to do here!
   if (!layoutPath) {
-    return source
+    return res.mdxNode.body
   }
 
-  // Remove frontmatter
-  const { content } = grayMatter(source)
-
   const { fromMarkdown } = await import(`mdast-util-from-markdown`)
   const { toMarkdown } = await import(`mdast-util-to-markdown`)
 
   const { mdxjs } = await import(`micromark-extension-mdxjs`)
   const { mdxFromMarkdown, mdxToMarkdown } = await import(`mdast-util-mdx`)
 
   // Parse MDX to AST
-  const tree = fromMarkdown(content, {
+  const tree = fromMarkdown(res.mdxNode.body, {
     extensions: [mdxjs()],
     mdastExtensions: [mdxFromMarkdown()],
   })
@@ -59,14 +61,13 @@ const gatsbyLayoutLoader: LoaderDefinition = async function (source) {
   )
 
   if (hasDefaultExport) {
-    return content
+    return res.mdxNode.body
   }
 
   tree.children.unshift({
     type: `mdxjsEsm` as `text`,
     value: `import GatsbyMDXLayout from "${layoutPath}"`,
   })
-
   tree.children.push({
     type: `mdxjsEsm` as `text`,
     value: `export default GatsbyMDXLayout`,

packages/gatsby-plugin-mdx/src/gatsby-mdx-loader.ts

@@ -5,7 +5,7 @@ import type { NodeMap } from "./types"
 
 import { getOptions } from "loader-utils"
 
-import compileMDX from "./compile-mdx"
+import { compileMDX } from "./compile-mdx"
 
 export interface IGatsbyMDXLoaderOptions {
   options: ProcessorOptions
@@ -25,7 +25,14 @@ const gatsbyMDXLoader: LoaderDefinition = async function (source) {
 
   const { mdxNode, fileNode } = res
 
-  return compileMDX(source, mdxNode, fileNode, options)
+  const { processedMDX } = await compileMDX(
+    // We want to work with the transformed source from our layout plugin
+    { ...mdxNode, body: source },
+    fileNode,
+    options
+  )
+
+  return processedMDX
 }
 
 export default gatsbyMDXLoader

packages/gatsby-plugin-mdx/src/gatsby-node.ts

@@ -1,5 +1,6 @@
 import type { GatsbyNode, NodeInput } from "gatsby"
 import type { FileSystemNode } from "gatsby-source-filesystem"
+import type { Options } from "rehype-infer-description-meta"
 import type { IFileNode, NodeMap } from "./types"
 
 import path from "path"
@@ -13,8 +14,9 @@ import {
   IMdxPluginOptions,
 } from "./plugin-options"
 import { IGatsbyLayoutLoaderOptions } from "./gatsby-layout-loader"
-import compileMDX from "./compile-mdx"
+import { compileMDX, compileMDXWithCustomOptions } from "./compile-mdx"
 import { IGatsbyMDXLoaderOptions } from "./gatsby-mdx-loader"
+import remarkInferTocMeta from "./remark-infer-toc-meta"
 
 /**
  * Add support for MDX files including using Gatsby layout components
@@ -119,13 +121,14 @@ export const preprocessSource: GatsbyNode["preprocessSource"] = async (
     return undefined
   }
 
-  const code = await compileMDX(
-    contents,
+  const { processedMDX } = await compileMDX(
     {
       id: ``,
       children: [],
       parent: ``,
       internal: { contentDigest: ``, owner: ``, type: `` },
+      body: contents,
+      rawBody: ``,
     },
     {
       id: ``,
@@ -138,11 +141,14 @@ export const preprocessSource: GatsbyNode["preprocessSource"] = async (
     mdxOptions
   )
 
-  return code.toString()
+  return processedMDX.toString()
 }
 
 export const createSchemaCustomization: GatsbyNode["createSchemaCustomization"] =
-  async ({ actions, schema }) => {
+  async (
+    { getNode, getNodesByType, pathPrefix, reporter, cache, actions, schema },
+    pluginOptions: IMdxPluginOptions
+  ) => {
     const { createTypes } = actions
     const typeDefs = [
       schema.buildObjectType({
@@ -155,9 +161,85 @@ export const createSchemaCustomization: GatsbyNode["createSchemaCustomization"]
         name: `Mdx`,
         fields: {
           rawBody: `String!`,
+          body: `String!`,
           frontmatter: `MdxFrontmatter!`,
           slug: `String`,
           title: `String`,
+          excerpt: {
+            type: `String`,
+            args: {
+              pruneLength: {
+                type: `Int`,
+                defaultValue: 140,
+              },
+            },
+            async resolve(mdxNode, { pruneLength }: { pruneLength: number }) {
+              const rehypeInferDescriptionMeta = (
+                await import(`rehype-infer-description-meta`)
+              ).default
+
+              const descriptionOptions: Options = { truncateSize: pruneLength }
+
+              const result = await compileMDXWithCustomOptions({
+                mdxNode,
+                pluginOptions,
+                customOptions: {
+                  mdxOptions: {
+                    rehypePlugins: [
+                      [rehypeInferDescriptionMeta, descriptionOptions],
+                    ],
+                  },
+                },
+                getNode,
+                getNodesByType,
+                pathPrefix,
+                reporter,
+                cache,
+              })
+
+              if (!result) {
+                return null
+              }
+
+              return result.metadata.description
+            },
+          },
+          tableOfContents: {
+            type: `JSON`,
+            args: {
+              maxDepth: {
+                type: `Int`,
+                default: 6,
+              },
+            },
+            async resolve(mdxNode, { maxDepth }) {
+              const { visit } = await import(`unist-util-visit`)
+              const { toc } = await import(`mdast-util-toc`)
+
+              const result = await compileMDXWithCustomOptions({
+                mdxNode,
+                pluginOptions,
+                customOptions: {
+                  mdxOptions: {
+                    remarkPlugins: [
+                      [remarkInferTocMeta, { visit, toc, maxDepth }],
+                    ],
+                  },
+                },
+                getNode,
+                getNodesByType,
+                pathPrefix,
+                reporter,
+                cache,
+              })
+
+              if (!result) {
+                return null
+              }
+
+              return result.metadata.toc
+            },
+          },
         },
         interfaces: [`Node`],
       }),
@@ -181,9 +263,9 @@ export const onCreateNode: GatsbyNode<FileSystemNode>["onCreateNode"] = async ({
   actions: { createNode, createParentChildLink },
   createNodeId,
 }) => {
-  const content = await loadNodeContent(node)
+  const rawBody = await loadNodeContent(node)
 
-  const { data: frontmatter } = grayMatter(content)
+  const { data: frontmatter, content: body } = grayMatter(rawBody)
 
   // Use slug from frontmatter, otherwise fall back to the file name and path
   const slug =
@@ -200,11 +282,12 @@ export const onCreateNode: GatsbyNode<FileSystemNode>["onCreateNode"] = async ({
     children: [],
     parent: node.id,
     internal: {
-      content: content,
+      content: rawBody,
       type: `Mdx`,
       contentDigest: node.internal.contentDigest,
     },
-    rawBody: content,
+    rawBody,
+    body,
     slug,
     title,
     frontmatter,