Rewrite how MDX is parsed

.eslintignore

@@ -16,6 +16,8 @@ packages/gatsby-theme-mdx/src/components/header.js
 packages/gatsby-theme-mdx/src/components/layout.js
 packages/gatsby-theme-mdx/src/components/search.js
 
+packages/remark-mdx
+
 # tmp
 examples/vue
 packages/vue

.gitignore

@@ -13,3 +13,5 @@ artifacts
 .*cache
 public
 .npmrc
+.nyc_output/
+coverage/

.remarkrc.js

@@ -5,6 +5,7 @@ const wooorm = require('retext-preset-wooorm')
 module.exports = {
   plugins: [
     './packages/remark-mdx',
+    './packages/remark-mdxjs',
     'preset-wooorm',
     'preset-prettier',
     [

docs/advanced/api.mdx

@@ -115,8 +115,6 @@ const mdxText = fs.readFileSync('hello.mdx', 'utf8')
 const jsx = mdx.sync(mdxText)
 ```
 
-<!-- TODO: links should be updated. Can we also inline this example? -->
-
 MDX’s [runtime][] package has [example][] usage.
 
 ## Compiler

docs/advanced/contributing.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/contributing" />

docs/advanced/custom-loader.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/guides/custom-loader" />

docs/advanced/retext-plugins.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/advanced/using-plugins#using-retext-plugins" />

docs/advanced/runtime.mdx

@@ -1,10 +1,5 @@
 import Doc from '@mdx-js/runtime/readme.md'
 
-<!--
-TODO: add a nicer, smaller intro to the runtime: when to use it, when not to.
-We can also move it to `api` maybe?
--->
-
 # Runtime
 
 <Doc />

docs/advanced/specification.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/advanced/ast" />

docs/advanced/sync-api.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/advanced/api#sync-api" />

docs/advanced/typescript.mdx

@@ -1,10 +1,5 @@
 # TypeScript
 
-<!--
-TODO: this feels like it’s placed weirdly.
-Where could it go?
--->
-
 Below are some basic typings to get started with:
 
 ```tsx

docs/advanced/writing-a-plugin.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/guides/custom-loader" />

docs/contributing.mdx

@@ -148,8 +148,6 @@ If you’re having issues installing locally you might need to run
 - [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
 - [GitHub Help](https://help.github.com)
 
-<!-- Definitions -->
-
 [coc]: https://github.com/mdx-js/.github/blob/master/code-of-conduct.md
 [support]: https://mdxjs.com/support
 [unit-test]: https://twitter.com/sindresorhus/status/579306280495357953

docs/editor-plugins.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/editors" />

docs/getting-started/typescript.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/advanced/typescript" />

docs/guides/wrapper-customization.mdx

@@ -125,5 +125,5 @@ or Brent Jackson’s [MDX Blocks](https://github.com/jxnblk/mdx-blocks)
 
 ---
 
-- <https://mxstbr.blog/2017/02/react-children-deepdive>
-- <https://github.com/jxnblk/mdx-blocks>
+- [React children deep dive](https://mxstbr.blog/2017/02/react-children-deepdive)
+- [MDX Blocks](https://github.com/jxnblk/mdx-blocks)

docs/plugins.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/advanced/plugins" />

docs/support.mdx

@@ -50,8 +50,6 @@ Spending the extra time up front can help save everyone time in the long run.
 
 [Read the Contributing guidelines on the MDX website][contributing].
 
-<!-- Definitions -->
-
 [coc]: https://github.com/mdx-js/.github/blob/master/code-of-conduct.md
 [rubberduck]: https://rubberduckdebugging.com
 [xy]: https://meta.stackexchange.com/questions/66377/what-is-the-xy-problem/66378#66378

docs/syntax.mdx

@@ -1,3 +1 @@
-import Redirect from 'gatsby-theme-mdx/src/components/redirect'
-
 <Redirect to="/getting-started#syntax" />

package.json

@@ -22,26 +22,26 @@
     "docs-build": "gatsby build",
     "docs-deploy": "now && now alias $(pbpaste) mdxjs.com && now alias $(pbpaste) www.mdxjs.com",
     "format": "yarn lint --fix",
-    "lint": "eslint . --cache --ext js,jsx,md,mdx -f friendly",
+    "lint": "eslint . --cache --ext js,jsx -f friendly",
     "now-build": "yarn build && yarn docs-build",
     "postinstall": "yarn-deduplicate || exit 0",
     "prepublishOnly": "yarn build",
     "pretest": "yarn build",
     "publish": "lerna publish --force-publish=\"*\"",
     "publish-ci": "lerna publish -y --canary --preid ci --pre-dist-tag ci",
     "publish-next": "lerna publish --force-publish=\"*\" --dist-tag next --preid rc",
-    "test": "jest --runInBand --detectOpenHandles"
+    "test": "jest --runInBand --detectOpenHandles && nyc --reporter lcov tape packages/remark-mdx/test/index.js"
   },
   "devDependencies": {
     "@babel/core": "7.9.6",
     "@babel/plugin-syntax-jsx": "7.8.3",
     "@babel/preset-env": "7.9.6",
     "@babel/preset-react": "7.9.4",
     "@mapbox/rehype-prism": "0.4.0",
-    "@mdx-js/mdx": "^1.6.2",
-    "@mdx-js/react": "^1.6.2",
-    "@mdx-js/test-util": "^1.6.2",
-    "@mdx-js/vue": "^1.6.2",
+    "@mdx-js/mdx": "^1.6.3",
+    "@mdx-js/react": "^1.6.3",
+    "@mdx-js/test-util": "^1.6.3",
+    "@mdx-js/vue": "^1.6.3",
     "@pkgr/rollup": "0.10.3",
     "@vue/babel-preset-jsx": "1.1.2",
     "@vue/test-utils": "1.0.3",
@@ -73,8 +73,9 @@
     "rehype-katex": "3.0.0",
     "remark-autolink-headings": "6.0.0",
     "remark-math": "2.0.1",
-    "remark-mdx": "^1.6.2",
-    "remark-parse": "8.0.2",
+    "remark-mdx": "1.6.3",
+    "remark-mdxjs": "1.6.3",
+    "remark-parse": "8.0.1",
     "remark-preset-prettier": "0.4.0",
     "remark-preset-wooorm": "7.0.0",
     "remark-slug": "6.0.0",
@@ -103,7 +104,7 @@
   },
   "husky": {
     "hooks": {
-      "pre-commit": "lint-staged"
+      "#pre-commit": "lint-staged"
     }
   },
   "jest": {

packages/gatsby-theme-mdx/package.json

@@ -47,8 +47,8 @@
     "react-helmet": "6.0.0",
     "react-live": "2.2.2",
     "remark-emoji": "2.1.0",
-    "remark-mdx-remove-exports": "^1.6.3",
-    "remark-mdx-remove-imports": "^1.6.3",
+    "remark-mdxjs-remove-exports": "^1.6.3",
+    "remark-mdxjs-remove-imports": "^1.6.3",
     "remark-slug": "6.0.0",
     "theme-ui": "0.3.1"
   }

packages/gatsby-theme-mdx/src/components/mdx-components.js

@@ -2,6 +2,7 @@ import React from 'react'
 import CodePreview from './code-preview'
 import CodeBlock from './code-block'
 import Link from './link'
+import Redirect from './redirect'
 
 const heading = Tag => props => (
   <Tag {...props}>
@@ -35,5 +36,6 @@ export default {
   h3: heading('h3'),
   h4: heading('h4'),
   h5: heading('h5'),
-  h6: heading('h6')
+  h6: heading('h6'),
+  Redirect
 }

packages/gatsby-theme-mdx/src/components/playground-editor.js

@@ -5,8 +5,8 @@ import {MDXProvider, mdx as createElement} from '@mdx-js/react'
 import * as Rebass from '@rebass/emotion'
 import {ThemeContext} from '@emotion/core'
 import {css} from 'theme-ui'
-import removeImports from 'remark-mdx-remove-imports'
-import removeExports from 'remark-mdx-remove-exports'
+import removeImports from 'remark-mdxjs-remove-imports'
+import removeExports from 'remark-mdxjs-remove-exports'
 
 import CodeBlock from './code-block'
 

packages/mdx/index.js

@@ -1,11 +1,11 @@
 const unified = require('unified')
 const toMDAST = require('remark-parse')
 const remarkMdx = require('remark-mdx')
+const remarkMdxJs = require('remark-mdxjs')
 const footnotes = require('remark-footnotes')
 const squeeze = require('remark-squeeze-paragraphs')
 const visit = require('unist-util-visit')
 const raw = require('hast-util-raw')
-const toMDXAST = require('./md-ast-to-mdx-ast')
 const mdxAstToMdxHast = require('./mdx-ast-to-mdx-hast')
 const mdxHastToJsx = require('./mdx-hast-to-jsx')
 
@@ -30,9 +30,9 @@ function createMdxAstCompiler(options) {
   const fn = unified()
     .use(toMDAST, options)
     .use(remarkMdx, options)
+    .use(remarkMdxJs, options)
     .use(footnotes, options)
     .use(squeeze, options)
-    .use(toMDXAST, options)
 
   plugins.forEach(plugin => {
     // Handle [plugin, pluginOptions] syntax

packages/mdx/mdx-ast-to-mdx-hast.js

@@ -1,4 +1,5 @@
 const toHAST = require('mdast-util-to-hast')
+const all = require('mdast-util-to-hast/lib/all')
 const detab = require('detab')
 const u = require('unist-builder')
 
@@ -61,32 +62,38 @@ function mdxAstToMdxHast() {
           h(node, 'code', props, [u('text', value)])
         ])
       },
+      // To do: rename to `mdxJsImport`
       import(h, node) {
         return Object.assign({}, node, {
           type: 'import'
         })
       },
+      // To do: rename to `mdxJsExport`
       export(h, node) {
         return Object.assign({}, node, {
           type: 'export'
         })
       },
-      comment(h, node) {
+      mdxBlockElement(h, node) {
+        return Object.assign({}, node, {children: all(h, node)})
+      },
+      mdxSpanElement(h, node) {
+        return Object.assign({}, node, {children: all(h, node)})
+      },
+      mdxBlockExpression(h, node) {
         return Object.assign({}, node, {
-          type: 'comment'
+          type: 'mdxBlockExpression'
         })
       },
-      jsx(h, node) {
+      mdxSpanExpression(h, node) {
         return Object.assign({}, node, {
-          type: 'jsx'
+          type: 'mdxSpanExpression'
         })
       }
     }
 
     const hast = toHAST(tree, {
-      handlers,
-      // Enable passing of HTML nodes to HAST as raw nodes
-      allowDangerousHtml: true
+      handlers
     })
 
     return hast