Merge pull request #52 from frencojobs/fix/ooms

fix: cache promises instead of highlighters
shikijs · Jun 30, 2021 · e7ee319 · e7ee319
2 parents 4df3f98 + 8b5e420
commit e7ee319
Show file tree

Hide file tree

Showing 5 changed files with 116 additions and 106 deletions.
diff --git a/examples/docusaurus/docusaurus.config.js b/examples/docusaurus/docusaurus.config.js
@@ -1,75 +1,75 @@
 /** @type {import('@docusaurus/types').DocusaurusConfig} */
 module.exports = {
-  title: 'My Site',
-  tagline: 'Dinosaurs are cool',
-  url: 'https://your-docusaurus-test-site.com',
-  baseUrl: '/',
-  onBrokenLinks: 'throw',
-  onBrokenMarkdownLinks: 'warn',
-  favicon: 'img/favicon.ico',
-  organizationName: 'facebook', // Usually your GitHub org/user name.
-  projectName: 'docusaurus', // Usually your repo name.
+  title: "My Site",
+  tagline: "Dinosaurs are cool",
+  url: "https://your-docusaurus-test-site.com",
+  baseUrl: "/",
+  onBrokenLinks: "throw",
+  onBrokenMarkdownLinks: "warn",
+  favicon: "img/favicon.ico",
+  organizationName: "facebook", // Usually your GitHub org/user name.
+  projectName: "docusaurus", // Usually your repo name.
   themeConfig: {
     navbar: {
-      title: 'My Site',
+      title: "My Site",
       logo: {
-        alt: 'My Site Logo',
-        src: 'img/logo.svg',
+        alt: "My Site Logo",
+        src: "img/logo.svg",
       },
       items: [
         {
-          type: 'doc',
-          docId: 'intro',
-          position: 'left',
-          label: 'Tutorial',
+          type: "doc",
+          docId: "intro",
+          position: "left",
+          label: "Tutorial",
         },
-        {to: '/blog', label: 'Blog', position: 'left'},
+        { to: "/blog", label: "Blog", position: "left" },
         {
-          href: 'https://github.com/facebook/docusaurus',
-          label: 'GitHub',
-          position: 'right',
+          href: "https://github.com/facebook/docusaurus",
+          label: "GitHub",
+          position: "right",
         },
       ],
     },
     footer: {
-      style: 'dark',
+      style: "dark",
       links: [
         {
-          title: 'Docs',
+          title: "Docs",
           items: [
             {
-              label: 'Tutorial',
-              to: '/docs/intro',
+              label: "Tutorial",
+              to: "/docs/intro",
             },
           ],
         },
         {
-          title: 'Community',
+          title: "Community",
           items: [
             {
-              label: 'Stack Overflow',
-              href: 'https://stackoverflow.com/questions/tagged/docusaurus',
+              label: "Stack Overflow",
+              href: "https://stackoverflow.com/questions/tagged/docusaurus",
             },
             {
-              label: 'Discord',
-              href: 'https://discordapp.com/invite/docusaurus',
+              label: "Discord",
+              href: "https://discordapp.com/invite/docusaurus",
             },
             {
-              label: 'Twitter',
-              href: 'https://twitter.com/docusaurus',
+              label: "Twitter",
+              href: "https://twitter.com/docusaurus",
             },
           ],
         },
         {
-          title: 'More',
+          title: "More",
           items: [
             {
-              label: 'Blog',
-              to: '/blog',
+              label: "Blog",
+              to: "/blog",
             },
             {
-              label: 'GitHub',
-              href: 'https://github.com/facebook/docusaurus',
+              label: "GitHub",
+              href: "https://github.com/facebook/docusaurus",
             },
           ],
         },
@@ -79,26 +79,24 @@ module.exports = {
   },
   presets: [
     [
-      '@docusaurus/preset-classic',
+      "@docusaurus/preset-classic",
       {
-
         docs: {
-          sidebarPath: require.resolve('./sidebars.js'),
+          remarkPlugins: [[require("remark-shiki-twoslash").default, { themes: ["min-light", "nord"] }]],
+          sidebarPath: require.resolve("./sidebars.js"),
           // Please change this to your repo.
-          editUrl:
-            'https://github.com/facebook/docusaurus/edit/master/website/',
+          editUrl: "https://github.com/facebook/docusaurus/edit/master/website/",
         },
         blog: {
+          remarkPlugins: [[require("remark-shiki-twoslash").default, { themes: ["min-light", "nord"] }]],
           showReadingTime: true,
           // Please change this to your repo.
-          editUrl:
-            'https://github.com/facebook/docusaurus/edit/master/website/blog/',
+          editUrl: "https://github.com/facebook/docusaurus/edit/master/website/blog/",
         },
         theme: {
-          customCss: require.resolve('./src/css/custom.css'),
+          customCss: require.resolve("./src/css/custom.css"),
         },
       },
     ],
-    ["docusaurus-preset-shiki-twoslash", { theme: "nord" }]
   ],
-};
+}
diff --git a/examples/docusaurus/package.json b/examples/docusaurus/package.json
@@ -19,7 +19,7 @@
     "@mdx-js/react": "^1.6.21",
     "@svgr/webpack": "^5.5.0",
     "clsx": "^1.1.1",
-    "docusaurus-preset-shiki-twoslash": "file:../../packages/docusaurus-preset-shiki-twoslash",
+    "remark-shiki-twoslash": "file:../../packages/remark-shiki-twoslash",
     "file-loader": "^6.2.0",
     "react": "^17.0.1",
     "react-dom": "^17.0.1",

diff --git a/examples/docusaurus/scripts/clone_docs.mjs b/examples/docusaurus/scripts/clone_docs.mjs
@@ -0,0 +1,13 @@
+// A script to generate a number of cloned ts files to test performance
+// Not part of the example
+
+import { promises as fs } from "fs"
+
+const base = await fs.readFile("./docs/intro.md", "utf-8")
+const count = parseInt(process.argv.slice(2)[0])
+
+for (let i = 0; i < count; i++) {
+  await fs.writeFile(`./docs/cloned_intro_${i}.md`, base, "utf-8")
+}
+
+console.log(`Generated ${count} file${count === 1 ? "" : "s"}.`)
diff --git a/examples/docusaurus/yarn.lock b/examples/docusaurus/yarn.lock
@@ -1814,10 +1814,10 @@
   resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.3.tgz#9c088679876f374eb5983f150d4787aa6fb32d7e"
   integrity sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==
 
-"@typescript/twoslash@1.1.7":
-  version "1.1.7"
-  resolved "https://registry.yarnpkg.com/@typescript/twoslash/-/twoslash-1.1.7.tgz#9fc5709f37940f2deda396b74e503c6c33c54d6e"
-  integrity sha512-+oASPajHbUpmwsZgf0/ioBn9vjIodAO4c0na2nMLWxBKDMGAo16m8uEFtxjIrEYp6l+h3rBvHchSFucv1/qcNQ==
+"@typescript/twoslash@2.0.1":
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/@typescript/twoslash/-/twoslash-2.0.1.tgz#bf6dbd571e832e8f2114810d562eec00632ce107"
+  integrity sha512-+TgjZp0d331A2A9nBBjX/EFax3LnR9MtziICqgo2l23hXV9JY4EiNPdcBSEDBRzaWQIMvSa5YvBENnE6uZ33EA==
   dependencies:
     "@typescript/vfs" "1.3.4"
     debug "^4.1.1"
@@ -3403,12 +3403,6 @@ dns-txt@^2.0.2:
   dependencies:
     buffer-indexof "^1.0.0"
 
-"docusaurus-preset-shiki-twoslash@file:../../packages/docusaurus-preset-shiki-twoslash":
-  version "1.0.2"
-  dependencies:
-    remark-shiki-twoslash "1.3.0"
-    typescript ">3"
-
 dom-converter@^0.2:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/dom-converter/-/dom-converter-0.2.0.tgz#6721a9daee2e293682955b6afe416771627bb768"
@@ -7036,7 +7030,7 @@ regenerate@^1.4.0:
   resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.2.tgz#b9346d8827e8f5a32f7ba29637d398b69014848a"
   integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==
 
-regenerator-runtime@^0.13.4:
+regenerator-runtime@^0.13.4, regenerator-runtime@^0.13.7:
   version "0.13.7"
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz#cac2dacc8a1ea675feaabaeb8ae833898ae46f55"
   integrity sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==
@@ -7175,17 +7169,16 @@ [email protected]:
     vfile-location "^3.0.0"
     xtend "^4.0.1"
 
-[email protected]:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/remark-shiki-twoslash/-/remark-shiki-twoslash-1.3.0.tgz#64b3646a849928abc599767d13d23f058a96f1df"
-  integrity sha512-2xnVRFhjJ+Q2w62PNzcZKpm7qJKAsmu5+sEwfDKCK3gE6CaKzUkIrv/RSxbnqLJ9rWLLhdF1o+4T8WvRwF+JmQ==
+"remark-shiki-twoslash@file:../../packages/remark-shiki-twoslash":
+  version "1.4.8"
   dependencies:
-    "@typescript/twoslash" "1.1.7"
+    "@typescript/twoslash" "2.0.1"
     "@typescript/vfs" "1.3.4"
-    shiki "^0.9.3"
-    shiki-twoslash "1.3.1"
+    regenerator-runtime "^0.13.7"
+    shiki "0.9.3"
+    shiki-twoslash "1.5.0"
     tslib "2.1.0"
-    typescript "*"
+    typescript ">3"
     unist-util-visit "^2.0.0"
 
 [email protected]:
@@ -7606,17 +7599,17 @@ shelljs@^0.8.4:
     interpret "^1.0.0"
     rechoir "^0.6.2"
 
-shiki-twoslash@1.3.1:
-  version "1.3.1"
-  resolved "https://registry.yarnpkg.com/shiki-twoslash/-/shiki-twoslash-1.3.1.tgz#64ee98895029d781fb898e66e4de1724539d4180"
-  integrity sha512-BasO1NrB8boky+EybyvQo0j/AMNea3FcUyrJXV0dCElLgmTVTTuNvUQ9DAk3RRNg818pjitU0xdDpopQqGEZbQ==
+shiki-twoslash@1.5.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/shiki-twoslash/-/shiki-twoslash-1.5.0.tgz#21625311bac5a4738df97476bfb529747d81210c"
+  integrity sha512-TCXLEreIeo6x3oUdkva41lv57wv/GjpgbofrFkh26iEcYQZvUva9decmH4PD7U13zRIuEYnQCOY/8mLZZlb8OA==
   dependencies:
-    "@typescript/twoslash" "1.1.7"
+    "@typescript/twoslash" "2.0.1"
     "@typescript/vfs" "1.3.4"
-    shiki "^0.9.3"
-    typescript "*"
+    shiki "0.9.3"
+    typescript ">3"
 
-shiki@^0.9.3:
+[email protected]:
   version "0.9.3"
   resolved "https://registry.yarnpkg.com/shiki/-/shiki-0.9.3.tgz#7bf7bcf3ed50ca525ec89cc09254abce4264d5ca"
   integrity sha512-NEjg1mVbAUrzRv2eIcUt3TG7X9svX7l3n3F5/3OdFq+/BxUdmBOeKGiH4icZJBLHy354Shnj6sfBTemea2e7XA==
@@ -8193,7 +8186,7 @@ typedarray-to-buffer@^3.1.5:
   dependencies:
     is-typedarray "^1.0.0"
 
-typescript@*, typescript@>3, typescript@^4.3.2:
+typescript@>3, typescript@^4.3.2:
   version "4.3.2"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.2.tgz#399ab18aac45802d6f2498de5054fcbbe716a805"
   integrity sha512-zZ4hShnmnoVnAHpVHWpTcxdv7dWP60S2FsydQLV8V5PbS3FifjWFFRiHSWpDJahly88PRyV5teTSLoq4eG7mKw==

diff --git a/packages/remark-shiki-twoslash/src/index.ts b/packages/remark-shiki-twoslash/src/index.ts
@@ -1,7 +1,7 @@
 import type { TwoSlashReturn } from "@typescript/twoslash"
 import type { Node } from "unist"
 import { UserConfigSettings, renderCodeToHTML } from "shiki-twoslash"
-import { Lang, Highlighter, getHighlighter, IThemeRegistration } from "shiki"
+import { Lang, Highlighter, getHighlighter } from "shiki"
 import visit from "unist-util-visit"
 
 import { addIncludes, replaceIncludesInCode } from "./includes"
@@ -59,14 +59,15 @@ export const runTwoSlashOnNode = (code: string, lang: string, meta: string, sett
 }
 
 // To make sure we only have one highlighter per theme in a process
-const highlighterCache = new WeakMap<UserConfigSettings, Highlighter[]>()
+const highlighterCache = new Map<UserConfigSettings, Promise<Highlighter[]>>()
 
 /** Sets up the highlighters, and cache's for recalls */
-export const highlightersFromSettings = async (settings: UserConfigSettings) => {
+export const highlightersFromSettings = (settings: UserConfigSettings) => {
+  // console.log("i should only log once per theme")
+  // ^ uncomment this to debug if required
   const themes = settings.themes || (settings.theme ? [settings.theme] : ["light-plus"])
-  if (highlighterCache.has(settings)) return highlighterCache.get(settings)!
 
-  const highlighters = await Promise.all(
+  return Promise.all(
     themes.map(async theme => {
       // You can put a string, a path, or the JSON theme obj
       const themeName = (theme as any).name || theme
@@ -77,9 +78,6 @@ export const highlightersFromSettings = async (settings: UserConfigSettings) =>
       return highlighter
     })
   )
-
-  highlighterCache.set(settings, highlighters)
-  return highlighters
 }
 
 const amendSettingsForDefaults = (settings: UserConfigSettings) => {
@@ -95,7 +93,7 @@ const amendSettingsForDefaults = (settings: UserConfigSettings) => {
 
 const parsingNewFile = () => includes.clear()
 
-////////////////// The Remark API
+// --- The Remark API ---
 
 /* A rich AST node for uninst with twoslash'd data */
 type RemarkCodeNode = Node & {
@@ -114,8 +112,12 @@ type RemarkCodeNode = Node & {
 function remarkTwoslash(settings: UserConfigSettings = {}) {
   amendSettingsForDefaults(settings)
 
+  if (!highlighterCache.has(settings)) {
+    highlighterCache.set(settings, highlightersFromSettings(settings))
+  }
+
   const transform = async (markdownAST: any) => {
-    const highlighters = await highlightersFromSettings(settings)
+    const highlighters = await highlighterCache.get(settings)!
     parsingNewFile()
     visit(markdownAST, "code", remarkVisitor(highlighters, settings))
   }
@@ -126,38 +128,42 @@ function remarkTwoslash(settings: UserConfigSettings = {}) {
 /**
  * The function doing the work of transforming any codeblock samples in a remark AST.
  */
-export const remarkVisitor = (highlighters: Highlighter[], twoslashSettings: UserConfigSettings = {}) => (
-  node: RemarkCodeNode
-) => {
-  let lang = node.lang
-  // The meta is the bit after lang in: ```lang [this bit]
-  const metaString = !node.meta ? "" : typeof node.meta === "string" ? node.meta : node.meta.join(" ")
-  const code = node.value
-
-  const twoslash = runTwoSlashOnNode(code, lang, metaString, twoslashSettings)
-  if (twoslash) {
-    node.value = twoslash.code
-    node.lang = twoslash.extension as Lang
-    node.twoslash = twoslash
+export const remarkVisitor =
+  (highlighters: Highlighter[], twoslashSettings: UserConfigSettings = {}) =>
+  (node: RemarkCodeNode) => {
+    let lang = node.lang
+    // The meta is the bit after lang in: ```lang [this bit]
+    const metaString = !node.meta ? "" : typeof node.meta === "string" ? node.meta : node.meta.join(" ")
+    const code = node.value
+
+    const twoslash = runTwoSlashOnNode(code, lang, metaString, twoslashSettings)
+    if (twoslash) {
+      node.value = twoslash.code
+      node.lang = twoslash.extension as Lang
+      node.twoslash = twoslash
+    }
+
+    const shikiHTML = getHTML(node.value, lang, metaString, highlighters, twoslash)
+    node.type = "html"
+    node.value = shikiHTML
+    node.children = []
   }
 
-  const shikiHTML = getHTML(node.value, lang, metaString, highlighters, twoslash)
-  node.type = "html"
-  node.value = shikiHTML
-  node.children = []
-}
-
 export default remarkTwoslash
 
-////////////////// The Markdown-it API
+// --- The Markdown-it API ---
 
 /** Only the inner function exposed as a synchronous API for markdown-it */
 
 export const setupForFile = async (settings: UserConfigSettings = {}) => {
   amendSettingsForDefaults(settings)
   parsingNewFile()
 
-  let highlighters = await highlightersFromSettings(settings)
+  if (!highlighterCache.has(settings)) {
+    highlighterCache.set(settings, highlightersFromSettings(settings))
+  }
+
+  let highlighters = await highlighterCache.get(settings)!
   return { settings, highlighters }
 }