From 5401506484a391d89a8b9132dcdb81cb9b43e9aa Mon Sep 17 00:00:00 2001 From: Frenco Date: Wed, 30 Jun 2021 18:18:41 +0630 Subject: [PATCH 1/2] fix: cache promises instead of highlighters Closes #35 --- packages/remark-shiki-twoslash/src/index.ts | 68 +++++++++++---------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/packages/remark-shiki-twoslash/src/index.ts b/packages/remark-shiki-twoslash/src/index.ts index 8b0f864..f4b2400 100755 --- a/packages/remark-shiki-twoslash/src/index.ts +++ b/packages/remark-shiki-twoslash/src/index.ts @@ -1,7 +1,7 @@ import type { TwoSlashReturn } from "@typescript/twoslash" import type { Node } from "unist" import { UserConfigSettings, renderCodeToHTML } from "shiki-twoslash" -import { Lang, Highlighter, getHighlighter, IThemeRegistration } from "shiki" +import { Lang, Highlighter, getHighlighter } from "shiki" import visit from "unist-util-visit" import { addIncludes, replaceIncludesInCode } from "./includes" @@ -59,14 +59,15 @@ export const runTwoSlashOnNode = (code: string, lang: string, meta: string, sett } // To make sure we only have one highlighter per theme in a process -const highlighterCache = new WeakMap() +const highlighterCache = new Map>() /** Sets up the highlighters, and cache's for recalls */ -export const highlightersFromSettings = async (settings: UserConfigSettings) => { +export const highlightersFromSettings = (settings: UserConfigSettings) => { + // console.log("i should only log once per theme") + // ^ uncomment this to debug if required const themes = settings.themes || (settings.theme ? [settings.theme] : ["light-plus"]) - if (highlighterCache.has(settings)) return highlighterCache.get(settings)! - const highlighters = await Promise.all( + return Promise.all( themes.map(async theme => { // You can put a string, a path, or the JSON theme obj const themeName = (theme as any).name || theme @@ -77,9 +78,6 @@ export const highlightersFromSettings = async (settings: UserConfigSettings) => return highlighter }) ) - - highlighterCache.set(settings, highlighters) - return highlighters } const amendSettingsForDefaults = (settings: UserConfigSettings) => { @@ -95,7 +93,7 @@ const amendSettingsForDefaults = (settings: UserConfigSettings) => { const parsingNewFile = () => includes.clear() -////////////////// The Remark API +// --- The Remark API --- /* A rich AST node for uninst with twoslash'd data */ type RemarkCodeNode = Node & { @@ -114,8 +112,12 @@ type RemarkCodeNode = Node & { function remarkTwoslash(settings: UserConfigSettings = {}) { amendSettingsForDefaults(settings) + if (!highlighterCache.has(settings)) { + highlighterCache.set(settings, highlightersFromSettings(settings)) + } + const transform = async (markdownAST: any) => { - const highlighters = await highlightersFromSettings(settings) + const highlighters = await highlighterCache.get(settings)! parsingNewFile() visit(markdownAST, "code", remarkVisitor(highlighters, settings)) } @@ -126,30 +128,30 @@ function remarkTwoslash(settings: UserConfigSettings = {}) { /** * The function doing the work of transforming any codeblock samples in a remark AST. */ -export const remarkVisitor = (highlighters: Highlighter[], twoslashSettings: UserConfigSettings = {}) => ( - node: RemarkCodeNode -) => { - let lang = node.lang - // The meta is the bit after lang in: ```lang [this bit] - const metaString = !node.meta ? "" : typeof node.meta === "string" ? node.meta : node.meta.join(" ") - const code = node.value - - const twoslash = runTwoSlashOnNode(code, lang, metaString, twoslashSettings) - if (twoslash) { - node.value = twoslash.code - node.lang = twoslash.extension as Lang - node.twoslash = twoslash +export const remarkVisitor = + (highlighters: Highlighter[], twoslashSettings: UserConfigSettings = {}) => + (node: RemarkCodeNode) => { + let lang = node.lang + // The meta is the bit after lang in: ```lang [this bit] + const metaString = !node.meta ? "" : typeof node.meta === "string" ? node.meta : node.meta.join(" ") + const code = node.value + + const twoslash = runTwoSlashOnNode(code, lang, metaString, twoslashSettings) + if (twoslash) { + node.value = twoslash.code + node.lang = twoslash.extension as Lang + node.twoslash = twoslash + } + + const shikiHTML = getHTML(node.value, lang, metaString, highlighters, twoslash) + node.type = "html" + node.value = shikiHTML + node.children = [] } - const shikiHTML = getHTML(node.value, lang, metaString, highlighters, twoslash) - node.type = "html" - node.value = shikiHTML - node.children = [] -} - export default remarkTwoslash -////////////////// The Markdown-it API +// --- The Markdown-it API --- /** Only the inner function exposed as a synchronous API for markdown-it */ @@ -157,7 +159,11 @@ export const setupForFile = async (settings: UserConfigSettings = {}) => { amendSettingsForDefaults(settings) parsingNewFile() - let highlighters = await highlightersFromSettings(settings) + if (!highlighterCache.has(settings)) { + highlighterCache.set(settings, highlightersFromSettings(settings)) + } + + let highlighters = await highlighterCache.get(settings)! return { settings, highlighters } } From 8b5e420fa90aeb705e0b5f70e2bca2688774d2f1 Mon Sep 17 00:00:00 2001 From: Frenco Date: Wed, 30 Jun 2021 18:21:48 +0630 Subject: [PATCH 2/2] feat: update docusaurus example --- examples/docusaurus/docusaurus.config.js | 90 +++++++++++----------- examples/docusaurus/package.json | 2 +- examples/docusaurus/scripts/clone_docs.mjs | 13 ++++ examples/docusaurus/yarn.lock | 49 +++++------- 4 files changed, 79 insertions(+), 75 deletions(-) create mode 100644 examples/docusaurus/scripts/clone_docs.mjs diff --git a/examples/docusaurus/docusaurus.config.js b/examples/docusaurus/docusaurus.config.js index 752c5d6..78af0e8 100644 --- a/examples/docusaurus/docusaurus.config.js +++ b/examples/docusaurus/docusaurus.config.js @@ -1,75 +1,75 @@ /** @type {import('@docusaurus/types').DocusaurusConfig} */ module.exports = { - title: 'My Site', - tagline: 'Dinosaurs are cool', - url: 'https://your-docusaurus-test-site.com', - baseUrl: '/', - onBrokenLinks: 'throw', - onBrokenMarkdownLinks: 'warn', - favicon: 'img/favicon.ico', - organizationName: 'facebook', // Usually your GitHub org/user name. - projectName: 'docusaurus', // Usually your repo name. + title: "My Site", + tagline: "Dinosaurs are cool", + url: "https://your-docusaurus-test-site.com", + baseUrl: "/", + onBrokenLinks: "throw", + onBrokenMarkdownLinks: "warn", + favicon: "img/favicon.ico", + organizationName: "facebook", // Usually your GitHub org/user name. + projectName: "docusaurus", // Usually your repo name. themeConfig: { navbar: { - title: 'My Site', + title: "My Site", logo: { - alt: 'My Site Logo', - src: 'img/logo.svg', + alt: "My Site Logo", + src: "img/logo.svg", }, items: [ { - type: 'doc', - docId: 'intro', - position: 'left', - label: 'Tutorial', + type: "doc", + docId: "intro", + position: "left", + label: "Tutorial", }, - {to: '/blog', label: 'Blog', position: 'left'}, + { to: "/blog", label: "Blog", position: "left" }, { - href: 'https://github.com/facebook/docusaurus', - label: 'GitHub', - position: 'right', + href: "https://github.com/facebook/docusaurus", + label: "GitHub", + position: "right", }, ], }, footer: { - style: 'dark', + style: "dark", links: [ { - title: 'Docs', + title: "Docs", items: [ { - label: 'Tutorial', - to: '/docs/intro', + label: "Tutorial", + to: "/docs/intro", }, ], }, { - title: 'Community', + title: "Community", items: [ { - label: 'Stack Overflow', - href: 'https://stackoverflow.com/questions/tagged/docusaurus', + label: "Stack Overflow", + href: "https://stackoverflow.com/questions/tagged/docusaurus", }, { - label: 'Discord', - href: 'https://discordapp.com/invite/docusaurus', + label: "Discord", + href: "https://discordapp.com/invite/docusaurus", }, { - label: 'Twitter', - href: 'https://twitter.com/docusaurus', + label: "Twitter", + href: "https://twitter.com/docusaurus", }, ], }, { - title: 'More', + title: "More", items: [ { - label: 'Blog', - to: '/blog', + label: "Blog", + to: "/blog", }, { - label: 'GitHub', - href: 'https://github.com/facebook/docusaurus', + label: "GitHub", + href: "https://github.com/facebook/docusaurus", }, ], }, @@ -79,26 +79,24 @@ module.exports = { }, presets: [ [ - '@docusaurus/preset-classic', + "@docusaurus/preset-classic", { - docs: { - sidebarPath: require.resolve('./sidebars.js'), + remarkPlugins: [[require("remark-shiki-twoslash").default, { themes: ["min-light", "nord"] }]], + sidebarPath: require.resolve("./sidebars.js"), // Please change this to your repo. - editUrl: - 'https://github.com/facebook/docusaurus/edit/master/website/', + editUrl: "https://github.com/facebook/docusaurus/edit/master/website/", }, blog: { + remarkPlugins: [[require("remark-shiki-twoslash").default, { themes: ["min-light", "nord"] }]], showReadingTime: true, // Please change this to your repo. - editUrl: - 'https://github.com/facebook/docusaurus/edit/master/website/blog/', + editUrl: "https://github.com/facebook/docusaurus/edit/master/website/blog/", }, theme: { - customCss: require.resolve('./src/css/custom.css'), + customCss: require.resolve("./src/css/custom.css"), }, }, ], - ["docusaurus-preset-shiki-twoslash", { theme: "nord" }] ], -}; +} diff --git a/examples/docusaurus/package.json b/examples/docusaurus/package.json index fdd7ce1..0c4796f 100644 --- a/examples/docusaurus/package.json +++ b/examples/docusaurus/package.json @@ -19,7 +19,7 @@ "@mdx-js/react": "^1.6.21", "@svgr/webpack": "^5.5.0", "clsx": "^1.1.1", - "docusaurus-preset-shiki-twoslash": "file:../../packages/docusaurus-preset-shiki-twoslash", + "remark-shiki-twoslash": "file:../../packages/remark-shiki-twoslash", "file-loader": "^6.2.0", "react": "^17.0.1", "react-dom": "^17.0.1", diff --git a/examples/docusaurus/scripts/clone_docs.mjs b/examples/docusaurus/scripts/clone_docs.mjs new file mode 100644 index 0000000..4baf819 --- /dev/null +++ b/examples/docusaurus/scripts/clone_docs.mjs @@ -0,0 +1,13 @@ +// A script to generate a number of cloned ts files to test performance +// Not part of the example + +import { promises as fs } from "fs" + +const base = await fs.readFile("./docs/intro.md", "utf-8") +const count = parseInt(process.argv.slice(2)[0]) + +for (let i = 0; i < count; i++) { + await fs.writeFile(`./docs/cloned_intro_${i}.md`, base, "utf-8") +} + +console.log(`Generated ${count} file${count === 1 ? "" : "s"}.`) diff --git a/examples/docusaurus/yarn.lock b/examples/docusaurus/yarn.lock index 341994d..f172912 100644 --- a/examples/docusaurus/yarn.lock +++ b/examples/docusaurus/yarn.lock @@ -1814,10 +1814,10 @@ resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.3.tgz#9c088679876f374eb5983f150d4787aa6fb32d7e" integrity sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ== -"@typescript/twoslash@1.1.7": - version "1.1.7" - resolved "https://registry.yarnpkg.com/@typescript/twoslash/-/twoslash-1.1.7.tgz#9fc5709f37940f2deda396b74e503c6c33c54d6e" - integrity sha512-+oASPajHbUpmwsZgf0/ioBn9vjIodAO4c0na2nMLWxBKDMGAo16m8uEFtxjIrEYp6l+h3rBvHchSFucv1/qcNQ== +"@typescript/twoslash@2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@typescript/twoslash/-/twoslash-2.0.1.tgz#bf6dbd571e832e8f2114810d562eec00632ce107" + integrity sha512-+TgjZp0d331A2A9nBBjX/EFax3LnR9MtziICqgo2l23hXV9JY4EiNPdcBSEDBRzaWQIMvSa5YvBENnE6uZ33EA== dependencies: "@typescript/vfs" "1.3.4" debug "^4.1.1" @@ -3403,12 +3403,6 @@ dns-txt@^2.0.2: dependencies: buffer-indexof "^1.0.0" -"docusaurus-preset-shiki-twoslash@file:../../packages/docusaurus-preset-shiki-twoslash": - version "1.0.2" - dependencies: - remark-shiki-twoslash "1.3.0" - typescript ">3" - dom-converter@^0.2: version "0.2.0" resolved "https://registry.yarnpkg.com/dom-converter/-/dom-converter-0.2.0.tgz#6721a9daee2e293682955b6afe416771627bb768" @@ -7036,7 +7030,7 @@ regenerate@^1.4.0: resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.2.tgz#b9346d8827e8f5a32f7ba29637d398b69014848a" integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A== -regenerator-runtime@^0.13.4: +regenerator-runtime@^0.13.4, regenerator-runtime@^0.13.7: version "0.13.7" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz#cac2dacc8a1ea675feaabaeb8ae833898ae46f55" integrity sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew== @@ -7175,17 +7169,16 @@ remark-parse@8.0.3: vfile-location "^3.0.0" xtend "^4.0.1" -remark-shiki-twoslash@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/remark-shiki-twoslash/-/remark-shiki-twoslash-1.3.0.tgz#64b3646a849928abc599767d13d23f058a96f1df" - integrity sha512-2xnVRFhjJ+Q2w62PNzcZKpm7qJKAsmu5+sEwfDKCK3gE6CaKzUkIrv/RSxbnqLJ9rWLLhdF1o+4T8WvRwF+JmQ== +"remark-shiki-twoslash@file:../../packages/remark-shiki-twoslash": + version "1.4.8" dependencies: - "@typescript/twoslash" "1.1.7" + "@typescript/twoslash" "2.0.1" "@typescript/vfs" "1.3.4" - shiki "^0.9.3" - shiki-twoslash "1.3.1" + regenerator-runtime "^0.13.7" + shiki "0.9.3" + shiki-twoslash "1.5.0" tslib "2.1.0" - typescript "*" + typescript ">3" unist-util-visit "^2.0.0" remark-squeeze-paragraphs@4.0.0: @@ -7606,17 +7599,17 @@ shelljs@^0.8.4: interpret "^1.0.0" rechoir "^0.6.2" -shiki-twoslash@1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/shiki-twoslash/-/shiki-twoslash-1.3.1.tgz#64ee98895029d781fb898e66e4de1724539d4180" - integrity sha512-BasO1NrB8boky+EybyvQo0j/AMNea3FcUyrJXV0dCElLgmTVTTuNvUQ9DAk3RRNg818pjitU0xdDpopQqGEZbQ== +shiki-twoslash@1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/shiki-twoslash/-/shiki-twoslash-1.5.0.tgz#21625311bac5a4738df97476bfb529747d81210c" + integrity sha512-TCXLEreIeo6x3oUdkva41lv57wv/GjpgbofrFkh26iEcYQZvUva9decmH4PD7U13zRIuEYnQCOY/8mLZZlb8OA== dependencies: - "@typescript/twoslash" "1.1.7" + "@typescript/twoslash" "2.0.1" "@typescript/vfs" "1.3.4" - shiki "^0.9.3" - typescript "*" + shiki "0.9.3" + typescript ">3" -shiki@^0.9.3: +shiki@0.9.3: version "0.9.3" resolved "https://registry.yarnpkg.com/shiki/-/shiki-0.9.3.tgz#7bf7bcf3ed50ca525ec89cc09254abce4264d5ca" integrity sha512-NEjg1mVbAUrzRv2eIcUt3TG7X9svX7l3n3F5/3OdFq+/BxUdmBOeKGiH4icZJBLHy354Shnj6sfBTemea2e7XA== @@ -8193,7 +8186,7 @@ typedarray-to-buffer@^3.1.5: dependencies: is-typedarray "^1.0.0" -typescript@*, typescript@>3, typescript@^4.3.2: +typescript@>3, typescript@^4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.2.tgz#399ab18aac45802d6f2498de5054fcbbe716a805" integrity sha512-zZ4hShnmnoVnAHpVHWpTcxdv7dWP60S2FsydQLV8V5PbS3FifjWFFRiHSWpDJahly88PRyV5teTSLoq4eG7mKw==