From 6fe1b0279fce5a7a0e90ff79746ea0b641da3e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Mart=C3=ADn=20Seery?= Date: Mon, 31 Jan 2022 19:14:07 -0300 Subject: [PATCH] Add Shiki as an alternative to Prism (#2497) * [ci] yarn format * Added shiki to markdown-remark * Upgraded astro shiki * Added minimal example * Changed defaults to match * Replace `shiki` with `astro` classes * Added documentation * Updated Astro code to use new `codeToHtml` * Added changesets * Added basic test * Updated tests a bit Co-authored-by: JuanM04 --- .changeset/rotten-planets-love.md | 5 ++ .changeset/tiny-owls-dress.md | 5 ++ docs/src/pages/en/guides/markdown-content.md | 22 ++++++- examples/with-markdown-shiki/.gitignore | 17 ++++++ examples/with-markdown-shiki/.npmrc | 2 + examples/with-markdown-shiki/.stackblitzrc | 6 ++ examples/with-markdown-shiki/README.md | 12 ++++ examples/with-markdown-shiki/astro.config.mjs | 22 +++++++ examples/with-markdown-shiki/package.json | 14 +++++ .../with-markdown-shiki/public/favicon.ico | Bin 0 -> 4286 bytes .../with-markdown-shiki/sandbox.config.json | 11 ++++ .../src/layouts/main.astro | 20 +++++++ .../with-markdown-shiki/src/pages/index.md | 14 +++++ .../with-markdown-shiki/src/styles/global.css | 54 ++++++++++++++++++ examples/with-markdown-shiki/tsconfig.json | 5 ++ packages/astro/components/Code.astro | 2 +- packages/astro/package.json | 2 +- .../astro/test/astro-markdown-shiki.test.js | 53 +++++++++++++++++ .../src/layouts/content.astro | 10 ++++ .../src/pages/astro.astro | 18 ++++++ .../astro-markdown-shiki/src/pages/index.md | 24 ++++++++ packages/markdown/remark/package.json | 1 + packages/markdown/remark/src/index.ts | 10 +++- packages/markdown/remark/src/remark-shiki.ts | 23 ++++++++ packages/markdown/remark/src/types.ts | 5 +- yarn.lock | 8 +-- 26 files changed, 356 insertions(+), 9 deletions(-) create mode 100644 .changeset/rotten-planets-love.md create mode 100644 .changeset/tiny-owls-dress.md create mode 100644 examples/with-markdown-shiki/.gitignore create mode 100644 examples/with-markdown-shiki/.npmrc create mode 100644 examples/with-markdown-shiki/.stackblitzrc create mode 100644 examples/with-markdown-shiki/README.md create mode 100644 examples/with-markdown-shiki/astro.config.mjs create mode 100644 examples/with-markdown-shiki/package.json create mode 100644 examples/with-markdown-shiki/public/favicon.ico create mode 100644 examples/with-markdown-shiki/sandbox.config.json create mode 100644 examples/with-markdown-shiki/src/layouts/main.astro create mode 100644 examples/with-markdown-shiki/src/pages/index.md create mode 100644 examples/with-markdown-shiki/src/styles/global.css create mode 100644 examples/with-markdown-shiki/tsconfig.json create mode 100644 packages/astro/test/astro-markdown-shiki.test.js create mode 100644 packages/astro/test/fixtures/astro-markdown-shiki/src/layouts/content.astro create mode 100644 packages/astro/test/fixtures/astro-markdown-shiki/src/pages/astro.astro create mode 100644 packages/astro/test/fixtures/astro-markdown-shiki/src/pages/index.md create mode 100644 packages/markdown/remark/src/remark-shiki.ts diff --git a/.changeset/rotten-planets-love.md b/.changeset/rotten-planets-love.md new file mode 100644 index 000000000000..aa67307ebf34 --- /dev/null +++ b/.changeset/rotten-planets-love.md @@ -0,0 +1,5 @@ +--- +'@astrojs/markdown-remark': patch +--- + +Add Shiki as an alternative to Prism diff --git a/.changeset/tiny-owls-dress.md b/.changeset/tiny-owls-dress.md new file mode 100644 index 000000000000..82e35f3a2694 --- /dev/null +++ b/.changeset/tiny-owls-dress.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Bumped Shiki version diff --git a/docs/src/pages/en/guides/markdown-content.md b/docs/src/pages/en/guides/markdown-content.md index 8233a8af47f7..b5e33d5b28f3 100644 --- a/docs/src/pages/en/guides/markdown-content.md +++ b/docs/src/pages/en/guides/markdown-content.md @@ -33,7 +33,6 @@ In addition to custom components inside the [`` component](/en/guides/ - [GitHub-flavored Markdown](https://github.com/remarkjs/remark-gfm) - [remark-smartypants](https://github.com/silvenon/remark-smartypants) - [rehype-slug](https://github.com/rehypejs/rehype-slug) -- [Prism](https://prismjs.com/) Also, Astro supports third-party plugins for Markdown. You can provide your plugins in `astro.config.mjs`. @@ -85,6 +84,27 @@ export default { }; ``` +### Syntax Highlighting + +Astro comes with built-in support for [Prism](https://prismjs.com/) and [Shiki](https://shiki.matsu.io/). By default, Prism is enabled. You can modify this behavior by updating the `@astrojs/markdown-remark` options: + +```js +// astro.config.mjs +export default { + markdownOptions: { + render: [ + '@astrojs/markdown-remark', + { + // Pick a syntax highlighter. Can be 'prism' (default), 'shiki' or false to disable any highlighting. + syntaxHighlight: 'prism', + // If you are using shiki, here you can define a global theme. + shikiTheme: 'github-dark', + }, + ], + }, +}; +``` + ## Markdown Pages Astro treats any `.md` files inside of the `/src/pages` directory as pages. These files can contain frontmatter, but are otherwise processed as plain markdown files and do not support components. If you're looking to embed rich components in your markdown, take a look at the [Markdown Component](#astros-markdown-component) section. diff --git a/examples/with-markdown-shiki/.gitignore b/examples/with-markdown-shiki/.gitignore new file mode 100644 index 000000000000..c82467453074 --- /dev/null +++ b/examples/with-markdown-shiki/.gitignore @@ -0,0 +1,17 @@ +# build output +dist + +# dependencies +node_modules/ + +# logs +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# environment variables +.env +.env.production + +# macOS-specific files +.DS_Store diff --git a/examples/with-markdown-shiki/.npmrc b/examples/with-markdown-shiki/.npmrc new file mode 100644 index 000000000000..0cc653b2c3ac --- /dev/null +++ b/examples/with-markdown-shiki/.npmrc @@ -0,0 +1,2 @@ +## force pnpm to hoist +shamefully-hoist = true \ No newline at end of file diff --git a/examples/with-markdown-shiki/.stackblitzrc b/examples/with-markdown-shiki/.stackblitzrc new file mode 100644 index 000000000000..43798ecff844 --- /dev/null +++ b/examples/with-markdown-shiki/.stackblitzrc @@ -0,0 +1,6 @@ +{ + "startCommand": "npm start", + "env": { + "ENABLE_CJS_IMPORTS": true + } +} \ No newline at end of file diff --git a/examples/with-markdown-shiki/README.md b/examples/with-markdown-shiki/README.md new file mode 100644 index 000000000000..d97d1855d89e --- /dev/null +++ b/examples/with-markdown-shiki/README.md @@ -0,0 +1,12 @@ +# Astro Example: Markdown with Shiki + +``` +npm init astro -- --template with-markdown-shiki +``` + +[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/withastro/astro/tree/latest/examples/with-markdown) + +This example showcases Astro's [built-in Markdown support](../../docs/markdown.md). + +- `src/pages/index.astro` uses Astro's `` component. +- `src/pages/other.md` is a treated as a page entrypoint and uses a `layout`. diff --git a/examples/with-markdown-shiki/astro.config.mjs b/examples/with-markdown-shiki/astro.config.mjs new file mode 100644 index 000000000000..bb2a783215d7 --- /dev/null +++ b/examples/with-markdown-shiki/astro.config.mjs @@ -0,0 +1,22 @@ +// Full Astro Configuration API Documentation: +// https://docs.astro.build/reference/configuration-reference + +// @type-check enabled! +// VSCode and other TypeScript-enabled text editors will provide auto-completion, +// helpful tooltips, and warnings if your exported object is invalid. +// You can disable this by removing "@ts-check" and `@type` comments below. +import astroRemark from '@astrojs/markdown-remark'; + +// @ts-check +export default /** @type {import('astro').AstroUserConfig} */ ({ + // Enable Custom Markdown options, plugins, etc. + markdownOptions: { + render: [ + astroRemark, + { + syntaxHighlight: 'shiki', + shikiTheme: 'dracula', + }, + ], + }, +}); diff --git a/examples/with-markdown-shiki/package.json b/examples/with-markdown-shiki/package.json new file mode 100644 index 000000000000..1573fe5863ab --- /dev/null +++ b/examples/with-markdown-shiki/package.json @@ -0,0 +1,14 @@ +{ + "name": "@example/with-markdown-shiki", + "version": "0.0.1", + "private": true, + "scripts": { + "dev": "astro dev", + "start": "astro dev", + "build": "astro build", + "preview": "astro preview" + }, + "devDependencies": { + "astro": "^0.22.20" + } +} diff --git a/examples/with-markdown-shiki/public/favicon.ico b/examples/with-markdown-shiki/public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..578ad458b8906c08fbed84f42b045fea04db89d1 GIT binary patch literal 4286 zcmchZF=!M)6ox0}Fc8GdTHG!cdIY>nA!3n2f|wxIl0rn}Hl#=uf>?-!2r&jMEF^_k zh**lGut*gwBmoNv7AaB&2~nbzULg{WBhPQ{ZVzvF_HL8Cb&hv$_s#qN|IO^o>?+mA zuTW6tU%k~z<&{z+7$G%*nRsTcEO|90xy<-G5&JTt%CgZZCDT4%R?+{Vd^wh>P8_)} z`+dF$HQb9!>1o`Ivn;GInlCw{9T@Rt%q+d^T3Ke%cxkk;$v`{s^zCB9nHAv6w$Vbn z8fb<+eQTNM`;rf9#obfGnV#3+OQEUv4gU;{oA@zol%keY9-e>4W>p7AHmH~&!P7f7!Uj` zwgFeQ=<3G4O;mwWO`L!=R-=y3_~-DPjH3W^3f&jjCfC$o#|oGaahSL`_=f?$&Aa+W z2h8oZ+@?NUcjGW|aWJfbM*ZzxzmCPY`b~RobNrrj=rd`=)8-j`iSW64@0_b6?;GYk zNB+-fzOxlqZ?`y{OA$WigtZXa8)#p#=DPYxH=VeC_Q5q9Cv`mvW6*zU&Gnp1;oPM6 zaK_B3j(l^FyJgYeE9RrmDyhE7W2}}nW%ic#0v@i1E!yTey$W)U>fyd+!@2hWQ!Wa==NAtKoj`f3tp4y$Al`e;?)76?AjdaRR>|?&r)~3Git> zb1)a?uiv|R0_{m#A9c;7)eZ1y6l@yQ#oE*>(Z2fG-&&smPa2QTW>m*^K65^~`coP$ z8y5Y?iS<4Gz{Zg##$1mk)u-0;X|!xu^FCr;ce~X<&UWE&pBgqfYmEJTzpK9I%vr%b z3Ksd6qlPJLI%HFfeXK_^|BXiKZC>Ocu(Kk6hD3G-8usLzVG^q00Qh gz)s7ge@$ApxGu7=(6IGIk+uG&HTev01^#CH3$(Wk5&!@I literal 0 HcmV?d00001 diff --git a/examples/with-markdown-shiki/sandbox.config.json b/examples/with-markdown-shiki/sandbox.config.json new file mode 100644 index 000000000000..9178af77d7de --- /dev/null +++ b/examples/with-markdown-shiki/sandbox.config.json @@ -0,0 +1,11 @@ +{ + "infiniteLoopProtection": true, + "hardReloadOnChange": false, + "view": "browser", + "template": "node", + "container": { + "port": 3000, + "startScript": "start", + "node": "14" + } +} diff --git a/examples/with-markdown-shiki/src/layouts/main.astro b/examples/with-markdown-shiki/src/layouts/main.astro new file mode 100644 index 000000000000..1c4441a1187d --- /dev/null +++ b/examples/with-markdown-shiki/src/layouts/main.astro @@ -0,0 +1,20 @@ +--- +const { content } = Astro.props; +--- + + + + + + + + {content.title} + + + + + + + diff --git a/examples/with-markdown-shiki/src/pages/index.md b/examples/with-markdown-shiki/src/pages/index.md new file mode 100644 index 000000000000..89e58184f42b --- /dev/null +++ b/examples/with-markdown-shiki/src/pages/index.md @@ -0,0 +1,14 @@ +--- +title: Shiki demo +layout: ../layouts/main.astro +--- + +# Shiki demo + +```js +var foo = 'bar'; + +function doSomething() { + return foo; +} +``` diff --git a/examples/with-markdown-shiki/src/styles/global.css b/examples/with-markdown-shiki/src/styles/global.css new file mode 100644 index 000000000000..e8b9d031421f --- /dev/null +++ b/examples/with-markdown-shiki/src/styles/global.css @@ -0,0 +1,54 @@ +pre, +code { + color: #d4d4d4; + font-size: 14px; + font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; + line-height: 1.5; + direction: ltr; + white-space: pre; + text-align: left; + text-shadow: none; + word-break: normal; + word-spacing: normal; + -moz-tab-size: 4; + -o-tab-size: 4; + tab-size: 4; + -webkit-hyphens: none; + -moz-hyphens: none; + -ms-hyphens: none; + hyphens: none; +} + +pre::selection, +code::selection { + text-shadow: none; + background: #b3d4fc; +} + +@media print { + pre, + code { + text-shadow: none; + } +} + +pre { + margin: 0.5rem 0 16px; + padding: 0.8rem 1rem 0.9rem; + overflow: auto; + background: #282a36; + border-radius: 4px; +} + +:not(pre) > code { + padding: 0.1em 0.3em; + color: #db4c69; + background: #f9f2f4; + border-radius: 0.3em; + white-space: pre-wrap; +} + +body { + max-width: 900px; + margin: auto; +} diff --git a/examples/with-markdown-shiki/tsconfig.json b/examples/with-markdown-shiki/tsconfig.json new file mode 100644 index 000000000000..8e881cf9c26c --- /dev/null +++ b/examples/with-markdown-shiki/tsconfig.json @@ -0,0 +1,5 @@ +{ + "compilerOptions": { + "moduleResolution": "node" + } +} diff --git a/packages/astro/components/Code.astro b/packages/astro/components/Code.astro index acebfce04381..37f23a99a594 100644 --- a/packages/astro/components/Code.astro +++ b/packages/astro/components/Code.astro @@ -43,7 +43,7 @@ function repairShikiTheme(html: string): string { } const highlighter = await shiki.getHighlighter({ theme }); -const _html = highlighter.codeToHtml(code, lang); +const _html = highlighter.codeToHtml(code, { lang }); const html = repairShikiTheme(_html); --- diff --git a/packages/astro/package.json b/packages/astro/package.json index 2891c1797f90..a1b3b11c6f27 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -97,7 +97,7 @@ "sass": "^1.43.4", "semver": "^7.3.5", "send": "^0.17.1", - "shiki": "^0.9.10", + "shiki": "^0.10.0", "shorthash": "^0.0.2", "slash": "^4.0.0", "sourcemap-codec": "^1.4.8", diff --git a/packages/astro/test/astro-markdown-shiki.test.js b/packages/astro/test/astro-markdown-shiki.test.js new file mode 100644 index 000000000000..f5e254e1b196 --- /dev/null +++ b/packages/astro/test/astro-markdown-shiki.test.js @@ -0,0 +1,53 @@ +import { expect } from 'chai'; +import cheerio from 'cheerio'; +import { loadFixture } from './test-utils.js'; +import markdownRemark from '@astrojs/markdown-remark'; + +describe('Astro Markdown Shiki', () => { + let fixture; + + before(async () => { + fixture = await loadFixture({ + projectRoot: './fixtures/astro-markdown-shiki/', + markdownOptions: { + render: [ + markdownRemark, + { + syntaxHighlight: 'shiki', + shikiTheme: 'github-light', + }, + ], + }, + buildOptions: { + sitemap: false, + }, + }); + await fixture.build(); + }); + + it('Can render markdown with shiki', async () => { + const html = await fixture.readFile('/index.html'); + const $ = cheerio.load(html); + + // There should be no HTML from Prism + expect($('.token')).to.have.lengthOf(0); + + expect($('pre')).to.have.lengthOf(1); + expect($('pre').hasClass('astro-code')).to.equal(true); + expect($('pre').attr().style).to.equal('background-color: #ffffff'); + }); + + it('Can render Astro with shiki', async () => { + const html = await fixture.readFile('/astro/index.html'); + const $ = cheerio.load(html); + + // There should be no HTML from Prism + expect($('.token')).to.have.lengthOf(0); + + expect($('pre')).to.have.lengthOf(2); + + expect($('span.line')).to.have.lengthOf(2); + expect($('span.line').get(0).children).to.have.lengthOf(1); + expect($('span.line').get(1).children).to.have.lengthOf(5); + }); +}); diff --git a/packages/astro/test/fixtures/astro-markdown-shiki/src/layouts/content.astro b/packages/astro/test/fixtures/astro-markdown-shiki/src/layouts/content.astro new file mode 100644 index 000000000000..925a243a9368 --- /dev/null +++ b/packages/astro/test/fixtures/astro-markdown-shiki/src/layouts/content.astro @@ -0,0 +1,10 @@ + + + + + +
+ +
+ + diff --git a/packages/astro/test/fixtures/astro-markdown-shiki/src/pages/astro.astro b/packages/astro/test/fixtures/astro-markdown-shiki/src/pages/astro.astro new file mode 100644 index 000000000000..d3a3493a65e5 --- /dev/null +++ b/packages/astro/test/fixtures/astro-markdown-shiki/src/pages/astro.astro @@ -0,0 +1,18 @@ +--- +import { Markdown } from 'astro/components'; +import Layout from '../layouts/content.astro'; +--- + + + + # Hello world + + ``` + plaintext + ``` + + ```js + console.log('JavaScript') + ``` + + diff --git a/packages/astro/test/fixtures/astro-markdown-shiki/src/pages/index.md b/packages/astro/test/fixtures/astro-markdown-shiki/src/pages/index.md new file mode 100644 index 000000000000..a75170537cce --- /dev/null +++ b/packages/astro/test/fixtures/astro-markdown-shiki/src/pages/index.md @@ -0,0 +1,24 @@ +--- +layout: ../layouts/content.astro +--- + +# Hello world + +```yaml +apiVersion: v3 +kind: Pod +metadata: + name: rss-site + labels: + app: web +spec: + containers: + - name: front-end + image: nginx + ports: + - containerPort: 80 + - name: rss-reader + image: nickchase/rss-php-nginx:v1 + ports: + - containerPort: 88 +``` diff --git a/packages/markdown/remark/package.json b/packages/markdown/remark/package.json index 6a0a69b45461..00a023d1d886 100644 --- a/packages/markdown/remark/package.json +++ b/packages/markdown/remark/package.json @@ -38,6 +38,7 @@ "remark-parse": "^10.0.1", "remark-rehype": "^10.0.1", "remark-smartypants": "^2.0.0", + "shiki": "^0.10.0", "unified": "^10.1.1", "unist-util-map": "^3.0.0", "unist-util-visit": "^4.1.0" diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index e8242279a631..78d64522724e 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -9,6 +9,7 @@ import { remarkJsx, loadRemarkJsx } from './remark-jsx.js'; import rehypeJsx from './rehype-jsx.js'; import rehypeEscape from './rehype-escape.js'; import remarkPrism from './remark-prism.js'; +import remarkShiki from './remark-shiki.js'; import remarkUnwrap from './remark-unwrap.js'; import { loadPlugins } from './load-plugins.js'; @@ -37,6 +38,8 @@ export async function renderMarkdown(content: string, opts?: MarkdownRenderingOp let { remarkPlugins = [], rehypePlugins = [] } = opts ?? {}; const scopedClassName = opts?.$?.scopedClassName; const mode = opts?.mode ?? 'mdx'; + const syntaxHighlight = opts?.syntaxHighlight ?? 'prism'; + const shikiTheme = opts?.shikiTheme ?? 'github-dark'; const isMDX = mode === 'mdx'; const { headers, rehypeCollectHeaders } = createCollectHeaders(); @@ -64,7 +67,12 @@ export async function renderMarkdown(content: string, opts?: MarkdownRenderingOp parser.use([scopedStyles(scopedClassName)]); } - parser.use([remarkPrism(scopedClassName)]); + if (syntaxHighlight === 'prism') { + parser.use([remarkPrism(scopedClassName)]); + } else if (syntaxHighlight === 'shiki') { + parser.use([await remarkShiki(shikiTheme)]); + } + parser.use([[markdownToHtml as any, { allowDangerousHtml: true, passThrough: ['raw', 'mdxTextExpression', 'mdxJsxTextElement', 'mdxJsxFlowElement'] }]]); loadedRehypePlugins.forEach(([plugin, opts]) => { diff --git a/packages/markdown/remark/src/remark-shiki.ts b/packages/markdown/remark/src/remark-shiki.ts new file mode 100644 index 000000000000..5becad76d7e1 --- /dev/null +++ b/packages/markdown/remark/src/remark-shiki.ts @@ -0,0 +1,23 @@ +import shiki from 'shiki'; +import { visit } from 'unist-util-visit'; + +const remarkShiki = async (theme: shiki.Theme) => { + const highlighter = await shiki.getHighlighter({ theme }); + + return () => (tree: any) => { + visit(tree, 'code', (node) => { + let html = highlighter.codeToHtml(node.value, { lang: node.lang ?? 'plaintext' }); + + // Replace "shiki" class naming with "astro". + html = html.replace('