Refactor createShikiHighlighter (#11825)

Co-authored-by: Sarah Rainsberger <[email protected]>

.changeset/breezy-colts-promise.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/markdoc': patch
+---
+
+Uses latest version of `@astrojs/markdown-remark` with updated Shiki APIs

.changeset/hungry-jokes-try.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/markdown-remark': major
+---
+
+Updates return object of `createShikiHighlighter` as `codeToHast` and `codeToHtml` to allow generating either the hast or html string directly

.changeset/large-zebras-sniff.md

@@ -0,0 +1,9 @@
+---
+'astro': major
+---
+
+Updates internal Shiki rehype plugin to highlight code blocks as hast (using Shiki's `codeToHast()` API). This allows a more direct Markdown and MDX processing, and improves the performance when building the project, but may cause issues with existing Shiki transformers.
+
+If you are using Shiki transformers passed to `markdown.shikiConfig.transformers`, you must make sure they do not use the `postprocess` hook as it no longer runs on code blocks in `.md` and `.mdx` files. (See [the Shiki documentation on transformer hooks](https://shiki.style/guide/transformers#transformer-hooks) for more information). 
+
+Code blocks in `.mdoc` files and `<Code />` component do not use the internal Shiki rehype plugin and are unaffected.

packages/astro/components/Code.astro

@@ -111,13 +111,13 @@ const highlighter = await getCachedHighlighter({
 	],
 	theme,
 	themes,
-	defaultColor,
-	wrap,
-	transformers,
 });
 
-const html = await highlighter.highlight(code, typeof lang === 'string' ? lang : lang.name, {
+const html = await highlighter.codeToHtml(code, typeof lang === 'string' ? lang : lang.name, {
+	defaultColor,
+	wrap,
 	inline,
+	transformers,
 	meta,
 	attributes: rest as any,
 });

packages/integrations/markdoc/src/extensions/shiki.ts

@@ -5,7 +5,11 @@ import { unescapeHTML } from 'astro/runtime/server/index.js';
 import type { AstroMarkdocConfig } from '../config.js';
 
 export default async function shiki(config?: ShikiConfig): Promise<AstroMarkdocConfig> {
-	const highlighter = await createShikiHighlighter(config);
+	const highlighter = await createShikiHighlighter({
+		langs: config?.langs,
+		theme: config?.theme,
+		themes: config?.themes,
+	});
 
 	return {
 		nodes: {
@@ -16,7 +20,11 @@ export default async function shiki(config?: ShikiConfig): Promise<AstroMarkdocC
 					// Only the `js` part is parsed as `attributes.language` and the rest is ignored. This means
 					// some Shiki transformers may not work correctly as it relies on the `meta`.
 					const lang = typeof attributes.language === 'string' ? attributes.language : 'plaintext';
-					const html = await highlighter.highlight(attributes.content, lang);
+					const html = await highlighter.codeToHtml(attributes.content, lang, {
+						wrap: config?.wrap,
+						defaultColor: config?.defaultColor,
+						transformers: config?.transformers,
+					});
 
 					// Use `unescapeHTML` to return `HTMLString` for Astro renderer to inline as HTML
 					return unescapeHTML(html) as any;

packages/markdown/remark/src/highlight.ts

@@ -4,7 +4,11 @@ import { toText } from 'hast-util-to-text';
 import { removePosition } from 'unist-util-remove-position';
 import { visitParents } from 'unist-util-visit-parents';
 
-type Highlighter = (code: string, language: string, options?: { meta?: string }) => Promise<string>;
+type Highlighter = (
+	code: string,
+	language: string,
+	options?: { meta?: string },
+) => Promise<Root | string>;
 
 const languagePattern = /\blanguage-(\S+)\b/;
 
@@ -73,12 +77,17 @@ export async function highlightCodeBlocks(tree: Root, highlighter: Highlighter)
 	for (const { node, language, grandParent, parent } of nodes) {
 		const meta = (node.data as any)?.meta ?? node.properties.metastring ?? undefined;
 		const code = toText(node, { whitespace: 'pre' });
-		// TODO: In Astro 5, have `highlighter()` return hast directly to skip expensive HTML parsing and serialization.
-		const html = await highlighter(code, language, { meta });
-		// The replacement returns a root node with 1 child, the `<pr>` element replacement.
-		const replacement = fromHtml(html, { fragment: true }).children[0] as Element;
-		// We just generated this node, so any positional information is invalid.
-		removePosition(replacement);
+		const result = await highlighter(code, language, { meta });
+
+		let replacement: Element;
+		if (typeof result === 'string') {
+			// The replacement returns a root node with 1 child, the `<pre>` element replacement.
+			replacement = fromHtml(result, { fragment: true }).children[0] as Element;
+			// We just generated this node, so any positional information is invalid.
+			removePosition(replacement);
+		} else {
+			replacement = result.children[0] as Element;
+		}
 
 		// We replace the parent in its parent with the new `<pre>` element.
 		const index = grandParent.children.indexOf(parent);

packages/markdown/remark/src/index.ts

@@ -26,7 +26,12 @@ export { rehypeHeadingIds } from './rehype-collect-headings.js';
 export { remarkCollectImages } from './remark-collect-images.js';
 export { rehypePrism } from './rehype-prism.js';
 export { rehypeShiki } from './rehype-shiki.js';
-export { createShikiHighlighter, type ShikiHighlighter } from './shiki.js';
+export {
+	createShikiHighlighter,
+	type ShikiHighlighter,
+	type CreateShikiHighlighterOptions,
+	type ShikiHighlighterHighlightOptions,
+} from './shiki.js';
 export * from './types.js';
 
 export const markdownConfigDefaults: Required<AstroMarkdownOptions> = {

packages/markdown/remark/src/rehype-shiki.ts

@@ -8,9 +8,20 @@ export const rehypeShiki: Plugin<[ShikiConfig?], Root> = (config) => {
 	let highlighterAsync: Promise<ShikiHighlighter> | undefined;
 
 	return async (tree) => {
-		highlighterAsync ??= createShikiHighlighter(config);
+		highlighterAsync ??= createShikiHighlighter({
+			langs: config?.langs,
+			theme: config?.theme,
+			themes: config?.themes,
+		});
 		const highlighter = await highlighterAsync;
 
-		await highlightCodeBlocks(tree, highlighter.highlight);
+		await highlightCodeBlocks(tree, (code, language, options) => {
+			return highlighter.codeToHast(code, language, {
+				meta: options?.meta,
+				wrap: config?.wrap,
+				defaultColor: config?.defaultColor,
+				transformers: config?.transformers,
+			});
+		});
 	};
 };