Bump shikiji, use transformers API, expose transformers API

.changeset/polite-dogs-join.md

@@ -0,0 +1,5 @@
+---
+"@astrojs/markdoc": patch
+---
+
+Removes unnecessary `shikiji` dependency

.changeset/six-scissors-worry.md

@@ -0,0 +1,6 @@
+---
+"@astrojs/markdown-remark": minor
+"astro": minor
+---
+
+Bumps shikiji and exposes a new `markdown.shikiConfig.transformers` config option. You can use this option to transform the shikiji hast (AST format of the generated HTML) to extend the features of the final HTML. See [shikiji's documentation](https://shikiji.netlify.app/guide/transformers) for more details.

packages/astro/package.json

@@ -166,7 +166,7 @@
     "resolve": "^1.22.4",
     "semver": "^7.5.4",
     "server-destroy": "^1.0.1",
-    "shikiji": "^0.6.13",
+    "shikiji": "^0.9.17",
     "string-width": "^7.0.0",
     "strip-ansi": "^7.1.0",
     "tsconfck": "^3.0.0",
@@ -224,6 +224,7 @@
     "remark-code-titles": "^0.1.2",
     "rollup": "^4.5.0",
     "sass": "^1.69.5",
+    "shikiji-core": "^0.9.17",
     "srcset-parse": "^1.1.0",
     "unified": "^11.0.4"
   },

packages/astro/src/core/config/schema.ts

@@ -17,9 +17,11 @@ import { appendForwardSlash, prependForwardSlash, removeTrailingForwardSlash } f
 // This import is required to appease TypeScript!
 // See https://github.com/withastro/astro/pull/8762
 import 'mdast-util-to-hast';
+import 'shikiji-core';
 
 type ShikiLangs = NonNullable<ShikiConfig['langs']>;
 type ShikiTheme = NonNullable<ShikiConfig['theme']>;
+type ShikiTransformers = NonNullable<ShikiConfig['transformers']>;
 
 const ASTRO_CONFIG_DEFAULTS = {
 	root: '.',
@@ -274,6 +276,10 @@ export const AstroConfigSchema = z.object({
 						)
 						.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.experimentalThemes!),
 					wrap: z.boolean().or(z.null()).default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.wrap!),
+					transformers: z
+						.custom<ShikiTransformers[number]>()
+						.array()
+						.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.transformers!),
 				})
 				.default({}),
 			remarkPlugins: z

packages/astro/src/core/errors/dev/vite.ts

@@ -1,7 +1,6 @@
-import { replaceCssVariables } from '@astrojs/markdown-remark';
 import * as fs from 'node:fs';
 import { fileURLToPath } from 'node:url';
-import { codeToHtml } from 'shikiji';
+import { codeToHtml, createCssVariablesTheme } from 'shikiji';
 import type { ErrorPayload } from 'vite';
 import type { ModuleLoader } from '../../module-loader/index.js';
 import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js';
@@ -124,7 +123,11 @@ export interface AstroErrorPayload {
 // Map these to `.js` during error highlighting.
 const ALTERNATIVE_JS_EXTS = ['cjs', 'mjs'];
 const ALTERNATIVE_MD_EXTS = ['mdoc'];
-const INLINE_STYLE_SELECTOR_GLOBAL = /style="(.*?)"/g;
+
+let _cssVariablesTheme: ReturnType<typeof createCssVariablesTheme>;
+const cssVariablesTheme = () =>
+	_cssVariablesTheme ??
+	(_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' }));
 
 /**
  * Generate a payload for Vite's error overlay
@@ -147,21 +150,15 @@ export async function getViteErrorPayload(err: ErrorWithMetadata): Promise<Astro
 	if (ALTERNATIVE_MD_EXTS.includes(highlighterLang ?? '')) {
 		highlighterLang = 'md';
 	}
-	let highlightedCode = err.fullCode
+	const highlightedCode = err.fullCode
 		? await codeToHtml(err.fullCode, {
 				// @ts-expect-error always assume that shiki can accept the lang string
 				lang: highlighterLang,
-				theme: 'css-variables',
+				theme: cssVariablesTheme(),
 				lineOptions: err.loc?.line ? [{ line: err.loc.line, classes: ['error-line'] }] : undefined,
 			})
 		: undefined;
 
-	if (highlightedCode) {
-		highlightedCode = highlightedCode.replace(INLINE_STYLE_SELECTOR_GLOBAL, (m) =>
-			replaceCssVariables(m)
-		);
-	}
-
 	return {
 		type: 'error',
 		err: {

packages/astro/src/core/errors/overlay.ts

@@ -68,7 +68,7 @@ const style = /* css */ `
     --toggle-border-color: #C3CADB;
 
   /* Syntax Highlighting */
-  --astro-code-color-text: #000000;
+  --astro-code-foreground: #000000;
   --astro-code-token-constant: #4ca48f;
   --astro-code-token-string: #9f722a;
   --astro-code-token-comment: #8490b5;
@@ -121,7 +121,7 @@ const style = /* css */ `
   --toggle-border-color: #3D4663;
 
   /* Syntax Highlighting */
-  --astro-code-color-text: #ffffff;
+  --astro-code-foreground: #ffffff;
   --astro-code-token-constant: #90f4e3;
   --astro-code-token-string: #f4cf90;
   --astro-code-token-comment: #8490b5;

packages/integrations/markdoc/package.json

@@ -71,7 +71,6 @@
     "gray-matter": "^4.0.3",
     "htmlparser2": "^9.0.0",
     "kleur": "^4.1.5",
-    "shikiji": "^0.6.13",
     "zod": "^3.22.4"
   },
   "peerDependencies": {

packages/markdown/remark/package.json

@@ -38,7 +38,7 @@
     "remark-parse": "^11.0.0",
     "remark-rehype": "^11.0.0",
     "remark-smartypants": "^2.0.0",
-    "shikiji": "^0.6.13",
+    "shikiji": "^0.9.17",
     "unified": "^11.0.4",
     "unist-util-visit": "^5.0.0",
     "vfile": "^6.0.1"

packages/markdown/remark/src/index.ts

@@ -36,6 +36,7 @@ export const markdownConfigDefaults: Required<AstroMarkdownOptions> = {
 		theme: 'github-dark',
 		experimentalThemes: {},
 		wrap: false,
+		transformers: [],
 	},
 	remarkPlugins: [],
 	rehypePlugins: [],

packages/markdown/remark/src/shiki.ts

@@ -1,37 +1,37 @@
-import { bundledLanguages, getHighlighter } from 'shikiji';
+import { bundledLanguages, createCssVariablesTheme, getHighlighter } from 'shikiji';
 import { visit } from 'unist-util-visit';
 import type { ShikiConfig } from './types.js';
 
 export interface ShikiHighlighter {
 	highlight(code: string, lang?: string, options?: { inline?: boolean }): string;
 }
 
+// TODO: Remove this special replacement in Astro 5
 const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
-	'#000001': 'var(--astro-code-color-text)',
-	'#000002': 'var(--astro-code-color-background)',
-	'#000004': 'var(--astro-code-token-constant)',
-	'#000005': 'var(--astro-code-token-string)',
-	'#000006': 'var(--astro-code-token-comment)',
-	'#000007': 'var(--astro-code-token-keyword)',
-	'#000008': 'var(--astro-code-token-parameter)',
-	'#000009': 'var(--astro-code-token-function)',
-	'#000010': 'var(--astro-code-token-string-expression)',
-	'#000011': 'var(--astro-code-token-punctuation)',
-	'#000012': 'var(--astro-code-token-link)',
+	'--astro-code-foreground': '--astro-code-color-text',
+	'--astro-code-background': '--astro-code-color-background',
 };
 const COLOR_REPLACEMENT_REGEX = new RegExp(
 	`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
 	'g'
 );
 
+let _cssVariablesTheme: ReturnType<typeof createCssVariablesTheme>;
+const cssVariablesTheme = () =>
+	_cssVariablesTheme ??
+	(_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' }));
+
 export async function createShikiHighlighter({
 	langs = [],
 	theme = 'github-dark',
 	experimentalThemes = {},
 	wrap = false,
+	transformers = [],
 }: ShikiConfig = {}): Promise<ShikiHighlighter> {
 	const themes = experimentalThemes;
 
+	theme = theme === 'css-variables' ? cssVariablesTheme() : theme;
+
 	const highlighter = await getHighlighter({
 		langs: langs.length ? langs : Object.keys(bundledLanguages),
 		themes: Object.values(themes).length ? Object.values(themes) : [theme],
@@ -53,74 +53,77 @@ export async function createShikiHighlighter({
 			return highlighter.codeToHtml(code, {
 				...themeOptions,
 				lang,
-				transforms: {
-					pre(node) {
-						// Swap to `code` tag if inline
-						if (inline) {
-							node.tagName = 'code';
-						}
-
-						// Cast to string as shikiji will always pass them as strings instead of any other types
-						const classValue = (node.properties.class as string) ?? '';
-						const styleValue = (node.properties.style as string) ?? '';
-
-						// Replace "shiki" class naming with "astro-code"
-						node.properties.class = classValue.replace(/shiki/g, 'astro-code');
-
-						// Handle code wrapping
-						// if wrap=null, do nothing.
-						if (wrap === false) {
-							node.properties.style = styleValue + '; overflow-x: auto;';
-						} else if (wrap === true) {
-							node.properties.style =
-								styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
-						}
-					},
-					line(node) {
-						// Add "user-select: none;" for "+"/"-" diff symbols.
-						// Transform `<span class="line"><span style="...">+ something</span></span>
-						// into      `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
-						if (lang === 'diff') {
-							const innerSpanNode = node.children[0];
-							const innerSpanTextNode =
-								innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
-
-							if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
-								const start = innerSpanTextNode.value[0];
-								if (start === '+' || start === '-') {
-									innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
-									innerSpanNode.children.unshift({
-										type: 'element',
-										tagName: 'span',
-										properties: { style: 'user-select: none;' },
-										children: [{ type: 'text', value: start }],
-									});
-								}
+				transformers: [
+					{
+						pre(node) {
+							// Swap to `code` tag if inline
+							if (inline) {
+								node.tagName = 'code';
 							}
-						}
-					},
-					code(node) {
-						if (inline) {
-							return node.children[0] as typeof node;
-						}
-					},
-					root(node) {
-						if (Object.values(experimentalThemes).length) {
-							return;
-						}
-
-						// theme.id for shiki -> shikiji compat
-						const themeName = typeof theme === 'string' ? theme : theme.name;
-						if (themeName === 'css-variables') {
-							// Replace special color tokens to CSS variables
-							visit(node as any, 'element', (child) => {
-								if (child.properties?.style) {
-									child.properties.style = replaceCssVariables(child.properties.style);
+
+							// Cast to string as shikiji will always pass them as strings instead of any other types
+							const classValue = (node.properties.class as string) ?? '';
+							const styleValue = (node.properties.style as string) ?? '';
+
+							// Replace "shiki" class naming with "astro-code"
+							node.properties.class = classValue.replace(/shiki/g, 'astro-code');
+
+							// Handle code wrapping
+							// if wrap=null, do nothing.
+							if (wrap === false) {
+								node.properties.style = styleValue + '; overflow-x: auto;';
+							} else if (wrap === true) {
+								node.properties.style =
+									styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
+							}
+						},
+						line(node) {
+							// Add "user-select: none;" for "+"/"-" diff symbols.
+							// Transform `<span class="line"><span style="...">+ something</span></span>
+							// into      `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
+							if (lang === 'diff') {
+								const innerSpanNode = node.children[0];
+								const innerSpanTextNode =
+									innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
+
+								if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
+									const start = innerSpanTextNode.value[0];
+									if (start === '+' || start === '-') {
+										innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
+										innerSpanNode.children.unshift({
+											type: 'element',
+											tagName: 'span',
+											properties: { style: 'user-select: none;' },
+											children: [{ type: 'text', value: start }],
+										});
+									}
 								}
-							});
-						}
+							}
+						},
+						code(node) {
+							if (inline) {
+								return node.children[0] as typeof node;
+							}
+						},
+						root(node) {
+							if (Object.values(experimentalThemes).length) {
+								return;
+							}
+
+							// theme.id for shiki -> shikiji compat
+							const themeName = typeof theme === 'string' ? theme : theme.name;
+							if (themeName === 'css-variables') {
+								// Replace special color tokens to CSS variables
+								visit(node as any, 'element', (child) => {
+									if (child.properties?.style) {
+										child.properties.style = replaceCssVariables(child.properties.style);
+									}
+								});
+							}
+						},
 					},
-				},
+					...transformers,
+				],
 			});
 		},
 	};

packages/markdown/remark/src/types.ts

@@ -4,6 +4,7 @@ import type { Options as RemarkRehypeOptions } from 'remark-rehype';
 import type {
 	BuiltinTheme,
 	LanguageRegistration,
+	ShikijiTransformer,
 	ThemeRegistration,
 	ThemeRegistrationRaw,
 } from 'shikiji';
@@ -32,11 +33,14 @@ export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlug
 
 export type RemarkRehype = RemarkRehypeOptions;
 
+export type ThemePresets = BuiltinTheme | 'css-variables';
+
 export interface ShikiConfig {
 	langs?: LanguageRegistration[];
-	theme?: BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw;
-	experimentalThemes?: Record<string, BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw>;
+	theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw;
+	experimentalThemes?: Record<string, ThemePresets | ThemeRegistration | ThemeRegistrationRaw>;
 	wrap?: boolean | null;
+	transformers?: ShikijiTransformer[];
 }
 
 export interface AstroMarkdownOptions {