From 76cbc7efacd766a4b4e0f56b9b86614caea648d3 Mon Sep 17 00:00:00 2001 From: Steven Levithan Date: Mon, 25 Nov 2024 06:31:32 +0100 Subject: [PATCH] feat: improve js engine support (and run tests) in pre-ES2024 envs (#847) --- docs/guide/regex-engines.md | 2 +- docs/references/engine-js-compat.md | 8 ++++---- packages/engine-javascript/test/compare.test.ts | 5 +---- packages/engine-javascript/test/general.test.ts | 4 +--- packages/shiki/test/core-sync.test.ts | 5 +---- pnpm-lock.yaml | 12 ++++++------ pnpm-workspace.yaml | 2 +- scripts/report-engine-js-compat.ts | 2 +- 8 files changed, 16 insertions(+), 24 deletions(-) diff --git a/docs/guide/regex-engines.md b/docs/guide/regex-engines.md index d6617c41a..d1ff03453 100644 --- a/docs/guide/regex-engines.md +++ b/docs/guide/regex-engines.md @@ -79,7 +79,7 @@ The JavaScript engine is best when running in the browser and in cases when you For the best result, [Oniguruma-To-ES](https://github.com/slevithan/oniguruma-to-es) uses the [RegExp `v` flag](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets), which is available in Node.js v20+ and ES2024 ([Browser compatibility](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets#browser_compatibility)). -For older environments, it can use the `u` flag but somewhat fewer grammars are supported. +For older environments, it can use the `u` flag but this results in a few less grammars being supported. By default, the runtime target is automatically detected. You can override this behavior by setting the `target` option: diff --git a/docs/references/engine-js-compat.md b/docs/references/engine-js-compat.md index e1c90c05e..8eccfc188 100644 --- a/docs/references/engine-js-compat.md +++ b/docs/references/engine-js-compat.md @@ -2,7 +2,7 @@ Compatibility reference of all built-in grammars with the [JavaScript RegExp engine](/guide/regex-engines#javascript-regexp-engine-experimental). -> Generated on Friday, November 22, 2024 +> Generated on Sunday, November 24, 2024 > > Version `1.23.1` > @@ -20,7 +20,7 @@ Compatibility reference of all built-in grammars with the [JavaScript RegExp eng ## Supported Languages Languages that work with the JavaScript RegExp engine, and will produce the same result as the WASM engine (with the [sample snippets in the registry](https://github.com/shikijs/textmate-grammars-themes/tree/main/samples)). -In some edge cases, it's not guaranteed that the the highlighting will be 100% the same. If that happens, please create an issue with the sample snippet. +In some edge cases, it's not guaranteed that the highlighting will be 100% the same. If that happens, please create an issue with the sample snippet. | Language | Highlight Match | Patterns Parsable | Patterns Failed | Diff | | ------------------ | :-------------- | ----------------: | --------------: | ---: | @@ -210,7 +210,7 @@ In some edge cases, it's not guaranteed that the the highlighting will be 100% t | wenyan | ✅ OK | 18 | - | | | wgsl | ✅ OK | 44 | - | | | wikitext | ✅ OK | 104 | - | | -| wolfram | ✅ OK | 501 | - | | +| wolfram | ✅ OK | 501 | - | 10 | | xml | ✅ OK | 169 | - | | | xsl | ✅ OK | 171 | - | | | yaml | ✅ OK | 48 | - | | @@ -254,7 +254,7 @@ Languages that throw with the JavaScript RegExp engine, either because they cont | rst | [🚧 Mismatch](https://textmate-grammars-themes.netlify.app/?grammar=rst) | 1855 | 2 | 62 | | hack | ❌ Error | 947 | 1 | 114 | | purescript | ❌ Error | 72 | 1 | 42 | -| cpp | ❌ Error | 510 | 2 | 27 | +| cpp | ❌ Error | 510 | 2 | 8 | | csharp | ❌ Error | 306 | 3 | 204 | | markdown | ❌ Error | 115 | 3 | 857 | | swift | ❌ Error | 326 | 3 | 40 | diff --git a/packages/engine-javascript/test/compare.test.ts b/packages/engine-javascript/test/compare.test.ts index dbae80803..b63f671e2 100644 --- a/packages/engine-javascript/test/compare.test.ts +++ b/packages/engine-javascript/test/compare.test.ts @@ -142,9 +142,7 @@ const cases: Cases[] = [ }, ] -describe.skipIf( - +process.versions.node.split('.')[0] < 20, -)('cases', async () => { +describe('cases', async () => { await loadWasm(import('@shikijs/core/wasm-inlined')) const resolved = await Promise.all(cases.map(async (c) => { @@ -163,7 +161,6 @@ describe.skipIf( const engineWasm = createWasmOnigLibWrapper() const engineJs = createJavaScriptRegexEngine({ forgiving: true, - target: 'ES2024', }) const shiki1 = await createHighlighterCore({ diff --git a/packages/engine-javascript/test/general.test.ts b/packages/engine-javascript/test/general.test.ts index 4d52e22f2..7fbe236c2 100644 --- a/packages/engine-javascript/test/general.test.ts +++ b/packages/engine-javascript/test/general.test.ts @@ -2,9 +2,7 @@ import { describe, expect, it } from 'vitest' import { createHighlighter } from '../../shiki/src/index' import { createJavaScriptRegexEngine } from '../src' -describe.skipIf( - +process.versions.node.split('.')[0] < 20, -)('should', () => { +describe('should', () => { it('works', async () => { const shiki = await createHighlighter({ themes: ['vitesse-light'], diff --git a/packages/shiki/test/core-sync.test.ts b/packages/shiki/test/core-sync.test.ts index d16cb776c..6afca6624 100644 --- a/packages/shiki/test/core-sync.test.ts +++ b/packages/shiki/test/core-sync.test.ts @@ -4,10 +4,7 @@ import { createHighlighterCoreSync } from '../src/core' import js from '../src/langs/javascript.mjs' import nord from '../src/themes/nord.mjs' -describe.skipIf( - // JavaScript engine requires Node v20+ - +process.versions.node.split('.')[0] < 20, -)('should', () => { +describe('should', () => { const engine = createJavaScriptRegexEngine() it('works', () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 83c797e7c..47241ab67 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -139,8 +139,8 @@ catalogs: specifier: ^1.4.1 version: 1.4.1 oniguruma-to-es: - specifier: 0.6.0 - version: 0.6.0 + specifier: 0.6.1 + version: 0.6.1 picocolors: specifier: ^1.1.1 version: 1.1.1 @@ -553,7 +553,7 @@ importers: version: 9.3.0 oniguruma-to-es: specifier: 'catalog:' - version: 0.6.0 + version: 0.6.1 packages/engine-oniguruma: dependencies: @@ -4250,8 +4250,8 @@ packages: resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} engines: {node: '>=18'} - oniguruma-to-es@0.6.0: - resolution: {integrity: sha512-YaRxSzO6b7hnCzpq3IiyImHPnTNnhqFvhRdhg68gFMAaTEiFm0Wa8vUKSWqVz89a691o14KBPVJTcybjdRA6aQ==} + oniguruma-to-es@0.6.1: + resolution: {integrity: sha512-YQmbeCaNmSrSwuMuTq1oYCvb+jkISsruMu2XQTJAgLvHQkFKEMqJI33NVtq8EGPuRvFa0CHcH32l2CpTS9Fs/Q==} optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} @@ -9502,7 +9502,7 @@ snapshots: dependencies: mimic-function: 5.0.1 - oniguruma-to-es@0.6.0: + oniguruma-to-es@0.6.1: dependencies: emoji-regex-xs: 1.0.0 regex: 5.0.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 73a7a4880..7a01016b8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -53,7 +53,7 @@ catalog: minimist: ^1.2.8 monaco-editor-core: ^0.52.0 ofetch: ^1.4.1 - oniguruma-to-es: 0.6.0 + oniguruma-to-es: 0.6.1 picocolors: ^1.1.1 pinia: ^2.2.6 pnpm: ^9.13.2 diff --git a/scripts/report-engine-js-compat.ts b/scripts/report-engine-js-compat.ts index fd919e99d..5f2dba331 100644 --- a/scripts/report-engine-js-compat.ts +++ b/scripts/report-engine-js-compat.ts @@ -223,7 +223,7 @@ async function run() { '## Supported Languages', '', 'Languages that work with the JavaScript RegExp engine, and will produce the same result as the WASM engine (with the [sample snippets in the registry](https://github.com/shikijs/textmate-grammars-themes/tree/main/samples)).', - 'In some edge cases, it\'s not guaranteed that the the highlighting will be 100% the same. If that happens, please create an issue with the sample snippet.', + 'In some edge cases, it\'s not guaranteed that the highlighting will be 100% the same. If that happens, please create an issue with the sample snippet.', '', createTable(reportOk), '',