Add monarch tokenizer as a dedicated package

cspell.yaml

@@ -87,6 +87,7 @@ words:
   - tspd
   - tspwebsitepr
   - tsvs
+  - typespec
   - typespecvs
   - Uhoh
   - uitestresults

packages/monarch/.eslintrc.cjs

@@ -0,0 +1,7 @@
+require("@typespec/eslint-config-typespec/patch/modern-module-resolution");
+
+module.exports = {
+  plugins: ["@typespec/eslint-plugin"],
+  extends: ["@typespec/eslint-config-typespec", "plugin:@typespec/eslint-plugin/recommended"],
+  parserOptions: { tsconfigRootDir: __dirname, project: "tsconfig.config.json" },
+};

packages/monarch/README.md

@@ -0,0 +1,9 @@
+# @typespec/monarch
+
+Provide tokenizer for Monarch for use in [monaco-editor](https://github.com/microsoft/monaco-editor).
+
+## Updating
+
+### Updating the tests
+
+You can copy `./temp/monaco-tests.json`(after running the test) into the `src/basic-languages/typespec/typespec.test.ts` test file in the monaco-editor repository.

packages/monarch/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@typespec/monarch",
+  "private": true,
+  "version": "0.54.0",
+  "author": "Microsoft Corporation",
+  "description": "TypeSpec library providing OpenAPI concepts",
+  "homepage": "https://typespec.io",
+  "readme": "https://github.com/microsoft/typespec/blob/main/README.md",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/microsoft/typespec.git"
+  },
+  "bugs": {
+    "url": "https://github.com/microsoft/typespec/issues"
+  },
+  "keywords": [
+    "typespec"
+  ],
+  "type": "module",
+  "main": "dist/src/index.js",
+  "exports": {
+    ".": "./dist/src/index.js"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "scripts": {
+    "clean": "rimraf ./dist ./temp",
+    "build": "tsc -p .",
+    "watch": "tsc -p . --watch",
+    "test": "vitest run",
+    "test:watch": "vitest -w",
+    "test:ui": "vitest --ui",
+    "test-official": "vitest run --coverage --reporter=junit --reporter=default --no-file-parallelism",
+    "lint": "eslint . --ext .ts --max-warnings=0",
+    "lint:fix": "eslint . --fix --ext .ts"
+  },
+  "files": [
+    "lib/*.tsp",
+    "dist/**",
+    "!dist/test/**"
+  ],
+  "devDependencies": {
+    "@types/node": "~18.11.19",
+    "@typespec/eslint-config-typespec": "workspace:~",
+    "@typespec/eslint-plugin": "workspace:~",
+    "@vitest/coverage-v8": "^1.4.0",
+    "@vitest/ui": "^1.4.0",
+    "c8": "^9.1.0",
+    "eslint": "^8.57.0",
+    "happy-dom": "^14.3.9",
+    "rimraf": "~5.0.5",
+    "typescript": "~5.4.2",
+    "vitest": "^1.4.0"
+  },
+  "dependencies": {
+    "monaco-editor-core": "^0.47.0"
+  }
+}

packages/monarch/src/index.ts

@@ -0,0 +1,3 @@
+import lang from "./typespec-monarch.js";
+
+export default lang;

packages/monarch/src/typespec-monarch.ts

@@ -0,0 +1,95 @@
+import type { languages } from "monaco-editor-core";
+
+const bounded = (text: string) => `\\b${text}\\b`;
+const notBefore = (regex: string) => `(?!${regex})`;
+
+const identifierStart = "[_a-zA-Z]";
+const identifierContinue = "[_a-zA-Z0-9]";
+const identifier = bounded(`${identifierStart}${identifierContinue}*`);
+const directive = bounded(`[_a-zA-Z-0-9]+`);
+
+const keywords = [
+  "import",
+  "model",
+  "scalar",
+  "namespace",
+  "op",
+  "interface",
+  "union",
+  "using",
+  "is",
+  "extends",
+  "enum",
+  "alias",
+  "return",
+  "void",
+  "if",
+  "else",
+  "projection",
+  "dec",
+  "extern",
+  "fn",
+];
+const namedLiterals = ["true", "false", "null", "unknown", "never"];
+const nonCommentWs = `[ \\t\\r\\n]`;
+const numericLiteral = `[0-9]+`;
+
+export default {
+  defaultToken: "",
+  tokenPostfix: ".tsp",
+  brackets: [
+    { open: "{", close: "}", token: "delimiter.curly" },
+    { open: "[", close: "]", token: "delimiter.square" },
+    { open: "(", close: ")", token: "delimiter.parenthesis" },
+  ],
+  symbols: /[=:;<>]+/,
+  keywords,
+  namedLiterals,
+  escapes: `\\\\(u{[0-9A-Fa-f]+}|n|r|t|\\\\|"|\\\${)`,
+  tokenizer: {
+    root: [{ include: "@expression" }, { include: "@whitespace" }],
+    stringVerbatim: [
+      { regex: `(|"|"")[^"]`, action: { token: "string" } },
+      { regex: `"""${notBefore(`"`)}`, action: { token: "string", next: "@pop" } },
+    ],
+    stringLiteral: [
+      { regex: `\\\${`, action: { token: "delimiter.bracket", next: "@bracketCounting" } },
+      { regex: `[^\\\\"$]+`, action: { token: "string" } },
+      { regex: "@escapes", action: { token: "string.escape" } },
+      { regex: `\\\\.`, action: { token: "string.escape.invalid" } },
+      { regex: `"`, action: { token: "string", next: "@pop" } },
+    ],
+    bracketCounting: [
+      { regex: `{`, action: { token: "delimiter.bracket", next: "@bracketCounting" } },
+      { regex: `}`, action: { token: "delimiter.bracket", next: "@pop" } },
+      { include: "@expression" },
+    ],
+    comment: [
+      { regex: `[^\\*]+`, action: { token: "comment" } },
+      { regex: `\\*\\/`, action: { token: "comment", next: "@pop" } },
+      { regex: `[\\/*]`, action: { token: "comment" } },
+    ],
+    whitespace: [
+      { regex: nonCommentWs },
+      { regex: `\\/\\*`, action: { token: "comment", next: "@comment" } },
+      { regex: `\\/\\/.*$`, action: { token: "comment" } },
+    ],
+    expression: [
+      { regex: `"""`, action: { token: "string", next: "@stringVerbatim" } },
+      { regex: `"${notBefore(`""`)}`, action: { token: "string", next: "@stringLiteral" } },
+      { regex: numericLiteral, action: { token: "number" } },
+      {
+        regex: identifier,
+        action: {
+          cases: {
+            "@keywords": { token: "keyword" },
+            "@namedLiterals": { token: "keyword" },
+            "@default": { token: "identifier" },
+          },
+        },
+      },
+      { regex: `@${identifier}`, action: { token: "tag" } },
+      { regex: `#${directive}`, action: { token: "directive" } },
+    ],
+  },
+} satisfies languages.IMonarchLanguage;