From 432ede2983a31bf033b8c3e3e632ea3ab0d08e3e Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 28 Mar 2024 12:58:54 -0700 Subject: [PATCH 01/10] Basics --- packages/monarch/.eslintrc.cjs | 7 ++ packages/monarch/README.md | 3 + packages/monarch/package.json | 59 ++++++++++++++ packages/monarch/src/index.ts | 110 ++++++++++++++++++++++++++ packages/monarch/tsconfig.config.json | 4 + packages/monarch/tsconfig.json | 10 +++ packages/monarch/vitest.config.ts | 4 + pnpm-lock.yaml | 108 +++++++++++++++++-------- tsconfig.ws.json | 1 + 9 files changed, 274 insertions(+), 32 deletions(-) create mode 100644 packages/monarch/.eslintrc.cjs create mode 100644 packages/monarch/README.md create mode 100644 packages/monarch/package.json create mode 100644 packages/monarch/src/index.ts create mode 100644 packages/monarch/tsconfig.config.json create mode 100644 packages/monarch/tsconfig.json create mode 100644 packages/monarch/vitest.config.ts diff --git a/packages/monarch/.eslintrc.cjs b/packages/monarch/.eslintrc.cjs new file mode 100644 index 0000000000..bed8471c3f --- /dev/null +++ b/packages/monarch/.eslintrc.cjs @@ -0,0 +1,7 @@ +require("@typespec/eslint-config-typespec/patch/modern-module-resolution"); + +module.exports = { + plugins: ["@typespec/eslint-plugin"], + extends: ["@typespec/eslint-config-typespec", "plugin:@typespec/eslint-plugin/recommended"], + parserOptions: { tsconfigRootDir: __dirname, project: "tsconfig.config.json" }, +}; diff --git a/packages/monarch/README.md b/packages/monarch/README.md new file mode 100644 index 0000000000..6e37151239 --- /dev/null +++ b/packages/monarch/README.md @@ -0,0 +1,3 @@ +# @typespec/monarch + +Provide tokenizer for Monarch for use in monaco-editor. diff --git a/packages/monarch/package.json b/packages/monarch/package.json new file mode 100644 index 0000000000..6dd67ae349 --- /dev/null +++ b/packages/monarch/package.json @@ -0,0 +1,59 @@ +{ + "name": "@typespec/monarch", + "private": true, + "version": "0.54.0", + "author": "Microsoft Corporation", + "description": "TypeSpec library providing OpenAPI concepts", + "homepage": "https://typespec.io", + "readme": "https://github.com/microsoft/typespec/blob/main/README.md", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/microsoft/typespec.git" + }, + "bugs": { + "url": "https://github.com/microsoft/typespec/issues" + }, + "keywords": [ + "typespec" + ], + "type": "module", + "main": "dist/src/index.js", + "exports": { + ".": "./dist/src/index.js" + }, + "engines": { + "node": ">=18.0.0" + }, + "scripts": { + "clean": "rimraf ./dist ./temp", + "build": "npm run gen-extern-signature && tsc -p . && npm run lint-typespec-library", + "watch": "tsc -p . --watch", + "test": "vitest run", + "test:ui": "vitest --ui", + "test-official": "vitest run --coverage --reporter=junit --reporter=default --no-file-parallelism", + "lint": "eslint . --ext .ts --max-warnings=0", + "lint:fix": "eslint . --fix --ext .ts", + "regen-docs": "tspd doc . --enable-experimental --output-dir ../../docs/libraries/openapi/reference" + }, + "files": [ + "lib/*.tsp", + "dist/**", + "!dist/test/**" + ], + "devDependencies": { + "@types/node": "~18.11.19", + "@typespec/eslint-config-typespec": "workspace:~", + "@typespec/eslint-plugin": "workspace:~", + "@vitest/coverage-v8": "^1.4.0", + "@vitest/ui": "^1.4.0", + "c8": "^9.1.0", + "eslint": "^8.57.0", + "rimraf": "~5.0.5", + "typescript": "~5.4.2", + "vitest": "^1.4.0" + }, + "dependencies": { + "monaco-editor-core": "^0.47.0" + } +} diff --git a/packages/monarch/src/index.ts b/packages/monarch/src/index.ts new file mode 100644 index 0000000000..dc299d1a2b --- /dev/null +++ b/packages/monarch/src/index.ts @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { languages } from "monaco-editor-core"; + +const bounded = (text: string) => `\\b${text}\\b`; +const notBefore = (regex: string) => `(?!${regex})`; + +const identifierStart = "[_a-zA-Z]"; +const identifierContinue = "[_a-zA-Z0-9]"; +const identifier = bounded(`${identifierStart}${identifierContinue}*`); +const directive = bounded(`[_a-zA-Z-0-9]+`); + +const keywords = [ + "metadata", + "targetScope", + "resource", + "module", + "param", + "var", + "output", + "for", + "in", + "if", + "existing", + "import", + "as", + "type", + "with", + "using", + "func", + "assert", + "provider", +]; + +const namedLiterals = ["true", "false", "null"]; + +const nonCommentWs = `[ \\t\\r\\n]`; + +const numericLiteral = `[0-9]+`; + +export const BicepLanguage: languages.IMonarchLanguage = { + defaultToken: "", + tokenPostfix: ".bicep", + + brackets: [ + { open: "{", close: "}", token: "delimiter.curly" }, + { open: "[", close: "]", token: "delimiter.square" }, + { open: "(", close: ")", token: "delimiter.parenthesis" }, + ], + + symbols: /[=>=6.0.0'} dependencies: '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.22 + '@jridgewell/trace-mapping': 0.3.25 /@azure/abort-controller@1.1.0: resolution: {integrity: sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==} @@ -1909,23 +1946,23 @@ packages: resolution: {integrity: sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.22 + '@jridgewell/trace-mapping': 0.3.25 jsesc: 2.5.2 /@babel/helper-annotate-as-pure@7.22.5: resolution: {integrity: sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 dev: false /@babel/helper-builder-binary-assignment-operator-visitor@7.22.15: resolution: {integrity: sha512-QkBXwGgaoC2GtGZRoma6kv7Szfv06khvhFav67ZExau2RaXzy8MpHSMO2PNoP2XtmQphJQRHFfg77Bq731Yizw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 dev: false /@babel/helper-compilation-targets@7.23.6: @@ -1992,26 +2029,26 @@ packages: engines: {node: '>=6.9.0'} dependencies: '@babel/template': 7.23.9 - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 /@babel/helper-hoist-variables@7.22.5: resolution: {integrity: sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 /@babel/helper-member-expression-to-functions@7.23.0: resolution: {integrity: sha512-6gfrPwh7OuT6gZyJZvd6WbTfrqAo7vm4xCzAXOusKqq/vWdKXphTpj5klHKNmRUU6/QRGlBsyU9mAIPaWHlqJA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 dev: false /@babel/helper-module-imports@7.22.15: resolution: {integrity: sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 /@babel/helper-module-transforms@7.23.3(@babel/core@7.24.0): resolution: {integrity: sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==} @@ -2030,7 +2067,7 @@ packages: resolution: {integrity: sha512-HBwaojN0xFRx4yIvpwGqxiV2tUfl7401jlok564NgB9EHS1y6QT17FmKWm4ztqjeVdXLuC4fSvHc5ePpQjoTbw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 dev: false /@babel/helper-plugin-utils@7.22.5: @@ -2071,14 +2108,14 @@ packages: resolution: {integrity: sha512-tK14r66JZKiC43p8Ki33yLBVJKlQDFoA8GYN67lWCDCqoL6EMMSuM9b+Iff2jHaM/RRFYl7K+iiru7hbRqNx8Q==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 dev: false /@babel/helper-split-export-declaration@7.22.6: resolution: {integrity: sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 /@babel/helper-string-parser@7.23.4: resolution: {integrity: sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==} @@ -2098,7 +2135,7 @@ packages: dependencies: '@babel/helper-function-name': 7.23.0 '@babel/template': 7.23.9 - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 dev: false /@babel/helpers@7.24.0: @@ -2124,7 +2161,8 @@ packages: engines: {node: '>=6.0.0'} hasBin: true dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 + dev: true /@babel/parser@7.24.0: resolution: {integrity: sha512-QuP/FxEAzMSjXygs8v4N9dvdXzEHN4W1oF3PxuWAtPo08UdM17u89RDMgjLn/mlc56iM0HlLmVkO/wgR+rDgHg==} @@ -2842,7 +2880,7 @@ packages: '@babel/helper-module-imports': 7.22.15 '@babel/helper-plugin-utils': 7.22.5 '@babel/plugin-syntax-jsx': 7.23.3(@babel/core@7.24.0) - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 dev: false /@babel/plugin-transform-react-pure-annotations@7.23.3(@babel/core@7.24.0): @@ -3099,7 +3137,7 @@ packages: dependencies: '@babel/core': 7.24.0 '@babel/helper-plugin-utils': 7.22.5 - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 esutils: 2.0.3 dev: false @@ -3163,8 +3201,8 @@ packages: engines: {node: '>=6.9.0'} dependencies: '@babel/code-frame': 7.23.5 - '@babel/parser': 7.23.9 - '@babel/types': 7.23.9 + '@babel/parser': 7.24.0 + '@babel/types': 7.24.0 /@babel/template@7.24.0: resolution: {integrity: sha512-Bkf2q8lMB0AFpX0NFEqSbx1OkTHf0f+0j82mkw+ZpzBnkk7e9Ql0891vlfgi+kHwOk8tQjiQHpqh4LaSa0fKEA==} @@ -3184,8 +3222,8 @@ packages: '@babel/helper-function-name': 7.23.0 '@babel/helper-hoist-variables': 7.22.5 '@babel/helper-split-export-declaration': 7.22.6 - '@babel/parser': 7.23.9 - '@babel/types': 7.23.9 + '@babel/parser': 7.24.0 + '@babel/types': 7.24.0 debug: 4.3.4 globals: 11.12.0 transitivePeerDependencies: @@ -3216,6 +3254,7 @@ packages: '@babel/helper-string-parser': 7.23.4 '@babel/helper-validator-identifier': 7.22.20 to-fast-properties: 2.0.0 + dev: true /@babel/types@7.24.0: resolution: {integrity: sha512-+j7a5c253RfKh8iABBhywc8NSfP5LURe7Uh4qpsh6jc+aLJguvmIUBdjSdEMQv2bENrCR5MfRdjGo7vzS/ob7w==} @@ -3717,7 +3756,7 @@ packages: react: ^18.0.0 react-dom: ^18.0.0 dependencies: - '@babel/parser': 7.23.9 + '@babel/parser': 7.24.0 '@babel/traverse': 7.23.9 '@docusaurus/logger': 3.1.1 '@docusaurus/utils': 3.1.1(@docusaurus/types@3.1.1)(@swc/core@1.4.8) @@ -6224,7 +6263,7 @@ packages: dependencies: '@jridgewell/set-array': 1.1.2 '@jridgewell/sourcemap-codec': 1.4.15 - '@jridgewell/trace-mapping': 0.3.22 + '@jridgewell/trace-mapping': 0.3.25 /@jridgewell/resolve-uri@3.1.1: resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} @@ -6238,7 +6277,7 @@ packages: resolution: {integrity: sha512-UTYAUj/wviwdsMfzoSJspJxbkH5o1snzwX0//0ENX1u/55kkZZkcTZP6u9bwKGkv+dkk9at4m1Cpt0uY80kcpQ==} dependencies: '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.22 + '@jridgewell/trace-mapping': 0.3.25 /@jridgewell/sourcemap-codec@1.4.15: resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} @@ -6248,13 +6287,13 @@ packages: dependencies: '@jridgewell/resolve-uri': 3.1.1 '@jridgewell/sourcemap-codec': 1.4.15 + dev: true /@jridgewell/trace-mapping@0.3.25: resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} dependencies: '@jridgewell/resolve-uri': 3.1.1 '@jridgewell/sourcemap-codec': 1.4.15 - dev: true /@jridgewell/trace-mapping@0.3.9: resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} @@ -7490,7 +7529,7 @@ packages: resolution: {integrity: sha512-1hnUxxjd83EAxbL4a0JDJoD3Dao3hmjvyvyEV8PzWmLK3B9m9NPlW7GKjFyoWE8nM7HnXzPcmmSyOW8yOddSXw==} engines: {node: '>=10'} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 entities: 4.5.0 dev: false @@ -7610,6 +7649,7 @@ packages: /@swagger-api/apidom-ns-json-schema-draft-4@0.97.0: resolution: {integrity: sha512-eBMIPxX4huNDGle6TOfSe1kKS1/HvL6w66GWWLFxZW2doCQHMADgjo7j/kVowrXiJtEoMgjBVp3W30WkcwBVug==} + requiresBuild: true dependencies: '@babel/runtime-corejs3': 7.24.0 '@swagger-api/apidom-ast': 0.97.0 @@ -8121,8 +8161,8 @@ packages: /@types/babel__core@7.20.5: resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} dependencies: - '@babel/parser': 7.23.9 - '@babel/types': 7.23.9 + '@babel/parser': 7.24.0 + '@babel/types': 7.24.0 '@types/babel__generator': 7.6.8 '@types/babel__template': 7.4.4 '@types/babel__traverse': 7.20.5 @@ -8131,20 +8171,20 @@ packages: /@types/babel__generator@7.6.8: resolution: {integrity: sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 dev: true /@types/babel__template@7.4.4: resolution: {integrity: sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==} dependencies: - '@babel/parser': 7.23.9 - '@babel/types': 7.23.9 + '@babel/parser': 7.24.0 + '@babel/types': 7.24.0 dev: true /@types/babel__traverse@7.20.5: resolution: {integrity: sha512-WXCyOcRtH37HAUkpXhUduaxdm82b4GSlyTqajXviN4EfiuPgNYR109xMCKvpl6zPIpua0DGlMEDCq+g8EdoheQ==} dependencies: - '@babel/types': 7.23.9 + '@babel/types': 7.24.0 dev: true /@types/body-parser@1.19.5: @@ -14810,6 +14850,10 @@ packages: pkg-types: 1.0.3 ufo: 1.4.0 + /monaco-editor-core@0.47.0: + resolution: {integrity: sha512-xho7dsmSYOIek6bkMKQApWHVyoOBQ1eaBIE6CIlER40paTN0r0efrig6Q9QkCacfzIuNNxNjyb1uMVoMpi8gcA==} + dev: false + /monaco-editor-webpack-plugin@7.1.0(monaco-editor@0.47.0)(webpack@5.90.1): resolution: {integrity: sha512-ZjnGINHN963JQkFqjjcBtn1XBtUATDZBMgNQhDQwd78w2ukRhFXAPNgWuacaQiDZsUr4h1rWv5Mv6eriKuOSzA==} peerDependencies: @@ -18171,7 +18215,7 @@ packages: uglify-js: optional: true dependencies: - '@jridgewell/trace-mapping': 0.3.22 + '@jridgewell/trace-mapping': 0.3.25 '@swc/core': 1.4.8 jest-worker: 27.5.1 schema-utils: 3.3.0 diff --git a/tsconfig.ws.json b/tsconfig.ws.json index 06502f6d90..8b7a86c97b 100644 --- a/tsconfig.ws.json +++ b/tsconfig.ws.json @@ -14,6 +14,7 @@ { "path": "packages/html-program-viewer/tsconfig.json" }, { "path": "packages/protobuf/tsconfig.json" }, { "path": "packages/openapi3/tsconfig.json" }, + { "path": "packages/monarch/tsconfig.json" }, { "path": "packages/bundler/tsconfig.json" }, { "path": "packages/playground-website/tsconfig.json" }, { "path": "packages/tspd/tsconfig.json" }, From 27c794107fd3e568718179f28a960e0ce3051814 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 28 Mar 2024 13:27:28 -0700 Subject: [PATCH 02/10] progress --- packages/monarch/package.json | 3 +- packages/monarch/src/generate-json.ts | 7 ++ packages/monarch/src/index.ts | 111 +---------------------- packages/monarch/src/typespec-monarch.ts | 92 +++++++++++++++++++ 4 files changed, 102 insertions(+), 111 deletions(-) create mode 100644 packages/monarch/src/generate-json.ts create mode 100644 packages/monarch/src/typespec-monarch.ts diff --git a/packages/monarch/package.json b/packages/monarch/package.json index 6dd67ae349..1a7f33fef6 100644 --- a/packages/monarch/package.json +++ b/packages/monarch/package.json @@ -27,14 +27,13 @@ }, "scripts": { "clean": "rimraf ./dist ./temp", - "build": "npm run gen-extern-signature && tsc -p . && npm run lint-typespec-library", + "build": "tsc -p . && node ./dist/src/generate-json.js", "watch": "tsc -p . --watch", "test": "vitest run", "test:ui": "vitest --ui", "test-official": "vitest run --coverage --reporter=junit --reporter=default --no-file-parallelism", "lint": "eslint . --ext .ts --max-warnings=0", "lint:fix": "eslint . --fix --ext .ts", - "regen-docs": "tspd doc . --enable-experimental --output-dir ../../docs/libraries/openapi/reference" }, "files": [ "lib/*.tsp", diff --git a/packages/monarch/src/generate-json.ts b/packages/monarch/src/generate-json.ts new file mode 100644 index 0000000000..48b3c445e4 --- /dev/null +++ b/packages/monarch/src/generate-json.ts @@ -0,0 +1,7 @@ +import { writeFile } from "fs/promises"; +import { dirname, resolve } from "path"; +import { fileURLToPath } from "url"; +import lang from "./typespec-monarch.js"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); +await writeFile(resolve(root, "out", "typespec-monarch.json"), JSON.stringify(lang, null, 2)); diff --git a/packages/monarch/src/index.ts b/packages/monarch/src/index.ts index dc299d1a2b..9d5b86db89 100644 --- a/packages/monarch/src/index.ts +++ b/packages/monarch/src/index.ts @@ -1,110 +1,3 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. +import lang from "./typespec-monarch.js"; -import { languages } from "monaco-editor-core"; - -const bounded = (text: string) => `\\b${text}\\b`; -const notBefore = (regex: string) => `(?!${regex})`; - -const identifierStart = "[_a-zA-Z]"; -const identifierContinue = "[_a-zA-Z0-9]"; -const identifier = bounded(`${identifierStart}${identifierContinue}*`); -const directive = bounded(`[_a-zA-Z-0-9]+`); - -const keywords = [ - "metadata", - "targetScope", - "resource", - "module", - "param", - "var", - "output", - "for", - "in", - "if", - "existing", - "import", - "as", - "type", - "with", - "using", - "func", - "assert", - "provider", -]; - -const namedLiterals = ["true", "false", "null"]; - -const nonCommentWs = `[ \\t\\r\\n]`; - -const numericLiteral = `[0-9]+`; - -export const BicepLanguage: languages.IMonarchLanguage = { - defaultToken: "", - tokenPostfix: ".bicep", - - brackets: [ - { open: "{", close: "}", token: "delimiter.curly" }, - { open: "[", close: "]", token: "delimiter.square" }, - { open: "(", close: ")", token: "delimiter.parenthesis" }, - ], - - symbols: /[=> `\\b${text}\\b`; +const notBefore = (regex: string) => `(?!${regex})`; + +const identifierStart = "[_a-zA-Z]"; +const identifierContinue = "[_a-zA-Z0-9]"; +const identifier = bounded(`${identifierStart}${identifierContinue}*`); +const directive = bounded(`[_a-zA-Z-0-9]+`); + +const keywords = [ + "import", + "model", + "scalar", + "namespace", + "op", + "interface", + "union", + "using", + "is", + "extends", + "enum", + "alias", + "return", + "void", + "never", + "if", + "else", + "projection", + "dec", + "extern", + "fn", +]; + +const namedLiterals = ["true", "false", "null"]; + +const nonCommentWs = `[ \\t\\r\\n]`; + +const numericLiteral = `[0-9]+`; + +export default { + defaultToken: "", + tokenPostfix: ".bicep", + brackets: [ + { open: "{", close: "}", token: "delimiter.curly" }, + { open: "[", close: "]", token: "delimiter.square" }, + { open: "(", close: ")", token: "delimiter.parenthesis" }, + ], + symbols: /[=:;]+/, + keywords, + namedLiterals, + escapes: `\\\\(u{[0-9A-Fa-f]+}|n|r|t|\\\\|"|\\\${)`, + tokenizer: { + root: [{ include: "@expression" }, { include: "@whitespace" }], + stringVerbatim: [ + { regex: `(|"|"")[^"]`, action: { token: "string" } }, + { regex: `"""${notBefore(`"`)}`, action: { token: "string.quote", next: "@pop" } }, + ], + stringLiteral: [ + { regex: `\\\${`, action: { token: "delimiter.bracket", next: "@bracketCounting" } }, + { regex: `[^\\\\"$]+`, action: { token: "string" } }, + { regex: "@escapes", action: { token: "string.escape" } }, + { regex: `\\\\.`, action: { token: "string.escape.invalid" } }, + { regex: `"`, action: { token: "string", next: "@pop" } }, + ], + bracketCounting: [ + { regex: `{`, action: { token: "delimiter.bracket", next: "@bracketCounting" } }, + { regex: `}`, action: { token: "delimiter.bracket", next: "@pop" } }, + { include: "@expression" }, + ], + comment: [ + { regex: `[^\\*]+`, action: { token: "comment" } }, + { regex: `\\*\\/`, action: { token: "comment", next: "@pop" } }, + { regex: `[\\/*]`, action: { token: "comment" } }, + ], + whitespace: [ + { regex: nonCommentWs }, + { regex: `\\/\\*`, action: { token: "comment", next: "@comment" } }, + { regex: `\\/\\/.*$`, action: { token: "comment" } }, + ], + expression: [ + { regex: `"""`, action: { token: "string.quote", next: "@stringVerbatim" } }, + { regex: `"${notBefore(`""`)}`, action: { token: "string.quote", next: "@stringLiteral" } }, + { regex: numericLiteral, action: { token: "number" } }, + { regex: `@${identifier}`, action: { token: "tag" } }, + { regex: `#${directive}`, action: { token: "directive" } }, + ], + }, +} satisfies languages.IMonarchLanguage; From 761d89494312d7c8c91c9b9f77015f3b445bc9b3 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 28 Mar 2024 13:36:28 -0700 Subject: [PATCH 03/10] Simple monarch --- cspell.yaml | 1 + packages/monarch/src/typespec-monarch.ts | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cspell.yaml b/cspell.yaml index a946e43b2e..8b73c934a7 100644 --- a/cspell.yaml +++ b/cspell.yaml @@ -87,6 +87,7 @@ words: - tspd - tspwebsitepr - tsvs + - typespec - typespecvs - Uhoh - uitestresults diff --git a/packages/monarch/src/typespec-monarch.ts b/packages/monarch/src/typespec-monarch.ts index 773e319ea9..581dbad23e 100644 --- a/packages/monarch/src/typespec-monarch.ts +++ b/packages/monarch/src/typespec-monarch.ts @@ -1,6 +1,3 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - import { languages } from "monaco-editor-core"; const bounded = (text: string) => `\\b${text}\\b`; @@ -26,7 +23,6 @@ const keywords = [ "alias", "return", "void", - "never", "if", "else", "projection", @@ -34,11 +30,8 @@ const keywords = [ "extern", "fn", ]; - -const namedLiterals = ["true", "false", "null"]; - +const namedLiterals = ["true", "false", "null", "unknown", "never"]; const nonCommentWs = `[ \\t\\r\\n]`; - const numericLiteral = `[0-9]+`; export default { @@ -49,7 +42,7 @@ export default { { open: "[", close: "]", token: "delimiter.square" }, { open: "(", close: ")", token: "delimiter.parenthesis" }, ], - symbols: /[=:;]+/, + symbols: /[=:;<>]+/, keywords, namedLiterals, escapes: `\\\\(u{[0-9A-Fa-f]+}|n|r|t|\\\\|"|\\\${)`, @@ -85,6 +78,16 @@ export default { { regex: `"""`, action: { token: "string.quote", next: "@stringVerbatim" } }, { regex: `"${notBefore(`""`)}`, action: { token: "string.quote", next: "@stringLiteral" } }, { regex: numericLiteral, action: { token: "number" } }, + { + regex: identifier, + action: { + cases: { + "@keywords": { token: "keyword" }, + "@namedLiterals": { token: "keyword" }, + "@default": { token: "identifier" }, + }, + }, + }, { regex: `@${identifier}`, action: { token: "tag" } }, { regex: `#${directive}`, action: { token: "directive" } }, ], From 9a9406a6b9002545274d8bc37e691f39d39d85d5 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 28 Mar 2024 13:55:49 -0700 Subject: [PATCH 04/10] progress --- packages/monarch/package.json | 3 ++- packages/monarch/src/typespec-monarch.ts | 2 +- packages/monarch/test/typespec-monarch.test.ts | 13 +++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 packages/monarch/test/typespec-monarch.test.ts diff --git a/packages/monarch/package.json b/packages/monarch/package.json index 1a7f33fef6..8d749c0ced 100644 --- a/packages/monarch/package.json +++ b/packages/monarch/package.json @@ -30,10 +30,11 @@ "build": "tsc -p . && node ./dist/src/generate-json.js", "watch": "tsc -p . --watch", "test": "vitest run", + "test:watch": "vitest -w", "test:ui": "vitest --ui", "test-official": "vitest run --coverage --reporter=junit --reporter=default --no-file-parallelism", "lint": "eslint . --ext .ts --max-warnings=0", - "lint:fix": "eslint . --fix --ext .ts", + "lint:fix": "eslint . --fix --ext .ts" }, "files": [ "lib/*.tsp", diff --git a/packages/monarch/src/typespec-monarch.ts b/packages/monarch/src/typespec-monarch.ts index 581dbad23e..0d541cc44d 100644 --- a/packages/monarch/src/typespec-monarch.ts +++ b/packages/monarch/src/typespec-monarch.ts @@ -36,7 +36,7 @@ const numericLiteral = `[0-9]+`; export default { defaultToken: "", - tokenPostfix: ".bicep", + tokenPostfix: ".tsp", brackets: [ { open: "{", close: "}", token: "delimiter.curly" }, { open: "[", close: "]", token: "delimiter.square" }, diff --git a/packages/monarch/test/typespec-monarch.test.ts b/packages/monarch/test/typespec-monarch.test.ts new file mode 100644 index 0000000000..39eed7992f --- /dev/null +++ b/packages/monarch/test/typespec-monarch.test.ts @@ -0,0 +1,13 @@ +import { editor, languages } from "monaco-editor-core"; +import { beforeAll, it } from "vitest"; +import lang from "../src/typespec-monarch.js"; + +beforeAll(() => { + languages.register({ id: "typespec" }); + languages.setMonarchTokensProvider("typespec", lang); +}); + +it("works", () => { + const tokensByLine = editor.tokenize(`model foo {}`, "typespec"); + console.log("LKines", tokensByLine); +}); From 978224408a9eabaf12f8239f586346fc2f9097a3 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 28 Mar 2024 15:26:38 -0700 Subject: [PATCH 05/10] fix test can run now --- packages/monarch/package.json | 1 + .../monarch/test/typespec-monarch.test.ts | 2 +- packages/monarch/vitest.config.ts | 9 +- pnpm-lock.yaml | 124 ++++++++++++++---- 4 files changed, 107 insertions(+), 29 deletions(-) diff --git a/packages/monarch/package.json b/packages/monarch/package.json index 8d749c0ced..0882f81744 100644 --- a/packages/monarch/package.json +++ b/packages/monarch/package.json @@ -49,6 +49,7 @@ "@vitest/ui": "^1.4.0", "c8": "^9.1.0", "eslint": "^8.57.0", + "happy-dom": "^14.3.9", "rimraf": "~5.0.5", "typescript": "~5.4.2", "vitest": "^1.4.0" diff --git a/packages/monarch/test/typespec-monarch.test.ts b/packages/monarch/test/typespec-monarch.test.ts index 39eed7992f..3695d14264 100644 --- a/packages/monarch/test/typespec-monarch.test.ts +++ b/packages/monarch/test/typespec-monarch.test.ts @@ -1,4 +1,4 @@ -import { editor, languages } from "monaco-editor-core"; +import { editor, languages } from "monaco-editor-core/esm/vs/editor/editor.api.js"; import { beforeAll, it } from "vitest"; import lang from "../src/typespec-monarch.js"; diff --git a/packages/monarch/vitest.config.ts b/packages/monarch/vitest.config.ts index 15eeaceb85..ac1408e733 100644 --- a/packages/monarch/vitest.config.ts +++ b/packages/monarch/vitest.config.ts @@ -1,4 +1,11 @@ import { defineConfig, mergeConfig } from "vitest/config"; import { defaultTypeSpecVitestConfig } from "../../vitest.workspace.js"; -export default mergeConfig(defaultTypeSpecVitestConfig, defineConfig({})); +export default mergeConfig( + defaultTypeSpecVitestConfig, + defineConfig({ + test: { + environment: "happy-dom", + }, + }) +); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9f99e04d59..e74eb3eaef 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,7 +43,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0 e2e: {} @@ -81,7 +81,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/bundle-uploader: dependencies: @@ -136,7 +136,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/bundler: dependencies: @@ -203,7 +203,7 @@ importers: version: 5.1.6(@types/node@18.11.19) vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/compiler: dependencies: @@ -309,7 +309,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) vscode-oniguruma: specifier: ~2.0.1 version: 2.0.1 @@ -388,7 +388,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/html-program-viewer: dependencies: @@ -446,7 +446,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/http: devDependencies: @@ -488,7 +488,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/internal-build-utils: dependencies: @@ -543,7 +543,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/json-schema: dependencies: @@ -598,7 +598,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/library-linter: devDependencies: @@ -631,7 +631,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/monarch: dependencies: @@ -660,6 +660,9 @@ importers: eslint: specifier: ^8.57.0 version: 8.57.0 + happy-dom: + specifier: ^14.3.9 + version: 14.3.9 rimraf: specifier: ~5.0.5 version: 5.0.5 @@ -668,7 +671,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/openapi: devDependencies: @@ -716,7 +719,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/openapi3: dependencies: @@ -774,7 +777,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/playground: dependencies: @@ -1016,7 +1019,7 @@ importers: version: 5.1.6(@types/node@18.11.19) vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/prettier-plugin-typespec: dependencies: @@ -1047,7 +1050,7 @@ importers: version: 4.13.0 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0 packages/protobuf: devDependencies: @@ -1092,7 +1095,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/rest: devDependencies: @@ -1137,7 +1140,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/samples: dependencies: @@ -1201,7 +1204,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/spec: devDependencies: @@ -1302,7 +1305,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/typespec-vs: devDependencies: @@ -1365,7 +1368,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) vscode-languageclient: specifier: ~9.0.1 version: 9.0.1 @@ -1410,7 +1413,7 @@ importers: version: 5.4.2 vitest: specifier: ^1.4.0 - version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + version: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) packages/website: dependencies: @@ -3293,7 +3296,7 @@ packages: semver: 7.6.0 source-map-support: 0.5.21 std-env: 3.7.0 - vitest: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + vitest: 1.4.0 yargs: 17.7.2 zod: 3.22.4 transitivePeerDependencies: @@ -8818,7 +8821,7 @@ packages: strip-literal: 2.0.0 test-exclude: 6.0.0 v8-to-istanbul: 9.2.0 - vitest: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + vitest: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) transitivePeerDependencies: - supports-color dev: true @@ -8861,7 +8864,8 @@ packages: pathe: 1.1.2 picocolors: 1.0.0 sirv: 2.0.4 - vitest: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + vitest: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9) + dev: true /@vitest/utils@1.4.0: resolution: {integrity: sha512-mx3Yd1/6e2Vt/PUC98DcqTirtfxUyAZ32uK82r8rZzbtBeBo+nqgnjx/LvqQdWsrvNtm14VmurNgcf4nqY5gJg==} @@ -11522,7 +11526,7 @@ packages: '@typescript-eslint/eslint-plugin': 7.2.0(@typescript-eslint/parser@7.2.0)(eslint@8.57.0)(typescript@5.4.2) '@typescript-eslint/utils': 7.2.0(eslint@8.57.0)(typescript@5.4.2) eslint: 8.57.0 - vitest: 1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0) + vitest: 1.4.0 transitivePeerDependencies: - supports-color - typescript @@ -11881,6 +11885,7 @@ packages: /fflate@0.8.2: resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} + dev: true /file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} @@ -12397,6 +12402,15 @@ packages: resolution: {integrity: sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==} dev: false + /happy-dom@14.3.9: + resolution: {integrity: sha512-0kPQchwthekcYpYN8CvCiq+/z5bqFYDLbTxZ+yDLwT8AFRVJDFadShHRxp3VAZRy7a5isOZ1j/LzsU1dtAIZMQ==} + engines: {node: '>=16.0.0'} + dependencies: + entities: 4.5.0 + webidl-conversions: 7.0.0 + whatwg-mimetype: 3.0.0 + dev: true + /has-flag@3.0.0: resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} engines: {node: '>=4'} @@ -18963,7 +18977,61 @@ packages: optionalDependencies: fsevents: 2.3.3 - /vitest@1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0): + /vitest@1.4.0: + resolution: {integrity: sha512-gujzn0g7fmwf83/WzrDTnncZt2UiXP41mHuFYFrdwaLRVQ6JYQEiME2IfEjU3vcFL3VKa75XhI3lFgn+hfVsQw==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/node': ^18.0.0 || >=20.0.0 + '@vitest/browser': 1.4.0 + '@vitest/ui': 1.4.0 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + dependencies: + '@vitest/expect': 1.4.0 + '@vitest/runner': 1.4.0 + '@vitest/snapshot': 1.4.0 + '@vitest/spy': 1.4.0 + '@vitest/utils': 1.4.0 + acorn-walk: 8.3.2 + chai: 4.4.1 + debug: 4.3.4 + execa: 8.0.1 + local-pkg: 0.5.0 + magic-string: 0.30.7 + pathe: 1.1.2 + picocolors: 1.0.0 + std-env: 3.7.0 + strip-literal: 2.0.0 + tinybench: 2.6.0 + tinypool: 0.8.2 + vite: 5.1.6(@types/node@18.11.19) + vite-node: 1.4.0(@types/node@18.11.19) + why-is-node-running: 2.2.2 + transitivePeerDependencies: + - less + - lightningcss + - sass + - stylus + - sugarss + - supports-color + - terser + + /vitest@1.4.0(@types/node@18.11.19)(@vitest/ui@1.4.0)(happy-dom@14.3.9): resolution: {integrity: sha512-gujzn0g7fmwf83/WzrDTnncZt2UiXP41mHuFYFrdwaLRVQ6JYQEiME2IfEjU3vcFL3VKa75XhI3lFgn+hfVsQw==} engines: {node: ^18.0.0 || >=20.0.0} hasBin: true @@ -18999,6 +19067,7 @@ packages: chai: 4.4.1 debug: 4.3.4 execa: 8.0.1 + happy-dom: 14.3.9 local-pkg: 0.5.0 magic-string: 0.30.7 pathe: 1.1.2 @@ -19018,6 +19087,7 @@ packages: - sugarss - supports-color - terser + dev: true /vscode-jsonrpc@8.2.0: resolution: {integrity: sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==} From 954d5d9de79e38a6f0333248012305cb6c36b017 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 28 Mar 2024 18:43:01 -0700 Subject: [PATCH 06/10] more tests --- packages/monarch/src/typespec-monarch.ts | 8 +- .../monarch/test/typespec-monarch.test.ts | 135 +++++++++++++++++- packages/monarch/tsconfig.json | 3 +- 3 files changed, 136 insertions(+), 10 deletions(-) diff --git a/packages/monarch/src/typespec-monarch.ts b/packages/monarch/src/typespec-monarch.ts index 0d541cc44d..6db4a323f0 100644 --- a/packages/monarch/src/typespec-monarch.ts +++ b/packages/monarch/src/typespec-monarch.ts @@ -1,4 +1,4 @@ -import { languages } from "monaco-editor-core"; +import type { languages } from "monaco-editor-core"; const bounded = (text: string) => `\\b${text}\\b`; const notBefore = (regex: string) => `(?!${regex})`; @@ -50,7 +50,7 @@ export default { root: [{ include: "@expression" }, { include: "@whitespace" }], stringVerbatim: [ { regex: `(|"|"")[^"]`, action: { token: "string" } }, - { regex: `"""${notBefore(`"`)}`, action: { token: "string.quote", next: "@pop" } }, + { regex: `"""${notBefore(`"`)}`, action: { token: "string", next: "@pop" } }, ], stringLiteral: [ { regex: `\\\${`, action: { token: "delimiter.bracket", next: "@bracketCounting" } }, @@ -75,8 +75,8 @@ export default { { regex: `\\/\\/.*$`, action: { token: "comment" } }, ], expression: [ - { regex: `"""`, action: { token: "string.quote", next: "@stringVerbatim" } }, - { regex: `"${notBefore(`""`)}`, action: { token: "string.quote", next: "@stringLiteral" } }, + { regex: `"""`, action: { token: "string", next: "@stringVerbatim" } }, + { regex: `"${notBefore(`""`)}`, action: { token: "string", next: "@stringLiteral" } }, { regex: numericLiteral, action: { token: "number" } }, { regex: identifier, diff --git a/packages/monarch/test/typespec-monarch.test.ts b/packages/monarch/test/typespec-monarch.test.ts index 3695d14264..a452dffb50 100644 --- a/packages/monarch/test/typespec-monarch.test.ts +++ b/packages/monarch/test/typespec-monarch.test.ts @@ -1,5 +1,7 @@ +import { deepStrictEqual } from "assert"; +import type { Token as MonacoToken } from "monaco-editor-core"; import { editor, languages } from "monaco-editor-core/esm/vs/editor/editor.api.js"; -import { beforeAll, it } from "vitest"; +import { beforeAll, it, type TestFunction } from "vitest"; import lang from "../src/typespec-monarch.js"; beforeAll(() => { @@ -7,7 +9,130 @@ beforeAll(() => { languages.setMonarchTokensProvider("typespec", lang); }); -it("works", () => { - const tokensByLine = editor.tokenize(`model foo {}`, "typespec"); - console.log("LKines", tokensByLine); -}); +interface Token { + text: string; + type: string; +} + +const Token = { + keyword: (text: string) => ({ type: "keyword.tsp", text }), + identifier: (text: string) => ({ type: "identifier.tsp", text }), + default: (text: string) => ({ type: "", text }), + quote: { type: "string.quote.tsp", text: `"` }, + stringQuoted: (text: string) => ({ type: "string.tsp", text: `"${text}"` }), + string: (text: string) => ({ type: "string.tsp", text }), +}; + +function simplifyTokens(text: string, tokens: MonacoToken[][]) { + const result = []; + + const lines = text.split("\n"); + + for (const [lineIndex, lineTokens] of tokens.entries()) { + const line = lines[lineIndex]; + for (const [index, token] of lineTokens.entries()) { + const nextOffset = lineTokens[index + 1]?.offset ?? line.length; + let tokenText = line.slice(token.offset, nextOffset); + if (token.type === "") { + tokenText = tokenText.trim(); + if (tokenText === "") continue; + } + result.push({ type: token.type, text: tokenText }); + } + } + + return result; +} + +function tokenize(text: string) { + const tokensByLine = editor.tokenize(text, "typespec"); + return simplifyTokens(text, tokensByLine); +} + +function tokenizeTo(expected: Token[]): TestFunction { + return (context) => { + const tokens = tokenize(context.task.name); + deepStrictEqual(tokens, expected); + }; +} + +it( + "model Foo {}", + tokenizeTo([Token.keyword("model"), Token.identifier("Foo"), Token.default("{}")]) +); + +it( + "interface Foo {}", + tokenizeTo([Token.keyword("interface"), Token.identifier("Foo"), Token.default("{}")]) +); + +it( + "union Foo {}", + tokenizeTo([Token.keyword("union"), Token.identifier("Foo"), Token.default("{}")]) +); + +it( + "scalar foo extends string;", + tokenizeTo([ + Token.keyword("scalar"), + Token.identifier("foo"), + Token.keyword("extends"), + Token.identifier("string"), + Token.default(";"), + ]) +); + +it( + "op test(): void;", + tokenizeTo([ + Token.keyword("op"), + Token.identifier("test"), + Token.default("():"), + Token.keyword("void"), + Token.default(";"), + ]) +); + +it( + "enum Direction { up, down }", + tokenizeTo([ + Token.keyword("enum"), + Token.identifier("Direction"), + Token.default("{"), + Token.identifier("up"), + Token.default(","), + Token.identifier("down"), + Token.default("}"), + ]) +); + +it( + `alias Foo = "a" | "b";`, + tokenizeTo([ + Token.keyword("alias"), + Token.identifier("Foo"), + Token.default("="), + Token.stringQuoted("a"), + Token.default("|"), + Token.stringQuoted("b"), + Token.default(";"), + ]) +); + +it( + `alias T = """ + this + is + multiline + """`, + tokenizeTo([ + Token.keyword("alias"), + Token.identifier("T"), + Token.default("="), + Token.string(`"""`), + Token.string(` this`), + Token.string(` is`), + Token.string(` multiline`), + Token.string(` """`), + ]) +); diff --git a/packages/monarch/tsconfig.json b/packages/monarch/tsconfig.json index 25e8b9b207..6e60792fb9 100644 --- a/packages/monarch/tsconfig.json +++ b/packages/monarch/tsconfig.json @@ -4,7 +4,8 @@ "outDir": "dist", "rootDir": ".", "tsBuildInfoFile": "temp/tsconfig.tsbuildinfo", - "lib": ["DOM", "ESNext"] + "lib": ["DOM", "ESNext"], + "verbatimModuleSyntax": true }, "include": ["src/**/*.ts", "test/**/*.ts"] } From 999812d3bc86e08a894ca773be0499ad0732ec11 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 29 Mar 2024 08:01:18 -0700 Subject: [PATCH 07/10] monarch --- .../monarch/test/typespec-monarch.test.ts | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/packages/monarch/test/typespec-monarch.test.ts b/packages/monarch/test/typespec-monarch.test.ts index a452dffb50..df7a028350 100644 --- a/packages/monarch/test/typespec-monarch.test.ts +++ b/packages/monarch/test/typespec-monarch.test.ts @@ -56,11 +56,66 @@ function tokenizeTo(expected: Token[]): TestFunction { }; } +it( + `import "@typespec/http";`, + tokenizeTo([Token.keyword("import"), Token.stringQuoted("@typespec/http"), Token.default(";")]) +); + +it( + "using TypeSpec.Http", + tokenizeTo([ + Token.keyword("using"), + Token.identifier("TypeSpec"), + Token.default("."), + Token.identifier("Http"), + ]) +); +it( + "namespace Foo {}", + tokenizeTo([Token.keyword("namespace"), Token.identifier("Foo"), Token.default("{}")]) +); + +it( + `namespace Foo { + model Bar {} + }`, + tokenizeTo([ + Token.keyword("namespace"), + Token.identifier("Foo"), + Token.default("{"), + Token.keyword("model"), + Token.identifier("Bar"), + Token.default("{}"), + Token.default("}"), + ]) +); + it( "model Foo {}", tokenizeTo([Token.keyword("model"), Token.identifier("Foo"), Token.default("{}")]) ); +it( + "model Foo is Bar;", + tokenizeTo([ + Token.keyword("model"), + Token.identifier("Foo"), + Token.keyword("is"), + Token.identifier("Bar"), + Token.default(";"), + ]) +); +it( + "model Foo extends Bar;", + tokenizeTo([ + Token.keyword("model"), + Token.identifier("Foo"), + Token.keyword("extends"), + Token.identifier("Bar"), + Token.default(";"), + ]) +); + it( "interface Foo {}", tokenizeTo([Token.keyword("interface"), Token.identifier("Foo"), Token.default("{}")]) From f168e7c97e7294de91e1634dafca22c36bb7aaa2 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 29 Mar 2024 08:03:07 -0700 Subject: [PATCH 08/10] Don't need json --- packages/monarch/package.json | 2 +- packages/monarch/src/generate-json.ts | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 packages/monarch/src/generate-json.ts diff --git a/packages/monarch/package.json b/packages/monarch/package.json index 0882f81744..f64dc817f6 100644 --- a/packages/monarch/package.json +++ b/packages/monarch/package.json @@ -27,7 +27,7 @@ }, "scripts": { "clean": "rimraf ./dist ./temp", - "build": "tsc -p . && node ./dist/src/generate-json.js", + "build": "tsc -p .", "watch": "tsc -p . --watch", "test": "vitest run", "test:watch": "vitest -w", diff --git a/packages/monarch/src/generate-json.ts b/packages/monarch/src/generate-json.ts deleted file mode 100644 index 48b3c445e4..0000000000 --- a/packages/monarch/src/generate-json.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { writeFile } from "fs/promises"; -import { dirname, resolve } from "path"; -import { fileURLToPath } from "url"; -import lang from "./typespec-monarch.js"; - -const root = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); -await writeFile(resolve(root, "out", "typespec-monarch.json"), JSON.stringify(lang, null, 2)); From fa57dfeecf2856b7f9d33adfcb956060826d3721 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 29 Mar 2024 08:04:31 -0700 Subject: [PATCH 09/10] . --- packages/monarch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/monarch/README.md b/packages/monarch/README.md index 6e37151239..06d26cc9c7 100644 --- a/packages/monarch/README.md +++ b/packages/monarch/README.md @@ -1,3 +1,3 @@ # @typespec/monarch -Provide tokenizer for Monarch for use in monaco-editor. +Provide tokenizer for Monarch for use in [monaco-editor](https://github.com/microsoft/monaco-editor). From a6df8a7930caf652a70b1351bfb49875ef972126 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 29 Mar 2024 09:10:53 -0700 Subject: [PATCH 10/10] Auto create test for monaco --- packages/monarch/README.md | 6 ++++ .../monarch/test/typespec-monarch.test.ts | 29 ++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/packages/monarch/README.md b/packages/monarch/README.md index 06d26cc9c7..4f20353499 100644 --- a/packages/monarch/README.md +++ b/packages/monarch/README.md @@ -1,3 +1,9 @@ # @typespec/monarch Provide tokenizer for Monarch for use in [monaco-editor](https://github.com/microsoft/monaco-editor). + +## Updating + +### Updating the tests + +You can copy `./temp/monaco-tests.json`(after running the test) into the `src/basic-languages/typespec/typespec.test.ts` test file in the monaco-editor repository. diff --git a/packages/monarch/test/typespec-monarch.test.ts b/packages/monarch/test/typespec-monarch.test.ts index df7a028350..16b12d1108 100644 --- a/packages/monarch/test/typespec-monarch.test.ts +++ b/packages/monarch/test/typespec-monarch.test.ts @@ -1,14 +1,32 @@ import { deepStrictEqual } from "assert"; +import { writeFile } from "fs/promises"; import type { Token as MonacoToken } from "monaco-editor-core"; import { editor, languages } from "monaco-editor-core/esm/vs/editor/editor.api.js"; -import { beforeAll, it, type TestFunction } from "vitest"; +import { resolve } from "path"; +import { fileURLToPath } from "url"; +import { afterAll, beforeAll, it, type TestFunction } from "vitest"; import lang from "../src/typespec-monarch.js"; +interface MonacoTestFormat { + line: string; + tokens: any[]; +} + +const lines: MonacoTestFormat[][] = []; beforeAll(() => { + lines.length = 0; languages.register({ id: "typespec" }); languages.setMonarchTokensProvider("typespec", lang); }); +afterAll(async () => { + // Write the test format expected in monaco-editor repo. You can then copy the content to the test file and update src/basic-languages/typespec/typespec.test.ts + await writeFile( + resolve(fileURLToPath(import.meta.url), "../../temp/monaco-tests.json"), + JSON.stringify(lines, null, 2) + ); +}); + interface Token { text: string; type: string; @@ -46,6 +64,15 @@ function simplifyTokens(text: string, tokens: MonacoToken[][]) { function tokenize(text: string) { const tokensByLine = editor.tokenize(text, "typespec"); + const textLines = text.split("\n"); + const group: MonacoTestFormat[] = []; + for (const [lineIndex, tokens] of tokensByLine.entries()) { + group.push({ + line: textLines[lineIndex], + tokens: tokens.map((x) => ({ startIndex: x.offset, type: x.type })), + }); + } + lines.push(group); return simplifyTokens(text, tokensByLine); }