Improve TM language for enums, unions and escaped identifiers (#3069)

resolves #3070 The tm language wasn't correctly defining the syntax for escaped identifiers ``` `North West` ``` Enums and unions syntax was also not very accurate and didn't tokenize everything correctly ## Example of issue before ![image](https://github.com/microsoft/typespec/assets/1031227/ee60f4fe-5eef-4f86-baa5-1cc96b665526) ## After ![image](https://github.com/microsoft/typespec/assets/1031227/38405f18-0fa4-44ba-82f6-594bcd4fcebf)
microsoft · Mar 27, 2024 · ce9c567 · ce9c567
1 parent 32e45d4
commit ce9c567
Show file tree

Hide file tree

Showing 5 changed files with 343 additions and 44 deletions.
diff --git a/.chronus/changes/tmlanguage-escpaed-identifiers-2024-2-27-16-22-28.md b/.chronus/changes/tmlanguage-escpaed-identifiers-2024-2-27-16-22-28.md
@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/compiler"
+---
+
+TmLanguage: Fix tokenization of escaped identifiers, enums and unions
diff --git a/grammars/typespec.json b/grammars/typespec.json
@@ -3,7 +3,7 @@
   "name": "TypeSpec",
   "scopeName": "source.tsp",
   "fileTypes": [
-    ".tsp"
+    "tsp"
   ],
   "patterns": [
     {
@@ -85,7 +85,7 @@
     },
     "decorator-declaration-statement": {
       "name": "meta.decorator-declaration-statement.typespec",
-      "begin": "(?:(extern)\\s+)?\\b(dec)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)",
+      "begin": "(?:(extern)\\s+)?\\b(dec)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)",
       "beginCaptures": {
         "1": {
           "name": "keyword.other.tsp"
@@ -160,7 +160,7 @@
     },
     "doc-comment-param": {
       "name": "comment.block.tsp",
-      "match": "(?x)((@)(?:param|template))\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)\\b",
+      "match": "(?x)((@)(?:param|template))\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)\\b",
       "captures": {
         "1": {
           "name": "keyword.tag.tspdoc"
@@ -187,7 +187,7 @@
     },
     "doc-comment-unknown-tag": {
       "name": "comment.block.tsp",
-      "match": "(?x)((@)(?:\\b[_$[:alpha:]][_$[:alnum:]]*\\b))\\b",
+      "match": "(?x)((@)(?:\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`))\\b",
       "captures": {
         "1": {
           "name": "entity.name.tag.tsp"
@@ -236,12 +236,68 @@
         }
       ]
     },
+    "enum-body": {
+      "name": "meta.enum-body.typespec",
+      "begin": "\\{",
+      "beginCaptures": {
+        "0": {
+          "name": "punctuation.curlybrace.open.tsp"
+        }
+      },
+      "end": "\\}",
+      "endCaptures": {
+        "0": {
+          "name": "punctuation.curlybrace.close.tsp"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#enum-member"
+        },
+        {
+          "include": "#token"
+        },
+        {
+          "include": "#directive"
+        },
+        {
+          "include": "#decorator"
+        },
+        {
+          "include": "#punctuation-comma"
+        }
+      ]
+    },
+    "enum-member": {
+      "name": "meta.enum-member.typespec",
+      "begin": "(?:(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)\\s*(:?))",
+      "beginCaptures": {
+        "1": {
+          "name": "variable.name.tsp"
+        },
+        "2": {
+          "name": "keyword.operator.type.annotation.tsp"
+        }
+      },
+      "end": "(?=,|;|@|\\)|\\}|\\b(?:extern)\\b|\\b(?:namespace|model|op|using|import|enum|alias|union|interface|dec|fn)\\b)",
+      "patterns": [
+        {
+          "include": "#token"
+        },
+        {
+          "include": "#type-annotation"
+        }
+      ]
+    },
     "enum-statement": {
       "name": "meta.enum-statement.typespec",
-      "begin": "\\b(enum)\\b",
+      "begin": "\\b(enum)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)",
       "beginCaptures": {
         "1": {
           "name": "keyword.other.tsp"
+        },
+        "2": {
+          "name": "entity.name.type.tsp"
         }
       },
       "end": "(?<=\\})|(?=,|;|@|\\)|\\}|\\b(?:extern)\\b|\\b(?:namespace|model|op|using|import|enum|alias|union|interface|dec|fn)\\b)",
@@ -250,7 +306,7 @@
           "include": "#token"
         },
         {
-          "include": "#expression"
+          "include": "#enum-body"
         }
       ]
     },
@@ -288,7 +344,7 @@
     },
     "function-call": {
       "name": "meta.function-call.typespec",
-      "begin": "(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)\\s*(\\()",
+      "begin": "(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)\\s*(\\()",
       "beginCaptures": {
         "1": {
           "name": "entity.name.function.tsp"
@@ -311,7 +367,7 @@
     },
     "function-declaration-statement": {
       "name": "meta.function-declaration-statement.typespec",
-      "begin": "(?:(extern)\\s+)?\\b(fn)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)",
+      "begin": "(?:(extern)\\s+)?\\b(fn)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)",
       "beginCaptures": {
         "1": {
           "name": "keyword.other.tsp"
@@ -338,7 +394,7 @@
     },
     "identifier-expression": {
       "name": "entity.name.type.tsp",
-      "match": "\\b[_$[:alpha:]][_$[:alnum:]]*\\b"
+      "match": "\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`"
     },
     "if-expression": {
       "name": "meta.if-expression.typespec",
@@ -425,7 +481,7 @@
     },
     "interface-member": {
       "name": "meta.interface-member.typespec",
-      "begin": "(?:\\b(op)\\b\\s+)?(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)",
+      "begin": "(?:\\b(op)\\b\\s+)?(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)",
       "beginCaptures": {
         "1": {
           "name": "keyword.other.tsp"
@@ -503,7 +559,7 @@
           "include": "#decorator"
         },
         {
-          "include": "#model-spread-property"
+          "include": "#spread-operator"
         },
         {
           "include": "#punctuation-semicolon"
@@ -530,7 +586,7 @@
     },
     "model-property": {
       "name": "meta.model-property.typespec",
-      "begin": "(?:(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)|(\\\"(?:[^\\\"\\\\]|\\\\.)*\\\"))",
+      "begin": "(?:(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)|(\\\"(?:[^\\\"\\\\]|\\\\.)*\\\"))",
       "beginCaptures": {
         "1": {
           "name": "variable.name.tsp"
@@ -555,21 +611,6 @@
         }
       ]
     },
-    "model-spread-property": {
-      "name": "meta.model-spread-property.typespec",
-      "begin": "\\.\\.\\.",
-      "beginCaptures": {
-        "0": {
-          "name": "keyword.operator.spread.tsp"
-        }
-      },
-      "end": "(?=,|;|@|\\)|\\}|\\b(?:extern)\\b|\\b(?:namespace|model|op|using|import|enum|alias|union|interface|dec|fn)\\b)",
-      "patterns": [
-        {
-          "include": "#expression"
-        }
-      ]
-    },
     "model-statement": {
       "name": "meta.model-statement.typespec",
       "begin": "\\b(model)\\b",
@@ -692,7 +733,7 @@
           "include": "#model-property"
         },
         {
-          "include": "#model-spread-property"
+          "include": "#spread-operator"
         },
         {
           "include": "#punctuation-comma"
@@ -717,7 +758,7 @@
     },
     "operation-statement": {
       "name": "meta.operation-statement.typespec",
-      "begin": "\\b(op)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)",
+      "begin": "\\b(op)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)",
       "beginCaptures": {
         "1": {
           "name": "keyword.other.tsp"
@@ -822,7 +863,7 @@
     },
     "projection-parameter": {
       "name": "meta.projection-parameter.typespec",
-      "begin": "(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)",
+      "begin": "(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)",
       "beginCaptures": {
         "1": {
           "name": "variable.name.tsp"
@@ -856,7 +897,7 @@
     },
     "projection-statement": {
       "name": "meta.projection-statement.typespec",
-      "begin": "\\b(projection)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)(#)(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)",
+      "begin": "\\b(projection)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)(#)(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)",
       "beginCaptures": {
         "1": {
           "name": "keyword.other.tsp"
@@ -952,6 +993,21 @@
         }
       ]
     },
+    "spread-operator": {
+      "name": "meta.spread-operator.typespec",
+      "begin": "\\.\\.\\.",
+      "beginCaptures": {
+        "0": {
+          "name": "keyword.operator.spread.tsp"
+        }
+      },
+      "end": "(?=,|;|@|\\)|\\}|\\b(?:extern)\\b|\\b(?:namespace|model|op|using|import|enum|alias|union|interface|dec|fn)\\b)",
+      "patterns": [
+        {
+          "include": "#expression"
+        }
+      ]
+    },
     "statement": {
       "patterns": [
         {
@@ -1150,7 +1206,7 @@
     },
     "type-parameter": {
       "name": "meta.type-parameter.typespec",
-      "begin": "(\\b[_$[:alpha:]][_$[:alnum:]]*\\b)",
+      "begin": "(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)",
       "beginCaptures": {
         "1": {
           "name": "entity.name.type.tsp"
@@ -1222,12 +1278,50 @@
         }
       ]
     },
+    "union-body": {
+      "name": "meta.union-body.typespec",
+      "begin": "\\{",
+      "beginCaptures": {
+        "0": {
+          "name": "punctuation.curlybrace.open.tsp"
+        }
+      },
+      "end": "\\}",
+      "endCaptures": {
+        "0": {
+          "name": "punctuation.curlybrace.close.tsp"
+        }
+      },
+      "patterns": [
+        {
+          "include": "#union-variant"
+        },
+        {
+          "include": "#token"
+        },
+        {
+          "include": "#directive"
+        },
+        {
+          "include": "#decorator"
+        },
+        {
+          "include": "#expression"
+        },
+        {
+          "include": "#punctuation-comma"
+        }
+      ]
+    },
     "union-statement": {
       "name": "meta.union-statement.typespec",
-      "begin": "\\b(union)\\b",
+      "begin": "\\b(union)\\b\\s+(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)",
       "beginCaptures": {
         "1": {
           "name": "keyword.other.tsp"
+        },
+        "2": {
+          "name": "entity.name.type.tsp"
         }
       },
       "end": "(?<=\\})|(?=,|;|@|\\)|\\}|\\b(?:extern)\\b|\\b(?:namespace|model|op|using|import|enum|alias|union|interface|dec|fn)\\b)",
@@ -1236,7 +1330,25 @@
           "include": "#token"
         },
         {
-          "include": "#type-parameters"
+          "include": "#union-body"
+        }
+      ]
+    },
+    "union-variant": {
+      "name": "meta.union-variant.typespec",
+      "begin": "(?:(\\b[_$[:alpha:]][_$[:alnum:]]*\\b|`(?:[^`\\\\]|\\\\.)*`)\\s*(:))",
+      "beginCaptures": {
+        "1": {
+          "name": "variable.name.tsp"
+        },
+        "2": {
+          "name": "keyword.operator.type.annotation.tsp"
+        }
+      },
+      "end": "(?=,|;|@|\\)|\\}|\\b(?:extern)\\b|\\b(?:namespace|model|op|using|import|enum|alias|union|interface|dec|fn)\\b)",
+      "patterns": [
+        {
+          "include": "#token"
         },
         {
           "include": "#expression"

diff --git a/packages/compiler/src/server/classify.ts b/packages/compiler/src/server/classify.ts
@@ -218,6 +218,9 @@ export function getSemanticTokens(ast: TypeSpecScriptNode): SemanticToken[] {
       case SyntaxKind.EnumStatement:
         classify(node.id, SemanticTokenKind.Enum);
         break;
+      case SyntaxKind.UnionStatement:
+        classify(node.id, SemanticTokenKind.Enum);
+        break;
       case SyntaxKind.EnumMember:
         classify(node.id, SemanticTokenKind.EnumMember);
         break;