From f5b732c774c1f52356b4a248787eb5b7753902f0 Mon Sep 17 00:00:00 2001 From: Praven Kuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 21 Aug 2024 22:23:28 -0400 Subject: [PATCH] .NET API review parser changes to use new token schema (#8850) * .NET API review parser changes to use new token schema --- src/dotnet/APIView/APIView.sln | 18 + src/dotnet/APIView/APIView/Model/CodeFile.cs | 68 +-- .../APIView/APIView/Model/V2/ReviewLine.cs | 143 +++++ .../APIView/APIView/Model/V2/ReviewToken.cs | 138 +++++ .../APIView/APIView/Model/V2/TokenKind.cs | 17 + .../CSharpAPIParser/Program.cs | 67 ++- .../TreeToken/CodeFileBuilder.cs | 517 ++++++++++-------- .../CSharpAPIParserTests.csproj | 3 + .../CSharpAPIParserTests/CodeFileTests.cs | 130 +++++ .../CSharpAPIParserTests/ProgramTests.cs | 2 - .../CSharpAPIParserTests/TestData.cs | 255 +++++++++ 11 files changed, 1059 insertions(+), 299 deletions(-) create mode 100644 src/dotnet/APIView/APIView/Model/V2/ReviewLine.cs create mode 100644 src/dotnet/APIView/APIView/Model/V2/ReviewToken.cs create mode 100644 src/dotnet/APIView/APIView/Model/V2/TokenKind.cs create mode 100644 tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/CodeFileTests.cs create mode 100644 tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/TestData.cs diff --git a/src/dotnet/APIView/APIView.sln b/src/dotnet/APIView/APIView.sln index 245fd0dd31b..1aa0a367db1 100644 --- a/src/dotnet/APIView/APIView.sln +++ b/src/dotnet/APIView/APIView.sln @@ -17,6 +17,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "APIViewIntegrationTests", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "APIViewUITests", "APIViewUITests\APIViewUITests.csproj", "{7246F62A-99BF-4C4F-B9AD-1996166E767E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSharpAPIParser", "..\..\..\tools\apiview\parsers\csharp-api-parser\CSharpAPIParser\CSharpAPIParser.csproj", "{0D5CD780-15F9-49F5-9C4F-126D8224A31E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSharpAPIParserTests", "..\..\..\tools\apiview\parsers\csharp-api-parser\CSharpAPIParserTests\CSharpAPIParserTests.csproj", "{4F057169-032D-4971-B7D9-71AF850D411E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestReferenceWithInternalsVisibleTo", "..\Azure.ClientSdk.Analyzers\TestReferenceWithInternalsVisibleTo\TestReferenceWithInternalsVisibleTo.csproj", "{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -51,6 +57,18 @@ Global {7246F62A-99BF-4C4F-B9AD-1996166E767E}.Debug|Any CPU.Build.0 = Debug|Any CPU {7246F62A-99BF-4C4F-B9AD-1996166E767E}.Release|Any CPU.ActiveCfg = Release|Any CPU {7246F62A-99BF-4C4F-B9AD-1996166E767E}.Release|Any CPU.Build.0 = Release|Any CPU + {0D5CD780-15F9-49F5-9C4F-126D8224A31E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0D5CD780-15F9-49F5-9C4F-126D8224A31E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0D5CD780-15F9-49F5-9C4F-126D8224A31E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0D5CD780-15F9-49F5-9C4F-126D8224A31E}.Release|Any CPU.Build.0 = Release|Any CPU + {4F057169-032D-4971-B7D9-71AF850D411E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4F057169-032D-4971-B7D9-71AF850D411E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4F057169-032D-4971-B7D9-71AF850D411E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4F057169-032D-4971-B7D9-71AF850D411E}.Release|Any CPU.Build.0 = Release|Any CPU + {0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/dotnet/APIView/APIView/Model/CodeFile.cs b/src/dotnet/APIView/APIView/Model/CodeFile.cs index 8fa07e7e43e..f5b7fe1c666 100644 --- a/src/dotnet/APIView/APIView/Model/CodeFile.cs +++ b/src/dotnet/APIView/APIView/Model/CodeFile.cs @@ -2,14 +2,16 @@ // Licensed under the MIT License. using APIView; +using APIView.Model.V2; using APIView.TreeToken; using System; using System.Collections.Generic; using System.IO; -using System.IO.Compression; using System.Linq; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.RegularExpressions; using System.Threading.Tasks; namespace ApiView @@ -19,17 +21,8 @@ public class CodeFile private static readonly JsonSerializerOptions _serializerOptions = new JsonSerializerOptions { AllowTrailingCommas = true, - ReadCommentHandling = JsonCommentHandling.Skip - }; - - private static readonly JsonSerializerOptions _treeStyleParserDeserializerOptions = new JsonSerializerOptions - { - Converters = { new StructuredTokenConverter() } - }; - - private static readonly JsonSerializerOptions _treeStyleParserSerializerOptions = new JsonSerializerOptions - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + ReadCommentHandling = JsonCommentHandling.Skip, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; private string _versionString; @@ -52,10 +45,18 @@ public string VersionString public string PackageVersion { get; set; } public string CrossLanguagePackageId { get; set; } public CodeFileToken[] Tokens { get; set; } = Array.Empty(); + // APIForest will be removed once server changes are added to dereference this property public List APIForest { get; set; } = new List(); public List LeafSections { get; set; } public NavigationItem[] Navigation { get; set; } public CodeDiagnostic[] Diagnostics { get; set; } + public string ParserVersion + { + get => _versionString; + set => _versionString = value; + } + public List ReviewLines { get; set; } = []; + public override string ToString() { return new CodeFileRenderer().Render(this).CodeLines.ToString(); @@ -64,23 +65,12 @@ public override string ToString() public static async Task DeserializeAsync(Stream stream, bool hasSections = false, bool doTreeStyleParserDeserialization = false) { - CodeFile codeFile = null; - if (doTreeStyleParserDeserialization) - { - using (var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true)) - { - codeFile = await JsonSerializer.DeserializeAsync(gzipStream, _treeStyleParserDeserializerOptions); - } - } - else - { - codeFile = await JsonSerializer.DeserializeAsync(stream, _serializerOptions); - } + var codeFile = await JsonSerializer.DeserializeAsync(stream, _serializerOptions); if (hasSections == false && codeFile.LeafSections == null && IsCollapsibleSectionSSupported(codeFile.Language)) hasSections = true; - // Spliting out the 'leafSections' of the codeFile is done so as not to have to render large codeFiles at once + // Splitting out the 'leafSections' of the codeFile is done so as not to have to render large codeFiles at once // Rendering sections in part helps to improve page load time if (hasSections) { @@ -151,23 +141,21 @@ public static async Task DeserializeAsync(Stream stream, bool hasSecti public async Task SerializeAsync(Stream stream) { - if (this.APIForest.Count > 0) - { - using (var tempStream = new MemoryStream()) - { - await JsonSerializer.SerializeAsync(tempStream, this, _treeStyleParserSerializerOptions); - tempStream.Position = 0; + await JsonSerializer.SerializeAsync(stream, this, _serializerOptions); + } - using (var compressionStream = new GZipStream(stream, CompressionMode.Compress, leaveOpen: true)) - { - await tempStream.CopyToAsync(compressionStream); - } - } - } - else + /// + /// Generates a complete text representation of API surface to help generating the content. + /// One use case of this function will be to support download request of entire API review surface. + /// + public string GetApiText() + { + StringBuilder sb = new(); + foreach (var line in ReviewLines) { - await JsonSerializer.SerializeAsync(stream, this, _serializerOptions); + line.AppendApiTextToBuilder(sb, 0, true); } - } + return sb.ToString(); + } } } diff --git a/src/dotnet/APIView/APIView/Model/V2/ReviewLine.cs b/src/dotnet/APIView/APIView/Model/V2/ReviewLine.cs new file mode 100644 index 00000000000..6b8663c82e3 --- /dev/null +++ b/src/dotnet/APIView/APIView/Model/V2/ReviewLine.cs @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Xml; +using APIView.TreeToken; + +namespace APIView.Model.V2 +{ + /// + /// Review line object corresponds to each line displayed on API review. If an empty line is required then add a review line object without any token. + /// + public class ReviewLine + { + /// + /// LineId is only required if we need to support commenting on a line that contains this token. + /// Usually code line for documentation or just punctuation is not required to have lineId.lineId should be a unique value within + /// the review token file to use it assign to review comments as well as navigation Id within the review page. /// for e.g Azure.Core.HttpHeader.Common, azure.template.template_main + /// + public string LineId { get; set; } + public string CrossLanguageId { get; set; } + /// + /// List of tokens that constructs a line in API review + /// + public List Tokens { get; set; } = []; + /// + /// Add any child lines as children. For e.g. all classes and namespace level methods are added as a children of namespace(module) level code line. + /// Similarly all method level code lines are added as children of it's class code line. + /// + public List Children { get; set; } = []; + /// + /// This is set if API is marked as hidden + /// + public bool? IsHidden { get; set; } + /// + /// This is set if a line is end of context. For e.g. end of a class or name space line "}" + /// + public bool? IsContextEndLine { get; set; } + // Following properties are helper methods that's used to render review lines to UI required format. + [JsonIgnore] + public DiffKind DiffKind { get; set; } = DiffKind.NoneDiff; + [JsonIgnore] + public bool IsActiveRevisionLine = true; + [JsonIgnore] + public bool IsDocumentation => Tokens.Count > 0 && Tokens[0].IsDocumentation == true; + [JsonIgnore] + public bool IsEmpty => Tokens.Count == 0 || !Tokens.Any( t => t.SkipDiff != true); + [JsonIgnore] + public bool Processed { get; set; } = false; + + public void AddToken(ReviewToken token) + { + Tokens.Add(token); + } + + public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDocs = true) + { + if (skipDocs && Tokens.Count > 0 && Tokens[0].IsDocumentation == true) + { + return; + } + + //Add empty line in case of review line without tokens + if (Tokens.Count == 0) + { + sb.Append(Environment.NewLine); + return; + } + //Add spaces for indentation + for (int i = 0; i < indent; i++) + { + sb.Append(" "); + } + //Process all tokens + sb.Append(ToString(true)); + + sb.Append(Environment.NewLine); + foreach (var child in Children) + { + child.AppendApiTextToBuilder(sb, indent + 1, skipDocs); + } + } + + private string ToString(bool includeAllTokens) + { + var filterdTokens = includeAllTokens ? Tokens: Tokens.Where(x => x.SkipDiff != true); + if (!filterdTokens.Any()) + { + return string.Empty; + } + StringBuilder sb = new(); + foreach (var token in filterdTokens) + { + sb.Append(token.Value); + sb.Append(token.HasSuffixSpace == true ? " " : string.Empty); + } + return sb.ToString(); + } + + + public override string ToString() + { + return ToString(false); + } + + public override bool Equals(object obj) + { + if(obj is ReviewLine other) + { + return ToString() == other.ToString(); + } + return false; + } + + public override int GetHashCode() + { + return ToString().GetHashCode(); + } + + public string GetTokenNodeIdHash(string parentNodeIdHash, int lineIndex) + { + var idPart = LineId; + var token = Tokens.FirstOrDefault(t => t.RenderClasses.Count > 0); + if (token != null) + { + idPart = $"{idPart}-{token.RenderClasses.First()}"; + } + idPart = $"{idPart}-{lineIndex}-{DiffKind}"; + var hash = CreateHashFromString(idPart); + return hash + parentNodeIdHash.Replace("nId", "").Replace("root", ""); // Append the parent node Id to ensure uniqueness + } + + private static string CreateHashFromString(string inputString) + { + int hash = inputString.GetHashCode(); + return "nId" + hash.ToString(); + } + } +} diff --git a/src/dotnet/APIView/APIView/Model/V2/ReviewToken.cs b/src/dotnet/APIView/APIView/Model/V2/ReviewToken.cs new file mode 100644 index 00000000000..fb45f0929eb --- /dev/null +++ b/src/dotnet/APIView/APIView/Model/V2/ReviewToken.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + + +using System; +using System.Collections.Generic; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +namespace APIView.Model.V2 +{ + /// + /// Token corresponds to each component within a code line. A separate token is required for keyword, punctuation, type name, text etc. + /// + public class ReviewToken + { + public ReviewToken() { } + public ReviewToken(string value, TokenKind kind) + { + Value = value; + Kind = kind; + } + public TokenKind Kind { get; set; } + public string Value { get; set; } + + /// + /// NavigationDisplayName property is to add a short name for the token that will be displayed in the navigation object. + /// + public string NavigationDisplayName { get; set; } + + /// + /// navigateToId should be set if the underlying token is required to be displayed as HREF to another type within the review. + /// + public string NavigateToId { get; set; } + + /// + /// set skipDiff to true if underlying token needs to be ignored from diff calculation. + /// For e.g. package metadata or dependency versions are usually not required to be excluded when comparing two revisions to avoid reporting them as API changes + /// + public bool? SkipDiff { get; set; } + + /// + /// This is set if API is marked as deprecated + /// + public bool? IsDeprecated { get; set; } + + /// + /// Set this to false if there is no suffix space required before next token. For e.g, punctuation right after method name + /// + public bool HasSuffixSpace { get; set; } = true; + + /// + /// Set isDocumentation to true if current token is part of documentation + /// + public bool? IsDocumentation { get; set; } + + /// + /// Language specific style css class names + /// + public List RenderClasses = []; + + public static ReviewToken CreateTextToken(string value, string navigateToId = null, bool hasSuffixSpace = true) + { + var token = new ReviewToken(value, TokenKind.Text); + if (!string.IsNullOrEmpty(navigateToId)) + { + token.NavigateToId = navigateToId; + } + token.HasSuffixSpace = hasSuffixSpace; + return token; + } + + public static ReviewToken CreateKeywordToken(string value, bool hasSuffixSpace = true) + { + var token = new ReviewToken(value, TokenKind.Keyword); + token.HasSuffixSpace = hasSuffixSpace; + return token; + } + + public static ReviewToken CreateKeywordToken(SyntaxKind syntaxKind, bool hasSuffixSpace = true) + { + return CreateKeywordToken(SyntaxFacts.GetText(syntaxKind), hasSuffixSpace); + } + + public static ReviewToken CreateKeywordToken(Accessibility accessibility) + { + return CreateKeywordToken(SyntaxFacts.GetText(accessibility)); + } + + public static ReviewToken CreatePunctuationToken(string value, bool hasSuffixSpace = true) + { + var token = new ReviewToken(value, TokenKind.Punctuation); + token.HasSuffixSpace = hasSuffixSpace; + return token; + } + + public static ReviewToken CreatePunctuationToken(SyntaxKind syntaxKind, bool hasSuffixSpace = true) + { + var token = CreatePunctuationToken(SyntaxFacts.GetText(syntaxKind), hasSuffixSpace); + return token; + } + + public static ReviewToken CreateTypeNameToken(string value, bool hasSuffixSpace = true) + { + var token = new ReviewToken(value, TokenKind.TypeName); + token.HasSuffixSpace = hasSuffixSpace; + return token; + } + + public static ReviewToken CreateMemberNameToken(string value, bool hasSuffixSpace = true) + { + var token = new ReviewToken(value, TokenKind.MemberName); + token.HasSuffixSpace = hasSuffixSpace; + return token; + } + + public static ReviewToken CreateLiteralToken(string value, bool hasSuffixSpace = true) + { + var token = new ReviewToken(value, TokenKind.Literal); + token.HasSuffixSpace = hasSuffixSpace; + return token; + } + + public static ReviewToken CreateStringLiteralToken(string value, bool hasSuffixSpace = true) + { + var token = new ReviewToken(value, TokenKind.StringLiteral); + token.HasSuffixSpace = hasSuffixSpace; + return token; + } + + public static ReviewToken CreateCommentToken(string value, bool hasSuffixSpace = true) + { + var token = new ReviewToken(value, TokenKind.Comment); + token.HasSuffixSpace = hasSuffixSpace; + return token; + } + } +} diff --git a/src/dotnet/APIView/APIView/Model/V2/TokenKind.cs b/src/dotnet/APIView/APIView/Model/V2/TokenKind.cs new file mode 100644 index 00000000000..b95d7378616 --- /dev/null +++ b/src/dotnet/APIView/APIView/Model/V2/TokenKind.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace APIView.Model.V2 +{ + public enum TokenKind + { + Text = 0, + Punctuation = 1, + Keyword = 2, + TypeName = 3, + MemberName = 4, + StringLiteral = 5, + Literal = 6, + Comment = 7 + } +} diff --git a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParser/Program.cs b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParser/Program.cs index e2f1215434e..4367439b95d 100644 --- a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParser/Program.cs +++ b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParser/Program.cs @@ -1,9 +1,10 @@ using System.CommandLine; using System.IO.Compression; -using System.Text.Json; -using System.Text.Json.Serialization; +using System.Text; +using System.Text.RegularExpressions; using System.Xml.Linq; using ApiView; +using APIView.Model.V2; using NuGet.Common; using NuGet.Packaging; using NuGet.Protocol; @@ -13,6 +14,12 @@ public static class Program { + // Regex parser nuget file name without extension to two groups + // package name and package version + // for e.g Azure.Core.1.0.0 to ["Azure.Core", "1.0.0"] + // or Azure.Storage.Blobs.12.0.0 to ["Azure.Storage.Blobs", "12.0.0"] + private static Regex _packageNameParser = new Regex("([A-Za-z.]*[a-z]).([\\S]*)", RegexOptions.Compiled); + public static int Main(string[] args) { var inputOption = new Option("--packageFilePath", "C# Package (.nupkg) file").ExistingOnly(); @@ -69,7 +76,10 @@ static async Task HandlePackageFileParsing(Stream stream, FileInfo packageFilePa if (dllEntries.Length == 0) { - Console.Error.WriteLine($"PackageFile {packageFilePath.FullName} contains no dlls."); + Console.Error.WriteLine($"PackageFile {packageFilePath.FullName} contains no dll. Creating a meta package API review file."); + var codeFile = CreateDummyCodeFile(packageFilePath.FullName, $"Package {packageFilePath.Name} does not contain any dll to create API review."); + outputFileName = string.IsNullOrEmpty(outputFileName) ? nuspecEntry.Name : outputFileName; + await CreateOutputFile(OutputDirectory.FullName, outputFileName, codeFile); return; } @@ -120,24 +130,15 @@ static async Task HandlePackageFileParsing(Stream stream, FileInfo packageFilePa if (assemblySymbol == null) { Console.Error.WriteLine($"PackageFile {packageFilePath.FullName} contains no Assembly Symbol."); + var codeFile = CreateDummyCodeFile(packageFilePath.FullName, $"Package {packageFilePath.Name} does not contain any assembly symbol to create API review."); + outputFileName = string.IsNullOrEmpty(outputFileName) ? packageFilePath.Name : outputFileName; + await CreateOutputFile(OutputDirectory.FullName, outputFileName, codeFile); return; } var parsedFileName = string.IsNullOrEmpty(outputFileName) ? assemblySymbol.Name : outputFileName; var treeTokenCodeFile = new CSharpAPIParser.TreeToken.CodeFileBuilder().Build(assemblySymbol, runAnalysis, dependencies); - var gzipJsonTokenFilePath = Path.Combine(OutputDirectory.FullName, $"{parsedFileName}.json.tgz"); - - - var options = new JsonSerializerOptions() - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - - await using FileStream gzipFileStream = new FileStream(gzipJsonTokenFilePath, FileMode.Create, FileAccess.Write); - await using GZipStream gZipStream = new GZipStream(gzipFileStream, CompressionLevel.Optimal); - await JsonSerializer.SerializeAsync(gZipStream, treeTokenCodeFile, options); - Console.WriteLine($"TokenCodeFile File {gzipJsonTokenFilePath} Generated Successfully."); - Console.WriteLine(); + await CreateOutputFile(OutputDirectory.FullName, parsedFileName, treeTokenCodeFile); } catch (Exception ex) { @@ -154,6 +155,40 @@ static async Task HandlePackageFileParsing(Stream stream, FileInfo packageFilePa } } + static async Task CreateOutputFile(string outputPath, string outputFileNamePrefix, CodeFile apiViewFile) + { + var jsonTokenFilePath = Path.Combine(outputPath, $"{outputFileNamePrefix}.json"); + await using FileStream fileStream = new(jsonTokenFilePath, FileMode.Create, FileAccess.Write); + await apiViewFile.SerializeAsync(fileStream); + Console.WriteLine($"TokenCodeFile File {jsonTokenFilePath} Generated Successfully."); + Console.WriteLine(); + } + + /*** Creates dummy API review file to support meta packages.*/ + static CodeFile CreateDummyCodeFile(string originalName, string text) + { + var packageName = Path.GetFileNameWithoutExtension(originalName); + var packageNameMatch = _packageNameParser.Match(packageName); + var packageVersion = ""; + if (packageNameMatch.Success) + { + packageName = packageNameMatch.Groups[1].Value; + packageVersion = $"{packageNameMatch.Groups[2].Value}"; + } + + var codeFile = new CodeFile(); + codeFile.PackageName = packageName; + codeFile.PackageVersion = packageVersion; + codeFile.ReviewLines.Add(new ReviewLine + { + Tokens = new List + { + ReviewToken.CreateTextToken(text) + } + }); + return codeFile; + } + static bool IsNuget(string name) { return name.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase); diff --git a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParser/TreeToken/CodeFileBuilder.cs b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParser/TreeToken/CodeFileBuilder.cs index dba59ffc0a9..5f9481ba541 100644 --- a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParser/TreeToken/CodeFileBuilder.cs +++ b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParser/TreeToken/CodeFileBuilder.cs @@ -1,18 +1,18 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using ApiView; using APIView.Analysis; -using APIView.TreeToken; +using APIView.Model.V2; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.SymbolDisplay; using System.Collections.Immutable; using System.ComponentModel; +using ApiView; namespace CSharpAPIParser.TreeToken { - internal class CodeFileBuilder + public class CodeFileBuilder { private static readonly char[] _newlineChars = new char[] { '\r', '\n' }; @@ -47,7 +47,7 @@ internal class CodeFileBuilder public ICodeFileBuilderSymbolOrderProvider SymbolOrderProvider { get; set; } = new CodeFileBuilderSymbolOrderProvider(); - public const string CurrentVersion = "28"; + public const string CurrentVersion = "29"; private IEnumerable EnumerateNamespaces(IAssemblySymbol assemblySymbol) { @@ -76,16 +76,19 @@ public CodeFile Build(IAssemblySymbol assemblySymbol, bool runAnalysis, List() { apiTreeNode }, - VersionString = CurrentVersion, - Diagnostics = analyzer.Results.ToArray(), - PackageName = assemblySymbol.Name, - PackageVersion = assemblySymbol.Identity.Version.ToString() - }; - - return treeTokenCodeFile; + codeFile.Diagnostics = analyzer.Results.ToArray(); + return codeFile; } - public static void BuildInternalsVisibleToAttributes(List apiTree, IAssemblySymbol assemblySymbol) + public static void BuildInternalsVisibleToAttributes(List reviewLines, IAssemblySymbol assemblySymbol) { var assemblyAttributes = assemblySymbol.GetAttributes() .Where(a => @@ -129,10 +119,13 @@ public static void BuildInternalsVisibleToAttributes(List apiTree, !a.ConstructorArguments[0].Value?.ToString()?.Contains("DynamicProxyGenAssembly2") == true); if (assemblyAttributes != null && assemblyAttributes.Any()) { - var apiTreeNode = new APITreeNode(); - apiTreeNode.Kind = apiTreeNode.Name = apiTreeNode.Id = APITreeNode.INTERNALS_VISIBLE_TO; - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateTextToken(value: "Exposes internals to:")); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateLineBreakToken()); + reviewLines.Add(new ReviewLine() + { + LineId = "InternalsVisibleTo", + Tokens = [ + ReviewToken.CreateStringLiteralToken("Exposes internals to:") + ] + }); foreach (AttributeData attribute in assemblyAttributes) { @@ -143,84 +136,107 @@ public static void BuildInternalsVisibleToAttributes(List apiTree, { var firstComma = param?.IndexOf(','); param = firstComma > 0 ? param?[..(int)firstComma] : param; - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateTextToken(value: param, id: attribute.AttributeClass?.Name)); + reviewLines.Add(new ReviewLine() + { + LineId = attribute.AttributeClass?.Name, + Tokens = [ + ReviewToken.CreateStringLiteralToken(param) + ] + }); } } } - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreateEmptyToken()); - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreateLineBreakToken()); - apiTree.Add(apiTreeNode); } } - public static void BuildDependencies(List apiTree, List dependencies) + public static void BuildDependencies(List reviewLines, List dependencies) { if (dependencies != null && dependencies.Any()) { - var apiTreeNode = new APITreeNode(); - apiTreeNode.Kind = apiTreeNode.Name = apiTreeNode.Id = APITreeNode.DEPENDENCIES; - - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateLineBreakToken()); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateTextToken(value: "Dependencies:")); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateLineBreakToken()); + //Dependencies + var headerLine = new ReviewLine() + { + LineId = "Dependencies" + }; + var depToken = ReviewToken.CreateStringLiteralToken("Dependencies:"); + depToken.NavigationDisplayName = "Dependencies"; + depToken.RenderClasses.Add("dependencies"); + headerLine.Tokens.Add(depToken); + reviewLines.Add(headerLine); foreach (DependencyInfo dependency in dependencies) { - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateTextToken(value: dependency.Name, id: dependency.Name)); - var dependencyVersionToken = StructuredToken.CreateTextToken(value: $"-{dependency.Version}"); - dependencyVersionToken.TagsObj.Add(StructuredToken.SKIPP_DIFF); - apiTreeNode.TopTokensObj.Add(dependencyVersionToken); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateLineBreakToken()); + var versionToken = ReviewToken.CreateStringLiteralToken($"-{dependency.Version}"); + versionToken.SkipDiff = true; + var dependencyLine = new ReviewLine() + { + LineId = dependency.Name, + Tokens = [ + ReviewToken.CreateStringLiteralToken(dependency.Name, false), + versionToken + ] + }; + reviewLines.Add(dependencyLine); } - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreateEmptyToken()); - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreateLineBreakToken()); - apiTree.Add(apiTreeNode); } } - private void BuildNamespace(List apiTree, INamespaceSymbol namespaceSymbol) + private void BuildNamespace(List reviewLines, INamespaceSymbol namespaceSymbol) { bool isHidden = HasOnlyHiddenTypes(namespaceSymbol); + //Add an empty review line to add empty line in the review before current name space begins. + if (reviewLines.Count > 0) + { + reviewLines.Add(new ReviewLine() { IsHidden = isHidden }); + } - var apiTreeNode = new APITreeNode(); - apiTreeNode.Id = namespaceSymbol.GetId(); - apiTreeNode.Name = namespaceSymbol.ToDisplayString(); - apiTreeNode.Kind = APITreeNode.NAMESPACE; + var namespaceLine = new ReviewLine() + { + LineId = namespaceSymbol.GetId(), + Tokens = [ + ReviewToken.CreateKeywordToken("namespace") + ], + IsHidden = isHidden + }; - if (isHidden) + BuildNamespaceName(namespaceLine, namespaceSymbol); + var nameSpaceToken = namespaceLine.Tokens.LastOrDefault(); + if (nameSpaceToken != null) { - apiTreeNode.TagsObj.Add(APITreeNode.HIDDEN); + nameSpaceToken.RenderClasses.Add("namespace"); + nameSpaceToken.NavigationDisplayName = namespaceSymbol.ToDisplayString(); } - - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateKeywordToken(SyntaxKind.NamespaceKeyword)); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateSpaceToken()); - BuildNamespaceName(apiTreeNode.TopTokensObj, namespaceSymbol); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateSpaceToken()); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.OpenBraceToken)); + namespaceLine.Tokens.Last().HasSuffixSpace = true; + namespaceLine.Tokens.Add(ReviewToken.CreatePunctuationToken("{")); + // Add each members in the namespace foreach (var namedTypeSymbol in SymbolOrderProvider.OrderTypes(namespaceSymbol.GetTypeMembers())) { - BuildType(apiTreeNode.ChildrenObj, namedTypeSymbol, isHidden); + BuildType(namespaceLine.Children, namedTypeSymbol, isHidden); } - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CloseBraceToken)); - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreateLineBreakToken()); - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreateEmptyToken()); - - apiTree.Add(apiTreeNode); + reviewLines.Add(namespaceLine); + reviewLines.Add(new ReviewLine() + { + Tokens = [ + ReviewToken.CreateStringLiteralToken("}") + ], + IsHidden = isHidden, + IsContextEndLine = true + }); } - private void BuildNamespaceName(List tokenList, INamespaceSymbol namespaceSymbol) + private void BuildNamespaceName(ReviewLine namespaceLine, INamespaceSymbol namespaceSymbol) { if (!namespaceSymbol.ContainingNamespace.IsGlobalNamespace) { - BuildNamespaceName(tokenList, namespaceSymbol.ContainingNamespace); - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.DotToken)); + BuildNamespaceName(namespaceLine, namespaceSymbol.ContainingNamespace); + var punctuation = ReviewToken.CreatePunctuationToken(".", false); + namespaceLine.Tokens.Add(punctuation); } - DisplayName(tokenList, namespaceSymbol, namespaceSymbol); + DisplayName(namespaceLine, namespaceSymbol, namespaceSymbol); } - private bool HasAnyPublicTypes(INamespaceSymbol subNamespaceSymbol) { return subNamespaceSymbol.GetTypeMembers().Any(IsAccessible); @@ -231,72 +247,81 @@ private bool HasOnlyHiddenTypes(INamespaceSymbol namespaceSymbol) return namespaceSymbol.GetTypeMembers().All(t => IsHiddenFromIntellisense(t) || !IsAccessible(t)); } - private void BuildType(List apiTree, INamedTypeSymbol namedType, bool inHiddenScope) + private void BuildType(List reviewLines, INamedTypeSymbol namedType, bool inHiddenScope) { if (!IsAccessible(namedType)) { return; } - bool isHidden = IsHiddenFromIntellisense(namedType); - var apiTreeNode = new APITreeNode(); - apiTreeNode.Kind = APITreeNode.TYPE; - apiTreeNode.PropertiesObj.Add(APITreeNode.SUB_KIND, namedType.TypeKind.ToString().ToLowerInvariant()); - apiTreeNode.Id = namedType.GetId(); - apiTreeNode.Name = namedType.ToDisplayString(SymbolDisplayFormat.MinimallyQualifiedFormat); - - if (isHidden && !inHiddenScope) + bool isHidden = IsHiddenFromIntellisense(namedType) || inHiddenScope; + //Add empty line to separate types before current node begins + if (reviewLines.Count > 0) { - apiTreeNode.TagsObj.Add(APITreeNode.HIDDEN); + reviewLines.Add(new ReviewLine() + { + IsHidden = isHidden + }); } - BuildDocumentation(apiTreeNode.TopTokensObj, namedType); - BuildAttributes(apiTreeNode.TopTokensObj, namedType.GetAttributes()); - BuildVisibility(apiTreeNode.TopTokensObj, namedType); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateSpaceToken()); + var reviewLine = new ReviewLine() + { + LineId = namedType.GetId(), + IsHidden = isHidden + }; + + // Build documentation, attributes, visibility, and name + BuildDocumentation(reviewLines, namedType, isHidden); + BuildAttributes(reviewLines, namedType.GetAttributes(), isHidden); + BuildVisibility(reviewLine.Tokens, namedType); switch (namedType.TypeKind) { case TypeKind.Class: - BuildClassModifiers(apiTreeNode.TopTokensObj, namedType); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateKeywordToken(SyntaxKind.ClassKeyword)); + BuildClassModifiers(reviewLine.Tokens, namedType); + reviewLine.Tokens.Add(ReviewToken.CreateKeywordToken(SyntaxKind.ClassKeyword)); break; case TypeKind.Delegate: - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateKeywordToken(SyntaxKind.DelegateKeyword)); + reviewLine.Tokens.Add(ReviewToken.CreateKeywordToken(SyntaxKind.DelegateKeyword)); break; case TypeKind.Enum: - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateKeywordToken(SyntaxKind.EnumKeyword)); + reviewLine.Tokens.Add(ReviewToken.CreateKeywordToken(SyntaxKind.EnumKeyword)); break; case TypeKind.Interface: - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateKeywordToken(SyntaxKind.InterfaceKeyword)); + reviewLine.Tokens.Add(ReviewToken.CreateKeywordToken(SyntaxKind.InterfaceKeyword)); break; case TypeKind.Struct: if (namedType.IsReadOnly) { - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateKeywordToken(SyntaxKind.ReadOnlyKeyword)); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateSpaceToken()); + reviewLine.Tokens.Add(ReviewToken.CreateKeywordToken(SyntaxKind.ReadOnlyKeyword)); } - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateKeywordToken(SyntaxKind.StructKeyword)); + reviewLine.Tokens.Add(ReviewToken.CreateKeywordToken(SyntaxKind.StructKeyword)); break; } - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateSpaceToken()); - DisplayName(apiTreeNode.TopTokensObj, namedType, namedType); + DisplayName(reviewLine, namedType, namedType); + + // Add navigation short name and render classes to Type name token. Navigation tree is built dynamically based on these properties + var typeToken = reviewLine.Tokens.FirstOrDefault(t => t.Kind == TokenKind.TypeName && string.IsNullOrEmpty(t.NavigateToId)); + if (typeToken != null) + { + typeToken.NavigationDisplayName = namedType.ToDisplayString(SymbolDisplayFormat.MinimallyQualifiedFormat); + typeToken.RenderClasses.Add(namedType.TypeKind.ToString().ToLowerInvariant()); + typeToken.HasSuffixSpace = true; + } if (namedType.TypeKind == TypeKind.Delegate) { - apiTreeNode.TopTokensObj.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.SemicolonToken)); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateLineBreakToken()); + reviewLine.Tokens.Last().HasSuffixSpace = false; + reviewLine.Tokens.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.SemicolonToken)); return; } - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateSpaceToken()); - BuildBaseType(apiTreeNode.TopTokensObj, namedType); - apiTreeNode.TopTokensObj.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.OpenBraceToken)); - + BuildBaseType(reviewLine, namedType); + reviewLine.Tokens.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.OpenBraceToken)); foreach (var namedTypeSymbol in SymbolOrderProvider.OrderTypes(namedType.GetTypeMembers())) { - BuildType(apiTreeNode.ChildrenObj, namedTypeSymbol, inHiddenScope || isHidden); + BuildType(reviewLine.Children, namedTypeSymbol, isHidden); } foreach (var member in SymbolOrderProvider.OrderMembers(namedType.GetMembers())) @@ -313,16 +338,20 @@ private void BuildType(List apiTree, INamedTypeSymbol namedType, bo continue; } } - BuildMember(apiTreeNode.ChildrenObj, member, inHiddenScope); + BuildMember(reviewLine.Children, member, isHidden); } - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CloseBraceToken)); - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreateLineBreakToken()); - apiTreeNode.BottomTokensObj.Add(StructuredToken.CreateEmptyToken()); - - apiTree.Add(apiTreeNode); + reviewLines.Add(reviewLine); + reviewLines.Add(new ReviewLine() + { + Tokens = [ + ReviewToken.CreateStringLiteralToken("}") + ], + IsHidden = isHidden, + IsContextEndLine = true + }); } - private void BuildDocumentation(List tokenList, ISymbol symbol) + private void BuildDocumentation(List reviewLines, ISymbol symbol, bool isHidden) { var lines = symbol.GetDocumentationCommentXml()?.Trim().Split(_newlineChars); @@ -334,48 +363,46 @@ private void BuildDocumentation(List tokenList, ISymbol symbol) } foreach (var line in lines) { - var docToken = new StructuredToken("// " + line.Trim()); - docToken.RenderClassesObj.Add("comment"); - docToken.PropertiesObj.Add(StructuredToken.GROUP_ID, StructuredToken.DOCUMENTATION); - tokenList.Add(docToken); - tokenList.Add(StructuredToken.CreateLineBreakToken()); + var docToken = ReviewToken.CreateCommentToken("// " + line.Trim()); + docToken.IsDocumentation = true; + reviewLines.Add(new ReviewLine() + { + Tokens = [docToken], + IsHidden = isHidden + }); } } } - private static void BuildClassModifiers(List tokenList, INamedTypeSymbol namedType) + private static void BuildClassModifiers(List tokenList, INamedTypeSymbol namedType) { if (namedType.IsAbstract) { - tokenList.Add(StructuredToken.CreateKeywordToken(SyntaxKind.AbstractKeyword)); - tokenList.Add(StructuredToken.CreateSpaceToken()); + tokenList.Add(ReviewToken.CreateKeywordToken(SyntaxKind.AbstractKeyword)); } if (namedType.IsStatic) { - tokenList.Add(StructuredToken.CreateKeywordToken(SyntaxKind.StaticKeyword)); - tokenList.Add(StructuredToken.CreateSpaceToken()); + tokenList.Add(ReviewToken.CreateKeywordToken(SyntaxKind.StaticKeyword)); } if (namedType.IsSealed) { - tokenList.Add(StructuredToken.CreateKeywordToken(SyntaxKind.SealedKeyword)); - tokenList.Add(StructuredToken.CreateSpaceToken()); + tokenList.Add(ReviewToken.CreateKeywordToken(SyntaxKind.SealedKeyword)); } } - private void BuildBaseType(List tokenList, INamedTypeSymbol namedType) + private void BuildBaseType(ReviewLine reviewLine, INamedTypeSymbol namedType) { bool first = true; if (namedType.BaseType != null && namedType.BaseType.SpecialType == SpecialType.None) { - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.ColonToken)); - tokenList.Add(StructuredToken.CreateSpaceToken()); + reviewLine.AddToken(ReviewToken.CreatePunctuationToken(SyntaxKind.ColonToken)); first = false; - - DisplayName(tokenList, namedType.BaseType); + DisplayName(reviewLine, namedType.BaseType); + reviewLine.Tokens.Last().HasSuffixSpace = true; } foreach (var typeInterface in namedType.Interfaces) @@ -384,58 +411,52 @@ private void BuildBaseType(List tokenList, INamedTypeSymbol nam if (!first) { - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CommaToken)); - tokenList.Add(StructuredToken.CreateSpaceToken()); + reviewLine.Tokens.Last().HasSuffixSpace = false; + reviewLine.AddToken(ReviewToken.CreatePunctuationToken(SyntaxKind.CommaToken)); } else { - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.ColonToken)); - tokenList.Add(StructuredToken.CreateSpaceToken()); + reviewLine.AddToken(ReviewToken.CreatePunctuationToken(SyntaxKind.ColonToken)); first = false; } - - DisplayName(tokenList, typeInterface); - } - - if (!first) - { - tokenList.Add(StructuredToken.CreateSpaceToken()); + DisplayName(reviewLine, typeInterface); + reviewLine.Tokens.Last().HasSuffixSpace = true; } } - private void BuildMember(List apiTree, ISymbol member, bool inHiddenScope) + private void BuildMember(List reviewLines, ISymbol member, bool inHiddenScope) { - bool isHidden = IsHiddenFromIntellisense(member); - var apiTreeNode = new APITreeNode(); - apiTreeNode.Kind = APITreeNode.MEMBER; - apiTreeNode.PropertiesObj.Add(APITreeNode.SUB_KIND, member.Kind.ToString()); - apiTreeNode.Id = member.GetId(); - apiTreeNode.Name = member.ToDisplayString(); - apiTreeNode.TagsObj.Add(APITreeNode.HIDE_FROM_NAV); + bool isHidden = IsHiddenFromIntellisense(member) || inHiddenScope; + var reviewLine = new ReviewLine() + { + LineId = member.GetId(), + IsHidden = isHidden + }; - if (isHidden && !inHiddenScope) + BuildDocumentation(reviewLines, member, isHidden); + BuildAttributes(reviewLines, member.GetAttributes(), isHidden); + reviewLines.Add(reviewLine); + DisplayName(reviewLine, member); + reviewLine.Tokens.Last().HasSuffixSpace = false; + + // Set member sub kind class for render class styling + var memToken = reviewLine.Tokens.FirstOrDefault(m => m.Kind == TokenKind.MemberName); + if (memToken != null) { - apiTreeNode.TagsObj.Add(APITreeNode.HIDDEN); + memToken.RenderClasses.Add(member.Kind.ToString().ToLowerInvariant()); } - BuildDocumentation(apiTreeNode.TopTokensObj, member); - BuildAttributes(apiTreeNode.TopTokensObj, member.GetAttributes()); - DisplayName(apiTreeNode.TopTokensObj, member); - if (member.Kind == SymbolKind.Field && member.ContainingType.TypeKind == TypeKind.Enum) { - apiTreeNode.TopTokensObj.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CommaToken)); + reviewLine.AddToken(ReviewToken.CreatePunctuationToken(SyntaxKind.CommaToken)); } else if (member.Kind != SymbolKind.Property) { - apiTreeNode.TopTokensObj.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.SemicolonToken)); + reviewLine.AddToken(ReviewToken.CreatePunctuationToken(SyntaxKind.SemicolonToken)); } - - apiTreeNode.TopTokensObj.Add(StructuredToken.CreateLineBreakToken()); - apiTree.Add(apiTreeNode); } - private void BuildAttributes(List tokenList, ImmutableArray attributes) + private void BuildAttributes(List reviewLines, ImmutableArray attributes, bool isHidden) { const string attributeSuffix = "Attribute"; foreach (var attribute in attributes) @@ -450,59 +471,61 @@ private void BuildAttributes(List tokenList, ImmutableArray private bool IsDecoratedWithAttribute(ISymbol member, string attributeName) => member.GetAttributes().Any(d => d.AttributeClass?.Name == attributeName); - private void BuildTypedConstant(List tokenList, TypedConstant typedConstant) + private void BuildTypedConstant(ReviewLine reviewLine, TypedConstant typedConstant) { + var tokenList = reviewLine.Tokens; if (typedConstant.IsNull) { - tokenList.Add(StructuredToken.CreateKeywordToken(SyntaxKind.NullKeyword)); + tokenList.Add(ReviewToken.CreateKeywordToken(SyntaxKind.NullKeyword, false)); } else if (typedConstant.Kind == TypedConstantKind.Enum) { @@ -544,18 +568,18 @@ private void BuildTypedConstant(List tokenList, TypedConstant t } else if (typedConstant.Kind == TypedConstantKind.Type) { - tokenList.Add(StructuredToken.CreateKeywordToken(SyntaxKind.TypeOfKeyword)); - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.OpenParenToken)); - DisplayName(tokenList, (ITypeSymbol)typedConstant.Value!); - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CloseParenToken)); + tokenList.Add(ReviewToken.CreateKeywordToken(SyntaxKind.TypeOfKeyword, false)); + tokenList.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.OpenParenToken, false)); + DisplayName(reviewLine, (ITypeSymbol)typedConstant.Value!); + reviewLine.Tokens.Last().HasSuffixSpace = false; + tokenList.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.CloseParenToken, false)); } else if (typedConstant.Kind == TypedConstantKind.Array) { - tokenList.Add(StructuredToken.CreateKeywordToken(SyntaxKind.NewKeyword)); - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.OpenBracketToken)); - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CloseBracketToken)); - tokenList.Add(StructuredToken.CreateSpaceToken()); - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.OpenBraceToken)); + tokenList.Add(ReviewToken.CreateKeywordToken(SyntaxKind.NewKeyword, false)); + tokenList.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.OpenBracketToken, false)); + tokenList.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.CloseBracketToken)); + tokenList.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.OpenBraceToken)); bool first = true; @@ -563,43 +587,43 @@ private void BuildTypedConstant(List tokenList, TypedConstant t { if (!first) { - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CommaToken)); - tokenList.Add(StructuredToken.CreateSpaceToken()); + tokenList.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.CommaToken)); } else { first = false; } - BuildTypedConstant(tokenList, value); + BuildTypedConstant(reviewLine, value); + reviewLine.Tokens.Last().HasSuffixSpace = false; } - tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CloseBraceToken)); + tokenList.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.CloseBraceToken, false)); } else { if (typedConstant.Value is string s) { - tokenList.Add(StructuredToken.CreateStringLiteralToken(ObjectDisplay.FormatLiteral(s, ObjectDisplayOptions.UseQuotes | ObjectDisplayOptions.EscapeNonPrintableCharacters))); + tokenList.Add(ReviewToken.CreateStringLiteralToken(ObjectDisplay.FormatLiteral(s, ObjectDisplayOptions.UseQuotes | ObjectDisplayOptions.EscapeNonPrintableCharacters), false)); } else { - tokenList.Add(StructuredToken.CreateLiteralToken(ObjectDisplay.FormatPrimitive(typedConstant.Value, ObjectDisplayOptions.None))); + tokenList.Add(ReviewToken.CreateLiteralToken(ObjectDisplay.FormatPrimitive(typedConstant.Value, ObjectDisplayOptions.None), false)); } } } - private void BuildVisibility(List tokenList, ISymbol symbol) + private void BuildVisibility(List tokenList, ISymbol symbol) { - tokenList.Add(StructuredToken.CreateKeywordToken(ToEffectiveAccessibility(symbol.DeclaredAccessibility))); + tokenList.Add(ReviewToken.CreateKeywordToken(ToEffectiveAccessibility(symbol.DeclaredAccessibility))); } - private void DisplayName(List tokenList, ISymbol symbol, ISymbol? definedSymbol = null) + private void DisplayName(ReviewLine reviewLine, ISymbol symbol, ISymbol? definedSymbol = null) { - tokenList.Add(StructuredToken.CreateEmptyToken(id: symbol.GetId())); + var reviewLineTokens = reviewLine.Tokens; + if (NeedsAccessibility(symbol)) { - tokenList.Add(StructuredToken.CreateKeywordToken(ToEffectiveAccessibility(symbol.DeclaredAccessibility))); - tokenList.Add(StructuredToken.CreateSpaceToken()); + reviewLineTokens.Add(ReviewToken.CreateKeywordToken(ToEffectiveAccessibility(symbol.DeclaredAccessibility))); } if (symbol is IPropertySymbol propSymbol && propSymbol.DeclaredAccessibility != Accessibility.Internal) { @@ -615,8 +639,23 @@ private void DisplayName(List tokenList, ISymbol symbol, ISymbo i++; } } - tokenList.Add(MapToken(definedSymbol: definedSymbol!, symbolDisplayPart: parts[i], - previousSymbolDisplayPart: previous)); + var previousToken = reviewLine.Tokens.LastOrDefault(); + //Add a new code line as child if there is a line break + if (parts[i].Kind == SymbolDisplayPartKind.LineBreak) + { + var subLine = new ReviewLine() + { + LineId = definedSymbol.GetId(), + }; + reviewLine.Children.Add(subLine); + reviewLineTokens = subLine.Tokens; + } + var token = MapToken(definedSymbol: definedSymbol!, symbolDisplayPart: parts[i], + previousSymbolDisplayPart: previous, previousToken); + if (token != null) + { + reviewLineTokens.Add(token); + } previous = parts[i]; } } @@ -625,8 +664,13 @@ private void DisplayName(List tokenList, ISymbol symbol, ISymbo SymbolDisplayPart previous = default(SymbolDisplayPart); foreach (var symbolDisplayPart in symbol.ToDisplayParts(_defaultDisplayFormat)) { - tokenList.Add(MapToken(definedSymbol: definedSymbol!, symbolDisplayPart: symbolDisplayPart, - previousSymbolDisplayPart: previous)); + var previousToken = reviewLine.Tokens.LastOrDefault(); + var token = MapToken(definedSymbol: definedSymbol!, symbolDisplayPart: symbolDisplayPart, + previousSymbolDisplayPart: previous, previousToken: previousToken); + if (token != null) + { + reviewLineTokens.Add(token); + } previous = symbolDisplayPart; } } @@ -647,7 +691,7 @@ private bool NeedsAccessibility(ISymbol symbol) }; } - private StructuredToken MapToken(ISymbol definedSymbol, SymbolDisplayPart symbolDisplayPart, SymbolDisplayPart previousSymbolDisplayPart) + private ReviewToken? MapToken(ISymbol definedSymbol, SymbolDisplayPart symbolDisplayPart, SymbolDisplayPart previousSymbolDisplayPart, ReviewToken? previousToken) { string? navigateToId = null; var symbol = symbolDisplayPart.Symbol; @@ -659,10 +703,9 @@ private StructuredToken MapToken(ISymbol definedSymbol, SymbolDisplayPart symbol navigateToId = symbol.GetId(); } - var definitionId = (definedSymbol != null && SymbolEqualityComparer.Default.Equals(definedSymbol, symbol)) ? definedSymbol.GetId() : null; var tokenValue = symbolDisplayPart.ToString(); - StructuredToken? token = null; + ReviewToken? token = null; switch (symbolDisplayPart.Kind) { @@ -675,28 +718,21 @@ private StructuredToken MapToken(ISymbol definedSymbol, SymbolDisplayPart symbol case SymbolDisplayPartKind.ErrorTypeName: case SymbolDisplayPartKind.InterfaceName: case SymbolDisplayPartKind.StructName: - token = StructuredToken.CreateTypeNameToken(tokenValue); + token = ReviewToken.CreateTypeNameToken(tokenValue, false); break; case SymbolDisplayPartKind.Keyword: - token = StructuredToken.CreateKeywordToken(tokenValue); - break; - case SymbolDisplayPartKind.LineBreak: - token = StructuredToken.CreateLineBreakToken(); + token = ReviewToken.CreateKeywordToken(tokenValue, false); break; case SymbolDisplayPartKind.StringLiteral: - token = StructuredToken.CreateStringLiteralToken(tokenValue); + token = ReviewToken.CreateStringLiteralToken(tokenValue, false); break; case SymbolDisplayPartKind.Punctuation: - token = StructuredToken.CreatePunctuationToken(tokenValue); + token = ReviewToken.CreatePunctuationToken(tokenValue, false); break; case SymbolDisplayPartKind.Space: - if (previousSymbolDisplayPart.Kind == SymbolDisplayPartKind.Punctuation && previousSymbolDisplayPart.ToString().Equals(",")) - { - token = StructuredToken.CreateParameterSeparatorToken(); - } - else + if (previousToken != null) { - token = StructuredToken.CreateSpaceToken(); + previousToken.HasSuffixSpace = true; } break; case SymbolDisplayPartKind.PropertyName: @@ -707,23 +743,18 @@ private StructuredToken MapToken(ISymbol definedSymbol, SymbolDisplayPart symbol case SymbolDisplayPartKind.EnumMemberName: case SymbolDisplayPartKind.ExtensionMethodName: case SymbolDisplayPartKind.ConstantName: - token = StructuredToken.CreateMemberNameToken(tokenValue); + token = ReviewToken.CreateMemberNameToken(tokenValue, false); break; default: - token = StructuredToken.CreateTextToken(tokenValue); + token = ReviewToken.CreateTextToken(tokenValue, hasSuffixSpace: false); break; } - if (!String.IsNullOrWhiteSpace(definitionId)) + if (token != null && !String.IsNullOrWhiteSpace(navigateToId)) { - token.Id = definitionId!; + token.NavigateToId = navigateToId!; } - if (!String.IsNullOrWhiteSpace(navigateToId)) - { - token.PropertiesObj.Add(StructuredToken.NAVIGATE_TO_ID, navigateToId!); - } - return token; } @@ -767,9 +798,9 @@ private bool IsAccessibleExplicitInterfaceImplementation(ISymbol s) internal class CodeFileBuilderEnumFormatter : AbstractSymbolDisplayVisitor { - private readonly List _tokenList; + private readonly List _tokenList; - public CodeFileBuilderEnumFormatter(List tokenList) : base(null, SymbolDisplayFormat.FullyQualifiedFormat, false, null, 0, false) + public CodeFileBuilderEnumFormatter(List tokenList) : base(null, SymbolDisplayFormat.FullyQualifiedFormat, false, null, 0, false) { _tokenList = tokenList; } @@ -781,29 +812,33 @@ protected override AbstractSymbolDisplayVisitor MakeNotFirstVisitor(bool inNames protected override void AddLiteralValue(SpecialType type, object value) { - _tokenList.Add(StructuredToken.CreateLiteralToken(ObjectDisplay.FormatPrimitive(value, ObjectDisplayOptions.None))); + _tokenList.Add(ReviewToken.CreateLiteralToken(ObjectDisplay.FormatPrimitive(value, ObjectDisplayOptions.None))); } protected override void AddExplicitlyCastedLiteralValue(INamedTypeSymbol namedType, SpecialType type, object value) { - _tokenList.Add(StructuredToken.CreateLiteralToken(ObjectDisplay.FormatPrimitive(value, ObjectDisplayOptions.None))); + _tokenList.Add(ReviewToken.CreateLiteralToken(ObjectDisplay.FormatPrimitive(value, ObjectDisplayOptions.None))); } protected override void AddSpace() { - _tokenList.Add(StructuredToken.CreateSpaceToken()); + var lastToken = _tokenList.LastOrDefault(); + if (lastToken != null) + { + lastToken.HasSuffixSpace = true; + } } protected override void AddBitwiseOr() { - _tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.BarToken)); + _tokenList.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.BarToken)); } public override void VisitField(IFieldSymbol symbol) { - _tokenList.Add(StructuredToken.CreateTypeNameToken(symbol.Type.Name)); - _tokenList.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.DotToken)); - _tokenList.Add(StructuredToken.CreateMemberNameToken(symbol.Name)); + _tokenList.Add(ReviewToken.CreateTypeNameToken(symbol.Type.Name)); + _tokenList.Add(ReviewToken.CreatePunctuationToken(SyntaxKind.DotToken)); + _tokenList.Add(ReviewToken.CreateMemberNameToken(symbol.Name)); } public void Format(ITypeSymbol? type, object? typedConstantValue) diff --git a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/CSharpAPIParserTests.csproj b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/CSharpAPIParserTests.csproj index c7269e20c2e..761cb1f297b 100644 --- a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/CSharpAPIParserTests.csproj +++ b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/CSharpAPIParserTests.csproj @@ -10,8 +10,11 @@ + + + diff --git a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/CodeFileTests.cs b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/CodeFileTests.cs new file mode 100644 index 00000000000..59aea555c7d --- /dev/null +++ b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/CodeFileTests.cs @@ -0,0 +1,130 @@ +using System.Reflection; +using ApiView; +using System; +using System.Text.Json; +using System.Text.Json.Nodes; +using Newtonsoft.Json.Schema; +using Newtonsoft.Json.Linq; +using System.Text.Json.Serialization; + + +namespace CSharpAPIParserTests +{ + public class CodeFileTests + { + private readonly CodeFile codeFile; + public Assembly assembly { get; set; } + + public CodeFileTests() + { + assembly = Assembly.Load("Azure.Template"); + var dllStream = assembly.GetFile("Azure.Template.dll"); + var assemblySymbol = CompilationFactory.GetCompilation(dllStream, null); + this.codeFile = new CSharpAPIParser.TreeToken.CodeFileBuilder().Build(assemblySymbol, true, null); + } + + [Fact] + public void TestPackageName() + { + Assert.Equal("Azure.Template", codeFile.PackageName); + } + + [Fact] + public void TestTopLevelReviewLineCount() + { + Assert.Equal(8, codeFile.ReviewLines.Count()); + } + + [Fact] + public void TestAPIReviewContent() + { + string expected = @"namespace Azure.Template { + public class TemplateClient { + public TemplateClient(string vaultBaseUrl, TokenCredential credential); + public TemplateClient(string vaultBaseUrl, TokenCredential credential, TemplateClientOptions options); + protected TemplateClient(); + public virtual HttpPipeline Pipeline { get; } + public virtual Response GetSecret(string secretName, RequestContext context); + public virtual Task GetSecretAsync(string secretName, RequestContext context); + public virtual Response GetSecretValue(string secretName, CancellationToken cancellationToken = default); + public virtual Task> GetSecretValueAsync(string secretName, CancellationToken cancellationToken = default); + } + + public class TemplateClientOptions : ClientOptions { + public enum ServiceVersion { + V7_0 = 1, + } + public TemplateClientOptions(ServiceVersion version = V7_0); + } +} + +namespace Azure.Template.Models { + public class SecretBundle { + public string ContentType { get; } + public string Id { get; } + public string Kid { get; } + public bool? Managed { get; } + public IReadOnlyDictionary Tags { get; } + public string Value { get; } + } +} + +namespace Microsoft.Extensions.Azure { + public static class TemplateClientBuilderExtensions { + public static IAzureClientBuilder AddTemplateClient(this TBuilder builder, string vaultBaseUrl) where TBuilder : IAzureClientFactoryBuilderWithCredential; + public static IAzureClientBuilder AddTemplateClient(this TBuilder builder, TConfiguration configuration) where TBuilder : IAzureClientFactoryBuilderWithConfiguration; + } +} +"; + Assert.Equal(expected, codeFile.GetApiText()); + } + + [Fact] + public void TestCodeFileJsonSchema() + { + //Verify JSON file generated for Azure.Template + var isValid = validateSchema(codeFile); + Assert.True(isValid); + } + + [Fact] + public void TestCodeFileJsonSchema2() + { + //Verify JSON file generated for Azure.Storage.Blobs + var storageAssembly = Assembly.Load("Azure.Storage.Blobs"); + var dllStream = storageAssembly.GetFile("Azure.Storage.Blobs.dll"); + var assemblySymbol = CompilationFactory.GetCompilation(dllStream, null); + var storageCodeFile = new CSharpAPIParser.TreeToken.CodeFileBuilder().Build(assemblySymbol, true, null); + var isValid = validateSchema(storageCodeFile); + Assert.True(isValid); + } + + private bool validateSchema(CodeFile codeFile) + { + var json = JsonSerializer.Serialize(codeFile, new JsonSerializerOptions + { + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }); + var schema = JSchema.Parse(TestData.TokenJsonSchema); + var jsonObject = JObject.Parse(json); + + IList validationErrors = new List(); + bool isValid = jsonObject.IsValid(schema, out validationErrors); + if (isValid) + { + Console.WriteLine("JSON is valid."); + } + else + { + Console.WriteLine("JSON is invalid. Errors:"); + foreach (string error in validationErrors) + { + Console.WriteLine(error); + } + } + return isValid; + } + } +} diff --git a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/ProgramTests.cs b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/ProgramTests.cs index 05070930d47..c259b9eace7 100644 --- a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/ProgramTests.cs +++ b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/ProgramTests.cs @@ -1,5 +1,3 @@ -using CSharpAPIParser; - namespace CSharpAPIParserTests { public class ProgramTests diff --git a/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/TestData.cs b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/TestData.cs new file mode 100644 index 00000000000..a0b29de7d8c --- /dev/null +++ b/tools/apiview/parsers/csharp-api-parser/CSharpAPIParserTests/TestData.cs @@ -0,0 +1,255 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CSharpAPIParserTests +{ + internal class TestData + { + public static string TokenJsonSchema = @"{ + ""$schema"": ""https://json-schema.org/draft/2020-12/schema"", + ""$id"": ""CodeFile.json"", + ""type"": ""object"", + ""properties"": { + ""PackageName"": { + ""type"": ""string"" + }, + ""PackageVersion"": { + ""type"": ""string"" + }, + ""ParserVersion"": { + ""type"": ""string"", + ""description"": ""version of the APIview language parser used to create token file"" + }, + ""Language"": { + ""anyOf"": [ + { + ""type"": ""string"", + ""const"": ""C"" + }, + { + ""type"": ""string"", + ""const"": ""C++"" + }, + { + ""type"": ""string"", + ""const"": ""C#"" + }, + { + ""type"": ""string"", + ""const"": ""Go"" + }, + { + ""type"": ""string"", + ""const"": ""Java"" + }, + { + ""type"": ""string"", + ""const"": ""JavaScript"" + }, + { + ""type"": ""string"", + ""const"": ""Kotlin"" + }, + { + ""type"": ""string"", + ""const"": ""Python"" + }, + { + ""type"": ""string"", + ""const"": ""Swagger"" + }, + { + ""type"": ""string"", + ""const"": ""Swift"" + }, + { + ""type"": ""string"", + ""const"": ""TypeSpec"" + } + ] + }, + ""LanguageVariant"": { + ""anyOf"": [ + { + ""type"": ""string"", + ""const"": ""None"" + }, + { + ""type"": ""string"", + ""const"": ""Spring"" + }, + { + ""type"": ""string"", + ""const"": ""Android"" + } + ], + ""default"": ""None"", + ""description"": ""Language variant is applicable only for java variants"" + }, + ""CrossLanguagePackageId"": { + ""type"": ""string"" + }, + ""ReviewLines"": { + ""type"": ""array"", + ""items"": { + ""$ref"": ""#/$defs/ReviewLine"" + } + }, + ""Diagnostics"": { + ""type"": ""array"", + ""items"": { + ""$ref"": ""#/$defs/CodeDiagnostic"" + }, + ""description"": ""Add any system generated comments. Each comment is linked to review line ID"" + } + }, + ""required"": [ + ""PackageName"", + ""PackageVersion"", + ""ParserVersion"", + ""Language"", + ""ReviewLines"" + ], + ""description"": ""ReviewFile represents entire API review object. This will be processed to render review lines."", + ""$defs"": { + ""ReviewLine"": { + ""type"": ""object"", + ""properties"": { + ""LineId"": { + ""type"": ""string"", + ""description"": ""lineId is only required if we need to support commenting on a line that contains this token. \nUsually code line for documentation or just punctuation is not required to have lineId. lineId should be a unique value within \nthe review token file to use it assign to review comments as well as navigation Id within the review page.\nfor e.g Azure.Core.HttpHeader.Common, azure.template.template_main"" + }, + ""CrossLanguageId"": { + ""type"": ""string"" + }, + ""Tokens"": { + ""type"": ""array"", + ""items"": { + ""$ref"": ""#/$defs/ReviewToken"" + }, + ""description"": ""list of tokens that constructs a line in API review"" + }, + ""Children"": { + ""type"": ""array"", + ""items"": { + ""$ref"": ""#/$defs/ReviewLine"" + }, + ""description"": ""Add any child lines as children. For e.g. all classes and namespace level methods are added as a children of namespace(module) level code line. \nSimilarly all method level code lines are added as children of it's class code line."" + }, + ""IsHidden"": { + ""type"": ""boolean"", + ""description"": ""Set current line as hidden code line by default. .NET has hidden APIs and architects does not want to see them by default."" + } + }, + ""required"": [ + ""Tokens"" + ], + ""description"": ""ReviewLine object corresponds to each line displayed on API review. If an empty line is required then add a code line object without any token."" + }, + ""CodeDiagnostic"": { + ""type"": ""object"", + ""properties"": { + ""DiagnosticId"": { + ""type"": ""string"" + }, + ""TargetId"": { + ""type"": ""string"", + ""description"": ""Id of ReviewLine object where this diagnostic needs to be displayed(Options"" + }, + ""Text"": { + ""type"": ""string"" + }, + ""Level"": { + ""$ref"": ""#/$defs/CodeDiagnosticLevel"" + }, + ""HelpLinkUri"": { + ""type"": ""string"" + } + }, + ""required"": [ + ""TargetId"", + ""Text"", + ""Level"" + ], + ""description"": ""System comment object is to add system generated comment. It can be one of the 4 different types of system comments."" + }, + ""ReviewToken"": { + ""type"": ""object"", + ""properties"": { + ""Kind"": { + ""$ref"": ""#/$defs/TokenKind"" + }, + ""Value"": { + ""type"": ""string"" + }, + ""NavigationDisplayName"": { + ""type"": ""string"", + ""description"": ""NavigationDisplayName can be used set navigation dispaly name to be used in navigation tree.\nThis value will be used in the navigation if NavigationDisplayName is not set."" + }, + ""NavigateToId"": { + ""type"": ""string"", + ""description"": ""navigateToId should be set if the underlying token is required to be displayed as HREF to another type within the review.\nFor e.g. a param type which is class name in the same package"" + }, + ""SkipDiff"": { + ""type"": ""boolean"", + ""default"": false, + ""description"": ""set skipDiff to true if underlying token needs to be ignored from diff calculation. For e.g. package metadata or dependency versions \nare usually excluded when comparing two revisions to avoid reporting them as API changes"" + }, + ""IsDeprecated"": { + ""type"": ""boolean"", + ""default"": false, + ""description"": ""This is set if API is marked as deprecated"" + }, + ""HasSuffixSpace"": { + ""type"": ""boolean"", + ""default"": true, + ""description"": ""Set this to false if there is no suffix space required before next token. For e.g, punctuation right after method name"" + }, + ""IsDocumentation"": { + ""type"": ""boolean"", + ""default"": false, + ""description"": ""Set isDocumentation to true if current token is part of documentation"" + }, + ""RenderClasses"": { + ""type"": ""array"", + ""items"": { + ""type"": ""string"" + }, + ""description"": ""Language specific style css class names"" + } + }, + ""required"": [ + ""Kind"", + ""Value"" + ], + ""description"": ""Token corresponds to each component within a code line. A separate token is required for keyword, punctuation, type name, text etc."" + }, + ""CodeDiagnosticLevel"": { + ""type"": ""number"", + ""enum"": [ + 1, + 2, + 3, + 4 + ] + }, + ""TokenKind"": { + ""type"": ""number"", + ""enum"": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + } + } +}"; + } +}