.NET API review parser changes to use new token schema

src/dotnet/APIView/APIView.sln

@@ -17,6 +17,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "APIViewIntegrationTests", "
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "APIViewUITests", "APIViewUITests\APIViewUITests.csproj", "{7246F62A-99BF-4C4F-B9AD-1996166E767E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSharpAPIParser", "..\..\..\tools\apiview\parsers\csharp-api-parser\CSharpAPIParser\CSharpAPIParser.csproj", "{0D5CD780-15F9-49F5-9C4F-126D8224A31E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSharpAPIParserTests", "..\..\..\tools\apiview\parsers\csharp-api-parser\CSharpAPIParserTests\CSharpAPIParserTests.csproj", "{4F057169-032D-4971-B7D9-71AF850D411E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestReferenceWithInternalsVisibleTo", "..\Azure.ClientSdk.Analyzers\TestReferenceWithInternalsVisibleTo\TestReferenceWithInternalsVisibleTo.csproj", "{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -51,6 +57,18 @@ Global
 		{7246F62A-99BF-4C4F-B9AD-1996166E767E}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{7246F62A-99BF-4C4F-B9AD-1996166E767E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7246F62A-99BF-4C4F-B9AD-1996166E767E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0D5CD780-15F9-49F5-9C4F-126D8224A31E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0D5CD780-15F9-49F5-9C4F-126D8224A31E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0D5CD780-15F9-49F5-9C4F-126D8224A31E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0D5CD780-15F9-49F5-9C4F-126D8224A31E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4F057169-032D-4971-B7D9-71AF850D411E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4F057169-032D-4971-B7D9-71AF850D411E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4F057169-032D-4971-B7D9-71AF850D411E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4F057169-032D-4971-B7D9-71AF850D411E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/dotnet/APIView/APIView/Model/CodeFile.cs

@@ -2,14 +2,16 @@
 // Licensed under the MIT License.
 
 using APIView;
+using APIView.Model.V2;
 using APIView.TreeToken;
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.IO.Compression;
 using System.Linq;
+using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 
 namespace ApiView
@@ -19,17 +21,8 @@ public class CodeFile
         private static readonly JsonSerializerOptions _serializerOptions = new JsonSerializerOptions
         {
             AllowTrailingCommas = true,
-            ReadCommentHandling = JsonCommentHandling.Skip
-        };
-
-        private static readonly JsonSerializerOptions _treeStyleParserDeserializerOptions = new JsonSerializerOptions
-        {
-            Converters = { new StructuredTokenConverter() }
-        };
-
-        private static readonly JsonSerializerOptions _treeStyleParserSerializerOptions = new JsonSerializerOptions
-        {
-            DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+            ReadCommentHandling = JsonCommentHandling.Skip,
+            DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
         };
 
         private string _versionString;
@@ -52,10 +45,18 @@ public string VersionString
         public string PackageVersion { get; set; }
         public string CrossLanguagePackageId { get; set; }
         public CodeFileToken[] Tokens { get; set; } = Array.Empty<CodeFileToken>();
+        // APIForest will be removed once server changes are added to dereference this property
         public List<APITreeNode> APIForest { get; set; } = new List<APITreeNode>();
         public List<CodeFileToken[]> LeafSections { get; set; }
         public NavigationItem[] Navigation { get; set; }
         public CodeDiagnostic[] Diagnostics { get; set; }
+        public string ParserVersion
+        {
+            get => _versionString;
+            set => _versionString = value;
+        }
+        public List<ReviewLine> ReviewLines { get; set; } = [];
+
         public override string ToString()
         {
             return new CodeFileRenderer().Render(this).CodeLines.ToString();
@@ -64,23 +65,12 @@ public override string ToString()
 
         public static async Task<CodeFile> DeserializeAsync(Stream stream, bool hasSections = false, bool doTreeStyleParserDeserialization = false)
         {
-            CodeFile codeFile = null;
-            if (doTreeStyleParserDeserialization)
-            {
-                using (var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true))
-                {
-                    codeFile = await JsonSerializer.DeserializeAsync<CodeFile>(gzipStream, _treeStyleParserDeserializerOptions);
-                }
-            }
-            else
-            {
-                codeFile = await JsonSerializer.DeserializeAsync<CodeFile>(stream, _serializerOptions);
-            }
+            var codeFile = await JsonSerializer.DeserializeAsync<CodeFile>(stream, _serializerOptions);
 
             if (hasSections == false && codeFile.LeafSections == null && IsCollapsibleSectionSSupported(codeFile.Language))
                 hasSections = true;
 
-            // Spliting out the 'leafSections' of the codeFile is done so as not to have to render large codeFiles at once
+            // Splitting out the 'leafSections' of the codeFile is done so as not to have to render large codeFiles at once
             // Rendering sections in part helps to improve page load time
             if (hasSections)
             {
@@ -151,23 +141,21 @@ public static async Task<CodeFile> DeserializeAsync(Stream stream, bool hasSecti
 
         public async Task SerializeAsync(Stream stream)
         {
-            if (this.APIForest.Count > 0)
-            {
-                using (var tempStream = new MemoryStream())
-                {
-                    await JsonSerializer.SerializeAsync(tempStream, this, _treeStyleParserSerializerOptions);
-                    tempStream.Position = 0;
+            await JsonSerializer.SerializeAsync(stream, this, _serializerOptions);
+        }
 
-                    using (var compressionStream = new GZipStream(stream, CompressionMode.Compress, leaveOpen: true))
-                    {
-                        await tempStream.CopyToAsync(compressionStream);
-                    }
-                }
-            }
-            else
+        /// <summary>
+        /// Generates a complete text representation of API surface to help generating the content.
+        /// One use case of this function will be to support download request of entire API review surface.
+        /// </summary>
+        public string GetApiText()
+        {
+            StringBuilder sb = new();
+            foreach (var line in ReviewLines)
             {
-                await JsonSerializer.SerializeAsync(stream, this, _serializerOptions);
+                line.AppendApiTextToBuilder(sb, 0, true);
             }
-        }
+            return sb.ToString();
+        }       
     }
 }

src/dotnet/APIView/APIView/Model/V2/ReviewLine.cs

@@ -0,0 +1,140 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Xml;
+using APIView.TreeToken;
+
+namespace APIView.Model.V2
+{
+    /// <summary>
+    /// Review line object corresponds to each line displayed on API review. If an empty line is required then add a review line object without any token.
+    /// </summary>
+    public class ReviewLine
+    {
+        /// <summary>
+        /// LineId is only required if we need to support commenting on a line that contains this token. 
+        /// Usually code line for documentation or just punctuation is not required to have lineId.lineId should be a unique value within
+        /// the review token file to use it assign to review comments as well as navigation Id within the review page.        /// for e.g Azure.Core.HttpHeader.Common, azure.template.template_main
+        /// </summary>
+        public string LineId { get; set; }
+        public string CrossLanguageId { get; set; }
+        /// <summary>
+        /// List of tokens that constructs a line in API review
+        /// </summary>
+        public List<ReviewToken> Tokens { get; set; } = [];
+        /// <summary>
+        /// Add any child lines as children. For e.g. all classes and namespace level methods are added as a children of namespace(module) level code line.
+        /// Similarly all method level code lines are added as children of it's class code line.
+        /// </summary>
+        public List<ReviewLine> Children { get; set; } = [];
+        /// <summary>
+        /// This is set if API is marked as hidden
+        /// </summary>
+        public bool? IsHidden { get; set; }
+
+        // Following properties are helper methods that's used to render review lines to UI required format.
+        [JsonIgnore]
+        public DiffKind DiffKind { get; set; } = DiffKind.NoneDiff;
+        [JsonIgnore]
+        public bool IsActiveRevisionLine = true;
+        [JsonIgnore]
+        public bool IsDocumentation => Tokens.Count > 0 && Tokens[0].IsDocumentation == true;
+        [JsonIgnore]
+        public bool IsEmpty => Tokens.Count == 0 || !Tokens.Any( t => t.SkipDiff != true);
+        [JsonIgnore]
+        public bool Processed { get; set; } = false;
+
+        public void AddToken(ReviewToken token)
+        {
+            Tokens.Add(token);
+        }
+
+        public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDocs = true)
+        {
+            if (skipDocs && Tokens.Count > 0 && Tokens[0].IsDocumentation == true)
+            {
+                return;
+            }
+
+            //Add empty line in case of review line without tokens
+            if (Tokens.Count == 0)
+            {
+                sb.Append(Environment.NewLine);
+                return;
+            }
+            //Add spaces for indentation
+            for (int i = 0; i < indent; i++)
+            {
+                sb.Append("    ");
+            }
+            //Process all tokens
+            sb.Append(ToString(true));
+
+            sb.Append(Environment.NewLine);
+            foreach (var child in Children)
+            {
+                child.AppendApiTextToBuilder(sb, indent + 1, skipDocs);
+            }
+        }
+
+        private string ToString(bool includeAllTokens)
+        {
+            var filterdTokens = includeAllTokens ? Tokens: Tokens.Where(x => x.SkipDiff != true);
+            if (!filterdTokens.Any())
+            {
+                return string.Empty;
+            }
+            StringBuilder sb = new();
+            foreach (var token in filterdTokens)
+            {
+                sb.Append(token.Value);
+                sb.Append(token.HasSuffixSpace == true ? " " : string.Empty);
+            }
+            return sb.ToString();
+        }
+
+
+        public override string ToString()
+        {
+            return ToString(false);
+        }
+
+        public override bool Equals(object obj)
+        {
+            if(obj is ReviewLine other)
+            {
+                return ToString() == other.ToString();
+            }
+            return false;
+        }
+
+        public override int GetHashCode()
+        {
+            return ToString().GetHashCode();
+        }
+
+        public string GetTokenNodeIdHash(string parentNodeIdHash, int lineIndex)
+        {
+            var idPart = LineId;
+            var token = Tokens.FirstOrDefault(t => t.RenderClasses.Count > 0);
+            if (token != null)
+            {
+                idPart = $"{idPart}-{token.RenderClasses.First()}";
+            }
+            idPart = $"{idPart}-{lineIndex}-{DiffKind}";
+            var hash = CreateHashFromString(idPart);
+            return hash + parentNodeIdHash.Replace("nId", "").Replace("root", ""); // Append the parent node Id to ensure uniqueness
+        }
+
+        private string CreateHashFromString(string inputString)
+        {
+            int hash = HashCode.Combine(inputString);
+            return "nId" + hash.ToString();
+        }
+    }
+}