Feature/Improved Tokens (#7980)

* CSharp Parser Conversion from Flat to Tree Tokens

Parse C# Assemblly to Token Tree

Add Token Tree Documentation

Add LineDefinitionId and HideFromNavigation

Undo internals visible to setting

Sort NameSpaces and Types

Ensure Each node has Name and Id

Serialize to Message Pack

Add Tags to StructuredToken

update tree Token Parser Documentation

Attempt to Improve Serialized JSON Size

Properties for Serialization and Deserialization

Use shorter names for diagnostics

update APIView Parser Contributing guide

Delete TreeTokenCodeFile

Update APIView C# Parser Project Name

Add MessagePack Serialization

Switch to System.Json.Text working

Update Contributing Doc

Add Navigate to ID

Make Parser a dotnet Tool

Update Parser Version for CodeFile

App Parser Style Property

Output GZipped TokenFile

Switch to Human Readable Json Property Names, use compression

Fix some issues raised in pull request

Updating Contributing.md

Update Contributing Docs

* CSharp Parser Re-Write

Parse C# Assemblly to Token Tree

Add Token Tree Documentation

Add LineDefinitionId and HideFromNavigation

Undo internals visible to setting

Sort NameSpaces and Types

Ensure Each node has Name and Id

Serialize to Message Pack

Add Tags to StructuredToken

update tree Token Parser Documentation

Attempt to Improve Serialized JSON Size

Properties for Serialization and Deserialization

Use shorter names for diagnostics

update APIView Parser Contributing guide

Delete TreeTokenCodeFile

Update APIView C# Parser Project Name

Add MessagePack Serialization

Switch to System.Json.Text working

Update Contributing Doc

Add Navigate to ID

Make Parser a dotnet Tool

Update Parser Version for CodeFile

App Parser Style Property

Output GZipped TokenFile

Switch to Human Readable Json Property Names, use compression

Fix some issues raised in pull request

Updating Contributing.md

Update Contributing Docs

initial pass to documentation

fix

pr feedback

one missed

* Convert Magic strings to static properties

* Build Parameter Separator

* Add Image Folder

* Update Page Navigation

* Update APIView Parser Contributing.md

src/dotnet/APIView/APIView/Analysis/Analyzer.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
 using ApiView;
@@ -9,7 +9,7 @@
 
 namespace APIView.Analysis
 {
-    internal class Analyzer : SymbolVisitor
+    public class Analyzer : SymbolVisitor
     {
         public List<CodeDiagnostic> Results { get; } = new List<CodeDiagnostic>();
 

src/dotnet/APIView/APIView/CodeFileTokensBuilder.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
 using APIView;
@@ -78,4 +78,4 @@ public void Comment(string text)
             Append(text, CodeFileTokenKind.Comment);
         }
     }
-}
+}

src/dotnet/APIView/APIView/Languages/CodeFileBuilderSymbolSorter.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
 using Microsoft.CodeAnalysis;
@@ -8,7 +8,7 @@
 
 namespace ApiView
 {
-    class CodeFileBuilderSymbolOrderProvider : ICodeFileBuilderSymbolOrderProvider
+    public class CodeFileBuilderSymbolOrderProvider : ICodeFileBuilderSymbolOrderProvider
     {
         public IEnumerable<T> OrderTypes<T>(IEnumerable<T> symbols) where T : ITypeSymbol
         {
@@ -92,4 +92,4 @@ private static int GetMemberOrder(ISymbol symbol)
             }
         }
     }
-}
+}

src/dotnet/APIView/APIView/Model/CodeDiagnostic.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
 namespace APIView
@@ -17,15 +17,10 @@ public CodeDiagnostic(string diagnosticId, string targetId, string text, string
             HelpLinkUri = helpLinkUri;
             Level = level;
         }
-
         public string DiagnosticId { get; set; }
-
         public string Text { get; set; }
-
         public string HelpLinkUri { get; set; }
-
         public string TargetId { get; set; }
-
         public CodeDiagnosticLevel Level { get; set; }
     }
 }

src/dotnet/APIView/APIView/Model/CodeFile.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using APIView;
+using APIView.TreeToken;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -20,51 +21,34 @@ public class CodeFile
         };
 
         private string _versionString;
-
         private static HashSet<string> _collapsibleLanguages = new HashSet<string>(new string[] { "Swagger" });
-
         [Obsolete("This is only for back compat, VersionString should be used")]
         public int Version { get; set; }
-
         public string VersionString
         {
 #pragma warning disable 618
             get => _versionString ?? Version.ToString();
 #pragma warning restore 618
             set => _versionString = value;
         }
-
         public string Name { get; set; }
-
         public string Language { get; set; }
-
         public string LanguageVariant { get; set; }
-
         public string PackageName { get; set; }
-
         public string ServiceName { get; set; }
-
         public string PackageDisplayName { get; set; }
-
         public string PackageVersion { get; set; }
-
         public string CrossLanguagePackageId { get; set; }
-
         public CodeFileToken[] Tokens { get; set; } = Array.Empty<CodeFileToken>();
-
+        public List<APITreeNode> APIForest { get; set; } = new List<APITreeNode>();
         public List<CodeFileToken[]> LeafSections { get; set; }
-
         public NavigationItem[] Navigation { get; set; }
-
         public CodeDiagnostic[] Diagnostics { get; set; }
-
         public override string ToString()
         {
             return new CodeFileRenderer().Render(this).CodeLines.ToString();
-        }
-
+        }  
         public static bool IsCollapsibleSectionSSupported(string language) => _collapsibleLanguages.Contains(language);
-
         public static async Task<CodeFile> DeserializeAsync(Stream stream, bool hasSections = false)
         {
             var codeFile = await JsonSerializer.DeserializeAsync<CodeFile>(