Add Token Tree Documentation

src/dotnet/APIView/APIView/Model/TokenTreeModel.cs

@@ -34,6 +34,13 @@ public static StructuredToken CreateLineBreakToken()
             return token;
         }
 
+        public static StructuredToken CreateEmptyToken()
+        {
+            var token = new StructuredToken("");
+            token.Kind = StructuredTokenKind.Content;
+            return token;
+        }
+
         public static StructuredToken CreateSpaceToken()
         {
             var token = new StructuredToken("\u0020");

tools/apiview/parsers/CONTRIBUTING.md

@@ -0,0 +1,62 @@
+# Contributing
+
+This page describes how to contribute to [APIView](../../../src//dotnet/APIView/APIViewWeb/CONTRIBUTING.md) language level parsers.
+Specifically how to create or update a language parser to produce tree style tokens for APIView.
+
+## Creating Tree Style Tokens
+The main idea is to capture the hierachy of the API using a tree data structure, then maintain a flat list of tokens for each node of the tree.
+
+![APITree](APITree.svg)
+
+Each tree node has top tokens which should be used to capture the main data on the node. If your language requires it use the bottom tokens to capture data that closed out the node.
+
+- Here are the models needed
+  ```
+  object APITreeNode
+    string Name
+    string Id
+    string Kind
+    Set<string> Tags
+    Dictionary<string, string> Properties
+    List<StructuredToken> TopTokens
+    List<StructuredToken> BottomTokens
+    List<APITreeNode> Children
+
+  object StructuredToken
+    string Value
+    string Id
+    StructuredTokenKind Kind
+    Dictionary<string, string> Properties 
+    Set<string> RenderClasses 
+
+  enum StructuredTokenKind
+    LineBreak
+    NoneBreakingSpace
+    TabSpace
+    ParameterSeparator
+    Content
+  ```
+### APITreeNode
+- `Name` : The name of the tree node which will be used for page navigation.
+- `Id` : Id of the node, which should be unique at the node level. i.e. unique among its siblings.
+- `Kind` : What kind of node is it. (namespace, class, module, method e.t.c)
+- `Tags` : Use this for thigs like `Deprecated`, `Hidden`
+- `Properties` : If the node needs more specification e.g. Use `SubKind` entry to make the node kind more specific. Feel free to push any other data that you think will be useful here, then file an issue for further implementation in APIView.
+- `TopTokens` : The main data of the node.
+- `BottomToken` : Data that closes out the node.
+- `Childrens` : Node immediate descendant
+
+### StructuredToken
+- `Value` : The token value which will be dispalyed.
+- `Id` : which will be used to navigate and find token on page.
+- `Kind` : Could be `LineBreak` `NoneBreakingSpace` `TabSpace` `ParameterSeparator` `Content`
+  All tokens should be content except for spacing tokens. ParameterSeparator should be used between method or function parameters. Spacing token dont need to have value.
+- `Properties` : Capture any other interesting data here.
+- `RenderClasses` : Add css classes for how the tokesn will be rendred.
+  To avoid collision between languages use a language prefix for you classes. e.g `csKeyword `, `jsModule`, `pyModule`
+
+If you want to have space between the API nodes add an empty token and lineBreak at the end of bottom tokens to simulate one empty line.
+
+Dont worry about indentation that will be handeled by the tree structure, unless you want to have indentation between the tokens than use `Tab` token kind.
+
+

tools/apiview/parsers/csharp-api-parser/TreeToken/CodeFileBuilder.cs

@@ -3,14 +3,11 @@
 
 using ApiView;
 using APIView.Analysis;
-using APIView.Model;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.SymbolDisplay;
 using System.Collections.Immutable;
 using System.ComponentModel;
-using System.Security.Cryptography;
-using System.Text;
 
 namespace csharp_api_parser.TreeToken
 {
@@ -151,6 +148,7 @@ public static void BuildInternalsVisibleToAttributes(List<APITreeNode> apiTree,
                         }
                     }
                 }
+                apiTreeNode.BottomTokens.Add(StructuredToken.CreateEmptyToken());
                 apiTreeNode.BottomTokens.Add(StructuredToken.CreateLineBreakToken());
                 apiTree.Add(apiTreeNode);
             }
@@ -173,6 +171,7 @@ public static void BuildDependencies(List<APITreeNode> apiTree, List<DependencyI
                     apiTreeNode.TopTokens.Add(StructuredToken.CreateTextToken(value: $"-{dependency.Version}"));
                     apiTreeNode.TopTokens.Add(StructuredToken.CreateLineBreakToken());
                 }
+                apiTreeNode.BottomTokens.Add(StructuredToken.CreateEmptyToken());
                 apiTreeNode.BottomTokens.Add(StructuredToken.CreateLineBreakToken());
                 apiTree.Add(apiTreeNode);
             }
@@ -204,7 +203,9 @@ private void BuildNamespace(List<APITreeNode> apiTree, INamespaceSymbol namespac
             }
 
             apiTreeNode.BottomTokens.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CloseBraceToken));
-            apiTreeNode.BottomTokens.Add(StructuredToken.CreateSpaceToken());
+            apiTreeNode.BottomTokens.Add(StructuredToken.CreateLineBreakToken());
+            apiTreeNode.BottomTokens.Add(StructuredToken.CreateEmptyToken());
+
             apiTree.Add(apiTreeNode);
         }
 
@@ -314,7 +315,9 @@ private void BuildType(List<APITreeNode> apiTree, INamedTypeSymbol namedType, bo
                 BuildMember(apiTreeNode.Children, member, inHiddenScope);
             }
             apiTreeNode.BottomTokens.Add(StructuredToken.CreatePunctuationToken(SyntaxKind.CloseBraceToken));
-            apiTreeNode.BottomTokens.Add(StructuredToken.CreateSpaceToken());
+            apiTreeNode.BottomTokens.Add(StructuredToken.CreateLineBreakToken());
+            apiTreeNode.BottomTokens.Add(StructuredToken.CreateEmptyToken());
+
             apiTree.Add(apiTreeNode);
         }