Skip to content

Commit

Permalink
Apiview token utility to dump API text and to convert to new model (#…
Browse files Browse the repository at this point in the history 
…8990)

* Add a dotnet tool to create API review text from input APIview json token file
  • Loading branch information
@praveenkuttappan
praveenkuttappan authored Sep 19, 2024
1 parent 09dcddb commit 8f44ab8
Show file tree
Hide file tree
Showing 10 changed files with 445 additions and 39 deletions.
6 changes: 6 additions & 0 deletions src/dotnet/APIView/APIView.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSharpAPIParserTests", "..\
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestReferenceWithInternalsVisibleTo", "..\Azure.ClientSdk.Analyzers\TestReferenceWithInternalsVisibleTo\TestReferenceWithInternalsVisibleTo.csproj", "{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "APIViewJsonUtility", "APIViewJsonUtility\APIViewJsonUtility.csproj", "{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -69,6 +71,10 @@ Global
{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.Build.0 = Release|Any CPU
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
200 changes: 196 additions & 4 deletions src/dotnet/APIView/APIView/Model/CodeFile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace ApiView
Expand Down Expand Up @@ -148,14 +147,207 @@ public async Task SerializeAsync(Stream stream)
/// Generates a complete text representation of API surface to help generating the content.
/// One use case of this function will be to support download request of entire API review surface.
/// </summary>
public string GetApiText()
public string GetApiText(bool skipDocs = true)
{
StringBuilder sb = new();
foreach (var line in ReviewLines)
{
line.AppendApiTextToBuilder(sb, 0, true);
line.AppendApiTextToBuilder(sb, 0, skipDocs, GetIndentationForLanguage(Language));
}
return sb.ToString();
}
}

public static int GetIndentationForLanguage(string language)
{
switch (language)
{
case "C++":
case "C":
return 2;
default:
return 4;
}
}

public void ConvertToTreeTokenModel()
{
Dictionary<string, string> navigationItems = new Dictionary<string, string>();
ReviewLine reviewLine = new ReviewLine();
ReviewLine previousLine = null;
bool isDocumentation = false;
bool isHidden = false;
bool skipDiff = false;
bool isDeprecated = false;
bool skipIndent = false;
string className = "";
//Process all navigation items in old model to generate a map
GetNavigationMap(navigationItems, Navigation);

List<ReviewToken> currentLineTokens = new List<ReviewToken>();
foreach(var oldToken in Tokens)
{
ReviewToken token = null;
switch(oldToken.Kind)
{
case CodeFileTokenKind.DocumentRangeStart:
isDocumentation = true; break;
case CodeFileTokenKind.DocumentRangeEnd:
isDocumentation = false; break;
case CodeFileTokenKind.DeprecatedRangeStart:
isDeprecated = true; break;
case CodeFileTokenKind.DeprecatedRangeEnd:
isDeprecated = false; break;
case CodeFileTokenKind.SkipDiffRangeStart:
skipDiff = true; break;
case CodeFileTokenKind.SkipDiffRangeEnd:
skipDiff = false; break;
case CodeFileTokenKind.HiddenApiRangeStart:
isHidden = true; break;
case CodeFileTokenKind.HiddenApiRangeEnd:
isHidden = false; break;
case CodeFileTokenKind.Keyword:
token = ReviewToken.CreateKeywordToken(oldToken.Value, false);
var keywordValue = oldToken.Value.ToLower();
if (keywordValue == "class" || keywordValue == "enum" || keywordValue == "struct" || keywordValue == "interface" || keywordValue == "type" || keywordValue == "namespace")
className = keywordValue;
break;
case CodeFileTokenKind.Comment:
token = ReviewToken.CreateCommentToken(oldToken.Value, false);
break;
case CodeFileTokenKind.Text:
token = ReviewToken.CreateTextToken(oldToken.Value, oldToken.NavigateToId, false);
break;
case CodeFileTokenKind.Punctuation:
token = ReviewToken.CreatePunctuationToken(oldToken.Value, false);
break;
case CodeFileTokenKind.TypeName:
token = ReviewToken.CreateTypeNameToken(oldToken.Value, false);
if (currentLineTokens.Any(t => t.Kind == TokenKind.Keyword && t.Value.ToLower() == className))
token.RenderClasses.Add(className);
className = "";
break;
case CodeFileTokenKind.MemberName:
token = ReviewToken.CreateMemberNameToken(oldToken.Value, false);
break;
case CodeFileTokenKind.StringLiteral:
token = ReviewToken.CreateStringLiteralToken(oldToken.Value, false);
break;
case CodeFileTokenKind.Literal:
token = ReviewToken.CreateLiteralToken(oldToken.Value, false);
break;
case CodeFileTokenKind.ExternalLinkStart:
token = ReviewToken.CreateStringLiteralToken(oldToken.Value, false);
break;
case CodeFileTokenKind.Whitespace:
if (currentLineTokens.Count > 0) {
currentLineTokens.Last().HasSuffixSpace = true;
}
else if (!skipIndent) {
reviewLine.Indent += oldToken.Value.Length;
}
break;
case CodeFileTokenKind.Newline:
var parent = previousLine;
skipIndent = false;
if (currentLineTokens.Count > 0)
{
while (parent != null && parent.Indent >= reviewLine.Indent)
parent = parent.parentLine;
}
else
{
//If current line is empty line then add it as an empty line under previous line's parent
parent = previousLine?.parentLine;
}

if (parent == null)
{
this.ReviewLines.Add(reviewLine);
}
else
{
parent.Children.Add(reviewLine);
reviewLine.parentLine = parent;
}

if (currentLineTokens.Count == 0)
{
//Empty line. So just add previous line as related line
reviewLine.RelatedToLine = previousLine?.LineId;
}
else
{
reviewLine.Tokens = currentLineTokens;
previousLine = reviewLine;
}

reviewLine = new ReviewLine();
// If previous line ends with "," then next line will be sub line to show split content in multiple lines.
// Set next line's indent same as current line
// This is required to convert C++ tokens correctly
if (previousLine != null && previousLine.Tokens.LastOrDefault()?.Value == "," && Language == "C++")
{
reviewLine.Indent = previousLine.Indent;
skipIndent = true;
}
currentLineTokens = new List<ReviewToken>();
break;
case CodeFileTokenKind.LineIdMarker:
if (string.IsNullOrEmpty(reviewLine.LineId))
reviewLine.LineId = oldToken.Value;
break;
default:
Console.WriteLine($"Unsupported token kind to convert to new model, Kind: {oldToken.Kind}, value: {oldToken.Value}, Line Id: {oldToken.DefinitionId}");
break;
}

if (token != null)
{
currentLineTokens.Add(token);

if (oldToken.Equals("}") || oldToken.Equals("};"))
reviewLine.IsContextEndLine = true;
if (isHidden)
reviewLine.IsHidden = true;
if (oldToken.DefinitionId != null)
reviewLine.LineId = oldToken.DefinitionId;
if (oldToken.CrossLanguageDefinitionId != null)
reviewLine.CrossLanguageId = oldToken.CrossLanguageDefinitionId;
if (isDeprecated)
token.IsDeprecated = true;
if (skipDiff)
token.SkipDiff = true;
if (isDocumentation)
token.IsDocumentation = true;
}
}

//Process last line
if (currentLineTokens.Count > 0)
{
reviewLine.Tokens = currentLineTokens;
var parent = previousLine;
while (parent != null && parent.Indent >= reviewLine.Indent)
parent = parent.parentLine;

if (parent == null)
this.ReviewLines.Add(reviewLine);
else
parent.Children.Add(reviewLine);
}
}

private static void GetNavigationMap(Dictionary<string, string> navigationItems, NavigationItem[] items)
{
if (items == null)
return;

foreach (var item in items)
{
var key = string.IsNullOrEmpty(item.NavigationId) ? item.Text : item.NavigationId;
navigationItems.Add(key, item.Text);
GetNavigationMap(navigationItems, item.ChildItems);
}
}
}
}
13 changes: 10 additions & 3 deletions src/dotnet/APIView/APIView/Model/V2/ReviewLine.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,16 @@ public class ReviewLine
[JsonIgnore]
public bool Processed { get; set; } = false;

[JsonIgnore]
public int Indent { get; set; }
[JsonIgnore]
public ReviewLine parentLine { get; set; }
public void AddToken(ReviewToken token)
{
Tokens.Add(token);
}

public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDocs = true)
public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDocs = true, int lineIndentSpaces = 4)
{
if (skipDocs && Tokens.Count > 0 && Tokens[0].IsDocumentation == true)
{
Expand All @@ -77,15 +81,18 @@ public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDo
//Add spaces for indentation
for (int i = 0; i < indent; i++)
{
sb.Append(" ");
for(int j = 0; j < lineIndentSpaces; j++)
{
sb.Append(" ");
}
}
//Process all tokens
sb.Append(ToString(true));

sb.Append(Environment.NewLine);
foreach (var child in Children)
{
child.AppendApiTextToBuilder(sb, indent + 1, skipDocs);
child.AppendApiTextToBuilder(sb, indent + 1, skipDocs, lineIndentSpaces);
}
}

Expand Down
20 changes: 20 additions & 0 deletions src/dotnet/APIView/APIViewJsonUtility/APIViewJsonUtility.csproj
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<PackAsTool>true</PackAsTool>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<PackageId>APIViewJsonTool</PackageId>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\APIView\APIView.csproj" />
</ItemGroup>

</Project>
Loading

0 comments on commit 8f44ab8

Please sign in to comment.