Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Apiview token utility to dump API text and to convert to new model #8990

Merged
merged 7 commits into from
Sep 19, 2024
Merged

Apiview token utility to dump API text and to convert to new model #8990

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
37c5a78
Add a dotnet tool to create API review text from input APIview json t…
praveenkuttappan Sep 6, 2024
9a894be
Add utility to convert API view token to new tree token model
praveenkuttappan Sep 17, 2024
55c8ab8
Support existing navigation panel tokens for backward compatibility f…
praveenkuttappan Sep 18, 2024
3c52ad4
Apply suggestions from code review
praveenkuttappan Sep 18, 2024
44f4dd0
Merge branch 'main' into apiview_token_converter
praveenkuttappan Sep 18, 2024
1b4b8ed
Add CI to publish the utility
praveenkuttappan Sep 18, 2024
e99f537
Fix build warnings
praveenkuttappan Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/dotnet/APIView/APIView.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSharpAPIParserTests", "..\
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestReferenceWithInternalsVisibleTo", "..\Azure.ClientSdk.Analyzers\TestReferenceWithInternalsVisibleTo\TestReferenceWithInternalsVisibleTo.csproj", "{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "APIViewJsonUtility", "APIViewJsonUtility\APIViewJsonUtility.csproj", "{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -69,6 +71,10 @@ Global
{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.Build.0 = Release|Any CPU
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
200 changes: 196 additions & 4 deletions src/dotnet/APIView/APIView/Model/CodeFile.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace ApiView
Expand Down Expand Up @@ -148,14 +147,207 @@ public async Task SerializeAsync(Stream stream)
/// Generates a complete text representation of API surface to help generating the content.
/// One use case of this function will be to support download request of entire API review surface.
/// </summary>
public string GetApiText()
public string GetApiText(bool skipDocs = true)
{
StringBuilder sb = new();
foreach (var line in ReviewLines)
{
line.AppendApiTextToBuilder(sb, 0, true);
line.AppendApiTextToBuilder(sb, 0, skipDocs, GetIndentationForLanguage(Language));
}
return sb.ToString();
}
}

public static int GetIndentationForLanguage(string language)
maririos marked this conversation as resolved.
Show resolved Hide resolved
{
switch (language)
{
case "C++":
case "C":
return 2;
default:
return 4;
}
}

public void ConvertToTreeTokenModel()
{
Dictionary<string, string> navigationItems = new Dictionary<string, string>();
ReviewLine reviewLine = new ReviewLine();
ReviewLine previousLine = null;
bool isDocumentation = false;
bool isHidden = false;
bool skipDiff = false;
bool isDeprecated = false;
bool skipIndent = false;
string className = "";
//Process all navigation items in old model to generate a map
GetNavigationMap(navigationItems, Navigation);

List<ReviewToken> currentLineTokens = new List<ReviewToken>();
foreach(var oldToken in Tokens)
{
ReviewToken token = null;
switch(oldToken.Kind)
{
case CodeFileTokenKind.DocumentRangeStart:
isDocumentation = true; break;
case CodeFileTokenKind.DocumentRangeEnd:
isDocumentation = false; break;
case CodeFileTokenKind.DeprecatedRangeStart:
isDeprecated = true; break;
case CodeFileTokenKind.DeprecatedRangeEnd:
isDeprecated = false; break;
case CodeFileTokenKind.SkipDiffRangeStart:
skipDiff = true; break;
case CodeFileTokenKind.SkipDiffRangeEnd:
skipDiff = false; break;
case CodeFileTokenKind.HiddenApiRangeStart:
isHidden = true; break;
case CodeFileTokenKind.HiddenApiRangeEnd:
isHidden = false; break;
case CodeFileTokenKind.Keyword:
token = ReviewToken.CreateKeywordToken(oldToken.Value, false);
var keywordValue = oldToken.Value.ToLower();
if (keywordValue == "class" || keywordValue == "enum" || keywordValue == "struct" || keywordValue == "interface" || keywordValue == "type" || keywordValue == "namespace")
className = keywordValue;
break;
case CodeFileTokenKind.Comment:
token = ReviewToken.CreateCommentToken(oldToken.Value, false);
break;
case CodeFileTokenKind.Text:
token = ReviewToken.CreateTextToken(oldToken.Value, oldToken.NavigateToId, false);
break;
case CodeFileTokenKind.Punctuation:
token = ReviewToken.CreatePunctuationToken(oldToken.Value, false);
break;
case CodeFileTokenKind.TypeName:
token = ReviewToken.CreateTypeNameToken(oldToken.Value, false);
if (currentLineTokens.Any(t => t.Kind == TokenKind.Keyword && t.Value.ToLower() == className))
token.RenderClasses.Add(className);
className = "";
break;
case CodeFileTokenKind.MemberName:
token = ReviewToken.CreateMemberNameToken(oldToken.Value, false);
break;
case CodeFileTokenKind.StringLiteral:
token = ReviewToken.CreateStringLiteralToken(oldToken.Value, false);
break;
case CodeFileTokenKind.Literal:
token = ReviewToken.CreateLiteralToken(oldToken.Value, false);
break;
case CodeFileTokenKind.ExternalLinkStart:
token = ReviewToken.CreateStringLiteralToken(oldToken.Value, false);
break;
case CodeFileTokenKind.Whitespace:
if (currentLineTokens.Count > 0) {
currentLineTokens.Last().HasSuffixSpace = true;
}
else if (!skipIndent) {
reviewLine.Indent += oldToken.Value.Length;
}
break;
case CodeFileTokenKind.Newline:
var parent = previousLine;
skipIndent = false;
if (currentLineTokens.Count > 0)
{
while (parent != null && parent.Indent >= reviewLine.Indent)
parent = parent.parentLine;
}
else
{
//If current line is empty line then add it as an empty line under previous line's parent
parent = previousLine?.parentLine;
}

if (parent == null)
{
this.ReviewLines.Add(reviewLine);
}
else
{
parent.Children.Add(reviewLine);
reviewLine.parentLine = parent;
}

if (currentLineTokens.Count == 0)
{
//Empty line. So just add previous line as related line
reviewLine.RelatedToLine = previousLine?.LineId;
}
else
{
reviewLine.Tokens = currentLineTokens;
previousLine = reviewLine;
}

reviewLine = new ReviewLine();
// If previous line ends with "," then next line will be sub line to show split content in multiple lines.
// Set next line's indent same as current line
// This is required to convert C++ tokens correctly
praveenkuttappan marked this conversation as resolved.
Show resolved Hide resolved
if (previousLine != null && previousLine.Tokens.LastOrDefault()?.Value == "," && Language == "C++")
{
reviewLine.Indent = previousLine.Indent;
skipIndent = true;
}
currentLineTokens = new List<ReviewToken>();
break;
case CodeFileTokenKind.LineIdMarker:
if (string.IsNullOrEmpty(reviewLine.LineId))
reviewLine.LineId = oldToken.Value;
break;
default:
Console.WriteLine($"Unsupported token kind to convert to new model, Kind: {oldToken.Kind}, value: {oldToken.Value}, Line Id: {oldToken.DefinitionId}");
maririos marked this conversation as resolved.
Show resolved Hide resolved
break;
}

if (token != null)
{
currentLineTokens.Add(token);

if (oldToken.Equals("}") || oldToken.Equals("};"))
reviewLine.IsContextEndLine = true;
if (isHidden)
reviewLine.IsHidden = true;
if (oldToken.DefinitionId != null)
reviewLine.LineId = oldToken.DefinitionId;
if (oldToken.CrossLanguageDefinitionId != null)
reviewLine.CrossLanguageId = oldToken.CrossLanguageDefinitionId;
if (isDeprecated)
token.IsDeprecated = true;
if (skipDiff)
token.SkipDiff = true;
if (isDocumentation)
token.IsDocumentation = true;
}
}

//Process last line
if (currentLineTokens.Count > 0)
{
reviewLine.Tokens = currentLineTokens;
var parent = previousLine;
while (parent != null && parent.Indent >= reviewLine.Indent)
parent = parent.parentLine;

if (parent == null)
this.ReviewLines.Add(reviewLine);
else
parent.Children.Add(reviewLine);
}
}

private static void GetNavigationMap(Dictionary<string, string> navigationItems, NavigationItem[] items)
{
if (items == null)
return;

foreach (var item in items)
{
var key = string.IsNullOrEmpty(item.NavigationId) ? item.Text : item.NavigationId;
navigationItems.Add(key, item.Text);
GetNavigationMap(navigationItems, item.ChildItems);
}
}
}
}
13 changes: 10 additions & 3 deletions src/dotnet/APIView/APIView/Model/V2/ReviewLine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,16 @@ public class ReviewLine
[JsonIgnore]
public bool Processed { get; set; } = false;

[JsonIgnore]
public int Indent { get; set; }
[JsonIgnore]
public ReviewLine parentLine { get; set; }
public void AddToken(ReviewToken token)
{
Tokens.Add(token);
}

public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDocs = true)
public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDocs = true, int lineIndentSpaces = 4)
{
if (skipDocs && Tokens.Count > 0 && Tokens[0].IsDocumentation == true)
{
Expand All @@ -77,15 +81,18 @@ public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDo
//Add spaces for indentation
for (int i = 0; i < indent; i++)
{
sb.Append(" ");
for(int j = 0; j < lineIndentSpaces; j++)
{
sb.Append(" ");
}
}
//Process all tokens
sb.Append(ToString(true));

sb.Append(Environment.NewLine);
foreach (var child in Children)
{
child.AppendApiTextToBuilder(sb, indent + 1, skipDocs);
child.AppendApiTextToBuilder(sb, indent + 1, skipDocs, lineIndentSpaces);
}
}

Expand Down
20 changes: 20 additions & 0 deletions src/dotnet/APIView/APIViewJsonUtility/APIViewJsonUtility.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<PackAsTool>true</PackAsTool>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<PackageId>APIViewJsonTool</PackageId>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\APIView\APIView.csproj" />
</ItemGroup>

</Project>
Loading