Skip to content

Commit

Permalink
APIView server and client side changes for new token schema
Browse files Browse the repository at this point in the history
  • Loading branch information
praveenkuttappan committed Aug 18, 2024
1 parent b30bd53 commit 9d4ccc1
Show file tree
Hide file tree
Showing 23 changed files with 1,060 additions and 1,093 deletions.
6 changes: 6 additions & 0 deletions src/dotnet/APIView/APIView.sln
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "APIViewIntegrationTests", "
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "APIViewUITests", "APIViewUITests\APIViewUITests.csproj", "{7246F62A-99BF-4C4F-B9AD-1996166E767E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSharpAPIParser", "..\..\..\tools\apiview\parsers\csharp-api-parser\CSharpAPIParser\CSharpAPIParser.csproj", "{9A36F1D1-2510-43E6-B23F-16B9F1B52F93}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -51,6 +53,10 @@ Global
{7246F62A-99BF-4C4F-B9AD-1996166E767E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7246F62A-99BF-4C4F-B9AD-1996166E767E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7246F62A-99BF-4C4F-B9AD-1996166E767E}.Release|Any CPU.Build.0 = Release|Any CPU
{9A36F1D1-2510-43E6-B23F-16B9F1B52F93}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9A36F1D1-2510-43E6-B23F-16B9F1B52F93}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9A36F1D1-2510-43E6-B23F-16B9F1B52F93}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9A36F1D1-2510-43E6-B23F-16B9F1B52F93}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
64 changes: 22 additions & 42 deletions src/dotnet/APIView/APIView/Model/CodeFile.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
// Licensed under the MIT License.

using APIView;
using APIView.TreeToken;
using APIView.Model.V2;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
Expand All @@ -19,17 +19,8 @@ public class CodeFile
private static readonly JsonSerializerOptions _serializerOptions = new JsonSerializerOptions
{
AllowTrailingCommas = true,
ReadCommentHandling = JsonCommentHandling.Skip
};

private static readonly JsonSerializerOptions _treeStyleParserDeserializerOptions = new JsonSerializerOptions
{
Converters = { new StructuredTokenConverter() }
};

private static readonly JsonSerializerOptions _treeStyleParserSerializerOptions = new JsonSerializerOptions
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
ReadCommentHandling = JsonCommentHandling.Skip,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};

private string _versionString;
Expand All @@ -51,11 +42,17 @@ public string VersionString
public string PackageDisplayName { get; set; }
public string PackageVersion { get; set; }
public string CrossLanguagePackageId { get; set; }
public CodeFileToken[] Tokens { get; set; } = Array.Empty<CodeFileToken>();
public List<APITreeNode> APIForest { get; set; } = new List<APITreeNode>();
public CodeFileToken[] Tokens { get; set; } = [];
public List<CodeFileToken[]> LeafSections { get; set; }
public NavigationItem[] Navigation { get; set; }
public CodeDiagnostic[] Diagnostics { get; set; }
public string ParserVersion
{
get => _versionString;
set => _versionString = value;
}
public List<ReviewLine> ReviewLines { get; set; } = [];

public override string ToString()
{
return new CodeFileRenderer().Render(this).CodeLines.ToString();
Expand All @@ -64,23 +61,12 @@ public override string ToString()

public static async Task<CodeFile> DeserializeAsync(Stream stream, bool hasSections = false, bool doTreeStyleParserDeserialization = false)
{
CodeFile codeFile = null;
if (doTreeStyleParserDeserialization)
{
using (var gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true))
{
codeFile = await JsonSerializer.DeserializeAsync<CodeFile>(gzipStream, _treeStyleParserDeserializerOptions);
}
}
else
{
codeFile = await JsonSerializer.DeserializeAsync<CodeFile>(stream, _serializerOptions);
}
var codeFile = await JsonSerializer.DeserializeAsync<CodeFile>(stream, _serializerOptions);

if (hasSections == false && codeFile.LeafSections == null && IsCollapsibleSectionSSupported(codeFile.Language))
hasSections = true;

// Spliting out the 'leafSections' of the codeFile is done so as not to have to render large codeFiles at once
// Splitting out the 'leafSections' of the codeFile is done so as not to have to render large codeFiles at once
// Rendering sections in part helps to improve page load time
if (hasSections)
{
Expand Down Expand Up @@ -151,23 +137,17 @@ public static async Task<CodeFile> DeserializeAsync(Stream stream, bool hasSecti

public async Task SerializeAsync(Stream stream)
{
if (this.APIForest.Count > 0)
{
using (var tempStream = new MemoryStream())
{
await JsonSerializer.SerializeAsync(tempStream, this, _treeStyleParserSerializerOptions);
tempStream.Position = 0;
await JsonSerializer.SerializeAsync(stream, this, _serializerOptions);
}

using (var compressionStream = new GZipStream(stream, CompressionMode.Compress, leaveOpen: true))
{
await tempStream.CopyToAsync(compressionStream);
}
}
}
else
public string GetApiText()
{
StringBuilder sb = new();
foreach (var line in ReviewLines)
{
await JsonSerializer.SerializeAsync(stream, this, _serializerOptions);
line.GetApiText(sb, 0, true);
}
return sb.ToString();
}
}
}
88 changes: 51 additions & 37 deletions src/dotnet/APIView/APIView/Model/StructuredTokenModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Text.Json.Serialization;
using System.Text.Json;
using System;
using APIView.Model.V2;

namespace APIView.TreeToken
{
Expand Down Expand Up @@ -45,43 +46,6 @@ public enum DiffKind
Removed = 3
}

public class StructuredTokenConverter : JsonConverter<StructuredToken>
{
private readonly string _parameterSeparator;

public StructuredTokenConverter(string parameterSeparator = "\u00A0")
{
_parameterSeparator = parameterSeparator;
}

public override StructuredToken Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var jObject = JsonDocument.ParseValue(ref reader).RootElement;
var myObject = JsonSerializer.Deserialize<StructuredToken>(jObject.GetRawText());

switch (myObject.Kind)
{
case StructuredTokenKind.LineBreak:
myObject.Value = "\u000A";
break;
case StructuredTokenKind.NonBreakingSpace:
myObject.Value = "\u00A0";
break;
case StructuredTokenKind.TabSpace:
myObject.Value = "\u00A0\u00A0\u00A0\u00A0";
break;
case StructuredTokenKind.ParameterSeparator:
myObject.Value = _parameterSeparator;
break;
}
return myObject;
}

public override void Write(Utf8JsonWriter writer, StructuredToken value, JsonSerializerOptions options)
{
JsonSerializer.Serialize(writer, value, options);
}
}

/// <summary>
/// Represents an APIView token its properties and tags for APIView parsers.
Expand Down Expand Up @@ -260,6 +224,56 @@ public StructuredToken(StructuredToken token)
}
}

public StructuredToken(ReviewToken token)
{
Value = token.Value;
RenderClassesObj = new HashSet<string>(token.RenderClasses);

if (token.IsDeprecated == true)
{
TagsObj.Add("Deprecated");
}

if (!string.IsNullOrEmpty(token.NavigateToId))
{
PropertiesObj.Add("NavigateToId", token.NavigateToId);
}

if (token.IsDocumentation == true)
{
TagsObj.Add(StructuredToken.DOCUMENTATION);
}
string className = StructuredToken.TEXT;
switch (token.Kind)
{
case TokenKind.Text:
className = StructuredToken.TEXT;
break;
case TokenKind.Punctuation:
className = StructuredToken.PUNCTUATION;
break;
case TokenKind.Keyword:
className = StructuredToken.KEYWORD;
break;
case TokenKind.TypeName:
className = StructuredToken.TYPE_NAME;
break;
case TokenKind.MemberName:
className = StructuredToken.MEMBER_NAME;
break;
case TokenKind.Comment:
className = StructuredToken.COMMENT;
break;
case TokenKind.StringLiteral:
className = StructuredToken.STRING_LITERAL;
break;
case TokenKind.Literal:
className = StructuredToken.LITERAL;
break;
}
RenderClassesObj.Add(className);
}


public static StructuredToken CreateLineBreakToken()
{
Expand Down
148 changes: 148 additions & 0 deletions src/dotnet/APIView/APIView/Model/V2/ReviewLine.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using APIView.TreeToken;

namespace APIView.Model.V2
{
/*** Review line object corresponds to each line displayed on API review. If an empty line is required then
* add a review line object without any token. */
public class ReviewLine
{
/*** LineId is only required if we need to support commenting on a line that contains this token.
* Usually code line for documentation or just punctuation is not required to have lineId. lineId should be a unique value within
* the review token file to use it assign to review comments as well as navigation Id within the review page.
* for e.g Azure.Core.HttpHeader.Common, azure.template.template_main
*/
public string LineId { get; set; }
public string CrossLanguageId { get; set; }
/*** list of tokens that constructs a line in API review */
public List<ReviewToken> Tokens { get; set; } = [];
/*** Add any child lines as children. For e.g. all classes and namespace level methods are added as a children of namespace(module) level code line.
* Similarly all method level code lines are added as children of it's class code line.*/
public List<ReviewLine> Children { get; set; } = [];
/*** This is set if API is marked as hidden */
public bool? IsHidden { get; set; }

// Following properties are helper methods that's used to render review lines to UI required format.
[JsonIgnore]
public DiffKind DiffKind { get; set; } = DiffKind.NoneDiff;
[JsonIgnore]
public bool IsActiveRevisionLine = true;
[JsonIgnore]
public bool IsDocumentation => Tokens.Count > 0 && Tokens[0].IsDocumentation == true;
[JsonIgnore]
public bool IsEmpty => Tokens.Count == 0 || !Tokens.Any( t => t.SkipDiff != true);
[JsonIgnore]
public bool Processed { get; set; } = false;

public void Add(ReviewToken token)
{
Tokens.Add(token);
}

public void RemoveSuffixSpace()
{
if (Tokens.Count > 0)
{
Tokens[Tokens.Count - 1].HasSuffixSpace = false;
}
}

public void AddSuffixSpace()
{
if (Tokens.Count > 0)
{
Tokens[Tokens.Count - 1].HasSuffixSpace = true;
}
}

public void GetApiText(StringBuilder sb, int indent = 0, bool skipDocs = true)
{
if (skipDocs && Tokens.Count > 0 && Tokens[0].IsDocumentation == true)
{
return;
}

//Add empty line in case of review line without tokens
if (Tokens.Count == 0)
{
sb.Append(Environment.NewLine);
return;
}
//Add spaces for indentation
for (int i = 0; i < indent; i++)
{
sb.Append(" ");
}
//Process all tokens
sb.Append(ToString(true));

sb.Append(Environment.NewLine);
foreach (var child in Children)
{
child.GetApiText(sb, indent + 1, skipDocs);
}
}

private string ToString(bool includeAllTokens)
{
var filterdTokens = Tokens.Where(x => includeAllTokens || x.SkipDiff != true);
if (!filterdTokens.Any())
{
return "";
}
StringBuilder sb = new();
foreach (var token in filterdTokens)
{
sb.Append(token.Value);
sb.Append(token.HasSuffixSpace == true ? " " : "");
}
return sb.ToString();
}


public override string ToString()
{
return ToString(false);
}

public override bool Equals(object obj)
{
if(obj is ReviewLine other)
{
return ToString() == other.ToString();
}
return false;
}

public override int GetHashCode()
{
return ToString().GetHashCode();
}

public string GetTokenNodeIdHash(string parentNodeIdHash, int lineIndex)
{
var idPart = LineId;
var token = Tokens.FirstOrDefault(t => t.RenderClasses.Count > 0);
if (token != null)
{
idPart = $"{idPart}-{token.RenderClasses.First()}";
}
idPart = $"{idPart}-{lineIndex}-{DiffKind}";
var hash = CreateHashFromString(idPart);
return hash + parentNodeIdHash.Replace("nId", "").Replace("root", ""); // Append the parent node Id to ensure uniqueness
}

private string CreateHashFromString(string inputString)
{
int hash = HashCode.Combine(inputString);
return "nId" + hash.ToString();
}
}
}
Loading

0 comments on commit 9d4ccc1

Please sign in to comment.