From c8648eb04fad8310c6ad1472e3d3d776dbb6e916 Mon Sep 17 00:00:00 2001 From: Chidozie Ononiwu Date: Wed, 12 Jun 2024 15:44:00 -0700 Subject: [PATCH] Deserialize using gzip --- src/dotnet/APIView/APIView/Model/CodeFile.cs | 39 ++++++- .../Client/css/shared/theme-colors.scss | 4 +- .../Languages/JsonLanguageService.cs | 10 +- .../LeanControllers/APIRevisionsController.cs | 4 +- .../LeanControllers/CommentsController.cs | 2 +- .../LeanControllers/ReviewsController.cs | 10 +- .../Repositories/BlobCodeFileRepository.cs | 2 +- .../api-revision-options.component.ts | 15 ++- .../code-panel/code-panel.component.scss | 36 ++++-- .../reviews-list/reviews-list.component.ts | 4 +- .../revisions-list.component.ts | 2 +- tools/apiview/parsers/CONTRIBUTING.md | 109 ++++++++++++++++++ 12 files changed, 203 insertions(+), 34 deletions(-) create mode 100644 tools/apiview/parsers/CONTRIBUTING.md diff --git a/src/dotnet/APIView/APIView/Model/CodeFile.cs b/src/dotnet/APIView/APIView/Model/CodeFile.cs index e1973899822..77051ce6dbb 100644 --- a/src/dotnet/APIView/APIView/Model/CodeFile.cs +++ b/src/dotnet/APIView/APIView/Model/CodeFile.cs @@ -6,8 +6,10 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Tasks; namespace ApiView @@ -20,11 +22,16 @@ public class CodeFile ReadCommentHandling = JsonCommentHandling.Skip }; - private static readonly JsonSerializerOptions _deSerializerOptions = new JsonSerializerOptions + private static readonly JsonSerializerOptions _treeStyleParserDeserializerOptions = new JsonSerializerOptions { Converters = { new StructuredTokenConverter() } }; + private static readonly JsonSerializerOptions _treeStyleParserSerializerOptions = new JsonSerializerOptions + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + private string _versionString; private static HashSet _collapsibleLanguages = new HashSet(new string[] { "Swagger" }); @@ -73,10 +80,18 @@ public override string ToString() public static bool IsCollapsibleSectionSSupported(string language) => _collapsibleLanguages.Contains(language); #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public static async Task DeserializeAsync(Stream stream, bool hasSections = false) + public static async Task DeserializeAsync(Stream stream, bool hasSections = false, bool useTreeStyleParserDeserializerOptions = false) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { - CodeFile codeFile = await JsonSerializer.DeserializeAsync(stream, _deSerializerOptions); + CodeFile codeFile = null; + if (useTreeStyleParserDeserializerOptions) + { + codeFile = await JsonSerializer.DeserializeAsync(stream, _treeStyleParserDeserializerOptions); + } + else + { + codeFile = await JsonSerializer.DeserializeAsync(stream, _serializerOptions); + } if (hasSections == false && codeFile.LeafSections == null && IsCollapsibleSectionSSupported(codeFile.Language)) hasSections = true; @@ -152,7 +167,23 @@ public static async Task DeserializeAsync(Stream stream, bool hasSecti public async Task SerializeAsync(Stream stream) { - await JsonSerializer.SerializeAsync(stream, this, _serializerOptions); + if (this.APIForest.Count > 0) + { + using (var tempStream = new MemoryStream()) + { + await JsonSerializer.SerializeAsync(tempStream, this, _treeStyleParserSerializerOptions); + tempStream.Position = 0; + + using (var compressionStream = new GZipStream(stream, CompressionMode.Compress, leaveOpen: true)) + { + await tempStream.CopyToAsync(compressionStream); + } + } + } + else + { + await JsonSerializer.SerializeAsync(stream, this, _serializerOptions); + } } } } diff --git a/src/dotnet/APIView/APIViewWeb/Client/css/shared/theme-colors.scss b/src/dotnet/APIView/APIViewWeb/Client/css/shared/theme-colors.scss index 2128453b68d..05cdc5a85e2 100644 --- a/src/dotnet/APIView/APIViewWeb/Client/css/shared/theme-colors.scss +++ b/src/dotnet/APIView/APIViewWeb/Client/css/shared/theme-colors.scss @@ -47,12 +47,14 @@ --name-color: #{$base-text-color}; --code-color: #d1377e; --code-comment: green; + --literal-color: #008509; + --enum-color: #8700BD; --java-doc-color: #8c8c8c; --java-comment-color: #8c8c8c; --java-field-name-color: #871094; --java-method-name-color: #00627A; --java-keyword-color: #0033B3; - --java-anotation-name-color: 9E880D; + --java-anotation-name-color: #9E880D; --java-string-literal-color: #067D17; --java-number-color: #1750EB; --alert-success-color: #0a3622; diff --git a/src/dotnet/APIView/APIViewWeb/Languages/JsonLanguageService.cs b/src/dotnet/APIView/APIViewWeb/Languages/JsonLanguageService.cs index b7d071fa8d4..53f4004c685 100644 --- a/src/dotnet/APIView/APIViewWeb/Languages/JsonLanguageService.cs +++ b/src/dotnet/APIView/APIViewWeb/Languages/JsonLanguageService.cs @@ -3,6 +3,7 @@ using System; using System.IO; +using System.IO.Compression; using System.Threading.Tasks; using ApiView; @@ -11,7 +12,7 @@ namespace APIViewWeb public class JsonLanguageService : LanguageService { public override string Name { get; } = "Json"; - public override string[] Extensions { get; } = { ".json" }; + public override string[] Extensions { get; } = { ".json", ".json.tgz" }; public override bool CanUpdate(string versionString) => false; @@ -23,6 +24,13 @@ public override bool IsSupportedFile(string name) public override async Task GetCodeFileAsync(string originalName, Stream stream, bool runAnalysis) { + if (originalName.EndsWith(".tgz", StringComparison.OrdinalIgnoreCase)) + { + using (GZipStream gzipStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true)) + { + return await CodeFile.DeserializeAsync(gzipStream, useTreeStyleParserDeserializerOptions: true); + } + } return await CodeFile.DeserializeAsync(stream, true); } } diff --git a/src/dotnet/APIView/APIViewWeb/LeanControllers/APIRevisionsController.cs b/src/dotnet/APIView/APIViewWeb/LeanControllers/APIRevisionsController.cs index d34c7bb0f7c..d0e1531a5a0 100644 --- a/src/dotnet/APIView/APIViewWeb/LeanControllers/APIRevisionsController.cs +++ b/src/dotnet/APIView/APIViewWeb/LeanControllers/APIRevisionsController.cs @@ -41,7 +41,7 @@ public async Task>> GetAPIRevis /// /// /// - [HttpPut("/delete", Name = "DeleteAPIRevisions")] + [HttpPut("delete", Name = "DeleteAPIRevisions")] public async Task DeleteAPIRevisionsAsync([FromBody] APIRevisionSoftDeleteParam deleteParams) { foreach (var apiRevisionId in deleteParams.apiRevisionIds) @@ -55,7 +55,7 @@ public async Task DeleteAPIRevisionsAsync([FromBody] APIRevisionSoftDeleteParam /// /// /// - [HttpPut("/restore", Name = "RestoreAPIRevisions")] + [HttpPut("restore", Name = "RestoreAPIRevisions")] public async Task RestoreAPIRevisionsAsync([FromBody] APIRevisionSoftDeleteParam deleteParams) { foreach (var apiRevisionId in deleteParams.apiRevisionIds) diff --git a/src/dotnet/APIView/APIViewWeb/LeanControllers/CommentsController.cs b/src/dotnet/APIView/APIViewWeb/LeanControllers/CommentsController.cs index 717be59bcd5..814d932e5cd 100644 --- a/src/dotnet/APIView/APIViewWeb/LeanControllers/CommentsController.cs +++ b/src/dotnet/APIView/APIViewWeb/LeanControllers/CommentsController.cs @@ -31,7 +31,7 @@ public CommentsController(ILogger logger, /// /// /// - [HttpGet("/{reviewId}", Name = "GetComments")] + [HttpGet("{reviewId}", Name = "GetComments")] public async Task>> GetCommentsAsync(string reviewId) { var comments = await _commentsManager.GetCommentsAsync(reviewId); diff --git a/src/dotnet/APIView/APIViewWeb/LeanControllers/ReviewsController.cs b/src/dotnet/APIView/APIViewWeb/LeanControllers/ReviewsController.cs index 58c0d7f8f38..c6a229810e7 100644 --- a/src/dotnet/APIView/APIViewWeb/LeanControllers/ReviewsController.cs +++ b/src/dotnet/APIView/APIViewWeb/LeanControllers/ReviewsController.cs @@ -13,6 +13,8 @@ using Microsoft.AspNetCore.SignalR; using System.Diagnostics; using System; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; namespace APIViewWeb.LeanControllers { @@ -27,12 +29,13 @@ public class ReviewsController : BaseApiController public readonly UserPreferenceCache _preferenceCache; private readonly ICosmosUserProfileRepository _userProfileRepository; private readonly IHubContext _signalRHubContext; + private readonly IWebHostEnvironment _env; public ReviewsController(ILogger logger, IAPIRevisionsManager reviewRevisionsManager, IReviewManager reviewManager, ICommentsManager commentManager, IBlobCodeFileRepository codeFileRepository, IConfiguration configuration, UserPreferenceCache preferenceCache, - ICosmosUserProfileRepository userProfileRepository, IHubContext signalRHub) + ICosmosUserProfileRepository userProfileRepository, IHubContext signalRHub, IWebHostEnvironment env) { _logger = logger; _apiRevisionsManager = reviewRevisionsManager; @@ -43,6 +46,7 @@ public ReviewsController(ILogger logger, _preferenceCache = preferenceCache; _userProfileRepository = userProfileRepository; _signalRHubContext = signalRHub; + _env = env; } @@ -113,7 +117,7 @@ public async Task> GetReviewContentAsync(string revi { var comments = await _commentsManager.GetCommentsAsync(reviewId); - var activeRevisionReviewCodeFile = await _codeFileRepository.GetCodeFileWithCompressionAsync(activeAPIRevision.Id, activeAPIRevision.Files[0].FileId, false); + var activeRevisionReviewCodeFile = await _codeFileRepository.GetCodeFileWithCompressionAsync(activeAPIRevision.Id, activeAPIRevision.Files[0].FileId, _env.IsProduction()); var result = new CodePanelData(); @@ -127,7 +131,7 @@ public async Task> GetReviewContentAsync(string revi if (!string.IsNullOrEmpty(diffApiRevisionId)) { var diffAPIRevision = await _apiRevisionsManager.GetAPIRevisionAsync(User, diffApiRevisionId); - var diffRevisionReviewCodeFile = await _codeFileRepository.GetCodeFileWithCompressionAsync(diffAPIRevision.Id, diffAPIRevision.Files[0].FileId); + var diffRevisionReviewCodeFile = await _codeFileRepository.GetCodeFileWithCompressionAsync(diffAPIRevision.Id, diffAPIRevision.Files[0].FileId, _env.IsProduction()); codePanelRawData.APIForest = CodeFileHelpers.ComputeAPIForestDiff(activeRevisionReviewCodeFile.APIForest, diffRevisionReviewCodeFile.APIForest); } diff --git a/src/dotnet/APIView/APIViewWeb/Repositories/BlobCodeFileRepository.cs b/src/dotnet/APIView/APIViewWeb/Repositories/BlobCodeFileRepository.cs index e9b724095f5..1b9a5825798 100644 --- a/src/dotnet/APIView/APIViewWeb/Repositories/BlobCodeFileRepository.cs +++ b/src/dotnet/APIView/APIViewWeb/Repositories/BlobCodeFileRepository.cs @@ -66,7 +66,7 @@ public async Task GetCodeFileWithCompressionAsync(string revisionId, s } var info = await client.DownloadAsync(); using var gzipStream = new GZipStream(info.Value.Content, CompressionMode.Decompress); - codeFile = await CodeFile.DeserializeAsync(gzipStream); + codeFile = await CodeFile.DeserializeAsync(gzipStream, useTreeStyleParserDeserializerOptions: true); if (updateCache) { using var _ = _cache.CreateEntry(key) diff --git a/src/dotnet/APIView/ClientSPA/src/app/_components/api-revision-options/api-revision-options.component.ts b/src/dotnet/APIView/ClientSPA/src/app/_components/api-revision-options/api-revision-options.component.ts index 512caf9d1dc..1810eb27cf8 100644 --- a/src/dotnet/APIView/ClientSPA/src/app/_components/api-revision-options/api-revision-options.component.ts +++ b/src/dotnet/APIView/ClientSPA/src/app/_components/api-revision-options/api-revision-options.component.ts @@ -204,12 +204,15 @@ export class ApiRevisionOptionsComponent implements OnChanges { let apiRevision = mappedApiRevisions.shift(); if (latestGAApiRevision === null) { - let versionParts = apiRevision.version.match(semVarRegex); - if (versionParts.groups?.prelabel === undefined && versionParts.groups?.prenumber === undefined && - versionParts.groups?.prenumsep === undefined && versionParts.groups?.presep === undefined) { - apiRevision.isLatestGA = true; - latestGAApiRevision = apiRevision; - continue; + if (apiRevision.version) { + let versionParts = apiRevision.version.match(semVarRegex); + + if (versionParts.groups?.prelabel === undefined && versionParts.groups?.prenumber === undefined && + versionParts.groups?.prenumsep === undefined && versionParts.groups?.presep === undefined) { + apiRevision.isLatestGA = true; + latestGAApiRevision = apiRevision; + continue; + } } } diff --git a/src/dotnet/APIView/ClientSPA/src/app/_components/code-panel/code-panel.component.scss b/src/dotnet/APIView/ClientSPA/src/app/_components/code-panel/code-panel.component.scss index ec3b1821d17..98223ceea6a 100644 --- a/src/dotnet/APIView/ClientSPA/src/app/_components/code-panel/code-panel.component.scss +++ b/src/dotnet/APIView/ClientSPA/src/app/_components/code-panel/code-panel.component.scss @@ -121,6 +121,7 @@ .code-line-content { grid-column: 2; position: relative; + white-space: pre; } .line-number { @@ -164,20 +165,31 @@ } } - .csharp { - .comment { - color: var(--code-comment); - } - - .keyword { - color: var(--keyword-color); - } - - .tname { - color: var(--class-color); - } + .comment { + color: var(--code-comment); + } + + .keyword { + color: var(--keyword-color); } + .tname { + color: var(--class-color); + } + + .mname { + color: var(--name-color); + } + + .enum { + color: var(--enum-color); + } + + .literal .sliteral { + color: var(--literal-color); + } + + .java { .javadoc { color: var(--java-doc-color); diff --git a/src/dotnet/APIView/ClientSPA/src/app/_components/reviews-list/reviews-list.component.ts b/src/dotnet/APIView/ClientSPA/src/app/_components/reviews-list/reviews-list.component.ts index fbb32e181ee..2838a8a3481 100644 --- a/src/dotnet/APIView/ClientSPA/src/app/_components/reviews-list/reviews-list.component.ts +++ b/src/dotnet/APIView/ClientSPA/src/app/_components/reviews-list/reviews-list.component.ts @@ -336,7 +336,7 @@ export class ReviewsListComponent implements OnInit, AfterViewInit { `Use api-extractor to generate a docModel file`, `Upload generated api.json file` ]; - this.acceptedFilesForReviewUpload = ".json"; + this.acceptedFilesForReviewUpload = ".api.json"; this.createReviewForm.get('selectedFile')?.enable(); this.createReviewForm.get('filePath')?.disable(); break; @@ -381,7 +381,7 @@ export class ReviewsListComponent implements OnInit, AfterViewInit { this.createReviewInstruction = [ `Upload JSON API review token file.` ]; - this.acceptedFilesForReviewUpload = ".json"; + this.acceptedFilesForReviewUpload = ".json, .tgz"; this.createReviewForm.get('selectedFile')?.enable(); this.createReviewForm.get('filePath')?.disable(); break; diff --git a/src/dotnet/APIView/ClientSPA/src/app/_components/revisions-list/revisions-list.component.ts b/src/dotnet/APIView/ClientSPA/src/app/_components/revisions-list/revisions-list.component.ts index f96821c39dc..52fe7eb6334 100644 --- a/src/dotnet/APIView/ClientSPA/src/app/_components/revisions-list/revisions-list.component.ts +++ b/src/dotnet/APIView/ClientSPA/src/app/_components/revisions-list/revisions-list.component.ts @@ -342,7 +342,7 @@ export class RevisionsListComponent implements OnInit, OnChanges { this.showDiffButton = (value.length == 2) ? true : false; let canDelete = (value.length > 0)? true : false; for (const revision of value) { - if (revision.createdBy != this.userProfile?.userName || revision.apiRevisionType != "Manual") + if (revision.createdBy != this.userProfile?.userName || revision.apiRevisionType != "manual") { canDelete = false; break; diff --git a/tools/apiview/parsers/CONTRIBUTING.md b/tools/apiview/parsers/CONTRIBUTING.md new file mode 100644 index 00000000000..81a5786e356 --- /dev/null +++ b/tools/apiview/parsers/CONTRIBUTING.md @@ -0,0 +1,109 @@ +# Contributing + +## Overview +This page describes how to contribute to [APIView](../../../src//dotnet/APIView/APIViewWeb/APIViewWeb.csproj) language level parsers. +Specifically how to create or update a language parser to produce tree style tokens for APIView. + +Previously APIview tokens were created as a flat list assigned to the `CodeFileToken[] Tokens` property of the [CodeFile](../../../src/dotnet/APIView/APIView/Model/CodeFile.cs). Then the page navigation is created and assinged to `NavigationItem[] Navigation`. For tree style tokens these two properties are no longer required, instread a `List APIForest` property will be used to capture the generated tree of tokens. + +The main idea is to capture the hierachy of the API using a tree data structure, then maintain a flat list of tokens for each node of the tree. + +![APITree](APITree.svg) + +Each module of the API (namespace, class, methods) should be its own node. Members of a module (methods, in a class), (classes in a namespace) should be added as child nodes of its parent module. +Each tree node has top tokens which should be used to capture the main tokens on the node, these can span multiple lines. Module name, decorators, and prameters should be modeled as toptokens. If the language requires it use the bottom tokens to capture tokens that closes out the node, this is usually just the closing bracket and/or empty lines. + +## Object Definitions + +- Here are the models needed + ``` + object APITreeNode + string Name + string Id + string Kind + Set Tags + Dictionary Properties + List TopTokens + List BottomTokens + List Children + + object StructuredToken + string Value + string Id + StructuredTokenKind Kind + Set Tags + Dictionary Properties + Set RenderClasses + + enum StructuredTokenKind + Content + LineBreak + NoneBreakingSpace + TabSpace + ParameterSeparator + Url + ``` + +### APITreeNode +- `Name`: *(Required)* The name of the tree node which will be used as label for the API Navigation. Generally use the name of the module (class, method). +- `Id`: *(Required)* Id of the node, which should be unique at the node level. i.e. unique among its siblings. Use whatever your existing parser is assigning to DefinitionId for the main Token of the node. Each node must have an Id. +- `Kind` *(Required)* : What kind of node is it. Please ensure you set a Kind for each node. Using any of the following `assembly`, `class`, `delegate`, `enum`, `interface`, `method` , `namespace`, `package`, `struct`, `type` will get you the corresponding default icons for the page navigation but feel free to use something language specific then reach out to APIView to support any new name used. +- `Tags` : Use this for opt in or opt out boolean properties. The currently supported tags are + - `Deprecated` Mark a node as deprecated + - `Hidden` Mark a node as Hidden + - `HideFromNavigation` Indicate that anode should be hidden from the page navigation. + - `SkipDiff` Indicate that a node should not be used in computatation of diff. + - `CrossLangDefId` The cross language definitionId for the node. +- `Properties` : Use this for other properties of the node. The currently supported keys are + - `SubKind` Similar to kind, use this to make the node more specific. e.g. `Kind = 'Type'`, and `SubKind = 'class'` or somethign specific to you language. We also use this to make the navigation icon it will overide kind. + - `IconName` Use this only if you are looking to add a custom icon different from language wide defaults. New addtions will need to be supported APIView side. +- `TopTokens` : The main data of the node. This is all the tokens that actually define the node. e.g. For a class this would include the access modifier, the class name, any attributes or decorators e.t.c. For a method this would include the return type, method name, parameters e.t.c. See StructuredToken description below for more info. +- `BottomToken` : Data that closes out the node. Depending on the language this would include the closing curly brace and/or empty lines. You can simulate an empty line by adding an empty token (content token with no value) and a lineBreak token. +- `Children` : The nodes immediate children. For a namespace this would be classes, for a class this would be the class constructors and methods. + + +Sort each node at each level of the tree by your desired property, this is to ensure that difference in node order does not result in diff. + +Ensure each node has an Id and Kind. The combination of `Id`, `Kind` and `SubKind` should make the node unique across all nodes in the tree. This is very important. For example a class and a method can potentally have the same Id, but the kind should diffrentiate them from each other. + + +### StructuredToken +- `Value` : The token value which will be dispalyed. Spacing tokens don't need to have value. +- `Id` : This is essentially whatever existing parser was asigning to the token DefinitionId. You dont have to assign a tokenId to everytoken. +- `Kind` *(Required)* : An enum + - `Content` Specifies that the token is content + - `LineBreak` Space token indicating switch to new line. + - `NonBreakingSpace` Regular single space + - `TabSpace` 4 NonBreakingSpaces + - `ParameterSeparator` Use this between method parameters. Depending on user setting this would result in a singlespace or new line + - `Url` A url token should have `LinkText` property i.e `token.Properties["LinkText"]` and the url/link should be the token value. + All tokens should be content except for spacing tokens and url. ParameterSeparator should be used between method or function parameters. +- `Tags` : Use this for opt in or opt out boolean properties.The currently supported tags are + - `SkippDiff` Indicate that a token should not be used in computatation of diff. + - `Deprecated` Mark a token as deprecated +- `Properties` : Properties of the token. + - `GroupId` : `doc` to group consecutive comment tokens as documentation. + - `NavigateToId` Id for navigating to where the node where the token is defined. Should match the definitionId of the referenced node. +- `RenderClasses` : Add css classes for how the tokens will be rendred. Classes currently being used are `text` `keyword` `punc` `tname` `mname` `literal` `sliteral` `comment` Feel free to add your own custom class. Whatever custom classes you use please provide us the appriopriate css for the class so we can update APIView. + +Dont worry about indentation that will be handeled by the tree structure, unless you want to have indentation between the tokens then use `TabSpace` token kind. + +Assign the final parsed value to a `List APIForest` property of the `CodeFile` + +## Serilizations + +Serialize the generated code file to JSON them compress the file using Gzip compression. Try to make the json as small as possible by ignoring null values and empty collections. Bellow is example serialization setting in C# and Java. + +## How to handle commons Scenarios +- TEXT, KEYWORD, COMMENT : Add `text`, `keyword`, `comment` to RenderClasses of the token +- NEW_LINE : Create a token with `Kind = LineBreak` +- WHITE_SPACE : Create token with `Kind = NonBreakingSpace` +- PUNCTUATION : Create a token with `Kind = Content` and the `Value = the punctuation` +- DOCUMENTATION : Add `GroupId = doc` in the properties of the token. This identifies a range of consecutive tokens as belonging to a group. +- SKIP_DIFF : Add `SkipDiff` to the Tag to indicate that node or token should not be included in diff computation +- LINE_ID_MARKER : You can add a empty token. `Kind = Content` and `Value = ""` then give it an `Id` to make it commentable. +- EXTERNAL_LINK : Create a single token set `Kind = Url`, `Value = link` then add the link text as a properties `LinkText`; +- Common Tags: `Deprecated`, `Hidden`, `HideFromNav`, `SkipDiff` +- Cross Language Id: Use `CrossLangId` as key with value in the node properties. + +Please reach out at [APIView Teams Channel](https://teams.microsoft.com/l/channel/19%3A3adeba4aa1164f1c889e148b1b3e3ddd%40thread.skype/APIView?groupId=3e17dcb0-4257-4a30-b843-77f47f1d4121&tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47) if you need more infomation. \ No newline at end of file