diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index a7c8e64..05b26b6 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -1,116 +1,124 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Build Schema", - "$ref": "#/definitions/build", - "definitions": { - "build": { - "type": "object", - "properties": { - "Continue": { - "type": "boolean", - "description": "Indicates to continue a previously failed build attempt" - }, - "CurrentListingUrl": { - "type": "string", - "description": "Path to existing index.json file, typically https://{owner}.github.io/{repo}/index.json" - }, - "CurrentPackageName": { - "type": "string", - "description": "PackageName" - }, - "Help": { - "type": "boolean", - "description": "Shows the help text for this build assembly" - }, - "Host": { - "type": "string", - "description": "Host for execution. Default is 'automatic'", - "enum": [ - "AppVeyor", - "AzurePipelines", - "Bamboo", - "Bitbucket", - "Bitrise", - "GitHubActions", - "GitLab", - "Jenkins", - "Rider", - "SpaceAutomation", - "TeamCity", - "Terminal", - "TravisCI", - "VisualStudio", - "VSCode" - ] - }, - "ListPublishDirectory": { - "type": "string", - "description": "Directory to save index into" - }, - "NoLogo": { - "type": "boolean", - "description": "Disables displaying the NUKE logo" - }, - "PackageListingSourceFilename": { - "type": "string", - "description": "Filename of source json" - }, - "PackageListingSourceFolder": { - "type": "string", - "description": "Path to Target Listing Root" - }, - "Partition": { - "type": "string", - "description": "Partition to use on CI" - }, - "Plan": { - "type": "boolean", - "description": "Shows the execution plan (HTML)" - }, - "Profile": { - "type": "array", - "description": "Defines the profiles to load", - "items": { - "type": "string" - } - }, - "Root": { - "type": "string", - "description": "Root directory during build execution" - }, - "Skip": { - "type": "array", - "description": "List of targets to be skipped. Empty list skips all dependencies", - "items": { - "type": "string", - "enum": [ - "BuildMultiPackageListing", - "BuildRepoListing" - ] - } - }, - "Target": { - "type": "array", - "description": "List of targets to be invoked. Default is '{default_target}'", - "items": { - "type": "string", - "enum": [ - "BuildMultiPackageListing", - "BuildRepoListing" - ] - } - }, - "Verbosity": { - "type": "string", - "description": "Logging verbosity during build execution. Default is 'Normal'", - "enum": [ - "Minimal", - "Normal", - "Quiet", - "Verbose" - ] - } - } - } - } +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Build Schema", + "$ref": "#/definitions/build", + "definitions": { + "build": { + "type": "object", + "properties": { + "Continue": { + "type": "boolean", + "description": "Indicates to continue a previously failed build attempt" + }, + "CurrentListingUrl": { + "type": "string", + "description": "Path to existing index.json file, typically https://{owner}.github.io/{repo}/index.json" + }, + "CurrentPackageName": { + "type": "string", + "description": "PackageName" + }, + "Help": { + "type": "boolean", + "description": "Shows the help text for this build assembly" + }, + "Host": { + "type": "string", + "description": "Host for execution. Default is 'automatic'", + "enum": [ + "AppVeyor", + "AzurePipelines", + "Bamboo", + "Bitbucket", + "Bitrise", + "GitHubActions", + "GitLab", + "Jenkins", + "Rider", + "SpaceAutomation", + "TeamCity", + "Terminal", + "TravisCI", + "VisualStudio", + "VSCode" + ] + }, + "ListPublishDirectory": { + "type": "string", + "description": "Directory to save index into" + }, + "LocalTestPackagesPath": { + "type": "string", + "description": "Path to Target Package" + }, + "NoLogo": { + "type": "boolean", + "description": "Disables displaying the NUKE logo" + }, + "PackageListingSourceFilename": { + "type": "string", + "description": "Filename of source json" + }, + "PackageListingSourceFolder": { + "type": "string", + "description": "Path to Target Listing Root" + }, + "PackageListingSuffix": { + "type": "string", + "description": "Suffix to append to the listing package name and ID" + }, + "Partition": { + "type": "string", + "description": "Partition to use on CI" + }, + "Plan": { + "type": "boolean", + "description": "Shows the execution plan (HTML)" + }, + "Profile": { + "type": "array", + "description": "Defines the profiles to load", + "items": { + "type": "string" + } + }, + "Root": { + "type": "string", + "description": "Root directory during build execution" + }, + "Skip": { + "type": "array", + "description": "List of targets to be skipped. Empty list skips all dependencies", + "items": { + "type": "string", + "enum": [ + "BuildMultiPackageListing", + "BuildRepoListing" + ] + } + }, + "Target": { + "type": "array", + "description": "List of targets to be invoked. Default is '{default_target}'", + "items": { + "type": "string", + "enum": [ + "BuildMultiPackageListing", + "BuildRepoListing" + ] + } + }, + "Verbosity": { + "type": "string", + "description": "Logging verbosity during build execution. Default is 'Normal'", + "enum": [ + "Minimal", + "Normal", + "Quiet", + "Verbose" + ] + } + } + } + } } \ No newline at end of file diff --git a/PackageBuilder/Build.cs b/PackageBuilder/Build.cs index 7b50690..5da337a 100644 --- a/PackageBuilder/Build.cs +++ b/PackageBuilder/Build.cs @@ -38,7 +38,7 @@ partial class Build : NukeBuild const string PackageListingPublishFilename = "index.json"; const string WebPageAppFilename = "app.js"; - [Parameter("Directory to save index into")] + [Parameter("Directory to save index into")] AbsolutePath ListPublishDirectory = RootDirectory / "docs"; [Parameter("PackageName")] @@ -46,21 +46,30 @@ partial class Build : NukeBuild [Parameter("Filename of source json")] string PackageListingSourceFilename = "source.json"; - + // assumes that "template-package-listings" repo is checked out in sibling dir for local testing, can be overriden - [Parameter("Path to Target Listing Root")] + [Parameter("Path to Target Listing Root")] AbsolutePath PackageListingSourceFolder = IsServerBuild ? RootDirectory.Parent - : RootDirectory.Parent / "template-package-listing"; + : RootDirectory.Parent / "package-list-action"; + private string _currentListingUrl; [Parameter("Path to existing index.json file, typically https://{owner}.github.io/{repo}/index.json")] - string CurrentListingUrl => - $"https://{GitHubActions.RepositoryOwner}.github.io/{GitHubActions.Repository.Split('/')[1]}/{PackageListingPublishFilename}"; - + public string CurrentListingUrl + { + get => _currentListingUrl ?? + $"https://{GitHubActions.RepositoryOwner}.github.io/{GitHubActions.Repository.Split('/')[1]}/{PackageListingPublishFilename}"; + + set => _currentListingUrl = value; + } + // assumes that "template-package" repo is checked out in sibling dir to this repo, can be overridden - [Parameter("Path to Target Package")] - AbsolutePath LocalTestPackagesPath => RootDirectory.Parent / "template-package" / "Packages"; - + [Parameter("Path to Target Package")] + AbsolutePath LocalTestPackagesPath => RootDirectory.Parent / "template-package" / "Packages"; + + [Parameter("Suffix to append to the listing package name and ID")] + string PackageListingSuffix = "Listing"; + AbsolutePath PackageListingSourcePath => PackageListingSourceFolder / PackageListingSourceFilename; AbsolutePath WebPageSourcePath => PackageListingSourceFolder / "Website"; @@ -84,8 +93,12 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) { var result = new ListingSource() { - name = $"{manifest.displayName} Listing", - id = $"{manifest.name}.listing", + name = PackageListingSuffix != "" + ? $"{manifest.displayName} {PackageListingSuffix}" + : $"{manifest.displayName}", + id = PackageListingSuffix != "" + ? $"{manifest.name}.{PackageListingSuffix.ToLower()}" + : $"{manifest.name}", author = new VRC.PackageManagement.Automation.Multi.Author() { name = manifest.author.name ?? "", @@ -102,21 +115,21 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) }; return result; } - + Target BuildRepoListing => _ => _ .Executes(async () => { ListingSource listSource; - + if (!FileSystemTasks.FileExists(PackageListingSourcePath)) { - AbsolutePath packagePath = RootDirectory.Parent / "Packages" / CurrentPackageName / PackageManifestFilename; + AbsolutePath packagePath = RootDirectory.Parent / "Packages" / CurrentPackageName / PackageManifestFilename; if (!FileSystemTasks.FileExists(packagePath)) { Serilog.Log.Error($"Could not find Listing Source at {PackageListingSourcePath} or Package Manifest at {packagePath}, you need at least one of them."); return; } - + // Deserialize manifest from packagePath var manifest = JsonConvert.DeserializeObject(File.ReadAllText(packagePath), JsonReadOptions); listSource = MakeListingSourceFromManifest(manifest); @@ -138,7 +151,7 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) listSource.id = $"io.github.{GetRepoOwner()}.{GetRepoName()}"; Serilog.Log.Warning($"Your listing needs an id. We've autogenerated one for you: {listSource.id}. If you want to change it, edit {PackageListingSourcePath}."); } - + // Get existing RepoList URLs or create empty one, so we can skip existing packages var currentRepoListString = IsServerBuild ? await GetAuthenticatedString(CurrentListingUrl) : null; var currentPackageUrls = currentRepoListString == null @@ -149,7 +162,7 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) // Make collection for constructed packages var packages = new List(); var possibleReleaseUrls = new List(); - + // Add packages from listing source if included if (listSource.packages != null) { @@ -168,6 +181,7 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) } // Add each release url to the packages collection if it's not already in the listing, and its zip is valid + List> tasks = new List>(); foreach (string url in possibleReleaseUrls) { Serilog.Log.Information($"Looking at {url}"); @@ -176,8 +190,10 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) Serilog.Log.Information($"Current listing already contains {url}, skipping"); continue; } - - var manifest = await HashZipAndReturnManifest(url); + + tasks.Add(HashZipAndReturnManifest(url)); + + /* var manifest = await HashZipAndReturnManifest(url); if (manifest == null) { Serilog.Log.Information($"Could not find manifest in zip file {url}, skipping."); @@ -185,7 +201,30 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) } // Add package with updated manifest to collection - Serilog.Log.Information($"Found {manifest.Id} ({manifest.name}) {manifest.Version}, adding to listing."); + Serilog.Log.Information($"Found {manifest.Id} ({manifest.name}) {manifest.Version}, adding to listing. Currently {packages.Count} packages."); + packages.Add(manifest); */ + } + + Serilog.Log.Information($"Waiting for {tasks.Count} tasks to finish."); + System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); + sw.Start(); + + VRCPackageManifest[] finishedTasks = await Task.WhenAll(tasks); + + sw.Stop(); + Serilog.Log.Information($"All tasks finished in {sw.ElapsedMilliseconds}ms, processing results..."); + + //process any finished tasks + foreach (var manifest in finishedTasks) + { + if (manifest == null) + { + Serilog.Log.Information($"Could not find manifest in zip file, skipping."); + continue; + } + + // Add package with updated manifest to collection + Serilog.Log.Information($"Found {manifest.Id} ({manifest.name}) {manifest.Version}, adding to listing. Currently {packages.Count} packages."); packages.Add(manifest); } @@ -201,7 +240,8 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) // Server builds write into the source directory itself // So we dont need to clear it out - if (!IsServerBuild) { + if (!IsServerBuild) + { FileSystemTasks.EnsureCleanDirectory(ListPublishDirectory); } @@ -215,15 +255,18 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) string indexTemplateContent = File.ReadAllText(indexReadPath); - var listingInfo = new { + var listingInfo = new + { Name = listSource.name, Url = listSource.url, Description = listSource.description, - InfoLink = new { + InfoLink = new + { Text = listSource.infoLink?.text, Url = listSource.infoLink?.url, }, - Author = new { + Author = new + { Name = listSource.author.name, Url = listSource.author.url, Email = listSource.author.email @@ -231,14 +274,16 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) BannerImage = !string.IsNullOrEmpty(listSource.bannerUrl), BannerImageUrl = listSource.bannerUrl, }; - + Serilog.Log.Information($"Made listingInfo {JsonConvert.SerializeObject(listingInfo, JsonWriteOptions)}"); var latestPackages = packages.OrderByDescending(p => p.Version).DistinctBy(p => p.Id).ToList(); Serilog.Log.Information($"LatestPackages: {JsonConvert.SerializeObject(latestPackages, JsonWriteOptions)}"); - var formattedPackages = latestPackages.ConvertAll(p => new { + var formattedPackages = latestPackages.ConvertAll(p => new + { Name = p.Id, - Author = new { + Author = new + { Name = p.author?.name, Url = p.author?.url, }, @@ -250,17 +295,18 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) p.Description, DisplayName = p.Title, p.Version, - Dependencies = p.VPMDependencies.Select(dep => new { - Name = dep.Key, - Version = dep.Value - } + Dependencies = p.VPMDependencies.Select(dep => new + { + Name = dep.Key, + Version = dep.Value + } ).ToList(), }); - + var rendered = Scriban.Template.Parse(indexTemplateContent).Render( new { listingInfo, packages = formattedPackages }, member => member.Name ); - + File.WriteAllText(indexWritePath, rendered); var appJsRendered = Scriban.Template.Parse(File.ReadAllText(appReadPath)).Render( @@ -268,10 +314,11 @@ ListingSource MakeListingSourceFromManifest(VRCPackageManifest manifest) ); File.WriteAllText(indexAppWritePath, appJsRendered); - if (!IsServerBuild) { + if (!IsServerBuild) + { FileSystemTasks.CopyDirectoryRecursively(WebPageSourcePath, ListPublishDirectory, DirectoryExistsPolicy.Merge, FileExistsPolicy.Skip); } - + Serilog.Log.Information($"Saved Listing to {savePath}."); }); @@ -292,7 +339,7 @@ GitHubClient Client return _client; } } - + async Task> GetReleaseZipUrlsFromGitHubRepo(string ownerSlashName) { // Split string into owner and repo, or skip if invalid. @@ -311,7 +358,7 @@ async Task> GetReleaseZipUrlsFromGitHubRepo(string ownerSlashName) Assert.Fail($"Could not get remote repo {owner}/{name}."); return null; } - + // Go through each release var releases = await Client.Repository.Release.GetAll(owner, name); if (releases.Count == 0) @@ -321,7 +368,7 @@ async Task> GetReleaseZipUrlsFromGitHubRepo(string ownerSlashName) } var result = new List(); - + foreach (Octokit.Release release in releases) { result.AddRange(release.Assets.Where(asset => asset.Name.EndsWith(".zip")).Select(asset => asset.BrowserDownloadUrl)); @@ -339,37 +386,44 @@ string GetPackageType(IVRCPackage p) string result = "Any"; var manifest = p as VRCPackageManifest; if (manifest == null) return result; - + if (manifest.ContainsAvatarDependencies()) result = "Avatar"; else if (manifest.ContainsWorldDependencies()) result = "World"; - + return result; } async Task HashZipAndReturnManifest(string url) { - using (var response = await Http.GetAsync(url)) + using (var response = await Http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead)) { if (!response.IsSuccessStatusCode) { Assert.Fail($"Could not find valid zip file at {url}"); } - // Get manifest or return null - var bytes = await response.Content.ReadAsByteArrayAsync(); - var manifestBytes = GetFileFromZip(bytes, PackageManifestFilename); - if (manifestBytes == null) return null; - - var manifestString = Encoding.UTF8.GetString(manifestBytes); - var manifest = VRCPackageManifest.FromJson(manifestString); - var hash = GetHashForBytes(bytes); - manifest.zipSHA256 = hash; // putting the hash in here for now - // Point manifest towards release - manifest.url = url; - return manifest; + try + { + // Get manifest or return null + var bytes = await response.Content.ReadAsByteArrayAsync(); + var manifestBytes = GetFileFromZip(bytes, PackageManifestFilename); + if (manifestBytes == null) return null; + + var manifestString = Encoding.UTF8.GetString(manifestBytes); + var manifest = VRCPackageManifest.FromJson(manifestString); + var hash = GetHashForBytes(bytes); + manifest.zipSHA256 = hash; // putting the hash in here for now + // Point manifest towards release + manifest.url = url; + return manifest; + } + finally + { + response.Dispose(); + } } } - + static byte[] GetFileFromZip(byte[] bytes, string fileName) { byte[] ret = null; @@ -383,7 +437,6 @@ static byte[] GetFileFromZip(byte[] bytes, string fileName) ret = new byte[ze.Size]; s.Read(ret, 0, ret.Length); } - return ret; } @@ -438,12 +491,16 @@ static HttpClient Http return _http; } - _http = new HttpClient(); + HttpClientHandler handler = new HttpClientHandler(); + handler.Proxy = null; + handler.UseProxy = false; + + _http = new HttpClient(handler); _http.DefaultRequestHeaders.UserAgent.ParseAdd(VRCAgent); return _http; } } - + // https://www.newtonsoft.com/json/help/html/T_Newtonsoft_Json_JsonSerializerSettings.htm static JsonSerializerSettings JsonWriteOptions = new() { @@ -455,7 +512,7 @@ static HttpClient Http new VersionListConverter() }, }; - + static JsonSerializerSettings JsonReadOptions = new() { NullValueHandling = NullValueHandling.Ignore, @@ -466,4 +523,4 @@ static HttpClient Http }, }; } -} \ No newline at end of file +} diff --git a/PackageBuilder/PackageBuilder.csproj b/PackageBuilder/PackageBuilder.csproj index 653d38a..7e90ca1 100644 --- a/PackageBuilder/PackageBuilder.csproj +++ b/PackageBuilder/PackageBuilder.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net6.0;net8.0 CS0649;CS0169 .. diff --git a/build.sh b/build.sh index 0837e11..588b4c8 100755 --- a/build.sh +++ b/build.sh @@ -56,7 +56,13 @@ else export DOTNET_EXE="$DOTNET_DIRECTORY/dotnet" fi +FRAMEWORK="net6.0" +if dotnet --list-sdks | grep ^8 > /dev/null; then + FRAMEWORK="net8.0" +fi + echo "Microsoft (R) .NET Core SDK version $("$DOTNET_EXE" --version)" +echo "Using Framework $FRAMEWORK" "$DOTNET_EXE" build "$BUILD_PROJECT_FILE" /nodeReuse:false /p:UseSharedCompilation=false -nologo -clp:NoSummary --verbosity quiet -"$DOTNET_EXE" run --project "$BUILD_PROJECT_FILE" --no-build -- "$@" +"$DOTNET_EXE" run --project "$BUILD_PROJECT_FILE" --no-build --framework "$FRAMEWORK" -- "$@" diff --git a/package-list-action.sln b/package-list-action.sln new file mode 100644 index 0000000..a8a0e6e --- /dev/null +++ b/package-list-action.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PackageBuilder", "PackageBuilder\PackageBuilder.csproj", "{E468AF9D-C3CC-4294-859D-CAE62EB25815}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {E468AF9D-C3CC-4294-859D-CAE62EB25815}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E468AF9D-C3CC-4294-859D-CAE62EB25815}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E468AF9D-C3CC-4294-859D-CAE62EB25815}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E468AF9D-C3CC-4294-859D-CAE62EB25815}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {D213547B-F86B-42E5-BD20-EDA312F203E9} + EndGlobalSection +EndGlobal