diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index 0eeb812f55b..0b93ed41931 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -74,7 +74,7 @@ Text="No manifest file was found in the provided path: $(ManifestsBasePath)" /> - - - + + - - - /// Full path to the assets to publish manifest. + /// Full path to the assets to publish manifest(s) /// [Required] - public string AssetManifestPath { get; set; } + public ITaskItem[] AssetManifestPaths { get; set; } /// /// Full path to the folder containing blob assets. @@ -127,7 +127,7 @@ public class PublishArtifactsInManifest : MSBuild.Task /// /// Maximum number of parallel uploads for the upload tasks /// - public int MaxClients { get; set; } = 8; + public int MaxClients { get; set; } = 16; /// /// Directory where "nuget.exe" is installed. This will be used to publish packages. @@ -181,14 +181,25 @@ public async Task ExecuteAsync() { try { - Log.LogMessage(MessageImportance.High, "Publishing artifacts to feed."); - - if (string.IsNullOrWhiteSpace(AssetManifestPath) || !File.Exists(AssetManifestPath)) + List buildModels = new List(); + foreach (var assetManifestPath in AssetManifestPaths) { - Log.LogError($"Problem reading asset manifest path from '{AssetManifestPath}'"); + Log.LogMessage(MessageImportance.High, $"Publishing artifacts in {assetManifestPath.ItemSpec}."); + string fileName = assetManifestPath.ItemSpec; + if (string.IsNullOrWhiteSpace(fileName) || !File.Exists(fileName)) + { + Log.LogError($"Problem reading asset manifest path from '{fileName}'"); + } + else + { + buildModels.Add(BuildManifestUtil.ManifestFileToModel(assetManifestPath.ItemSpec, Log)); + } } - var buildModel = BuildManifestUtil.ManifestFileToModel(AssetManifestPath, Log); + if (Log.HasLoggedErrors) + { + return false; + } // Parsing the manifest may fail for several reasons if (Log.HasLoggedErrors) @@ -200,6 +211,7 @@ public async Task ExecuteAsync() // of the assets being published so we can add a new location for them. IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken); Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId); + Dictionary> buildAssets = CreateBuildAssetDictionary(buildInformation); await ParseTargetFeedConfigAsync(); @@ -209,7 +221,10 @@ public async Task ExecuteAsync() return false; } - SplitArtifactsInCategories(buildModel); + foreach (var buildModel in buildModels) + { + SplitArtifactsInCategories(buildModel); + } // Return errors from the safety checks if (Log.HasLoggedErrors) @@ -217,9 +232,11 @@ public async Task ExecuteAsync() return false; } - await HandlePackagePublishingAsync(client, buildInformation); - - await HandleBlobPublishingAsync(client, buildInformation); + await Task.WhenAll(new Task[] { + HandlePackagePublishingAsync(client, buildAssets), + HandleBlobPublishingAsync(client, buildAssets) + } + ); } catch (Exception e) { @@ -229,6 +246,64 @@ public async Task ExecuteAsync() return !Log.HasLoggedErrors; } + /// + /// Lookup an asset in the build asset dictionary by name and version + /// + /// Name of asset + /// Version of asset + /// Asset if one with the name and version exists, null otherwise + private Asset LookupAsset(string name, string version, Dictionary> buildAssets) + { + if (!buildAssets.TryGetValue(name, out List assetsWithName)) + { + return null; + } + return assetsWithName.FirstOrDefault(asset => asset.Version == version); + } + + /// + /// Lookup an asset in the build asset dictionary by name only. + /// This is for blob lookup purposes. + /// + /// Name of asset + /// + /// Asset if one with the name exists and is the only asset with the name. + /// Throws if there is more than one asset with that name. + /// + private Asset LookupAsset(string name, Dictionary> buildAssets) + { + if (!buildAssets.TryGetValue(name, out List assetsWithName)) + { + return null; + } + return assetsWithName.Single(); + } + + /// + /// Build up a map of asset name -> asset list so that we can avoid n^2 lookups when processing assets. + /// We use name only because blobs are only looked up by id (version is not recorded in the manifest). + /// This could mean that there might be multiple versions of the same asset in the build. + /// + /// Build information + /// Map of asset name -> list of assets with that name. + private Dictionary> CreateBuildAssetDictionary(Maestro.Client.Models.Build buildInformation) + { + Dictionary> buildAssets = new Dictionary>(); + foreach (var asset in buildInformation.Assets) + { + if (buildAssets.TryGetValue(asset.Name, out List assetsWithName)) + { + assetsWithName.Add(asset); + } + else + { + buildAssets.Add(asset.Name, new List() { asset }); + } + } + + return buildAssets; + } + /// /// Parse out the input TargetFeedConfig into a dictionary of FeedConfig types /// @@ -456,8 +531,16 @@ private void CheckForInternalBuildsOnPublicFeeds(FeedConfig feedConfig) return isPublic; } - private async Task HandlePackagePublishingAsync(IMaestroApi client, Maestro.Client.Models.Build buildInformation) + /// + /// Handle package publishing for all the feed configs. + /// + /// Maestro API client + /// Assets information about build being published. + /// Task + private async Task HandlePackagePublishingAsync(IMaestroApi client, Dictionary> buildAssets) { + List publishTasks = new List(); + foreach (var packagesPerCategory in PackagesByCategory) { var category = packagesPerCategory.Key; @@ -480,10 +563,10 @@ private async Task HandlePackagePublishingAsync(IMaestroApi client, Maestro.Clie switch (feedConfig.Type) { case FeedType.AzDoNugetFeed: - await PublishPackagesToAzDoNugetFeedAsync(filteredPackages, client, buildInformation, feedConfig); + publishTasks.Add(PublishPackagesToAzDoNugetFeedAsync(filteredPackages, client, buildAssets, feedConfig)); break; case FeedType.AzureStorageFeed: - await PublishPackagesToAzureStorageNugetFeedAsync(filteredPackages, client, buildInformation, feedConfig); + publishTasks.Add(PublishPackagesToAzureStorageNugetFeedAsync(filteredPackages, client, buildAssets, feedConfig)); break; default: Log.LogError($"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); @@ -496,6 +579,8 @@ private async Task HandlePackagePublishingAsync(IMaestroApi client, Maestro.Clie Log.LogError($"No target feed configuration found for artifact category: '{category}'."); } } + + await Task.WhenAll(publishTasks); } private List FilterPackages(List packages, FeedConfig feedConfig) @@ -516,8 +601,10 @@ private List FilterPackages(List pac } } - private async Task HandleBlobPublishingAsync(IMaestroApi client, Maestro.Client.Models.Build buildInformation) + private async Task HandleBlobPublishingAsync(IMaestroApi client, Dictionary> buildAssets) { + List publishTasks = new List(); + foreach (var blobsPerCategory in BlobsByCategory) { var category = blobsPerCategory.Key; @@ -540,10 +627,10 @@ private async Task HandleBlobPublishingAsync(IMaestroApi client, Maestro.Client. switch (feedConfig.Type) { case FeedType.AzDoNugetFeed: - await PublishBlobsToAzDoNugetFeedAsync(filteredBlobs, client, buildInformation, feedConfig); + publishTasks.Add(PublishBlobsToAzDoNugetFeedAsync(filteredBlobs, client, buildAssets, feedConfig)); break; case FeedType.AzureStorageFeed: - await PublishBlobsToAzureStorageNugetFeedAsync(filteredBlobs, client, buildInformation, feedConfig); + publishTasks.Add(PublishBlobsToAzureStorageNugetFeedAsync(filteredBlobs, client, buildAssets, feedConfig)); break; default: Log.LogError($"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); @@ -556,6 +643,8 @@ private async Task HandleBlobPublishingAsync(IMaestroApi client, Maestro.Client. Log.LogError($"No target feed configuration found for artifact category: '{category}'."); } } + + await Task.WhenAll(publishTasks); } /// @@ -648,24 +737,9 @@ public void SplitArtifactsInCategories(BuildModel buildModel) private async Task PublishPackagesToAzDoNugetFeedAsync( List packagesToPublish, IMaestroApi client, - Maestro.Client.Models.Build buildInformation, + Dictionary> buildAssets, FeedConfig feedConfig) { - foreach (var package in packagesToPublish) - { - var assetRecord = buildInformation.Assets - .Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version)) - .FirstOrDefault(); - - if (assetRecord == null) - { - Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}"); - continue; - } - - await TryAddAssetLocationAsync(client, assetRecord, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); - } - await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, async (feed, httpClient, package, feedAccount, feedVisibility, feedName) => { @@ -676,7 +750,16 @@ await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClien return; } - await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, feedAccount, feedVisibility, feedName); + Asset assetRecord = LookupAsset(package.Id, package.Version, buildAssets); + if (assetRecord == null) + { + Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}"); + } + + await Task.WhenAll(new Task[] { + TryAddAssetLocationAsync(client, assetRecord, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed), + PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version, feedAccount, feedVisibility, feedName), + }); }); } @@ -1013,28 +1096,13 @@ public static async Task CompareStreamsAsync(Stream localFileStream, Strea private async Task PublishBlobsToAzDoNugetFeedAsync( List blobsToPublish, IMaestroApi client, - Maestro.Client.Models.Build buildInformation, + Dictionary> buildAssets, FeedConfig feedConfig) { List packagesToPublish = new List(); foreach (var blob in blobsToPublish) { - var assetRecord = buildInformation.Assets - .Where(a => a.Name.Equals(blob.Id)) - .FirstOrDefault(); - - if (assetRecord == null) - { - Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}"); - continue; - } - - if (await TryAddAssetLocationAsync(client, assetRecord, feedConfig, AddAssetLocationToAssetAssetLocationType.Container) == false) - { - continue; - } - // Applies to symbol packages and core-sdk's VS feed packages if (blob.Id.EndsWith(PackageSuffix, StringComparison.OrdinalIgnoreCase)) { @@ -1049,33 +1117,44 @@ private async Task PublishBlobsToAzDoNugetFeedAsync( await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients: MaxClients, async (feed, httpClient, blob, feedAccount, feedVisibility, feedName) => { - // Determine the local path to the blob - string fileName = Path.GetFileName(blob.Id); - string localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); - if (!File.Exists(localBlobPath)) + // Lookup just by name + Asset assetRecord = LookupAsset(blob.Id, buildAssets); + if (assetRecord == null) { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); - return; + Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}"); } - - string id; - string version; - // Determine package ID and version by asking the nuget libraries - using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) + // Only attempt to push if the package doesn't have + // that location already. + else if (await TryAddAssetLocationAsync(client, assetRecord, feedConfig, AddAssetLocationToAssetAssetLocationType.Container)) { - PackageIdentity packageIdentity = packageReader.GetIdentity(); - id = packageIdentity.Id; - version = packageIdentity.Version.ToString(); - } + // Determine the local path to the blob + string fileName = Path.GetFileName(blob.Id); + string localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); + if (!File.Exists(localBlobPath)) + { + Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + return; + } + + string id; + string version; + // Determine package ID and version by asking the nuget libraries + using (var packageReader = new NuGet.Packaging.PackageArchiveReader(localBlobPath)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + } - await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAccount, feedVisibility, feedName); + await PushNugetPackageAsync(feed, httpClient, localBlobPath, id, version, feedAccount, feedVisibility, feedName); + } }); } private async Task PublishPackagesToAzureStorageNugetFeedAsync( List packagesToPublish, IMaestroApi client, - Maestro.Client.Models.Build buildInformation, + Dictionary> buildAssets, FeedConfig feedConfig) { var packages = packagesToPublish.Select(p => @@ -1101,28 +1180,28 @@ private async Task PublishPackagesToAzureStorageNugetFeedAsync( PassIfExistingItemIdentical = true }; - foreach (var package in packagesToPublish) + await Task.WhenAll(new Task[] { - var assetRecord = buildInformation.Assets - .Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version)) - .FirstOrDefault(); - - if (assetRecord == null) + Task.WhenAll(packagesToPublish.Select(async package => { - Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}"); - continue; - } - - await TryAddAssetLocationAsync(client, assetRecord, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); - } - - await blobFeedAction.PushToFeedAsync(packages, pushOptions); + Asset assetRecord = LookupAsset(package.Id, package.Version, buildAssets); + if (assetRecord == null) + { + Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}"); + } + else + { + await TryAddAssetLocationAsync(client, assetRecord, feedConfig, AddAssetLocationToAssetAssetLocationType.NugetFeed); + } + })), + blobFeedAction.PushToFeedAsync(packages, pushOptions) + }); } private async Task PublishBlobsToAzureStorageNugetFeedAsync( List blobsToPublish, IMaestroApi client, - Maestro.Client.Models.Build buildInformation, + Dictionary> buildAssets, FeedConfig feedConfig) { var blobs = blobsToPublish @@ -1154,22 +1233,27 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync( PassIfExistingItemIdentical = true }; - foreach (BlobArtifactModel blob in blobsToPublish) - { - Asset assetRecord = buildInformation.Assets - .Where(a => a.Name.Equals(blob.Id)) - .SingleOrDefault(); - - if (assetRecord == null) + await Task.WhenAll(new Task[] { - Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}"); - continue; - } + Task.WhenAll(blobsToPublish.Select(async blob => + { + Asset assetRecord = LookupAsset(blob.Id, buildAssets); - await TryAddAssetLocationAsync(client, assetRecord, feedConfig, AddAssetLocationToAssetAssetLocationType.Container); - } + if (assetRecord == null) + { + Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}"); + } + else + { + await TryAddAssetLocationAsync(client, assetRecord, feedConfig, AddAssetLocationToAssetAssetLocationType.Container); + } + })), + blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions) + } + ); - await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions); + // The latest links should be updated only after the publishing is complete, to avoid + // dead links in the interim. await CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig); }