From bf41bf0c02d037dae9b74beacb170e489baf593b Mon Sep 17 00:00:00 2001 From: Alex Perovich Date: Thu, 9 Sep 2021 16:57:56 -0700 Subject: [PATCH 1/5] Add publishing to the dotnetbuilds storage account alongside dotnetcli --- .vault-config/dotnetbuildskeys.yaml | 18 ++ .vault-config/dotnetbuildstokens.yaml | 49 +++++ eng/publishing/v3/publish-assets.yml | 5 + .../SdkTasks/PublishArtifactsInManifest.proj | 7 +- .../PublishArtifactsInManifestTests.cs | 2 +- .../PublishToSymbolServerTest.cs | 7 +- .../SetupTargetFeedConfigV3Tests.cs | 12 ++ .../src/AssetPublisher.cs | 34 ++++ .../src/AssetPublisherFactory.cs | 35 ++++ .../src/AzureDevOpsNugetFeedAssetPublisher.cs | 90 +++++++++ .../src/AzureStorageAssetPublisher.cs | 50 +++++ .../AzureStorageContainerAssetPublisher.cs | 25 +++ .../src/AzureStorageExtensions.cs | 77 ++++++++ .../src/AzureStorageFeedAssetPublisher.cs | 34 ++++ .../src/BlobFeedAction.cs | 8 +- .../src/PublishArtifactsInManifest.cs | 53 +++++- .../src/PublishArtifactsInManifestBase.cs | 180 +++++++++--------- .../src/PublishArtifactsInManifestV2.cs | 6 +- .../src/PublishArtifactsInManifestV3.cs | 19 +- .../src/PublishSignedAssets.cs | 4 + .../src/PushOptions.cs | 4 +- .../src/SemaphoreLock.cs | 45 +++++ .../src/common/AzureStorageUtils.cs | 71 +------ .../src/common/LatestLinksManager.cs | 39 ++-- .../src/common/UploadToAzure.cs | 3 +- .../src/model/PublishingConstants.cs | 12 +- .../src/model/SetupTargetFeedConfigV3.cs | 72 +++++++ .../src/model/TargetFeedConfig.cs | 7 +- 28 files changed, 766 insertions(+), 202 deletions(-) create mode 100644 .vault-config/dotnetbuildskeys.yaml create mode 100644 .vault-config/dotnetbuildstokens.yaml create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisher.cs create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisherFactory.cs create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageAssetPublisher.cs create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageContainerAssetPublisher.cs create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageExtensions.cs create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageFeedAssetPublisher.cs create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/SemaphoreLock.cs diff --git a/.vault-config/dotnetbuildskeys.yaml b/.vault-config/dotnetbuildskeys.yaml new file mode 100644 index 00000000000..90a841b6763 --- /dev/null +++ b/.vault-config/dotnetbuildskeys.yaml @@ -0,0 +1,18 @@ +storageLocation: + type: azure-key-vault + parameters: + subscription: 11c6037b-227b-4d63-bee1-18c7b68c3a40 + name: dotnetbuildskeys + +secrets: + dotnetbuilds-access-key: + type: azure-storage-key + parameters: + subscription: 11c6037b-227b-4d63-bee1-18c7b68c3a40 + account: dotnetbuilds + + dotnetbuilds-connection-string: + type: azure-storage-connection-string + parameters: + storageKeySecret: dotnetbuilds-access-key + account: dotnetbuilds diff --git a/.vault-config/dotnetbuildstokens.yaml b/.vault-config/dotnetbuildstokens.yaml new file mode 100644 index 00000000000..dcb3cf9fd1a --- /dev/null +++ b/.vault-config/dotnetbuildstokens.yaml @@ -0,0 +1,49 @@ +storageLocation: + type: azure-key-vault + parameters: + subscription: 11c6037b-227b-4d63-bee1-18c7b68c3a40 + name: dotnetbuildstokens + +references: + dotnetbuildskeys: + type: azure-key-vault + parameters: + subscription: 11c6037b-227b-4d63-bee1-18c7b68c3a40 + name: dotnetbuildskeys + +secrets: + dotnetbuilds-internal-container-uri: + type: azure-storage-container-sas-uri + parameters: + connectionString: + name: dotnetbuilds-connection-string + location: dotnetbuildskeys + permissions: rlwc + container: internal + + dotnetbuilds-internal-container-checksum-uri: + type: azure-storage-container-sas-uri + parameters: + connectionString: + name: dotnetbuilds-connection-string + location: dotnetbuildskeys + permissions: rlwc + container: internal-checksums + + dotnetbuilds-public-container-uri: + type: azure-storage-container-sas-uri + parameters: + connectionString: + name: dotnetbuilds-connection-string + location: dotnetbuildskeys + permissions: rlwc + container: public + + dotnetbuilds-public-container-checksum-uri: + type: azure-storage-container-sas-uri + parameters: + connectionString: + name: dotnetbuilds-connection-string + location: dotnetbuildskeys + permissions: rlwc + container: public-checksums diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index 60dca2adcde..4681707042f 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -12,6 +12,7 @@ jobs: timeoutInMinutes: 120 variables: - group: DotNet-Symbol-Server-Pats + - group: DotNetBuilds storage account tokens - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: IsStableBuild @@ -98,6 +99,10 @@ jobs: /p:UseStreamingPublishing='true' /p:StreamingPublishingMaxClients=16 /p:NonStreamingPublishingMaxClients=12 + /p:DotNetBuildsPublicUri='$(dotnetbuilds-public-container-uri)' + /p:DotNetBuildsPublicChecksumsUri='$(dotnetbuilds-public-container-checksum-uri)' + /p:DotNetBuildsInternalUri='$(dotnetbuilds-internal-container-uri)' + /p:DotNetBuildsInternalChecksumsUri='$(dotnetbuilds-internal-container-checksum-uri)' - template: /eng/common/templates/steps/publish-logs.yml parameters: StageLabel: '${{ parameters.stageName }}' diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index 6aea96bc2a2..c24e75f7adf 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -164,7 +164,12 @@ AzureProject="$(AzureProject)" UseStreamingPublishing="$(UseStreamingPublishing)" StreamingPublishingMaxClients="$(StreamingPublishingMaxClients)" - NonStreamingPublishingMaxClients="$(NonStreamingPublishingMaxClients)"/> + NonStreamingPublishingMaxClients="$(NonStreamingPublishingMaxClients)" + DotNetBuildsPublicUri="$(DotNetBuildsPublicUri)" + DotNetBuildsPublicChecksumsUri="$(DotNetBuildsPublicChecksumsUri)" + DotNetBuildsInternalUri="$(DotNetBuildsInternalUri)" + DotNetBuildsInternalChecksumsUri="$(DotNetBuildsInternalChecksumsUri)" + /> diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishArtifactsInManifestTests.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishArtifactsInManifestTests.cs index f46aa8abea8..e6f9a3c66bf 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishArtifactsInManifestTests.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishArtifactsInManifestTests.cs @@ -136,7 +136,7 @@ public async Task FeedConfigParserTests2Async() await task.ParseTargetFeedConfigAsync(); task.Log.HasLoggedErrors.Should().BeTrue(); buildEngine.BuildErrorEvents.Should().Contain(e => - e.Message.Equals("Invalid feed config type 'MyUnknownFeedType'. Possible values are: AzDoNugetFeed, AzureStorageFeed")); + e.Message.Equals("Invalid feed config type 'MyUnknownFeedType'. Possible values are: AzDoNugetFeed, AzureStorageFeed, AzureStorageContainer")); } [Fact] diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs index cefc81c5c69..a197e5c6bdd 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs @@ -82,7 +82,7 @@ public void PublishToBothSymbolServerTest() } [Fact] - public void TemporarySymbolDirectoryDoesNotExists() + public async Task TemporarySymbolDirectoryDoesNotExists() { var buildEngine = new MockBuildEngine(); var task = new PublishArtifactsInManifestV3() @@ -91,7 +91,7 @@ public void TemporarySymbolDirectoryDoesNotExists() }; var path = TestInputs.GetFullPath("Symbol"); var buildAsset = new Dictionary>(); - var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, buildAsset, null, path); + await task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, buildAsset, null, path); Assert.True(task.Log.HasLoggedErrors); } @@ -113,7 +113,6 @@ public void TemporarySymbolsDirectoryTest() [Fact] public void PublishSymbolApiIsCalledTest() { - var publishTask = new PublishArtifactsInManifestV3(); var path = TestInputs.GetFullPath("Symbols"); string[] fileEntries = Directory.GetFiles(path); var feedConfigsForSymbols = new HashSet(); @@ -128,8 +127,6 @@ public void PublishSymbolApiIsCalledTest() @internal: false, allowOverwrite: true, SymbolTargetType.SymWeb)); - Dictionary test = - publishTask.GetTargetSymbolServers(feedConfigsForSymbols, MsdlToken, SymWebToken); Assert.True(PublishSymbolsHelper.PublishAsync(null, path, SymWebToken, diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/SetupTargetFeedConfigV3Tests.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/SetupTargetFeedConfigV3Tests.cs index db7559468e5..0e06d46bcf3 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/SetupTargetFeedConfigV3Tests.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/SetupTargetFeedConfigV3Tests.cs @@ -162,6 +162,10 @@ public void StableFeeds(bool publishInstallersAndChecksums, bool isInternalBuild AzureDevOpsFeedsKey, buildEngine, symbolTargetType, + "public", + "publicchecksums", + "internal", + "internalchecksums", StablePackageFeed, StableSymbolsFeed, filesToExclude: FilesToExclude @@ -275,6 +279,10 @@ public void NonStableAndInternal(bool publishInstallersAndChecksums) AzureDevOpsFeedsKey, buildEngine: buildEngine, symbolTargetType, + "public", + "publicchecksums", + "internal", + "internalchecksums", filesToExclude: FilesToExclude ); @@ -385,6 +393,10 @@ public void NonStableAndPublic(bool publishInstallersAndChecksums) AzureDevOpsFeedsKey, buildEngine: buildEngine, symbolTargetType, + "public", + "publicchecksums", + "internal", + "internalchecksums", filesToExclude: FilesToExclude ); diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisher.cs new file mode 100644 index 00000000000..055bfabb990 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisher.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.Maestro.Client.Models; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public abstract class AssetPublisher : IDisposable + { + public TaskLoggingHelper Log { get; } + public abstract AddAssetLocationToAssetAssetLocationType LocationType { get; } + + protected AssetPublisher(TaskLoggingHelper log) + { + Log = log; + } + + public abstract Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null); + + protected virtual void Dispose(bool disposing) + { + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisherFactory.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisherFactory.cs new file mode 100644 index 00000000000..6066432efd6 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisherFactory.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.Build.Tasks.Feed.Model; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public class AssetPublisherFactory + { + private readonly TaskLoggingHelper _log; + + public AssetPublisherFactory(TaskLoggingHelper log) + { + _log = log; + } + + public virtual AssetPublisher CreateAssetPublisher(TargetFeedConfig feedConfig, PublishArtifactsInManifestBase task) + { + switch (feedConfig.Type) + { + case FeedType.AzDoNugetFeed: + return new AzureDevOpsNugetFeedAssetPublisher(_log, feedConfig.TargetURL, feedConfig.Token, task); + case FeedType.AzureStorageFeed: + var action = new BlobFeedAction(feedConfig.TargetURL, feedConfig.Token, _log); + return new AzureStorageFeedAssetPublisher(action.AccountName, action.AccountKey, action.ContainerName, _log); + case FeedType.AzureStorageContainer: + return new AzureStorageContainerAssetPublisher(new Uri(feedConfig.TargetURL), _log); + default: + throw new NotImplementedException(); + } + } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs new file mode 100644 index 00000000000..4fa04b469d9 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs @@ -0,0 +1,90 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.Build.Tasks.Feed.Model; +using Microsoft.DotNet.Maestro.Client.Models; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public class AzureDevOpsNugetFeedAssetPublisher : AssetPublisher + { + private readonly string _targetUrl; + private readonly string _accessToken; + private readonly PublishArtifactsInManifestBase _task; + private readonly string _feedAccount; + private readonly string _feedVisibility; + private readonly string _feedName; + private readonly HttpClient _httpClient; + + public AzureDevOpsNugetFeedAssetPublisher(TaskLoggingHelper log, string targetUrl, string accessToken, PublishArtifactsInManifestBase task) : base(log) + { + _targetUrl = targetUrl; + _accessToken = accessToken; + _task = task; + + var parsedUri = Regex.Match(_targetUrl, PublishingConstants.AzDoNuGetFeedPattern); + if (!parsedUri.Success) + { + Log.LogError( + $"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); + return; + } + _feedAccount = parsedUri.Groups["account"].Value; + _feedVisibility = parsedUri.Groups["visibility"].Value; + _feedName = parsedUri.Groups["feed"].Value; + + _httpClient = new HttpClient(new HttpClientHandler {CheckCertificateRevocationList = true}) + { + Timeout = TimeSpan.FromSeconds(300), + DefaultRequestHeaders = + { + Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.ASCII.GetBytes($":{_accessToken}"))) + }, + }; + } + + protected override void Dispose(bool disposing) + { + _httpClient?.Dispose(); + } + + public override AddAssetLocationToAssetAssetLocationType LocationType => AddAssetLocationToAssetAssetLocationType.NugetFeed; + + public override async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null) + { + if (!file.EndsWith(GeneralUtils.PackageSuffix, StringComparison.OrdinalIgnoreCase)) + { + Log.LogWarning( + $"AzDO feed publishing not available for blobs. Blob '{file}' was not published."); + return; + } + + using var packageReader = new PackageArchiveReader(file); + PackageIdentity packageIdentity = packageReader.GetIdentity(); + string id = packageIdentity.Id; + string version = packageIdentity.Version.ToString(); + + try + { + var config = new TargetFeedConfig(default, _targetUrl, default, default); + await _task.PushNugetPackageAsync(config, _httpClient, file, id, version, _feedAccount, _feedVisibility, _feedName); + } + catch (Exception e) + { + Log.LogErrorFromException(e); + } + } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageAssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageAssetPublisher.cs new file mode 100644 index 00000000000..1064c1a72f6 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageAssetPublisher.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; +using Azure.Storage.Blobs; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.Maestro.Client.Models; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public abstract class AzureStorageAssetPublisher : AssetPublisher + { + protected AzureStorageAssetPublisher(TaskLoggingHelper log) + : base(log) + { + } + + public override AddAssetLocationToAssetAssetLocationType LocationType => AddAssetLocationToAssetAssetLocationType.Container; + + public abstract BlobClient CreateBlobClient(string blobPath); + + public override async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null) + { + using (await SemaphoreLock.LockAsync(clientThrottle)) + { + blobPath = blobPath.Replace("\\", "/"); + var blobClient = CreateBlobClient(blobPath); + if (!options.AllowOverwrite && await blobClient.ExistsAsync()) + { + if (options.PassIfExistingItemIdentical) + { + if (!await blobClient.IsFileIdenticalToBlobAsync(file)) + { + Log.LogError($"Asset '{file}' already exists with different contents at '{blobPath}'"); + } + + return; + } + + Log.LogError($"Asset '{file}' already exists at '{blobPath}'"); + return; + } + + Log.LogMessage($"Uploading '{file}' to '{blobPath}'"); + await blobClient.UploadAsync(file); + } + } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageContainerAssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageContainerAssetPublisher.cs new file mode 100644 index 00000000000..f03080c0fc9 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageContainerAssetPublisher.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Azure.Storage.Blobs; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public class AzureStorageContainerAssetPublisher : AzureStorageAssetPublisher + { + private readonly Uri _containerUri; + + public AzureStorageContainerAssetPublisher(Uri containerUri, TaskLoggingHelper log) : base(log) + { + _containerUri = containerUri; + } + + public override BlobClient CreateBlobClient(string blobPath) + { + var containerClient = new BlobContainerClient(_containerUri); + return containerClient.GetBlobClient(blobPath); + } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageExtensions.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageExtensions.cs new file mode 100644 index 00000000000..ae0ef8a32db --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageExtensions.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public static class AzureStorageExtensions + { + public static string CalculateMD5(string filename) + { + using var md5 = MD5.Create(); + using var stream = File.OpenRead(filename); + byte[] hash = md5.ComputeHash(stream); + return Convert.ToBase64String(hash); + } + + public static async Task IsFileIdenticalToBlobAsync(this BlobClient client, string file) + { + BlobProperties properties = await client.GetPropertiesAsync(); + if (properties.ContentHash != null) + { + var localMD5 = CalculateMD5(file); + var blobMD5 = Convert.ToBase64String(properties.ContentHash); + return blobMD5.Equals(localMD5, StringComparison.OrdinalIgnoreCase); + } + + int bytesPerMegabyte = 1 * 1024 * 1024; + if (properties.ContentLength < bytesPerMegabyte) + { + byte[] existingBytes = new byte[properties.ContentLength]; + byte[] localBytes = File.ReadAllBytes(file); + + using var stream = new MemoryStream(existingBytes, true); + await client.DownloadToAsync(stream).ConfigureAwait(false); + return localBytes.SequenceEqual(existingBytes); + } + + using Stream localFileStream = File.OpenRead(file); + byte[] localBuffer = new byte[bytesPerMegabyte]; + byte[] remoteBuffer = new byte[bytesPerMegabyte]; + int localBytesRead = 0; + + do + { + long start = localFileStream.Position; + localBytesRead = await localFileStream.ReadAsync(localBuffer, 0, bytesPerMegabyte); + + var range = new HttpRange(start, localBytesRead); + BlobDownloadInfo download = await client.DownloadAsync(range).ConfigureAwait(false); + if (download.ContentLength != localBytesRead) + { + return false; + } + + using (var stream = new MemoryStream(remoteBuffer, true)) + { + await download.Content.CopyToAsync(stream).ConfigureAwait(false); + } + if (!remoteBuffer.SequenceEqual(localBuffer)) + { + return false; + } + } + while (localBytesRead > 0); + + return true; + } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageFeedAssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageFeedAssetPublisher.cs new file mode 100644 index 00000000000..9cf5f645a54 --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageFeedAssetPublisher.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Azure.Storage; +using Azure.Storage.Blobs; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public class AzureStorageFeedAssetPublisher : AzureStorageAssetPublisher + { + private readonly string _accountName; + private readonly string _accountKey; + private readonly string _containerName; + + public AzureStorageFeedAssetPublisher(string accountName, string accountKey, string containerName, + TaskLoggingHelper log) : base(log) + { + _accountName = accountName; + _accountKey = accountKey; + _containerName = containerName; + } + + public override BlobClient CreateBlobClient(string blobPath) + { + var cred = new StorageSharedKeyCredential(_accountName, _accountKey); + var endpoint = new Uri($"https://{_accountName}.blob.core.windows.net"); + var service = new BlobServiceClient(endpoint, cred); + var container = service.GetBlobContainerClient(_containerName); + return container.GetBlobClient(blobPath); + } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs index 791b440ece5..c90253e0cfe 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/BlobFeedAction.cs @@ -30,10 +30,10 @@ sealed class BlobFeedAction private SleetSource source; private bool hasToken = false; - private string AccountName { get; set; } - public string AccountKey { get; set; } - private string ContainerName { get; set; } - public string RelativePath { get; set; } + public string AccountName { get; } + public string AccountKey { get; } + public string ContainerName { get; } + public string RelativePath { get; } public BlobFeedAction(string expectedFeedUrl, string accountKey, MSBuild.TaskLoggingHelper Log) { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index 753ed3262cf..0b048241756 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -12,6 +12,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using Task = System.Threading.Tasks.Task; namespace Microsoft.DotNet.Build.Tasks.Feed { @@ -200,6 +201,15 @@ public string BuildQuality public int NonStreamingPublishingMaxClients {get; set;} + [Required] + public string DotNetBuildsPublicUri { get; set; } + [Required] + public string DotNetBuildsPublicChecksumsUri { get; set; } + [Required] + public string DotNetBuildsInternalUri { get; set; } + [Required] + public string DotNetBuildsInternalChecksumsUri { get; set; } + /// /// Just an internal flag to keep track whether we published assets via a V3 manifest or not. /// @@ -322,7 +332,7 @@ public PublishArtifactsInManifestBase WhichPublishingTask(string manifestFullPat internal PublishArtifactsInManifestBase ConstructPublishingV2Task(BuildModel buildModel) { - return new PublishArtifactsInManifestV2() + return new PublishArtifactsInManifestV2(new AssetPublisherFactory(Log)) { BuildEngine = this.BuildEngine, TargetFeedConfig = this.TargetFeedConfig, @@ -341,7 +351,7 @@ internal PublishArtifactsInManifestBase ConstructPublishingV2Task(BuildModel bui AkaMSGroupOwner = this.AkaMSGroupOwner, AkaMsOwners = this.AkaMsOwners, AkaMSTenant = this.AkaMSTenant, - BuildQuality = this.BuildQuality + BuildQuality = this.BuildQuality, }; } @@ -349,7 +359,7 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui { PublishedV3Manifest = true; - return new PublishArtifactsInManifestV3() + return new PublishArtifactsInManifestV3(new AssetPublisherFactory(Log)) { BuildEngine = this.BuildEngine, TargetChannels = this.TargetChannels, @@ -395,8 +405,43 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui AzureDevOpsOrg = this.AzureDevOpsOrg, UseStreamingPublishing = this.UseStreamingPublishing, StreamingPublishingMaxClients = this.StreamingPublishingMaxClients, - NonStreamingPublishingMaxClients = this.NonStreamingPublishingMaxClients + NonStreamingPublishingMaxClients = this.NonStreamingPublishingMaxClients, + DotNetBuildsPublicUri = FixUpSasTokenForMSBuildEscaping(DotNetBuildsPublicUri), + DotNetBuildsPublicChecksumsUri = FixUpSasTokenForMSBuildEscaping(DotNetBuildsPublicChecksumsUri), + DotNetBuildsInternalUri = FixUpSasTokenForMSBuildEscaping(DotNetBuildsInternalUri), + DotNetBuildsInternalChecksumsUri = FixUpSasTokenForMSBuildEscaping(DotNetBuildsInternalChecksumsUri), }; } + + private string FixUpSasTokenForMSBuildEscaping(string uriWithSas) + { + if (uriWithSas == null) + { + return null; + } + uriWithSas = TransformSection(uriWithSas, "sig=", "&se=", section => section.Replace("+", "%2B").Replace("/", "%2F").Replace("=", "%3D")); + uriWithSas = TransformSection(uriWithSas, "se=", "&sp=", section => section.Replace(":", "%3A")); + return uriWithSas; + } + + private string TransformSection(string uri, string start, string end, Func transform) + { + var startIndex = uri.IndexOf(start, StringComparison.Ordinal); + if (startIndex < 0) + { + throw new InvalidOperationException($"Unable to find '{start}'"); + } + + var endIndex = uri.IndexOf(end, startIndex, StringComparison.Ordinal); + if (endIndex < 0) + { + throw new InvalidOperationException($"Unable to find '{end}' after '{start}'"); + } + + var first = uri.Substring(0, startIndex + start.Length); + var middle = transform(uri.Substring(startIndex + start.Length, endIndex - (startIndex + start.Length))); + var last = uri.Substring(endIndex); + return first + middle + last; + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 9e31325f149..6009bc1be98 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -39,6 +39,8 @@ namespace Microsoft.DotNet.Build.Tasks.Feed /// public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities.Task { + public AssetPublisherFactory AssetPublisherFactory { get; } + /// /// Full path to the folder containing blob assets. /// @@ -203,6 +205,11 @@ public enum ArtifactName private int TimeoutInSeconds = 300; + protected PublishArtifactsInManifestBase(AssetPublisherFactory assetPublisherFactory) + { + AssetPublisherFactory = assetPublisherFactory; + } + public override bool Execute() { return ExecuteAsync().GetAwaiter().GetResult(); @@ -992,35 +999,21 @@ protected async Task HandleBlobPublishingAsync(Dictionary foreach (var blob in filteredBlobs) { string isolatedString = feedConfig.Isolated ? "Isolated" : "Non-Isolated"; - string internalString = feedConfig.Internal ? $", Internal" : ", Public"; + string internalString = feedConfig.Internal ? ", Internal" : ", Public"; string shippingString = blob.NonShipping ? "NonShipping" : "Shipping"; Log.LogMessage(MessageImportance.High, - $"Blob {blob.Id} ({shippingString}) should go to {feedConfig.TargetURL} ({isolatedString}{internalString})"); + $"Blob {blob.Id} ({shippingString}) should go to {feedConfig.SafeTargetURL} ({isolatedString}{internalString})"); } - switch (feedConfig.Type) - { - case FeedType.AzDoNugetFeed: - publishTasks.Add( - PublishBlobsToAzDoNugetFeedAsync( - filteredBlobs, - buildAssets, - feedConfig, - clientThrottle)); - break; - case FeedType.AzureStorageFeed: - publishTasks.Add( - PublishBlobsToAzureStorageNugetFeedAsync( - filteredBlobs, - buildAssets, - feedConfig, - clientThrottle)); - break; - default: - Log.LogError( - $"Unknown target feed type for category '{category}': '{feedConfig.Type}'."); - break; - } + var assetsToPublish = new HashSet(filteredBlobs.Select(b => b.Id)); + var publisher = AssetPublisherFactory.CreateAssetPublisher(feedConfig, this); + publishTasks.Add( + PublishAssetsAsync( + publisher, + assetsToPublish, + buildAssets, + feedConfig, + clientThrottle)); } } else @@ -1077,6 +1070,7 @@ public void SplitArtifactsInCategories(BuildModel buildModel) if (!Enum.TryParse(category, ignoreCase: true, out TargetFeedContentType categoryKey)) { Log.LogError($"Invalid target feed config category '{category}'."); + continue; } if (PackagesByCategory.ContainsKey(categoryKey)) @@ -1104,6 +1098,7 @@ public void SplitArtifactsInCategories(BuildModel buildModel) if (!Enum.TryParse(category, ignoreCase: true, out TargetFeedContentType categoryKey)) { Log.LogError($"Invalid target feed config category '{category}'."); + continue; } if (BlobsByCategory.ContainsKey(categoryKey)) @@ -1639,42 +1634,48 @@ public string CreateTemporaryDirectory() return temporaryDirectory; } - private async Task PublishBlobsToAzureStorageNugetFeedAsync( - HashSet blobsToPublish, + private async Task PublishAssetsAsync(AssetPublisher assetPublisher, HashSet assetsToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig, SemaphoreSlim clientThrottle) { if (UseStreamingPublishing) { - await PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(blobsToPublish, buildAssets, feedConfig, clientThrottle); + await PublishAssetsUsingStreamingPublishingAsync(assetPublisher, assetsToPublish, buildAssets, feedConfig, clientThrottle); } else { - await PublishBlobsToAzureStorageNugetAsync(blobsToPublish, buildAssets, feedConfig); + await PublishAssetsWithoutStreamingPublishingAsync(assetPublisher, assetsToPublish, buildAssets, feedConfig); } - if (LinkManager == null) + if (feedConfig.Type == FeedType.AzureStorageContainer || + feedConfig.Type == FeedType.AzureStorageFeed) { - LinkManager = new LatestLinksManager( - AkaMSClientId, - AkaMSClientSecret, - AkaMSTenant, - AkaMSGroupOwner, - AkaMSCreatedBy, - AkaMsOwners, - Log); + + if (LinkManager == null) + { + LinkManager = new LatestLinksManager( + AkaMSClientId, + AkaMSClientSecret, + AkaMSTenant, + AkaMSGroupOwner, + AkaMSCreatedBy, + AkaMsOwners, + Log); + } + + // The latest links should be updated only after the publishing is complete, to avoid + // dead links in the interim. + await LinkManager.CreateOrUpdateLatestLinksAsync( + assetsToPublish, + feedConfig, + feedConfig.Type == FeedType.AzureStorageContainer ? 0 : PublishingConstants.ExpectedFeedUrlSuffix.Length); } - // The latest links should be updated only after the publishing is complete, to avoid - // dead links in the interim. - await LinkManager.CreateOrUpdateLatestLinksAsync( - blobsToPublish, - feedConfig, - PublishingConstants.ExpectedFeedUrlSuffix.Length); } - private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( - HashSet blobsToPublish, + private async Task PublishAssetsUsingStreamingPublishingAsync( + AssetPublisher assetPublisher, + HashSet assetsToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig, SemaphoreSlim clientThrottle) @@ -1686,21 +1687,19 @@ private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( { return; } - var blobFeedAction = CreateBlobFeedAction(feedConfig); var pushOptions = new PushOptions { AllowOverwrite = feedConfig.AllowOverwrite, - PassIfExistingItemIdentical = true + PassIfExistingItemIdentical = true, }; using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true, AzureProject); - await Task.WhenAll(blobsToPublish.Select(async blob => + await Task.WhenAll(assetsToPublish.Select(async asset => { - try + using (await SemaphoreLock.LockAsync(clientThrottle)) { - await clientThrottle.WaitAsync(); string temporaryBlobDirectory = CreateTemporaryDirectory(); - var fileName = Path.GetFileName(blob.Id); + var fileName = Path.GetFileName(asset); var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName); Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}"); @@ -1711,63 +1710,54 @@ await DownloadFileAsync( containerId, fileName, localBlobPath); + gatherBlobDownloadTime.Stop(); + Log.LogMessage(MessageImportance.Low, $"Time taken to download file to '{localBlobPath}' is {gatherBlobDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)"); if (!File.Exists(localBlobPath)) { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + Log.LogError($"Could not locate '{asset} at '{localBlobPath}'"); } - gatherBlobDownloadTime.Stop(); - Log.LogMessage(MessageImportance.Low, $"Time taken to download file to '{localBlobPath}' is {gatherBlobDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)"); - - Log.LogMessage(MessageImportance.Low, - $"Successfully downloaded blob : {fileName} to {localBlobPath}"); + else + { + Log.LogMessage(MessageImportance.Low, + $"Successfully downloaded blob : {fileName} to {localBlobPath}"); - var item = new Microsoft.Build.Utilities.TaskItem(localBlobPath, - new Dictionary - { - {"RelativeBlobPath", blob.Id} - }); + TryAddAssetLocation( + asset, + assetVersion: null, + buildAssets, + feedConfig, + assetPublisher.LocationType); - TryAddAssetLocation( - blob.Id, - assetVersion: null, - buildAssets, - feedConfig, - AddAssetLocationToAssetAssetLocationType.Container); + Stopwatch gatherBlobPublishingTime = Stopwatch.StartNew(); + await assetPublisher.PublishAssetAsync(localBlobPath, asset, pushOptions, null); + gatherBlobPublishingTime.Stop(); + Log.LogMessage(MessageImportance.Low,$"Publishing {localBlobPath} completed in {gatherBlobPublishingTime.ElapsedMilliseconds / 1000.0} (seconds)"); + } - Stopwatch gatherBlobPublishingTime = Stopwatch.StartNew(); - await blobFeedAction.UploadAssetAsync(item, pushOptions, null); - gatherBlobPublishingTime.Stop(); - Log.LogMessage(MessageImportance.Low,$"Publishing {item.ItemSpec} completed in {gatherBlobPublishingTime.ElapsedMilliseconds / 1000.0} (seconds)"); DeleteTemporaryDirectory(temporaryBlobDirectory); } - finally - { - clientThrottle.Release(); - } })); } - private async Task PublishBlobsToAzureStorageNugetAsync( - HashSet blobsToPublish, + private async Task PublishAssetsWithoutStreamingPublishingAsync( + AssetPublisher assetPublisher, + HashSet assetsToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig) { - var blobs = blobsToPublish - .Select(blob => + var assets = assetsToPublish + .Select(asset => { - var fileName = Path.GetFileName(blob.Id); + var fileName = Path.GetFileName(asset); var localBlobPath = Path.Combine(BlobAssetsBasePath, fileName); if (!File.Exists(localBlobPath)) { - Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'"); + Log.LogError($"Could not locate '{asset} at '{localBlobPath}'"); } - return new Microsoft.Build.Utilities.TaskItem(localBlobPath, new Dictionary - { - {"RelativeBlobPath", blob.Id} - }); + return (localBlobPath, id: asset); }) .ToArray(); @@ -1776,23 +1766,25 @@ private async Task PublishBlobsToAzureStorageNugetAsync( return; } - var blobFeedAction = CreateBlobFeedAction(feedConfig); var pushOptions = new PushOptions { AllowOverwrite = feedConfig.AllowOverwrite, PassIfExistingItemIdentical = true }; - blobsToPublish - .ToList() - .ForEach(blob => TryAddAssetLocation( - blob.Id, + foreach (var asset in assetsToPublish) + { + TryAddAssetLocation( + asset, assetVersion: null, buildAssets, feedConfig, - AddAssetLocationToAssetAssetLocationType.Container)); + AddAssetLocationToAssetAssetLocationType.Container); + } - await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions); + using var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients); + await Task.WhenAll(assets.Select(asset => + assetPublisher.PublishAssetAsync(asset.localBlobPath, asset.id, pushOptions, clientThrottle))); } private BlobFeedAction CreateBlobFeedAction(TargetFeedConfig feedConfig) diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV2.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV2.cs index 8cf077b000b..02687582d2b 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV2.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV2.cs @@ -93,7 +93,7 @@ public async Task ParseTargetFeedConfigAsync() string feedKey = fc.GetMetadata(nameof(Model.TargetFeedConfig.Token)); string type = fc.GetMetadata(nameof(Model.TargetFeedConfig.Type)); AssetSelection assetSelection = AssetSelection.All; - bool isInternalFeed = false; + bool isInternalFeed; bool isIsolatedFeed = false; bool isOverridableFeed = false; @@ -220,5 +220,9 @@ public async Task ParseTargetFeedConfigAsync() } } } + + public PublishArtifactsInManifestV2(AssetPublisherFactory assetPublisherFactory = null) : base(assetPublisherFactory) + { + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 1f8f61d3f2c..8b00d55419f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -66,6 +66,15 @@ public class PublishArtifactsInManifestV3 : PublishArtifactsInManifestBase public string PublicSymbolsFeedOverride { get; set; } + [Required] + public string DotNetBuildsPublicUri { get; set; } + [Required] + public string DotNetBuildsPublicChecksumsUri { get; set; } + [Required] + public string DotNetBuildsInternalUri { get; set; } + [Required] + public string DotNetBuildsInternalChecksumsUri { get; set; } + public override bool Execute() { ExecuteAsync().GetAwaiter().GetResult(); @@ -160,8 +169,12 @@ public override async Task ExecuteAsync() GetFeed(targetChannelConfig.SymbolsFeed, SymbolsFeedOverride), shortLinkUrl, AzureDevOpsFeedsKey, - BuildEngine = this.BuildEngine, + BuildEngine, targetChannelConfig.SymbolTargetType, + DotNetBuildsPublicUri, + DotNetBuildsPublicChecksumsUri, + DotNetBuildsInternalUri, + DotNetBuildsInternalChecksumsUri, azureDevOpsPublicStaticSymbolsFeed: GetFeed(null, PublicSymbolsFeedOverride), filesToExclude: targetChannelConfig.FilenamesToExclude, flatten: targetChannelConfig.Flatten); @@ -270,5 +283,9 @@ public string GetFeed(string feed, string feedOverride) { return (AllowFeedOverrides && !string.IsNullOrEmpty(feedOverride)) ? feedOverride : feed; } + + public PublishArtifactsInManifestV3(AssetPublisherFactory assetPublisherFactory = null) : base(assetPublisherFactory) + { + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishSignedAssets.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishSignedAssets.cs index 388af99c38b..6afdcd7f100 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishSignedAssets.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishSignedAssets.cs @@ -97,5 +97,9 @@ await PushNugetPackagesAsync(packagesToPublish, targetFeedConfi await PushNugetPackageAsync(feed, httpClient, localPackagePath, package.Id, package.Version.ToString(), feedAccount, feedVisibility, feedName); }); } + + public PublishSignedAssets(AssetPublisherFactory assetPublisherFactory) : base(assetPublisherFactory) + { + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PushOptions.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PushOptions.cs index ecae2eeba1e..54d04ce1947 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PushOptions.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PushOptions.cs @@ -1,6 +1,8 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; + namespace Microsoft.DotNet.Build.Tasks.Feed { public class PushOptions diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/SemaphoreLock.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/SemaphoreLock.cs new file mode 100644 index 00000000000..d825f88feae --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/SemaphoreLock.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public readonly struct SemaphoreLock : IDisposable + { + private readonly SemaphoreSlim _sem; + + private SemaphoreLock(SemaphoreSlim sem) + { + _sem = sem; + } + + public void Dispose() + { + _sem?.Release(); + } + + public static ValueTask LockAsync(SemaphoreSlim sem) + { + if (sem == null) + { + return new ValueTask(new SemaphoreLock(null)); + } + Task waitTask = sem.WaitAsync(); + if (waitTask.IsCompleted && !waitTask.IsFaulted && !waitTask.IsCanceled) + { + return new ValueTask(new SemaphoreLock(sem)); + } + + static async Task WaitForLock(Task waitTask, SemaphoreSlim sem) + { + await waitTask.ConfigureAwait(false); + return new SemaphoreLock(sem); + } + + return new ValueTask(WaitForLock(waitTask, sem)); + } + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/AzureStorageUtils.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/AzureStorageUtils.cs index 9a4c4033291..a2c63e40b3c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/AzureStorageUtils.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/AzureStorageUtils.cs @@ -16,6 +16,7 @@ using System.Threading.Tasks; using Azure.Core.Pipeline; using System.Net.Http; +using Microsoft.DotNet.Build.Tasks.Feed; namespace Microsoft.DotNet.Build.CloudTestTasks { @@ -111,75 +112,7 @@ await blob.UploadAsync( } public async Task IsFileIdenticalToBlobAsync(string localFileFullPath, string blobPath) => - await IsFileIdenticalToBlobAsync(localFileFullPath, GetBlob(blobPath)).ConfigureAwait(false); - - /// - /// Return a bool indicating whether a local file's content is the same as - /// the content of a given blob. - /// - /// If the blob has the ContentHash property set, the comparison is performed using - /// that (MD5 hash). All recently-uploaded blobs or those uploaded by these libraries - /// should; some blob clients older than ~2012 may upload without the property set. - /// - /// When the ContentHash property is unset, a byte-by-byte comparison is performed. - /// - public async Task IsFileIdenticalToBlobAsync(string localFileFullPath, BlobClient blob) - { - BlobProperties properties = await blob.GetPropertiesAsync(); - if (properties.ContentHash != null) - { - var localMD5 = CalculateMD5(localFileFullPath); - var blobMD5 = Convert.ToBase64String(properties.ContentHash); - return blobMD5.Equals(localMD5, StringComparison.OrdinalIgnoreCase); - } - else - { - int bytesPerMegabyte = 1 * 1024 * 1024; - if (properties.ContentLength < bytesPerMegabyte) - { - byte[] existingBytes = new byte[properties.ContentLength]; - byte[] localBytes = File.ReadAllBytes(localFileFullPath); - - using (MemoryStream stream = new MemoryStream(existingBytes, true)) - { - await blob.DownloadToAsync(stream).ConfigureAwait(false); - } - return localBytes.SequenceEqual(existingBytes); - } - else - { - using (Stream localFileStream = File.OpenRead(localFileFullPath)) - { - byte[] localBuffer = new byte[bytesPerMegabyte]; - byte[] remoteBuffer = new byte[bytesPerMegabyte]; - int bytesLocalFile = 0; - - do - { - long start = localFileStream.Position; - int localBytesRead = await localFileStream.ReadAsync(localBuffer, 0, bytesPerMegabyte); - - HttpRange range = new HttpRange(start, localBytesRead); - BlobDownloadInfo download = await blob.DownloadAsync(range).ConfigureAwait(false); - if (download.ContentLength != localBytesRead) - { - return false; - } - using (MemoryStream stream = new MemoryStream(remoteBuffer, true)) - { - await download.Content.CopyToAsync(stream).ConfigureAwait(false); - } - if (!remoteBuffer.SequenceEqual(localBuffer)) - { - return false; - } - } - while (bytesLocalFile > 0); - } - return true; - } - } - } + await GetBlob(blobPath).IsFileIdenticalToBlobAsync(localFileFullPath); public async Task CreateContainerAsync(PublicAccessType publicAccess) { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/LatestLinksManager.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/LatestLinksManager.cs index e1dbe6a3b99..03da99e14a9 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/LatestLinksManager.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/LatestLinksManager.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Microsoft.DotNet.Build.Tasks.Feed.Model; @@ -44,7 +45,7 @@ public LatestLinksManager( } public async System.Threading.Tasks.Task CreateOrUpdateLatestLinksAsync( - HashSet blobsToPublish, + HashSet assetsToPublish, TargetFeedConfig feedConfig, int expectedSuffixLength) { @@ -53,28 +54,37 @@ public async System.Threading.Tasks.Task CreateOrUpdateLatestLinksAsync( return; } + string feedBaseUrl = feedConfig.SafeTargetURL; + if (expectedSuffixLength != 0) + { + // Strip away the feed expected suffix (index.json) + feedBaseUrl = feedBaseUrl.Substring(0, feedBaseUrl.Length - expectedSuffixLength); + } + if (!feedBaseUrl.EndsWith("/", StringComparison.OrdinalIgnoreCase)) + { + feedBaseUrl += "/"; + } + Logger.LogMessage(MessageImportance.High, "\nThe following aka.ms links for blobs will be created:"); - IEnumerable linksToCreate = blobsToPublish - .Where(blob => !feedConfig.FilenamesToExclude.Contains(Path.GetFileName(blob.Id))) - .Select(blob => + IEnumerable linksToCreate = assetsToPublish + .Where(asset => !feedConfig.FilenamesToExclude.Contains(Path.GetFileName(asset))) + .Select(asset => { - // Strip away the feed expected suffix (index.json) and append on the // blob path. - string actualTargetUrl = feedConfig.TargetURL.Substring(0, - feedConfig.TargetURL.Length - expectedSuffixLength) + blob.Id; + string actualTargetUrl = feedBaseUrl + asset; - // The dotnetcli storage account is in a single datacenter in the US and thus download - // times can be painful elsewhere. The CDN helps with this thefore we point the target + // The storage accounts are in a single datacenter in the US and thus download + // times can be painful elsewhere. The CDN helps with this therefore we point the target // of the aka.ms links to the CDN. - actualTargetUrl = actualTargetUrl.Replace("//dotnetcli.blob.core.windows.net/", "//dotnetcli.azureedge.net/"); + actualTargetUrl = actualTargetUrl.Replace(".blob.core.windows.net/", ".azureedge.net/"); AkaMSLink newLink = new AkaMSLink { - ShortUrl = GetLatestShortUrlForBlob(feedConfig, blob, feedConfig.Flatten), + ShortUrl = GetLatestShortUrlForBlob(feedConfig, asset, feedConfig.Flatten), TargetUrl = actualTargetUrl }; - Logger.LogMessage(MessageImportance.High, $" {Path.GetFileName(blob.Id)}"); + Logger.LogMessage(MessageImportance.High, $" {Path.GetFileName(asset)}"); Logger.LogMessage(MessageImportance.High, $" aka.ms/{newLink.ShortUrl} -> {newLink.TargetUrl}"); @@ -90,10 +100,9 @@ public async System.Threading.Tasks.Task CreateOrUpdateLatestLinksAsync( /// Feed configuration /// Blob /// Short url prefix for the blob. - /// - public string GetLatestShortUrlForBlob(TargetFeedConfig feedConfig, BlobArtifactModel blob, bool flatten) + public string GetLatestShortUrlForBlob(TargetFeedConfig feedConfig, string asset, bool flatten) { - string blobIdWithoutVersions = VersionIdentifier.RemoveVersions(blob.Id); + string blobIdWithoutVersions = VersionIdentifier.RemoveVersions(asset); if (flatten) { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/UploadToAzure.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/UploadToAzure.cs index 347df7eb6df..1bb88928d89 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/UploadToAzure.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/UploadToAzure.cs @@ -9,6 +9,7 @@ using System.Collections.Generic; using Azure.Storage.Blobs.Specialized; using Azure.Storage.Blobs; +using Microsoft.DotNet.Build.Tasks.Feed; namespace Microsoft.DotNet.Build.CloudTestTasks { @@ -104,7 +105,7 @@ public async Task ExecuteAsync(CancellationToken ct) { if (PassIfExistingItemIdentical) { - if (await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, blobReference)) + if (await blobReference.IsFileIdenticalToBlobAsync(item.ItemSpec)) { return; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/PublishingConstants.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/PublishingConstants.cs index 9c778de99ef..0245bc0e431 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/PublishingConstants.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/PublishingConstants.cs @@ -52,11 +52,17 @@ public enum BuildQuality } #region Target Channel Configs - private const string FeedForChecksums = "https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json"; - private const string FeedForInstallers = "https://dotnetcli.blob.core.windows.net/dotnet/index.json"; + public const string FeedForChecksums = "https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json"; + public const string FeedForInstallers = "https://dotnetcli.blob.core.windows.net/dotnet/index.json"; private const string FeedInternalForChecksums = "https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json"; - private const string FeedInternalForInstallers = "https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json"; + public const string FeedInternalForInstallers = "https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json"; + + public const string FeedStagingForInstallers = "https://dotnetbuilds.blob.core.windows.net/public"; + public const string FeedStagingForChecksums = "https://dotnetbuilds.blob.core.windows.net/public-checksums"; + + public const string FeedStagingInternalForInstallers = "https://dotnetbuilds.blob.core.windows.net/internal"; + public const string FeedStagingInternalForChecksums = "https://dotnetbuilds.blob.core.windows.net/internal-checksums"; private const string FeedGeneralTesting = "https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json"; private const string FeedGeneralTestingSymbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json"; diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs index b98e91740c4..c72319ac358 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs @@ -37,6 +37,11 @@ public class SetupTargetFeedConfigV3 : SetupTargetFeedConfigBase private bool Flatten { get; } + private string DotNetBuildsPublicUri { get; } + private string DotNetBuildsPublicChecksumsUri { get; } + private string DotNetBuildsInternalUri { get; } + private string DotNetBuildsInternalChecksumsUri { get; } + public SetupTargetFeedConfigV3(bool isInternalBuild, bool isStableBuild, string repositoryName, @@ -54,6 +59,10 @@ public SetupTargetFeedConfigV3(bool isInternalBuild, string azureDevOpsFeedsKey, IBuildEngine buildEngine, SymbolTargetType symbolTargetType, + string dotNetBuildsPublicUri, + string dotNetBuildsPublicChecksumsUri, + string dotNetBuildsInternalUri, + string dotNetBuildsInternalChecksumsUri, string stablePackagesFeed = null, string stableSymbolsFeed = null, string azureDevOpsPublicStaticSymbolsFeed = null, @@ -65,6 +74,10 @@ public SetupTargetFeedConfigV3(bool isInternalBuild, StableSymbolsFeed = stableSymbolsFeed; StablePackagesFeed = stablePackagesFeed; SymbolTargetType = symbolTargetType; + DotNetBuildsPublicUri = dotNetBuildsPublicUri; + DotNetBuildsPublicChecksumsUri = dotNetBuildsPublicChecksumsUri; + DotNetBuildsInternalUri = dotNetBuildsInternalUri; + DotNetBuildsInternalChecksumsUri = dotNetBuildsInternalChecksumsUri; AzureDevOpsPublicStaticSymbolsFeed = azureDevOpsPublicStaticSymbolsFeed; FilesToExclude = filesToExclude ?? new List(); Flatten = flatten; @@ -100,6 +113,20 @@ private List NonStableFeeds() { foreach (var contentType in Installers) { + if (string.Equals(InstallersTargetStaticFeed, PublishingConstants.FeedForInstallers, StringComparison.OrdinalIgnoreCase)) + { + targetFeedConfigs.Add( + new TargetFeedConfig( + contentType, + DotNetBuildsPublicUri, + FeedType.AzureStorageContainer, + null, + latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, + @internal: IsInternalBuild, + symbolTargetType: SymbolTargetType, + filenamesToExclude: FilesToExclude, + flatten: Flatten)); + } targetFeedConfigs.Add( new TargetFeedConfig( contentType, @@ -113,6 +140,20 @@ private List NonStableFeeds() flatten: Flatten)); } + if (string.Equals(ChecksumsTargetStaticFeed, PublishingConstants.FeedForChecksums, StringComparison.OrdinalIgnoreCase)) + { + targetFeedConfigs.Add( + new TargetFeedConfig( + TargetFeedContentType.Checksum, + DotNetBuildsPublicChecksumsUri, + FeedType.AzureStorageContainer, + null, + latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, + @internal: IsInternalBuild, + symbolTargetType: SymbolTargetType, + filenamesToExclude: FilesToExclude, + flatten: Flatten)); + } targetFeedConfigs.Add( new TargetFeedConfig( TargetFeedContentType.Checksum, @@ -272,6 +313,21 @@ private List StableFeeds() { foreach (var contentType in Installers) { + if (string.Equals(InstallersTargetStaticFeed, PublishingConstants.FeedForInstallers, StringComparison.OrdinalIgnoreCase)) + { + targetFeedConfigs.Add( + new TargetFeedConfig( + contentType, + DotNetBuildsPublicUri, + FeedType.AzureStorageContainer, + null, + isolated: true, + symbolTargetType: SymbolTargetType, + latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, + @internal: false, + filenamesToExclude: FilesToExclude, + flatten: Flatten)); + } targetFeedConfigs.Add( new TargetFeedConfig( contentType, @@ -287,6 +343,22 @@ private List StableFeeds() flatten: Flatten)); } + if (string.Equals(ChecksumsTargetStaticFeed, PublishingConstants.FeedForChecksums, StringComparison.OrdinalIgnoreCase)) + { + targetFeedConfigs.Add( + new TargetFeedConfig( + TargetFeedContentType.Checksum, + DotNetBuildsPublicChecksumsUri, + FeedType.AzureStorageContainer, + null, + isolated: true, + symbolTargetType: SymbolTargetType, + latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, + @internal: false, + filenamesToExclude: FilesToExclude, + flatten: Flatten)); + } + targetFeedConfigs.Add( new TargetFeedConfig( TargetFeedContentType.Checksum, diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs index 93e85a203d1..60390312782 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs @@ -13,6 +13,8 @@ namespace Microsoft.DotNet.Build.Tasks.Feed.Model /// public class TargetFeedConfig { + public string SafeTargetURL => new UriBuilder(TargetURL) {Query = "", Fragment = ""}.Uri.AbsoluteUri; + public TargetFeedContentType ContentType { get; } public string TargetURL { get; } @@ -125,7 +127,7 @@ public override string ToString() $"\n Internal? '{Internal}' " + $"\n AllowOverwrite? '{AllowOverwrite}' " + $"\n ShortUrlPrefix: '{LatestLinkShortUrlPrefix}' " + - $"\n TargetURL: '{TargetURL}'" + + $"\n TargetURL: '{SafeTargetURL}'" + $"\n FilenamesToExclude: \n\t{string.Join("\n\t", FilenamesToExclude)}" + $"\n Flatten: '{Flatten}'"; } @@ -164,7 +166,8 @@ public enum SymbolTargetType public enum FeedType { AzDoNugetFeed, - AzureStorageFeed + AzureStorageFeed, + AzureStorageContainer, } /// From 6ba4be3b416fc708b17dbf4234d7b4b663c3b985 Mon Sep 17 00:00:00 2001 From: Alex Perovich Date: Fri, 10 Sep 2021 15:13:08 -0700 Subject: [PATCH 2/5] PR feedback --- .../src/AssetPublisher.cs | 34 ------------------- .../src/AssetPublisherFactory.cs | 2 +- .../src/AzureDevOpsNugetFeedAssetPublisher.cs | 19 ++++++----- .../src/AzureStorageAssetPublisher.cs | 16 +++++---- .../src/AzureStorageExtensions.cs | 10 ------ .../src/IAssetPublisher.cs | 18 ++++++++++ .../src/PublishArtifactsInManifestBase.cs | 6 ++-- 7 files changed, 41 insertions(+), 64 deletions(-) delete mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisher.cs create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/IAssetPublisher.cs diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisher.cs deleted file mode 100644 index 055bfabb990..00000000000 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisher.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Threading; -using Microsoft.Build.Utilities; -using Microsoft.DotNet.Maestro.Client.Models; -using Task = System.Threading.Tasks.Task; - -namespace Microsoft.DotNet.Build.Tasks.Feed -{ - public abstract class AssetPublisher : IDisposable - { - public TaskLoggingHelper Log { get; } - public abstract AddAssetLocationToAssetAssetLocationType LocationType { get; } - - protected AssetPublisher(TaskLoggingHelper log) - { - Log = log; - } - - public abstract Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null); - - protected virtual void Dispose(bool disposing) - { - } - - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - } -} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisherFactory.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisherFactory.cs index 6066432efd6..5753ab00f9f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisherFactory.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisherFactory.cs @@ -16,7 +16,7 @@ public AssetPublisherFactory(TaskLoggingHelper log) _log = log; } - public virtual AssetPublisher CreateAssetPublisher(TargetFeedConfig feedConfig, PublishArtifactsInManifestBase task) + public virtual IAssetPublisher CreateAssetPublisher(TargetFeedConfig feedConfig, PublishArtifactsInManifestBase task) { switch (feedConfig.Type) { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs index 4fa04b469d9..bc4b8f5ce11 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs @@ -16,8 +16,9 @@ namespace Microsoft.DotNet.Build.Tasks.Feed { - public class AzureDevOpsNugetFeedAssetPublisher : AssetPublisher + public class AzureDevOpsNugetFeedAssetPublisher : IAssetPublisher, IDisposable { + private readonly TaskLoggingHelper _log; private readonly string _targetUrl; private readonly string _accessToken; private readonly PublishArtifactsInManifestBase _task; @@ -26,8 +27,9 @@ public class AzureDevOpsNugetFeedAssetPublisher : AssetPublisher private readonly string _feedName; private readonly HttpClient _httpClient; - public AzureDevOpsNugetFeedAssetPublisher(TaskLoggingHelper log, string targetUrl, string accessToken, PublishArtifactsInManifestBase task) : base(log) + public AzureDevOpsNugetFeedAssetPublisher(TaskLoggingHelper log, string targetUrl, string accessToken, PublishArtifactsInManifestBase task) { + _log = log; _targetUrl = targetUrl; _accessToken = accessToken; _task = task; @@ -35,9 +37,8 @@ public AzureDevOpsNugetFeedAssetPublisher(TaskLoggingHelper log, string targetUr var parsedUri = Regex.Match(_targetUrl, PublishingConstants.AzDoNuGetFeedPattern); if (!parsedUri.Success) { - Log.LogError( + throw new ArgumentException( $"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'"); - return; } _feedAccount = parsedUri.Groups["account"].Value; _feedVisibility = parsedUri.Groups["visibility"].Value; @@ -55,18 +56,18 @@ public AzureDevOpsNugetFeedAssetPublisher(TaskLoggingHelper log, string targetUr }; } - protected override void Dispose(bool disposing) + public void Dispose() { _httpClient?.Dispose(); } - public override AddAssetLocationToAssetAssetLocationType LocationType => AddAssetLocationToAssetAssetLocationType.NugetFeed; + public AddAssetLocationToAssetAssetLocationType LocationType => AddAssetLocationToAssetAssetLocationType.NugetFeed; - public override async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null) + public async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null) { if (!file.EndsWith(GeneralUtils.PackageSuffix, StringComparison.OrdinalIgnoreCase)) { - Log.LogWarning( + _log.LogWarning( $"AzDO feed publishing not available for blobs. Blob '{file}' was not published."); return; } @@ -83,7 +84,7 @@ public override async Task PublishAssetAsync(string file, string blobPath, PushO } catch (Exception e) { - Log.LogErrorFromException(e); + _log.LogErrorFromException(e); } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageAssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageAssetPublisher.cs index 1064c1a72f6..62eb487b263 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageAssetPublisher.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageAssetPublisher.cs @@ -9,18 +9,20 @@ namespace Microsoft.DotNet.Build.Tasks.Feed { - public abstract class AzureStorageAssetPublisher : AssetPublisher + public abstract class AzureStorageAssetPublisher : IAssetPublisher { + private readonly TaskLoggingHelper _log; + protected AzureStorageAssetPublisher(TaskLoggingHelper log) - : base(log) { + _log = log; } - public override AddAssetLocationToAssetAssetLocationType LocationType => AddAssetLocationToAssetAssetLocationType.Container; + public AddAssetLocationToAssetAssetLocationType LocationType => AddAssetLocationToAssetAssetLocationType.Container; public abstract BlobClient CreateBlobClient(string blobPath); - public override async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null) + public async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null) { using (await SemaphoreLock.LockAsync(clientThrottle)) { @@ -32,17 +34,17 @@ public override async Task PublishAssetAsync(string file, string blobPath, PushO { if (!await blobClient.IsFileIdenticalToBlobAsync(file)) { - Log.LogError($"Asset '{file}' already exists with different contents at '{blobPath}'"); + _log.LogError($"Asset '{file}' already exists with different contents at '{blobPath}'"); } return; } - Log.LogError($"Asset '{file}' already exists at '{blobPath}'"); + _log.LogError($"Asset '{file}' already exists at '{blobPath}'"); return; } - Log.LogMessage($"Uploading '{file}' to '{blobPath}'"); + _log.LogMessage($"Uploading '{file}' to '{blobPath}'"); await blobClient.UploadAsync(file); } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageExtensions.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageExtensions.cs index ae0ef8a32db..75dc243162c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageExtensions.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageExtensions.cs @@ -33,16 +33,6 @@ public static async Task IsFileIdenticalToBlobAsync(this BlobClient client } int bytesPerMegabyte = 1 * 1024 * 1024; - if (properties.ContentLength < bytesPerMegabyte) - { - byte[] existingBytes = new byte[properties.ContentLength]; - byte[] localBytes = File.ReadAllBytes(file); - - using var stream = new MemoryStream(existingBytes, true); - await client.DownloadToAsync(stream).ConfigureAwait(false); - return localBytes.SequenceEqual(existingBytes); - } - using Stream localFileStream = File.OpenRead(file); byte[] localBuffer = new byte[bytesPerMegabyte]; byte[] remoteBuffer = new byte[bytesPerMegabyte]; diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/IAssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/IAssetPublisher.cs new file mode 100644 index 00000000000..905bd8d1eec --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/IAssetPublisher.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.Maestro.Client.Models; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.Build.Tasks.Feed +{ + public interface IAssetPublisher + { + AddAssetLocationToAssetAssetLocationType LocationType { get; } + + Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null); + } +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index 6009bc1be98..b1e5dcd28e6 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -1634,7 +1634,7 @@ public string CreateTemporaryDirectory() return temporaryDirectory; } - private async Task PublishAssetsAsync(AssetPublisher assetPublisher, HashSet assetsToPublish, + private async Task PublishAssetsAsync(IAssetPublisher assetPublisher, HashSet assetsToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig, SemaphoreSlim clientThrottle) @@ -1674,7 +1674,7 @@ await LinkManager.CreateOrUpdateLatestLinksAsync( } private async Task PublishAssetsUsingStreamingPublishingAsync( - AssetPublisher assetPublisher, + IAssetPublisher assetPublisher, HashSet assetsToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig, @@ -1742,7 +1742,7 @@ await DownloadFileAsync( } private async Task PublishAssetsWithoutStreamingPublishingAsync( - AssetPublisher assetPublisher, + IAssetPublisher assetPublisher, HashSet assetsToPublish, Dictionary> buildAssets, TargetFeedConfig feedConfig) From 1dfe763fa93c466c9ddfd41c501e33c398707a90 Mon Sep 17 00:00:00 2001 From: Alex Perovich Date: Mon, 13 Sep 2021 15:44:33 -0700 Subject: [PATCH 3/5] More PR feedback --- .vault-config/dotnetbuildstokens.yaml | 20 +++++++ eng/publishing/v3/publish-assets.yml | 8 +-- .../SdkTasks/PublishArtifactsInManifest.proj | 8 +-- .../src/AzureDevOpsNugetFeedAssetPublisher.cs | 14 +++-- .../src/PublishArtifactsInManifest.cs | 52 ++++++------------- .../src/PublishArtifactsInManifestBase.cs | 17 +++--- .../src/common/GeneralUtils.cs | 1 + .../src/common/LatestLinksManager.cs | 17 ++++-- .../src/model/SetupTargetFeedConfigV3.cs | 8 +-- .../src/model/TargetFeedConfig.cs | 3 ++ 10 files changed, 83 insertions(+), 65 deletions(-) diff --git a/.vault-config/dotnetbuildstokens.yaml b/.vault-config/dotnetbuildstokens.yaml index dcb3cf9fd1a..fe96c3a1e39 100644 --- a/.vault-config/dotnetbuildstokens.yaml +++ b/.vault-config/dotnetbuildstokens.yaml @@ -21,6 +21,11 @@ secrets: permissions: rlwc container: internal + dotnetbuilds-internal-container-uri-base64: + type: base64-encoder + parameters: + secret: dotnetbuilds-internal-container-uri + dotnetbuilds-internal-container-checksum-uri: type: azure-storage-container-sas-uri parameters: @@ -30,6 +35,11 @@ secrets: permissions: rlwc container: internal-checksums + dotnetbuilds-internal-container-checksum-uri-base64: + type: base64-encoder + parameters: + secret: dotnetbuilds-internal-container-checksum-uri + dotnetbuilds-public-container-uri: type: azure-storage-container-sas-uri parameters: @@ -39,6 +49,11 @@ secrets: permissions: rlwc container: public + dotnetbuilds-public-container-uri-base64: + type: base64-encoder + parameters: + secret: dotnetbuilds-public-container-uri + dotnetbuilds-public-container-checksum-uri: type: azure-storage-container-sas-uri parameters: @@ -47,3 +62,8 @@ secrets: location: dotnetbuildskeys permissions: rlwc container: public-checksums + + dotnetbuilds-public-container-checksum-uri-base64: + type: base64-encoder + parameters: + secret: dotnetbuilds-public-container-checksum-uri diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml index 4681707042f..7753ebf8db3 100644 --- a/eng/publishing/v3/publish-assets.yml +++ b/eng/publishing/v3/publish-assets.yml @@ -99,10 +99,10 @@ jobs: /p:UseStreamingPublishing='true' /p:StreamingPublishingMaxClients=16 /p:NonStreamingPublishingMaxClients=12 - /p:DotNetBuildsPublicUri='$(dotnetbuilds-public-container-uri)' - /p:DotNetBuildsPublicChecksumsUri='$(dotnetbuilds-public-container-checksum-uri)' - /p:DotNetBuildsInternalUri='$(dotnetbuilds-internal-container-uri)' - /p:DotNetBuildsInternalChecksumsUri='$(dotnetbuilds-internal-container-checksum-uri)' + /p:DotNetBuildsPublicUriBase64='$(dotnetbuilds-public-container-uri-base64)' + /p:DotNetBuildsPublicChecksumsUriBase64='$(dotnetbuilds-public-container-checksum-uri-base64)' + /p:DotNetBuildsInternalUriBase64='$(dotnetbuilds-internal-container-uri-base64)' + /p:DotNetBuildsInternalChecksumsUriBase64='$(dotnetbuilds-internal-container-checksum-uri-base64)' - template: /eng/common/templates/steps/publish-logs.yml parameters: StageLabel: '${{ parameters.stageName }}' diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index c24e75f7adf..d4f37052c40 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -165,10 +165,10 @@ UseStreamingPublishing="$(UseStreamingPublishing)" StreamingPublishingMaxClients="$(StreamingPublishingMaxClients)" NonStreamingPublishingMaxClients="$(NonStreamingPublishingMaxClients)" - DotNetBuildsPublicUri="$(DotNetBuildsPublicUri)" - DotNetBuildsPublicChecksumsUri="$(DotNetBuildsPublicChecksumsUri)" - DotNetBuildsInternalUri="$(DotNetBuildsInternalUri)" - DotNetBuildsInternalChecksumsUri="$(DotNetBuildsInternalChecksumsUri)" + DotNetBuildsPublicUriBase64="$(DotNetBuildsPublicUriBase64)" + DotNetBuildsPublicChecksumsUriBase64="$(DotNetBuildsPublicChecksumsUriBase64)" + DotNetBuildsInternalUriBase64="$(DotNetBuildsInternalUriBase64)" + DotNetBuildsInternalChecksumsUriBase64="$(DotNetBuildsInternalChecksumsUriBase64)" /> diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs index bc4b8f5ce11..d02eac83a5a 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs @@ -46,7 +46,7 @@ public AzureDevOpsNugetFeedAssetPublisher(TaskLoggingHelper log, string targetUr _httpClient = new HttpClient(new HttpClientHandler {CheckCertificateRevocationList = true}) { - Timeout = TimeSpan.FromSeconds(300), + Timeout = GeneralUtils.NugetFeedPublisherHttpClientTimeout, DefaultRequestHeaders = { Authorization = new AuthenticationHeaderValue( @@ -72,10 +72,14 @@ public async Task PublishAssetAsync(string file, string blobPath, PushOptions op return; } - using var packageReader = new PackageArchiveReader(file); - PackageIdentity packageIdentity = packageReader.GetIdentity(); - string id = packageIdentity.Id; - string version = packageIdentity.Version.ToString(); + string id; + string version; + using (var packageReader = new PackageArchiveReader(file)) + { + PackageIdentity packageIdentity = packageReader.GetIdentity(); + id = packageIdentity.Id; + version = packageIdentity.Version.ToString(); + } try { diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index 0b048241756..837566da9d0 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using System; using System.Linq; +using System.Text; using System.Threading.Tasks; using Task = System.Threading.Tasks.Task; @@ -201,14 +202,13 @@ public string BuildQuality public int NonStreamingPublishingMaxClients {get; set;} - [Required] - public string DotNetBuildsPublicUri { get; set; } - [Required] - public string DotNetBuildsPublicChecksumsUri { get; set; } - [Required] - public string DotNetBuildsInternalUri { get; set; } - [Required] - public string DotNetBuildsInternalChecksumsUri { get; set; } + public string DotNetBuildsPublicUriBase64 { get; set; } + + public string DotNetBuildsPublicChecksumsUriBase64 { get; set; } + + public string DotNetBuildsInternalUriBase64 { get; set; } + + public string DotNetBuildsInternalChecksumsUriBase64 { get; set; } /// /// Just an internal flag to keep track whether we published assets via a V3 manifest or not. @@ -406,42 +406,20 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui UseStreamingPublishing = this.UseStreamingPublishing, StreamingPublishingMaxClients = this.StreamingPublishingMaxClients, NonStreamingPublishingMaxClients = this.NonStreamingPublishingMaxClients, - DotNetBuildsPublicUri = FixUpSasTokenForMSBuildEscaping(DotNetBuildsPublicUri), - DotNetBuildsPublicChecksumsUri = FixUpSasTokenForMSBuildEscaping(DotNetBuildsPublicChecksumsUri), - DotNetBuildsInternalUri = FixUpSasTokenForMSBuildEscaping(DotNetBuildsInternalUri), - DotNetBuildsInternalChecksumsUri = FixUpSasTokenForMSBuildEscaping(DotNetBuildsInternalChecksumsUri), + DotNetBuildsPublicUri = ConvertFromBase64(DotNetBuildsPublicUriBase64), + DotNetBuildsPublicChecksumsUri = ConvertFromBase64(DotNetBuildsPublicChecksumsUriBase64), + DotNetBuildsInternalUri = ConvertFromBase64(DotNetBuildsInternalUriBase64), + DotNetBuildsInternalChecksumsUri = ConvertFromBase64(DotNetBuildsInternalChecksumsUriBase64), }; } - private string FixUpSasTokenForMSBuildEscaping(string uriWithSas) + private string ConvertFromBase64(string value) { - if (uriWithSas == null) + if (value == null) { return null; } - uriWithSas = TransformSection(uriWithSas, "sig=", "&se=", section => section.Replace("+", "%2B").Replace("/", "%2F").Replace("=", "%3D")); - uriWithSas = TransformSection(uriWithSas, "se=", "&sp=", section => section.Replace(":", "%3A")); - return uriWithSas; - } - - private string TransformSection(string uri, string start, string end, Func transform) - { - var startIndex = uri.IndexOf(start, StringComparison.Ordinal); - if (startIndex < 0) - { - throw new InvalidOperationException($"Unable to find '{start}'"); - } - - var endIndex = uri.IndexOf(end, startIndex, StringComparison.Ordinal); - if (endIndex < 0) - { - throw new InvalidOperationException($"Unable to find '{end}' after '{start}'"); - } - - var first = uri.Substring(0, startIndex + start.Length); - var middle = transform(uri.Substring(startIndex + start.Length, endIndex - (startIndex + start.Length))); - var last = uri.Substring(endIndex); - return first + middle + last; + return Encoding.UTF8.GetString(Convert.FromBase64String(value)); } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index b1e5dcd28e6..d139ae33ec4 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -1664,12 +1664,17 @@ private async Task PublishAssetsAsync(IAssetPublisher assetPublisher, HashSet TimeSpan.FromSeconds(300); public static ExponentialRetry CreateDefaultRetryHandler() => new ExponentialRetry diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/LatestLinksManager.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/LatestLinksManager.cs index 03da99e14a9..9d5f85227c8 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/LatestLinksManager.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/LatestLinksManager.cs @@ -25,6 +25,11 @@ public class LatestLinksManager private string AkaMSCreatedBy { get; } private string AkaMSGroupOwner { get; } + private static HashSet AccountsWithCdns { get; } = new() + { + "dotnetcli.blob.core.windows.net", "dotnetbuilds.blob.core.windows.net", + }; + public LatestLinksManager( string akaMSClientId, string akaMSClientSecret, @@ -64,6 +69,13 @@ public async System.Threading.Tasks.Task CreateOrUpdateLatestLinksAsync( { feedBaseUrl += "/"; } + if (AccountsWithCdns.Any(account => feedBaseUrl.Contains(account))) + { + // The storage accounts are in a single datacenter in the US and thus download + // times can be painful elsewhere. The CDN helps with this therefore we point the target + // of the aka.ms links to the CDN. + feedBaseUrl = feedBaseUrl.Replace(".blob.core.windows.net", ".azureedge.net"); + } Logger.LogMessage(MessageImportance.High, "\nThe following aka.ms links for blobs will be created:"); IEnumerable linksToCreate = assetsToPublish @@ -74,11 +86,6 @@ public async System.Threading.Tasks.Task CreateOrUpdateLatestLinksAsync( // blob path. string actualTargetUrl = feedBaseUrl + asset; - // The storage accounts are in a single datacenter in the US and thus download - // times can be painful elsewhere. The CDN helps with this therefore we point the target - // of the aka.ms links to the CDN. - actualTargetUrl = actualTargetUrl.Replace(".blob.core.windows.net/", ".azureedge.net/"); - AkaMSLink newLink = new AkaMSLink { ShortUrl = GetLatestShortUrlForBlob(feedConfig, asset, feedConfig.Flatten), diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs index c72319ac358..bd38b731fdc 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs @@ -113,7 +113,7 @@ private List NonStableFeeds() { foreach (var contentType in Installers) { - if (string.Equals(InstallersTargetStaticFeed, PublishingConstants.FeedForInstallers, StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrEmpty(DotNetBuildsPublicUri) && string.Equals(InstallersTargetStaticFeed, PublishingConstants.FeedForInstallers, StringComparison.OrdinalIgnoreCase)) { targetFeedConfigs.Add( new TargetFeedConfig( @@ -140,7 +140,7 @@ private List NonStableFeeds() flatten: Flatten)); } - if (string.Equals(ChecksumsTargetStaticFeed, PublishingConstants.FeedForChecksums, StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrEmpty(DotNetBuildsPublicChecksumsUri) && string.Equals(ChecksumsTargetStaticFeed, PublishingConstants.FeedForChecksums, StringComparison.OrdinalIgnoreCase)) { targetFeedConfigs.Add( new TargetFeedConfig( @@ -313,7 +313,7 @@ private List StableFeeds() { foreach (var contentType in Installers) { - if (string.Equals(InstallersTargetStaticFeed, PublishingConstants.FeedForInstallers, StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrEmpty(DotNetBuildsPublicUri) && string.Equals(InstallersTargetStaticFeed, PublishingConstants.FeedForInstallers, StringComparison.OrdinalIgnoreCase)) { targetFeedConfigs.Add( new TargetFeedConfig( @@ -343,7 +343,7 @@ private List StableFeeds() flatten: Flatten)); } - if (string.Equals(ChecksumsTargetStaticFeed, PublishingConstants.FeedForChecksums, StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrEmpty(DotNetBuildsPublicChecksumsUri) && string.Equals(ChecksumsTargetStaticFeed, PublishingConstants.FeedForChecksums, StringComparison.OrdinalIgnoreCase)) { targetFeedConfigs.Add( new TargetFeedConfig( diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs index 60390312782..f4a8a9aa961 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs @@ -13,6 +13,9 @@ namespace Microsoft.DotNet.Build.Tasks.Feed.Model /// public class TargetFeedConfig { + /// + /// Returns the TargetURL stripped of SAS token so it can be used for logging purposes. + /// public string SafeTargetURL => new UriBuilder(TargetURL) {Query = "", Fragment = ""}.Uri.AbsoluteUri; public TargetFeedContentType ContentType { get; } From d073ef7069709d7434d7b2a2821a2df40dc70d00 Mon Sep 17 00:00:00 2001 From: Alex Perovich Date: Fri, 24 Sep 2021 16:19:01 -0700 Subject: [PATCH 4/5] Refactor Channel Configs based on feedback --- .../SdkTasks/PublishArtifactsInManifest.proj | 35 +- .../GeneralTests.cs | 14 +- .../PublishArtifactsInManifestTests.cs | 27 - .../SetupTargetFeedConfigV3Tests.cs | 367 +++++----- .../Microsoft.DotNet.Build.Tasks.Feed.csproj | 3 + .../src/PublishArtifactsInManifest.cs | 77 +- .../src/PublishArtifactsInManifestV3.cs | 58 +- .../src/model/PublishingConstants.cs | 660 ++++++------------ .../src/model/SetupTargetFeedConfigV3.cs | 430 ++++-------- .../src/model/TargetChannelConfig.cs | 143 ++-- .../src/model/TargetFeedConfig.cs | 7 +- 11 files changed, 691 insertions(+), 1130 deletions(-) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index d4f37052c40..784933a84f8 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -113,6 +113,22 @@ 16 + + + + + + + + + + + + + + + + @@ -128,12 +144,6 @@ MaestroApiEndpoint="$(MaestroApiEndpoint)" BuildAssetRegistryToken="$(BuildAssetRegistryToken)" PublishInstallersAndChecksums="$(PublishInstallersAndChecksums)" - ChecksumsFeedKey="$(ChecksumsFeedKey)" - InstallersFeedKey="$(InstallersFeedKey)" - InternalChecksumsFeedKey="$(InternalChecksumsFeedKey)" - InternalInstallersFeedKey="$(InternalInstallersFeedKey)" - AzureDevOpsFeedsKey="$(AzureDevOpsFeedsKey)" - AzureStorageTargetFeedKey="$(AzureStorageTargetFeedPAT)" AssetManifestPaths="@(ManifestFiles)" BlobAssetsBasePath="$(BlobBasePath)" PackageAssetsBasePath="$(PackageBasePath)" @@ -151,12 +161,6 @@ PublishSpecialClrFiles="$(PublishSpecialClrFiles)" BuildQuality="$(BuildQuality)" AllowFeedOverrides="$(AllowFeedOverrides)" - InstallersFeedOverride="$(InstallersFeedOverride)" - ChecksumsFeedOverride="$(ChecksumsFeedOverride)" - TransportFeedOverride="$(TransportFeedOverride)" - ShippingFeedOverride="$(ShippingFeedOverride)" - SymbolsFeedOverride="$(SymbolsFeedOverride)" - PublicSymbolsFeedOverride="$(PublicSymbolsFeedOverride)" AzdoApiToken="$(AzdoApiToken)" ArtifactsBasePath="$(ArtifactsBasePath)" BuildId="$(BuildId)" @@ -165,10 +169,9 @@ UseStreamingPublishing="$(UseStreamingPublishing)" StreamingPublishingMaxClients="$(StreamingPublishingMaxClients)" NonStreamingPublishingMaxClients="$(NonStreamingPublishingMaxClients)" - DotNetBuildsPublicUriBase64="$(DotNetBuildsPublicUriBase64)" - DotNetBuildsPublicChecksumsUriBase64="$(DotNetBuildsPublicChecksumsUriBase64)" - DotNetBuildsInternalUriBase64="$(DotNetBuildsInternalUriBase64)" - DotNetBuildsInternalChecksumsUriBase64="$(DotNetBuildsInternalChecksumsUriBase64)" + FeedKeys="@(FeedKey)" + FeedSasUris="@(FeedSasUri)" + FeedOverrides="@(FeedOverride)" /> diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/GeneralTests.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/GeneralTests.cs index e9c8d7bbd0b..0eb9ae620a4 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/GeneralTests.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/GeneralTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.IO; using System.Net; using System.Net.Http; @@ -24,12 +25,13 @@ public void ChannelConfigsHaveAllConfigs() foreach (var channelConfig in PublishingConstants.ChannelInfos) { channelConfig.Id.Should().BeGreaterThan(0); - channelConfig.ShippingFeed.Should().NotBeNullOrEmpty(); - channelConfig.ShippingFeed.Should().NotBeNullOrEmpty(); - channelConfig.TransportFeed.Should().NotBeNullOrEmpty(); - channelConfig.SymbolsFeed.Should().NotBeNullOrEmpty(); - channelConfig.ChecksumsFeed.Should().NotBeNullOrEmpty(); - channelConfig.InstallersFeed.Should().NotBeNullOrEmpty(); + channelConfig.TargetFeeds.Should().NotBeEmpty(); + foreach (TargetFeedContentType type in Enum.GetValues(typeof(TargetFeedContentType))) + { + if (type == TargetFeedContentType.None) + continue; + channelConfig.TargetFeeds.Should().Contain(f => f.ContentTypes.Contains(type)); + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishArtifactsInManifestTests.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishArtifactsInManifestTests.cs index e6f9a3c66bf..3d2d18e68aa 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishArtifactsInManifestTests.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishArtifactsInManifestTests.cs @@ -28,33 +28,6 @@ public class PublishArtifactsInManifestTests private const string RandomToken = "abcd"; private const string BlobFeedUrl = "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"; - // This test should be refactored: https://github.com/dotnet/arcade/issues/6715 - [Fact] - public void ConstructV2PublishingTask() - { - var manifestFullPath = TestInputs.GetFullPath(Path.Combine("Manifests", "SampleV2.xml")); - - var buildEngine = new MockBuildEngine(); - var task = new PublishArtifactsInManifest() - { - BuildEngine = buildEngine, - TargetChannels = GeneralTestingChannelId - }; - - // Dependency Injection setup - var collection = new ServiceCollection() - .AddSingleton() - .AddSingleton(); - task.ConfigureServices(collection); - using var provider = collection.BuildServiceProvider(); - - // Act and Assert - task.InvokeExecute(provider); - - var which = task.WhichPublishingTask(manifestFullPath); - which.Should().BeOfType(); - } - // This test should be refactored: https://github.com/dotnet/arcade/issues/6715 [Fact] public void ConstructV3PublishingTask() diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/SetupTargetFeedConfigV3Tests.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/SetupTargetFeedConfigV3Tests.cs index 0e06d46bcf3..2e3327c7731 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/SetupTargetFeedConfigV3Tests.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/SetupTargetFeedConfigV3Tests.cs @@ -1,11 +1,15 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using FluentAssertions; using Microsoft.Arcade.Test.Common; using Microsoft.DotNet.Build.Tasks.Feed.Model; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; using Xunit; using Xunit.Abstractions; @@ -31,12 +35,12 @@ public class SetupTargetFeedConfigV3Tests private const string AzureDevOpsStaticTransportFeed = "AzureDevOpsStaticTransportFeed"; private const string AzureDevOpsStaticSymbolsFeed = "AzureDevOpsStaticSymbolsFeed"; - private static List FilesToExclude = new List() { + private static ImmutableList FilesToExclude = ImmutableList.Create( "filename", "secondfilename" - }; + ); - private readonly List Installers = new List() { + private readonly List InstallersAndSymbols = new List() { TargetFeedContentType.OSX, TargetFeedContentType.Deb, TargetFeedContentType.Rpm, @@ -46,9 +50,28 @@ public class SetupTargetFeedConfigV3Tests TargetFeedContentType.Maven, TargetFeedContentType.VSIX, TargetFeedContentType.Badge, + TargetFeedContentType.Symbols, TargetFeedContentType.Other }; + private readonly ITaskItem[] FeedKeys = GetFeedKeys(); + + private static ITaskItem[] GetFeedKeys() + { + var installersKey = new TaskItem(PublishingConstants.FeedForInstallers); + installersKey.SetMetadata("Key", InstallersTargetStaticFeedKey); + var checksumsKey = new TaskItem(PublishingConstants.FeedForChecksums); + checksumsKey.SetMetadata("Key", ChecksumsTargetStaticFeedKey); + var azureDevops = new TaskItem("https://pkgs.dev.azure.com/dnceng"); + azureDevops.SetMetadata("Key", AzureDevOpsFeedsKey); + return new ITaskItem[] + { + installersKey, + checksumsKey, + azureDevops, + }; + } + private const SymbolTargetType symbolTargetType = SymbolTargetType.None; private readonly ITestOutputHelper Output; @@ -67,50 +90,16 @@ public void StableFeeds(bool publishInstallersAndChecksums, bool isInternalBuild { var expectedFeeds = new List(); - if (publishInstallersAndChecksums) - { - expectedFeeds.Add( - new TargetFeedConfig( - TargetFeedContentType.Checksum, - ChecksumsTargetStaticFeed, - FeedType.AzureStorageFeed, - ChecksumsTargetStaticFeedKey, - latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", - assetSelection: AssetSelection.All, - isolated: true, - @internal: false, - allowOverwrite: true, - symbolTargetType: symbolTargetType, - filenamesToExclude: FilesToExclude)); - - foreach (var contentType in Installers) - { - expectedFeeds.Add( - new TargetFeedConfig( - contentType, - InstallersTargetStaticFeed, - FeedType.AzureStorageFeed, - InstallersTargetStaticFeedKey, - latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", - assetSelection: AssetSelection.All, - isolated: true, - @internal: false, - allowOverwrite: true, - symbolTargetType: symbolTargetType, - filenamesToExclude: FilesToExclude)); - } - } - expectedFeeds.Add( new TargetFeedConfig( TargetFeedContentType.Package, StablePackageFeed, FeedType.AzDoNugetFeed, AzureDevOpsFeedsKey, - latestLinkShortUrlPrefix: string.Empty, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", assetSelection: AssetSelection.ShippingOnly, isolated: true, - @internal: false, + @internal: isInternalBuild, allowOverwrite: false, symbolTargetType: symbolTargetType, filenamesToExclude: FilesToExclude)); @@ -121,10 +110,10 @@ public void StableFeeds(bool publishInstallersAndChecksums, bool isInternalBuild StableSymbolsFeed, FeedType.AzDoNugetFeed, AzureDevOpsFeedsKey, - latestLinkShortUrlPrefix: string.Empty, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", assetSelection: AssetSelection.All, isolated: true, - @internal: false, + @internal: isInternalBuild, allowOverwrite: false, symbolTargetType: symbolTargetType, filenamesToExclude: FilesToExclude)); @@ -132,40 +121,70 @@ public void StableFeeds(bool publishInstallersAndChecksums, bool isInternalBuild expectedFeeds.Add( new TargetFeedConfig( TargetFeedContentType.Package, - AzureDevOpsStaticTransportFeed, + PublishingConstants.FeedDotNet5Transport, FeedType.AzDoNugetFeed, AzureDevOpsFeedsKey, - latestLinkShortUrlPrefix: string.Empty, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", assetSelection: AssetSelection.NonShippingOnly, isolated: false, - @internal: false, + @internal: isInternalBuild, allowOverwrite: false, symbolTargetType: symbolTargetType, filenamesToExclude: FilesToExclude)); + if (publishInstallersAndChecksums) + { + foreach (var contentType in InstallersAndSymbols) + { + if (contentType == TargetFeedContentType.Symbols) + { + continue; + } + expectedFeeds.Add( + new TargetFeedConfig( + contentType, + PublishingConstants.FeedForInstallers, + FeedType.AzureStorageFeed, + InstallersTargetStaticFeedKey, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", + assetSelection: AssetSelection.All, + isolated: false, + @internal: isInternalBuild, + allowOverwrite: false, + symbolTargetType: symbolTargetType, + filenamesToExclude: FilesToExclude)); + } + expectedFeeds.Add( + new TargetFeedConfig( + TargetFeedContentType.Checksum, + PublishingConstants.FeedForChecksums, + FeedType.AzureStorageFeed, + ChecksumsTargetStaticFeedKey, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", + assetSelection: AssetSelection.All, + isolated: false, + @internal: isInternalBuild, + allowOverwrite: false, + symbolTargetType: symbolTargetType, + filenamesToExclude: FilesToExclude)); + } + + var buildEngine = new MockBuildEngine(); + var channelConfig = PublishingConstants.ChannelInfos.First(c => c.Id == 131); var config = new SetupTargetFeedConfigV3( + channelConfig, isInternalBuild, - isStableBuild: true, + true, repositoryName: "test-repo", commitSha: "c0c0c0c0", - AzureStorageTargetFeedPAT, publishInstallersAndChecksums, - InstallersTargetStaticFeed, - InstallersTargetStaticFeedKey, - ChecksumsTargetStaticFeed, - ChecksumsTargetStaticFeedKey, - AzureDevOpsStaticShippingFeed, - AzureDevOpsStaticTransportFeed, - AzureDevOpsStaticSymbolsFeed, + FeedKeys, + Array.Empty(), + Array.Empty(), $"{LatestLinkShortUrlPrefix}/{BuildQuality}", - AzureDevOpsFeedsKey, buildEngine, symbolTargetType, - "public", - "publicchecksums", - "internal", - "internalchecksums", StablePackageFeed, StableSymbolsFeed, filesToExclude: FilesToExclude @@ -183,14 +202,40 @@ public void NonStableAndInternal(bool publishInstallersAndChecksums) { var expectedFeeds = new List(); + expectedFeeds.Add(new TargetFeedConfig( + TargetFeedContentType.Package, + PublishingConstants.FeedDotNet5Shipping, + FeedType.AzDoNugetFeed, + AzureDevOpsFeedsKey, + $"{LatestLinkShortUrlPrefix}/{BuildQuality}", + AssetSelection.ShippingOnly, + false, + true, + false, + symbolTargetType: symbolTargetType, + filenamesToExclude: FilesToExclude)); + + expectedFeeds.Add(new TargetFeedConfig( + TargetFeedContentType.Package, + PublishingConstants.FeedDotNet5Transport, + FeedType.AzDoNugetFeed, + AzureDevOpsFeedsKey, + $"{LatestLinkShortUrlPrefix}/{BuildQuality}", + AssetSelection.NonShippingOnly, + false, + true, + false, + symbolTargetType: symbolTargetType, + filenamesToExclude: FilesToExclude)); + if (publishInstallersAndChecksums) { - foreach (var contentType in Installers) + foreach (var contentType in InstallersAndSymbols) { expectedFeeds.Add( new TargetFeedConfig( contentType, - InstallersTargetStaticFeed, + PublishingConstants.FeedForInstallers, FeedType.AzureStorageFeed, InstallersTargetStaticFeedKey, latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", @@ -205,7 +250,7 @@ public void NonStableAndInternal(bool publishInstallersAndChecksums) expectedFeeds.Add( new TargetFeedConfig( TargetFeedContentType.Checksum, - ChecksumsTargetStaticFeed, + PublishingConstants.FeedForChecksums, FeedType.AzureStorageFeed, ChecksumsTargetStaticFeedKey, latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", @@ -217,72 +262,38 @@ public void NonStableAndInternal(bool publishInstallersAndChecksums) filenamesToExclude: FilesToExclude)); } - - expectedFeeds.Add( - new TargetFeedConfig( - TargetFeedContentType.Package, - AzureDevOpsStaticShippingFeed, - FeedType.AzDoNugetFeed, - AzureDevOpsFeedsKey, - latestLinkShortUrlPrefix: string.Empty, - assetSelection: AssetSelection.ShippingOnly, - isolated: false, - @internal: true, - allowOverwrite: false, - symbolTargetType: symbolTargetType, - filenamesToExclude: FilesToExclude)); - - expectedFeeds.Add( - new TargetFeedConfig( - TargetFeedContentType.Package, - AzureDevOpsStaticTransportFeed, - FeedType.AzDoNugetFeed, - AzureDevOpsFeedsKey, - latestLinkShortUrlPrefix: string.Empty, - assetSelection: AssetSelection.NonShippingOnly, - isolated: false, - @internal: true, - allowOverwrite: false, - symbolTargetType: symbolTargetType, - filenamesToExclude: FilesToExclude)); - - expectedFeeds.Add( - new TargetFeedConfig( - TargetFeedContentType.Symbols, - AzureDevOpsStaticSymbolsFeed, - FeedType.AzDoNugetFeed, - AzureDevOpsFeedsKey, - latestLinkShortUrlPrefix: string.Empty, - assetSelection: AssetSelection.All, - isolated: false, - @internal: true, - allowOverwrite: false, - symbolTargetType: symbolTargetType, - filenamesToExclude: FilesToExclude)); + else + { + expectedFeeds.Add( + new TargetFeedConfig( + TargetFeedContentType.Symbols, + PublishingConstants.FeedForInstallers, + FeedType.AzureStorageFeed, + InstallersTargetStaticFeedKey, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", + assetSelection: AssetSelection.All, + isolated: false, + @internal: true, + allowOverwrite: false, + symbolTargetType: symbolTargetType, + filenamesToExclude: FilesToExclude)); + } var buildEngine = new MockBuildEngine(); + var channelConfig = PublishingConstants.ChannelInfos.First(c => c.Id == 131); var config = new SetupTargetFeedConfigV3( + channelConfig, isInternalBuild: true, isStableBuild: false, repositoryName: "test-repo", commitSha: "c0c0c0c0", - AzureStorageTargetFeedPAT, publishInstallersAndChecksums, - InstallersTargetStaticFeed, - InstallersTargetStaticFeedKey, - ChecksumsTargetStaticFeed, - ChecksumsTargetStaticFeedKey, - AzureDevOpsStaticShippingFeed, - AzureDevOpsStaticTransportFeed, - AzureDevOpsStaticSymbolsFeed, + FeedKeys, + Array.Empty(), + Array.Empty(), $"{LatestLinkShortUrlPrefix}/{BuildQuality}", - AzureDevOpsFeedsKey, buildEngine: buildEngine, symbolTargetType, - "public", - "publicchecksums", - "internal", - "internalchecksums", filesToExclude: FilesToExclude ); @@ -298,61 +309,13 @@ public void NonStableAndPublic(bool publishInstallersAndChecksums) { var expectedFeeds = new List(); - if (publishInstallersAndChecksums) - { - expectedFeeds.Add( - new TargetFeedConfig( - TargetFeedContentType.Checksum, - ChecksumsTargetStaticFeed, - FeedType.AzureStorageFeed, - ChecksumsTargetStaticFeedKey, - latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", - assetSelection: AssetSelection.All, - isolated: false, - @internal: false, - allowOverwrite: false, - symbolTargetType: symbolTargetType, - filenamesToExclude: FilesToExclude)); - - foreach (var contentType in Installers) - { - expectedFeeds.Add( - new TargetFeedConfig( - contentType, - InstallersTargetStaticFeed, - FeedType.AzureStorageFeed, - InstallersTargetStaticFeedKey, - latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", - assetSelection: AssetSelection.All, - isolated: false, - @internal: false, - allowOverwrite: false, - symbolTargetType: symbolTargetType, - filenamesToExclude: FilesToExclude)); - } - } - - expectedFeeds.Add( - new TargetFeedConfig( - TargetFeedContentType.Symbols, - PublishingConstants.LegacyDotNetBlobFeedURL, - FeedType.AzureStorageFeed, - AzureStorageTargetFeedPAT, - latestLinkShortUrlPrefix: string.Empty, - assetSelection: AssetSelection.All, - isolated: false, - @internal: false, - allowOverwrite: false, - symbolTargetType: symbolTargetType, - filenamesToExclude: FilesToExclude)); - expectedFeeds.Add( new TargetFeedConfig( TargetFeedContentType.Package, - AzureDevOpsStaticShippingFeed, + PublishingConstants.FeedDotNet5Shipping, FeedType.AzDoNugetFeed, AzureDevOpsFeedsKey, - latestLinkShortUrlPrefix: string.Empty, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", assetSelection: AssetSelection.ShippingOnly, isolated: false, @internal: false, @@ -363,10 +326,10 @@ public void NonStableAndPublic(bool publishInstallersAndChecksums) expectedFeeds.Add( new TargetFeedConfig( TargetFeedContentType.Package, - AzureDevOpsStaticTransportFeed, + PublishingConstants.FeedDotNet5Transport, FeedType.AzDoNugetFeed, AzureDevOpsFeedsKey, - latestLinkShortUrlPrefix: string.Empty, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", assetSelection: AssetSelection.NonShippingOnly, isolated: false, @internal: false, @@ -374,29 +337,71 @@ public void NonStableAndPublic(bool publishInstallersAndChecksums) symbolTargetType: symbolTargetType, filenamesToExclude: FilesToExclude)); + if (publishInstallersAndChecksums) + { + foreach (var contentType in InstallersAndSymbols) + { + expectedFeeds.Add( + new TargetFeedConfig( + contentType, + PublishingConstants.FeedForInstallers, + FeedType.AzureStorageFeed, + InstallersTargetStaticFeedKey, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", + assetSelection: AssetSelection.All, + isolated: false, + @internal: false, + allowOverwrite: false, + symbolTargetType: symbolTargetType, + filenamesToExclude: FilesToExclude)); + } + + expectedFeeds.Add( + new TargetFeedConfig( + TargetFeedContentType.Checksum, + PublishingConstants.FeedForChecksums, + FeedType.AzureStorageFeed, + ChecksumsTargetStaticFeedKey, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", + assetSelection: AssetSelection.All, + isolated: false, + @internal: false, + allowOverwrite: false, + symbolTargetType: symbolTargetType, + filenamesToExclude: FilesToExclude)); + } + else + { + expectedFeeds.Add( + new TargetFeedConfig( + TargetFeedContentType.Symbols, + PublishingConstants.FeedForInstallers, + FeedType.AzureStorageFeed, + InstallersTargetStaticFeedKey, + latestLinkShortUrlPrefix: $"{LatestLinkShortUrlPrefix}/{BuildQuality}", + assetSelection: AssetSelection.All, + isolated: false, + @internal: false, + allowOverwrite: false, + symbolTargetType: symbolTargetType, + filenamesToExclude: FilesToExclude)); + } + var buildEngine = new MockBuildEngine(); + var channelConfig = PublishingConstants.ChannelInfos.First(c => c.Id == 131); var config = new SetupTargetFeedConfigV3( + channelConfig, isInternalBuild: false, isStableBuild: false, repositoryName: "test-repo", commitSha: "c0c0c0c0", - AzureStorageTargetFeedPAT, publishInstallersAndChecksums, - InstallersTargetStaticFeed, - InstallersTargetStaticFeedKey, - ChecksumsTargetStaticFeed, - ChecksumsTargetStaticFeedKey, - AzureDevOpsStaticShippingFeed, - AzureDevOpsStaticTransportFeed, - AzureDevOpsStaticSymbolsFeed, + FeedKeys, + Array.Empty(), + Array.Empty(), $"{LatestLinkShortUrlPrefix}/{BuildQuality}", - AzureDevOpsFeedsKey, buildEngine: buildEngine, symbolTargetType, - "public", - "publicchecksums", - "internal", - "internalchecksums", filesToExclude: FilesToExclude ); diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj index 5d85ef6bc24..981502a5606 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj @@ -24,6 +24,9 @@ + + + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index 837566da9d0..4d58ddd2917 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -107,31 +107,12 @@ public class PublishArtifactsInManifest : MSBuildTaskBase public bool PublishInstallersAndChecksums { get; set; } = false; - public string AzureStorageTargetFeedKey { get; set; } - public bool AllowFeedOverrides { get; set; } - public string ChecksumsFeedOverride { get; set; } - - public string ChecksumsFeedKey { get; set; } - - public string InstallersFeedOverride { get; set; } - - public string InstallersFeedKey { get; set; } - - public string InternalInstallersFeedKey { get; set; } - - public string InternalCheckSumsFeedKey { get; set; } - - public string AzureDevOpsFeedsKey { get; set; } - - public string TransportFeedOverride { get; set; } - - public string ShippingFeedOverride { get; set; } - - public string SymbolsFeedOverride { get; set; } + public ITaskItem[] FeedKeys { get; set; } + public ITaskItem[] FeedSasUris { get; set; } - public string PublicSymbolsFeedOverride { get; set; } + public ITaskItem[] FeedOverrides { get; set; } /// /// Path to dll and pdb files @@ -202,14 +183,6 @@ public string BuildQuality public int NonStreamingPublishingMaxClients {get; set;} - public string DotNetBuildsPublicUriBase64 { get; set; } - - public string DotNetBuildsPublicChecksumsUriBase64 { get; set; } - - public string DotNetBuildsInternalUriBase64 { get; set; } - - public string DotNetBuildsInternalChecksumsUriBase64 { get; set; } - /// /// Just an internal flag to keep track whether we published assets via a V3 manifest or not. /// @@ -332,27 +305,8 @@ public PublishArtifactsInManifestBase WhichPublishingTask(string manifestFullPat internal PublishArtifactsInManifestBase ConstructPublishingV2Task(BuildModel buildModel) { - return new PublishArtifactsInManifestV2(new AssetPublisherFactory(Log)) - { - BuildEngine = this.BuildEngine, - TargetFeedConfig = this.TargetFeedConfig, - BuildModel = buildModel, - BlobAssetsBasePath = this.BlobAssetsBasePath, - PackageAssetsBasePath = this.PackageAssetsBasePath, - BARBuildId = this.BARBuildId, - MaestroApiEndpoint = this.MaestroApiEndpoint, - BuildAssetRegistryToken = this.BuildAssetRegistryToken, - NugetPath = this.NugetPath, - InternalBuild = this.InternalBuild, - SkipSafetyChecks = this.SkipSafetyChecks, - AkaMSClientId = this.AkaMSClientId, - AkaMSClientSecret = this.AkaMSClientSecret, - AkaMSCreatedBy = this.AkaMSCreatedBy, - AkaMSGroupOwner = this.AkaMSGroupOwner, - AkaMsOwners = this.AkaMsOwners, - AkaMSTenant = this.AkaMSTenant, - BuildQuality = this.BuildQuality, - }; + Log.LogError("V2 Publishing is no longer supported."); + return null; } internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel buildModel) @@ -379,25 +333,16 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui AkaMsOwners = this.AkaMsOwners, AkaMSTenant = this.AkaMSTenant, PublishInstallersAndChecksums = this.PublishInstallersAndChecksums, - AzureDevOpsFeedsKey = this.AzureDevOpsFeedsKey, - InstallersFeedKey = this.InstallersFeedKey, - CheckSumsFeedKey = this.ChecksumsFeedKey, - InternalCheckSumsFeedKey = this.InternalCheckSumsFeedKey, - InternalInstallersFeedKey = this.InternalInstallersFeedKey, - AzureStorageTargetFeedKey = this.AzureStorageTargetFeedKey, + FeedKeys = this.FeedKeys, + FeedSasUris = this.FeedSasUris, + FeedOverrides = this.FeedOverrides, + AllowFeedOverrides = this.AllowFeedOverrides, PdbArtifactsBasePath = this.PdbArtifactsBasePath, SymWebToken = this.SymWebToken, MsdlToken = this.MsdlToken, SymbolPublishingExclusionsFile = this.SymbolPublishingExclusionsFile, PublishSpecialClrFiles = this.PublishSpecialClrFiles, BuildQuality = this.BuildQuality, - AllowFeedOverrides = this.AllowFeedOverrides, - InstallersFeedOverride = this.InstallersFeedOverride, - ChecksumsFeedOverride = this.ChecksumsFeedOverride, - ShippingFeedOverride = this.ShippingFeedOverride, - TransportFeedOverride = this.TransportFeedOverride, - SymbolsFeedOverride = this.SymbolsFeedOverride, - PublicSymbolsFeedOverride = this.PublicSymbolsFeedOverride, ArtifactsBasePath = this.ArtifactsBasePath, AzdoApiToken = this.AzdoApiToken, BuildId = this.BuildId, @@ -406,10 +351,6 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui UseStreamingPublishing = this.UseStreamingPublishing, StreamingPublishingMaxClients = this.StreamingPublishingMaxClients, NonStreamingPublishingMaxClients = this.NonStreamingPublishingMaxClients, - DotNetBuildsPublicUri = ConvertFromBase64(DotNetBuildsPublicUriBase64), - DotNetBuildsPublicChecksumsUri = ConvertFromBase64(DotNetBuildsPublicChecksumsUriBase64), - DotNetBuildsInternalUri = ConvertFromBase64(DotNetBuildsInternalUriBase64), - DotNetBuildsInternalChecksumsUri = ConvertFromBase64(DotNetBuildsInternalChecksumsUriBase64), }; } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs index 8b00d55419f..d024b7d7478 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestV3.cs @@ -24,22 +24,6 @@ public class PublishArtifactsInManifestV3 : PublishArtifactsInManifestBase [Required] public string TargetChannels { get; set; } - [Required] - public string AzureDevOpsFeedsKey { get; set; } - - [Required] - public string AzureStorageTargetFeedKey { get; set; } - - [Required] - public string InstallersFeedKey { get; set; } - - [Required] - public string CheckSumsFeedKey { get; set; } - - public string InternalInstallersFeedKey { get; set; } - - public string InternalCheckSumsFeedKey { get; set; } - public bool PublishInstallersAndChecksums { get; set; } public string PdbArtifactsBasePath { get; set; } @@ -54,26 +38,10 @@ public class PublishArtifactsInManifestV3 : PublishArtifactsInManifestBase public bool AllowFeedOverrides { get; set; } - public string InstallersFeedOverride { get; set; } - - public string ChecksumsFeedOverride { get; set; } - - public string ShippingFeedOverride { get; set; } + public ITaskItem[] FeedKeys { get; set; } + public ITaskItem[] FeedSasUris { get; set; } - public string TransportFeedOverride { get; set; } - - public string SymbolsFeedOverride { get; set; } - - public string PublicSymbolsFeedOverride { get; set; } - - [Required] - public string DotNetBuildsPublicUri { get; set; } - [Required] - public string DotNetBuildsPublicChecksumsUri { get; set; } - [Required] - public string DotNetBuildsInternalUri { get; set; } - [Required] - public string DotNetBuildsInternalChecksumsUri { get; set; } + public ITaskItem[] FeedOverrides { get; set; } public override bool Execute() { @@ -154,35 +122,25 @@ public override async Task ExecuteAsync() : $"dotnet/{targetChannelConfig.AkaMSChannelName}/{BuildQuality}"; var targetFeedsSetup = new SetupTargetFeedConfigV3( + targetChannelConfig, targetChannelConfig.IsInternal, BuildModel.Identity.IsStable, BuildModel.Identity.Name, BuildModel.Identity.Commit, - AzureStorageTargetFeedKey, PublishInstallersAndChecksums, - GetFeed(targetChannelConfig.InstallersFeed, InstallersFeedOverride), - targetChannelConfig.IsInternal ? InternalInstallersFeedKey : InstallersFeedKey, - GetFeed(targetChannelConfig.ChecksumsFeed, ChecksumsFeedOverride), - targetChannelConfig.IsInternal ? InternalCheckSumsFeedKey : CheckSumsFeedKey, - GetFeed(targetChannelConfig.ShippingFeed, ShippingFeedOverride), - GetFeed(targetChannelConfig.TransportFeed, TransportFeedOverride), - GetFeed(targetChannelConfig.SymbolsFeed, SymbolsFeedOverride), + FeedKeys, + FeedSasUris, + AllowFeedOverrides ? FeedOverrides : Array.Empty(), shortLinkUrl, - AzureDevOpsFeedsKey, BuildEngine, targetChannelConfig.SymbolTargetType, - DotNetBuildsPublicUri, - DotNetBuildsPublicChecksumsUri, - DotNetBuildsInternalUri, - DotNetBuildsInternalChecksumsUri, - azureDevOpsPublicStaticSymbolsFeed: GetFeed(null, PublicSymbolsFeedOverride), filesToExclude: targetChannelConfig.FilenamesToExclude, flatten: targetChannelConfig.Flatten); var targetFeedConfigs = targetFeedsSetup.Setup(); // No target feeds to publish to, very likely this is an error - if (targetFeedConfigs.Count() == 0) + if (!targetFeedConfigs.Any()) { Log.LogError($"No target feeds were found to publish the assets to."); return false; diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/PublishingConstants.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/PublishingConstants.cs index 0245bc0e431..627a00e2436 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/PublishingConstants.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/PublishingConstants.cs @@ -33,6 +33,34 @@ public class PublishingConstants public static readonly string AzDoNuGetFeedPattern = @"https://pkgs.dev.azure.com/(?[a-zA-Z0-9-]+)/(?[a-zA-Z0-9-]+/)?_packaging/(?.+)/nuget/v3/index.json"; + public static readonly TargetFeedContentType[] InstallersAndSymbols = { + TargetFeedContentType.OSX, + TargetFeedContentType.Deb, + TargetFeedContentType.Rpm, + TargetFeedContentType.Node, + TargetFeedContentType.BinaryLayout, + TargetFeedContentType.Installer, + TargetFeedContentType.Maven, + TargetFeedContentType.VSIX, + TargetFeedContentType.Badge, + TargetFeedContentType.Symbols, + TargetFeedContentType.Other + }; + + public static readonly TargetFeedContentType[] InstallersAndChecksums = { + TargetFeedContentType.OSX, + TargetFeedContentType.Deb, + TargetFeedContentType.Rpm, + TargetFeedContentType.Node, + TargetFeedContentType.BinaryLayout, + TargetFeedContentType.Installer, + TargetFeedContentType.Maven, + TargetFeedContentType.VSIX, + TargetFeedContentType.Badge, + TargetFeedContentType.Checksum, + TargetFeedContentType.Other + }; + public enum BuildQuality { [Description("daily")] @@ -65,61 +93,42 @@ public enum BuildQuality public const string FeedStagingInternalForChecksums = "https://dotnetbuilds.blob.core.windows.net/internal-checksums"; private const string FeedGeneralTesting = "https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json"; - private const string FeedGeneralTestingSymbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json"; private const string FeedDotNetExperimental = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"; - private const string FeedDotNetExperimentalSymbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json"; - private const string FeedDotNetEngShipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"; - private const string FeedDotNetEngTransport = FeedDotNetEngShipping; - private const string FeedDotNetEngSymbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json"; + private const string FeedDotNetEng = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"; - private const string FeedDotNetToolsShipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"; - private const string FeedDotNetToolsTransport = FeedDotNetToolsShipping; - private const string FeedDotNetToolsSymbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json"; + private const string FeedDotNetTools = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"; - private const string FeedDotNetToolsInternalShipping = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json"; - private const string FeedDotNetToolsInternalTransport = FeedDotNetToolsInternalShipping; - private const string FeedDotNetToolsInternalSymbols = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json"; + private const string FeedDotNetToolsInternal = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json"; private const string FeedDotNet31Shipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json"; private const string FeedDotNet31Transport = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json"; - private const string FeedDotNet31Symbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json"; private const string FeedDotNet31InternalShipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-internal/nuget/v3/index.json"; private const string FeedDotNet31InternalTransport = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json"; - private const string FeedDotNet31InternalSymbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json"; - private const string FeedDotNet31BlazorShipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-blazor/nuget/v3/index.json"; - private const string FeedDotNet31BlazorTransport = FeedDotNet31BlazorShipping; - private const string FeedDotNet31BlazorSymbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-blazor-symbols/nuget/v3/index.json"; + private const string FeedDotNet31Blazor = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-blazor/nuget/v3/index.json"; - private const string FeedDotNet5Shipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json"; - private const string FeedDotNet5Transport = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json"; - private const string FeedDotNet5Symbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json"; + public const string FeedDotNet5Shipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json"; + public const string FeedDotNet5Transport = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json"; private const string FeedDotNet6Shipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json"; private const string FeedDotNet6Transport = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-transport/nuget/v3/index.json"; - private const string FeedDotNet6Symbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json"; private const string FeedDotNet7Shipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json"; private const string FeedDotNet7Transport = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7-transport/nuget/v3/index.json"; - private const string FeedDotNet7Symbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7-symbols/nuget/v3/index.json"; private const string FeedDotNet6InternalShipping = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v3/index.json"; private const string FeedDotNet6InternalTransport = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v3/index.json"; - private const string FeedDotNet6InternalSymbols = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-symbols/nuget/v3/index.json"; private const string FeedDotNet5InternalShipping = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json"; private const string FeedDotNet5InternalTransport = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json"; - private const string FeedDotNet5InternalSymbols = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json"; private const string FeedDotNetLibrariesShipping = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries/nuget/v3/index.json"; private const string FeedDotNetLibrariesTransport = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-transport/nuget/v3/index.json"; - private const string FeedDotNetLibrariesSymbols = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries-symbols/nuget/v3/index.json"; private const string FeedGeneralTestingInternal = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/general-testing-internal/nuget/v3/index.json"; - private const string FeedGeneralTestingInternalSymbols = "https://pkgs.dev.azure.com/dnceng/internal/_packaging/general-testing-internal/nuget/v3/index.json"; private const SymbolTargetType InternalSymbolTargets = SymbolTargetType.SymWeb; private const SymbolTargetType PublicAndInternalSymbolTargets = SymbolTargetType.Msdl | SymbolTargetType.SymWeb; @@ -128,6 +137,115 @@ public enum BuildQuality "MergedManifest.xml" }; + private static TargetFeedSpecification[] DotNet31Feeds = + { + (TargetFeedContentType.Package, FeedDotNet31Shipping, AssetSelection.ShippingOnly), + (TargetFeedContentType.Package, FeedDotNet31Transport, AssetSelection.NonShippingOnly), + (InstallersAndSymbols, FeedForInstallers), + (TargetFeedContentType.Checksum, FeedForChecksums), + }; + + private static TargetFeedSpecification[] DotNet31InternalFeeds = + { + (TargetFeedContentType.Package, FeedDotNet31InternalShipping, AssetSelection.ShippingOnly), + (TargetFeedContentType.Package, FeedDotNet31InternalTransport, AssetSelection.NonShippingOnly), + (InstallersAndSymbols, FeedInternalForInstallers), + (TargetFeedContentType.Checksum, FeedInternalForChecksums), + }; + + private static TargetFeedSpecification[] DotNet31BlazorFeeds = + { + (TargetFeedContentType.Package, FeedDotNet31Blazor), + (InstallersAndSymbols, FeedInternalForInstallers), + (TargetFeedContentType.Checksum, FeedInternalForChecksums), + }; + + private static TargetFeedSpecification[] DotNet5Feeds = + { + (TargetFeedContentType.Package, FeedDotNet5Shipping, AssetSelection.ShippingOnly), + (TargetFeedContentType.Package, FeedDotNet5Transport, AssetSelection.NonShippingOnly), + (InstallersAndSymbols, FeedForInstallers), + (TargetFeedContentType.Checksum, FeedForChecksums), + }; + + private static TargetFeedSpecification[] DotNet5InternalFeeds = + { + (TargetFeedContentType.Package, FeedDotNet5InternalShipping, AssetSelection.ShippingOnly), + (TargetFeedContentType.Package, FeedDotNet5InternalTransport, AssetSelection.NonShippingOnly), + (InstallersAndSymbols, FeedInternalForInstallers), + (TargetFeedContentType.Checksum, FeedInternalForChecksums), + }; + + private static TargetFeedSpecification[] DotNet6Feeds = + { + (TargetFeedContentType.Package, FeedDotNet6Shipping, AssetSelection.ShippingOnly), + (TargetFeedContentType.Package, FeedDotNet6Transport, AssetSelection.NonShippingOnly), + (InstallersAndSymbols, FeedForInstallers), + (TargetFeedContentType.Checksum, FeedForChecksums), + }; + + private static TargetFeedSpecification[] DotNet6InternalFeeds = + { + (TargetFeedContentType.Package, FeedDotNet6InternalShipping, AssetSelection.ShippingOnly), + (TargetFeedContentType.Package, FeedDotNet6InternalTransport, AssetSelection.NonShippingOnly), + (InstallersAndSymbols, FeedInternalForInstallers), + (TargetFeedContentType.Checksum, FeedInternalForChecksums), + }; + + private static TargetFeedSpecification[] DotNet7Feeds = + { + (TargetFeedContentType.Package, FeedDotNet7Shipping, AssetSelection.ShippingOnly), + (TargetFeedContentType.Package, FeedDotNet7Transport, AssetSelection.NonShippingOnly), + (InstallersAndSymbols, FeedForInstallers), + (TargetFeedContentType.Checksum, FeedForChecksums), + }; + + private static TargetFeedSpecification[] DotNetEngFeeds = + { + (TargetFeedContentType.Package, FeedDotNetEng), + (InstallersAndSymbols, FeedForInstallers), + (TargetFeedContentType.Checksum, FeedForChecksums), + }; + + private static TargetFeedSpecification[] DotNetToolsFeeds = + { + (TargetFeedContentType.Package, FeedDotNetTools), + (InstallersAndSymbols, FeedForInstallers), + (TargetFeedContentType.Checksum, FeedForChecksums), + }; + + private static TargetFeedSpecification[] DotNetToolsInternalFeeds = + { + (TargetFeedContentType.Package, FeedDotNetToolsInternal), + (InstallersAndSymbols, FeedInternalForInstallers), + (TargetFeedContentType.Checksum, FeedInternalForChecksums), + }; + + private static TargetFeedSpecification[] DotNetExperimentalFeeds = + { + (TargetFeedContentType.Package, FeedDotNetExperimental), + (InstallersAndSymbols, FeedForInstallers), + (TargetFeedContentType.Checksum, FeedForChecksums), + }; + + private static TargetFeedSpecification[] GeneralTestingFeeds = + { + (TargetFeedContentType.Package, FeedGeneralTesting), + (InstallersAndSymbols, FeedForInstallers), + (TargetFeedContentType.Checksum, FeedForChecksums), + (InstallersAndSymbols, FeedStagingForInstallers), + (TargetFeedContentType.Checksum, FeedStagingForChecksums), + }; + + private static TargetFeedSpecification[] GeneralTestingInternalFeeds = + { + (TargetFeedContentType.Package, FeedGeneralTestingInternal), + (InstallersAndSymbols, FeedInternalForInstallers), + (TargetFeedContentType.Checksum, FeedInternalForChecksums), + (InstallersAndSymbols, FeedStagingInternalForInstallers), + (TargetFeedContentType.Checksum, FeedStagingInternalForChecksums), + }; + public static readonly List ChannelInfos = new List() { // ".NET 5 Dev", new TargetChannelConfig( @@ -135,11 +253,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "5.0", - FeedDotNet5Shipping, - FeedDotNet5Transport, - FeedDotNet5Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet5Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -149,11 +263,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "7.0", - FeedDotNet7Shipping, - FeedDotNet7Transport, - FeedDotNet7Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet7Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -163,11 +273,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "7.0.1xx", - FeedDotNet7Shipping, - FeedDotNet7Transport, - FeedDotNet7Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet7Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -177,11 +283,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -191,11 +293,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -205,11 +303,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0.1xx", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -219,11 +313,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0.1xx", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -233,11 +323,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0-preview1", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -247,11 +333,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0-preview2", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -261,11 +343,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0.1xx-preview2", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -275,11 +353,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0-preview3", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -289,11 +363,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0.1xx-preview3", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -303,11 +373,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0-preview4", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -317,11 +383,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0.1xx-preview4", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -331,11 +393,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0-preview5", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -345,11 +403,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0.1xx-preview5", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -359,11 +413,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0-preview6", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -373,11 +423,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0-preview6", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -387,11 +433,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0.1xx-preview6", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -401,11 +443,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0.1xx-preview6", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -415,11 +453,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0-preview7", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -429,11 +463,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0-preview7", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -443,11 +473,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0.1xx-preview7", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -457,11 +483,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0.1xx-preview7", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -471,11 +493,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0-rc1", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -485,11 +503,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0-rc1", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -499,11 +513,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0.1xx-rc1", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -513,11 +523,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0.1xx-rc1", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -527,11 +533,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0-rc2", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -541,11 +543,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0-rc2", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -555,11 +553,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "6.0.1xx-rc2", - FeedDotNet6Shipping, - FeedDotNet6Transport, - FeedDotNet6Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet6Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -569,11 +563,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "internal/6.0.1xx-rc2", - FeedDotNet6InternalShipping, - FeedDotNet6InternalTransport, - FeedDotNet6InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet6InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -583,11 +573,7 @@ public enum BuildQuality false, PublishingInfraVersion.Next, akaMSChannelName: "5.0", - FeedDotNet5Shipping, - FeedDotNet5Transport, - FeedDotNet5Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet5Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -597,11 +583,7 @@ public enum BuildQuality true, PublishingInfraVersion.Next, akaMSChannelName: "internal/5.0", - FeedDotNet5InternalShipping, - FeedDotNet5InternalTransport, - FeedDotNet5InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet5InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -611,11 +593,7 @@ public enum BuildQuality false, PublishingInfraVersion.Next, akaMSChannelName: "5.0.1xx", - FeedDotNet5Shipping, - FeedDotNet5Transport, - FeedDotNet5Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet5Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -625,11 +603,7 @@ public enum BuildQuality true, PublishingInfraVersion.Next, akaMSChannelName: "internal/5.0.1xx", - FeedDotNet5InternalShipping, - FeedDotNet5InternalTransport, - FeedDotNet5InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet5InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -639,11 +613,7 @@ public enum BuildQuality false, PublishingInfraVersion.Next, akaMSChannelName: "5.0.2xx", - FeedDotNet5Shipping, - FeedDotNet5Transport, - FeedDotNet5Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet5Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -653,11 +623,7 @@ public enum BuildQuality true, PublishingInfraVersion.Next, akaMSChannelName: "internal/5.0.2xx", - FeedDotNet5InternalShipping, - FeedDotNet5InternalTransport, - FeedDotNet5InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet5InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -667,11 +633,7 @@ public enum BuildQuality false, PublishingInfraVersion.Next, akaMSChannelName: "5.0.3xx", - FeedDotNet5Shipping, - FeedDotNet5Transport, - FeedDotNet5Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet5Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -681,11 +643,7 @@ public enum BuildQuality true, PublishingInfraVersion.Next, akaMSChannelName: "internal/5.0.3xx", - FeedDotNet5InternalShipping, - FeedDotNet5InternalTransport, - FeedDotNet5InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet5InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -695,11 +653,7 @@ public enum BuildQuality false, PublishingInfraVersion.Next, akaMSChannelName: "5.0.4xx", - FeedDotNet5Shipping, - FeedDotNet5Transport, - FeedDotNet5Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet5Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -709,11 +663,7 @@ public enum BuildQuality true, PublishingInfraVersion.Next, akaMSChannelName: "internal/5.0.4xx", - FeedDotNet5InternalShipping, - FeedDotNet5InternalTransport, - FeedDotNet5InternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNet5InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -723,11 +673,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "eng", - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, flatten: false), @@ -737,11 +683,7 @@ public enum BuildQuality false, PublishingInfraVersion.Next, "eng/net5", - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, flatten: false), @@ -751,11 +693,7 @@ public enum BuildQuality false, PublishingInfraVersion.Next, "eng/net6", - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -765,11 +703,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "eng/validation", - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, flatten: false), @@ -779,11 +713,7 @@ public enum BuildQuality false, PublishingInfraVersion.Next, "eng/net5validation", - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, flatten: false), @@ -793,11 +723,7 @@ public enum BuildQuality false, PublishingInfraVersion.Next, "eng/net6validation", - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, flatten: false), @@ -807,11 +733,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, "generaltesting", - FeedGeneralTesting, - FeedGeneralTesting, - FeedGeneralTestingSymbols, - FeedForChecksums, - FeedForInstallers, + GeneralTestingFeeds, PublicAndInternalSymbolTargets), // "General Testing Internal", @@ -820,11 +742,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, "generaltestinginternal", - FeedGeneralTestingInternal, - FeedGeneralTestingInternal, - FeedGeneralTestingInternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + GeneralTestingInternalFeeds, InternalSymbolTargets), // ".NET Core Tooling Dev", @@ -833,11 +751,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -848,11 +762,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -863,11 +773,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetToolsInternalShipping, - FeedDotNetToolsInternalTransport, - FeedDotNetToolsInternalSymbols, - FeedInternalForChecksums, - FeedInternalForInstallers, + DotNetToolsInternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -878,11 +784,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetExperimental, - FeedDotNetExperimental, - FeedDotNetExperimentalSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetExperimentalFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -893,11 +795,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -908,11 +806,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -923,11 +817,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -938,11 +828,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetEngShipping, - FeedDotNetEngTransport, - FeedDotNetEngSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetEngFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -953,11 +839,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -968,11 +850,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -983,11 +861,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: "3.1", - FeedDotNet31Shipping, - FeedDotNet31Transport, - FeedDotNet31Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet31Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -997,11 +871,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: "3.1", - FeedDotNet31Shipping, - FeedDotNet31Transport, - FeedDotNet31Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet31Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1011,11 +881,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: "3.1.2xx", - FeedDotNet31Shipping, - FeedDotNet31Transport, - FeedDotNet31Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet31Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1025,11 +891,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: "3.1.1xx", - FeedDotNet31Shipping, - FeedDotNet31Transport, - FeedDotNet31Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet31Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1039,11 +901,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: "3.1.3xx", - FeedDotNet31Shipping, - FeedDotNet31Transport, - FeedDotNet31Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet31Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1053,11 +911,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: "3.1.4xx", - FeedDotNet31Shipping, - FeedDotNet31Transport, - FeedDotNet31Symbols, - FeedForChecksums, - FeedForInstallers, + DotNet31Feeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1067,11 +921,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, akaMSChannelName: "internal/3.1.3xx", - FeedDotNet31InternalShipping, - FeedDotNet31InternalTransport, - FeedDotNet31InternalSymbols, - FeedForChecksums, - FeedForInstallers, + DotNet31InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1081,11 +931,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, akaMSChannelName: "internal/3.1", - FeedDotNet31InternalShipping, - FeedDotNet31InternalTransport, - FeedDotNet31InternalSymbols, - FeedForChecksums, - FeedForInstallers, + DotNet31InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1095,11 +941,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, akaMSChannelName: "internal/3.1.2xx", - FeedDotNet31InternalShipping, - FeedDotNet31InternalTransport, - FeedDotNet31InternalSymbols, - FeedForChecksums, - FeedForInstallers, + DotNet31InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1109,11 +951,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, akaMSChannelName: "internal/3.1.1xx", - FeedDotNet31InternalShipping, - FeedDotNet31InternalTransport, - FeedDotNet31InternalSymbols, - FeedForChecksums, - FeedForInstallers, + DotNet31InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1123,11 +961,7 @@ public enum BuildQuality true, PublishingInfraVersion.All, akaMSChannelName: "internal/3.1.4xx", - FeedDotNet31InternalShipping, - FeedDotNet31InternalTransport, - FeedDotNet31InternalSymbols, - FeedForChecksums, - FeedForInstallers, + DotNet31InternalFeeds, InternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1137,11 +971,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNet31BlazorShipping, - FeedDotNet31BlazorTransport, - FeedDotNet31BlazorSymbols, - FeedForChecksums, - FeedForInstallers, + DotNet31BlazorFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude), @@ -1151,11 +981,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -1166,11 +992,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -1181,11 +1003,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -1196,11 +1014,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -1211,11 +1025,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -1226,11 +1036,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -1241,11 +1047,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -1256,11 +1058,7 @@ public enum BuildQuality false, PublishingInfraVersion.All, string.Empty, - FeedDotNetToolsShipping, - FeedDotNetToolsTransport, - FeedDotNetToolsSymbols, - FeedForChecksums, - FeedForInstallers, + DotNetToolsFeeds, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), @@ -1271,11 +1069,13 @@ public enum BuildQuality false, PublishingInfraVersion.All, akaMSChannelName: string.Empty, - FeedDotNetLibrariesShipping, - FeedDotNetLibrariesTransport, - FeedDotNetLibrariesSymbols, - FeedForChecksums, - FeedForInstallers, + new TargetFeedSpecification[] + { + (TargetFeedContentType.Package, FeedDotNetLibrariesShipping, AssetSelection.ShippingOnly), + (TargetFeedContentType.Package, FeedDotNetLibrariesTransport, AssetSelection.NonShippingOnly), + (InstallersAndSymbols, FeedForInstallers), + (TargetFeedContentType.Checksum, FeedForChecksums), + }, PublicAndInternalSymbolTargets, filenamesToExclude: FilenamesToExclude, flatten: false), diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs index bd38b731fdc..1c5e677ae61 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs @@ -3,6 +3,8 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; using Microsoft.Build.Framework; using Microsoft.DotNet.Build.Tasks.Feed.Model; @@ -10,18 +12,7 @@ namespace Microsoft.DotNet.Build.Tasks.Feed { public class SetupTargetFeedConfigV3 : SetupTargetFeedConfigBase { - private readonly List Installers = new List() { - TargetFeedContentType.OSX, - TargetFeedContentType.Deb, - TargetFeedContentType.Rpm, - TargetFeedContentType.Node, - TargetFeedContentType.BinaryLayout, - TargetFeedContentType.Installer, - TargetFeedContentType.Maven, - TargetFeedContentType.VSIX, - TargetFeedContentType.Badge, - TargetFeedContentType.Other - }; + private readonly TargetChannelConfig _targetChannelConfig; private IBuildEngine BuildEngine { get; } @@ -29,352 +20,213 @@ public class SetupTargetFeedConfigV3 : SetupTargetFeedConfigBase private string StableSymbolsFeed { get; set; } - private string AzureDevOpsPublicStaticSymbolsFeed { get; set; } + private SymbolTargetType SymbolTargetType { get; } - private SymbolTargetType SymbolTargetType { get; set; } - - private List FilesToExclude { get; } + private ImmutableList FilesToExclude { get; } private bool Flatten { get; } - private string DotNetBuildsPublicUri { get; } - private string DotNetBuildsPublicChecksumsUri { get; } - private string DotNetBuildsInternalUri { get; } - private string DotNetBuildsInternalChecksumsUri { get; } - - public SetupTargetFeedConfigV3(bool isInternalBuild, + public SetupTargetFeedConfigV3( + TargetChannelConfig targetChannelConfig, + bool isInternalBuild, bool isStableBuild, string repositoryName, string commitSha, - string azureStorageTargetFeedPAT, bool publishInstallersAndChecksums, - string installersTargetStaticFeed, - string installersAzureAccountKey, - string checksumsTargetStaticFeed, - string checksumsAzureAccountKey, - string azureDevOpsStaticShippingFeed, - string azureDevOpsStaticTransportFeed, - string azureDevOpsStaticSymbolsFeed, + ITaskItem[] feedKeys, + ITaskItem[] feedSasUris, + ITaskItem[] feedOverrides, string latestLinkShortUrlPrefix, - string azureDevOpsFeedsKey, IBuildEngine buildEngine, SymbolTargetType symbolTargetType, - string dotNetBuildsPublicUri, - string dotNetBuildsPublicChecksumsUri, - string dotNetBuildsInternalUri, - string dotNetBuildsInternalChecksumsUri, string stablePackagesFeed = null, string stableSymbolsFeed = null, - string azureDevOpsPublicStaticSymbolsFeed = null, - List filesToExclude = null, + ImmutableList filesToExclude = null, bool flatten = true) - : base(isInternalBuild, isStableBuild, repositoryName, commitSha, azureStorageTargetFeedPAT, publishInstallersAndChecksums, installersTargetStaticFeed, installersAzureAccountKey, checksumsTargetStaticFeed, checksumsAzureAccountKey, azureDevOpsStaticShippingFeed, azureDevOpsStaticTransportFeed, azureDevOpsStaticSymbolsFeed, latestLinkShortUrlPrefix, azureDevOpsFeedsKey) + : base(isInternalBuild, isStableBuild, repositoryName, commitSha, null, publishInstallersAndChecksums, null, null, null, null, null, null, null, latestLinkShortUrlPrefix, null) { + _targetChannelConfig = targetChannelConfig; BuildEngine = buildEngine; StableSymbolsFeed = stableSymbolsFeed; StablePackagesFeed = stablePackagesFeed; SymbolTargetType = symbolTargetType; - DotNetBuildsPublicUri = dotNetBuildsPublicUri; - DotNetBuildsPublicChecksumsUri = dotNetBuildsPublicChecksumsUri; - DotNetBuildsInternalUri = dotNetBuildsInternalUri; - DotNetBuildsInternalChecksumsUri = dotNetBuildsInternalChecksumsUri; - AzureDevOpsPublicStaticSymbolsFeed = azureDevOpsPublicStaticSymbolsFeed; - FilesToExclude = filesToExclude ?? new List(); + FilesToExclude = filesToExclude ?? ImmutableList.Empty; Flatten = flatten; + FeedKeys = feedKeys.ToImmutableDictionary(i => i.ItemSpec, i => i.GetMetadata("Key")); + FeedSasUris = feedSasUris.ToImmutableDictionary(i => i.ItemSpec, i => i.GetMetadata("Uri")); + FeedOverrides = feedOverrides.ToImmutableDictionary(i => i.ItemSpec, i => i.GetMetadata("Replacement")); + AzureDevOpsFeedsKey = FeedKeys.TryGetValue("https://pkgs.dev.azure.com/dnceng", out string key) ? key : null; } - public override List Setup() - { - if (string.IsNullOrEmpty(InstallersAzureAccountKey)) - { - throw new ArgumentException("Parameters 'InstallersAzureAccountKey' is empty."); - } + public ImmutableDictionary FeedOverrides { get; set; } - if (string.IsNullOrEmpty(ChecksumsAzureAccountKey)) - { - throw new ArgumentException("Parameters 'ChecksumsAzureAccountKey' is empty."); - } + public ImmutableDictionary FeedSasUris { get; set; } - if (IsStableBuild) - { - return StableFeeds(); - } - else - { - return NonStableFeeds(); - } - } + public ImmutableDictionary FeedKeys { get; set; } - private List NonStableFeeds() + public override List Setup() { - List targetFeedConfigs = new List(); + return Feeds().ToList(); + } - if (PublishInstallersAndChecksums) + private IEnumerable Feeds() + { + if (IsStableBuild) { - foreach (var contentType in Installers) + if (string.IsNullOrEmpty(StablePackagesFeed)) { - if (!string.IsNullOrEmpty(DotNetBuildsPublicUri) && string.Equals(InstallersTargetStaticFeed, PublishingConstants.FeedForInstallers, StringComparison.OrdinalIgnoreCase)) + var packagesFeedTask = new CreateAzureDevOpsFeed() + { + BuildEngine = BuildEngine, + IsInternal = IsInternalBuild, + AzureDevOpsPersonalAccessToken = AzureDevOpsFeedsKey, + RepositoryName = RepositoryName, + CommitSha = CommitSha + }; + + if (!packagesFeedTask.Execute()) { - targetFeedConfigs.Add( - new TargetFeedConfig( - contentType, - DotNetBuildsPublicUri, - FeedType.AzureStorageContainer, - null, - latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, - @internal: IsInternalBuild, - symbolTargetType: SymbolTargetType, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); + throw new Exception($"Problems creating an AzureDevOps feed for repository '{RepositoryName}' and commit '{CommitSha}'."); } - targetFeedConfigs.Add( - new TargetFeedConfig( - contentType, - InstallersTargetStaticFeed, - FeedType.AzureStorageFeed, - InstallersAzureAccountKey, - latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, - @internal: IsInternalBuild, - symbolTargetType: SymbolTargetType, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); + + StablePackagesFeed = packagesFeedTask.TargetFeedURL; } - if (!string.IsNullOrEmpty(DotNetBuildsPublicChecksumsUri) && string.Equals(ChecksumsTargetStaticFeed, PublishingConstants.FeedForChecksums, StringComparison.OrdinalIgnoreCase)) + if (string.IsNullOrEmpty(StableSymbolsFeed)) { - targetFeedConfigs.Add( - new TargetFeedConfig( - TargetFeedContentType.Checksum, - DotNetBuildsPublicChecksumsUri, - FeedType.AzureStorageContainer, - null, - latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, - @internal: IsInternalBuild, - symbolTargetType: SymbolTargetType, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); + var symbolsFeedTask = new CreateAzureDevOpsFeed() + { + BuildEngine = BuildEngine, + IsInternal = IsInternalBuild, + AzureDevOpsPersonalAccessToken = AzureDevOpsFeedsKey, + RepositoryName = RepositoryName, + CommitSha = CommitSha, + ContentIdentifier = "sym" + }; + + if (!symbolsFeedTask.Execute()) + { + throw new Exception($"Problems creating an AzureDevOps (symbols) feed for repository '{RepositoryName}' and commit '{CommitSha}'."); + } + + StableSymbolsFeed = symbolsFeedTask.TargetFeedURL; } - targetFeedConfigs.Add( - new TargetFeedConfig( - TargetFeedContentType.Checksum, - ChecksumsTargetStaticFeed, - FeedType.AzureStorageFeed, - ChecksumsAzureAccountKey, - latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, - @internal: IsInternalBuild, - symbolTargetType: SymbolTargetType, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); - } - targetFeedConfigs.Add( - new TargetFeedConfig( + yield return new TargetFeedConfig( TargetFeedContentType.Package, - AzureDevOpsStaticShippingFeed, + StablePackagesFeed, FeedType.AzDoNugetFeed, AzureDevOpsFeedsKey, + LatestLinkShortUrlPrefix, assetSelection: AssetSelection.ShippingOnly, - @internal: IsInternalBuild, symbolTargetType: SymbolTargetType, + isolated: true, + @internal: IsInternalBuild, filenamesToExclude: FilesToExclude, - flatten: Flatten)); + flatten: Flatten); - targetFeedConfigs.Add( - new TargetFeedConfig( - TargetFeedContentType.Package, - AzureDevOpsStaticTransportFeed, + yield return new TargetFeedConfig( + TargetFeedContentType.Symbols, + StableSymbolsFeed, FeedType.AzDoNugetFeed, AzureDevOpsFeedsKey, - assetSelection: AssetSelection.NonShippingOnly, - @internal: IsInternalBuild, + LatestLinkShortUrlPrefix, symbolTargetType: SymbolTargetType, + isolated: true, + @internal: IsInternalBuild, filenamesToExclude: FilesToExclude, - flatten: Flatten)); - - // For symbols, we don't have a blob location where internal symbols can go today, - // so a feed is used in this case. This would be a potential performance improvement for internal builds. - // This is pretty uncommon though, as non-stable internal builds are quite rare. - string symbolsFeed; - FeedType symbolsFeedType; - string symbolsFeedSecret; - - if (IsInternalBuild) - { - symbolsFeed = AzureDevOpsStaticSymbolsFeed; - symbolsFeedType = FeedType.AzDoNugetFeed; - symbolsFeedSecret = AzureDevOpsFeedsKey; - } - else if (!string.IsNullOrEmpty(AzureDevOpsPublicStaticSymbolsFeed)) - { - symbolsFeed = AzureDevOpsPublicStaticSymbolsFeed; - symbolsFeedType = FeedType.AzDoNugetFeed; - symbolsFeedSecret = AzureDevOpsFeedsKey; + flatten: Flatten); } - else + foreach (var spec in _targetChannelConfig.TargetFeeds) { - symbolsFeed = PublishingConstants.LegacyDotNetBlobFeedURL; - symbolsFeedType = FeedType.AzureStorageFeed; - symbolsFeedSecret = AzureStorageTargetFeedPAT; - } - - targetFeedConfigs.Add( - new TargetFeedConfig( - TargetFeedContentType.Symbols, - symbolsFeed, - symbolsFeedType, - symbolsFeedSecret, - symbolTargetType: SymbolTargetType, - @internal: IsInternalBuild, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); + foreach (var type in spec.ContentTypes) + { + if (!PublishInstallersAndChecksums) + { + if (PublishingConstants.InstallersAndChecksums.Contains(type)) + { + continue; + } + } + if (IsStableBuild && ((type is TargetFeedContentType.Package && spec.Assets == AssetSelection.ShippingOnly) || type is TargetFeedContentType.Symbols)) + { + // stable build shipping packages and symbols were handled above + continue; + } - return targetFeedConfigs; + var feed = spec.FeedUrl; + feed = GetFeedOverride(feed); + if (type is TargetFeedContentType.Package && + spec.Assets == AssetSelection.ShippingOnly && + FeedOverrides.TryGetValue("transport-packages", out string newFeed)) + { + feed = newFeed; + } + else if (type is TargetFeedContentType.Package && + spec.Assets == AssetSelection.NonShippingOnly && + FeedOverrides.TryGetValue("shipping-packages", out newFeed)) + { + feed = newFeed; + } + var key = GetFeedKey(feed); + var sasUri = GetFeedSasUri(feed); + var feedType = feed.StartsWith("https://pkgs.dev.azure.com") + ? FeedType.AzDoNugetFeed + : (sasUri != null ? FeedType.AzureStorageContainer : FeedType.AzureStorageFeed); + yield return new TargetFeedConfig( + type, + sasUri ?? feed, + feedType, + key, + LatestLinkShortUrlPrefix, + spec.Assets, + false, + IsInternalBuild, + false, + filenamesToExclude: FilesToExclude, + flatten: Flatten + ); + } + } } - private List StableFeeds() + private string GetFeedOverride(string feed) { - List targetFeedConfigs = new List(); - - if (string.IsNullOrEmpty(StablePackagesFeed)) + foreach (var prefix in FeedOverrides.Keys) { - var packagesFeedTask = new CreateAzureDevOpsFeed() + if (feed.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) { - BuildEngine = BuildEngine, - IsInternal = IsInternalBuild, - AzureDevOpsPersonalAccessToken = AzureDevOpsFeedsKey, - RepositoryName = RepositoryName, - CommitSha = CommitSha - }; - - if (!packagesFeedTask.Execute()) - { - throw new Exception($"Problems creating an AzureDevOps feed for repository '{RepositoryName}' and commit '{CommitSha}'."); + return FeedOverrides[prefix]; } - - StablePackagesFeed = packagesFeedTask.TargetFeedURL; } - if (string.IsNullOrEmpty(StableSymbolsFeed)) - { - var symbolsFeedTask = new CreateAzureDevOpsFeed() - { - BuildEngine = BuildEngine, - IsInternal = IsInternalBuild, - AzureDevOpsPersonalAccessToken = AzureDevOpsFeedsKey, - RepositoryName = RepositoryName, - CommitSha = CommitSha, - ContentIdentifier = "sym" - }; + return feed; + } - if (!symbolsFeedTask.Execute()) + private string GetFeedSasUri(string feed) + { + foreach (var prefix in FeedSasUris.Keys) + { + if (feed.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) { - throw new Exception($"Problems creating an AzureDevOps (symbols) feed for repository '{RepositoryName}' and commit '{CommitSha}'."); + return FeedSasUris[prefix]; } - - StableSymbolsFeed = symbolsFeedTask.TargetFeedURL; } - targetFeedConfigs.Add( - new TargetFeedConfig( - TargetFeedContentType.Package, - StablePackagesFeed, - FeedType.AzDoNugetFeed, - AzureDevOpsFeedsKey, - assetSelection: AssetSelection.ShippingOnly, - symbolTargetType: SymbolTargetType, - isolated: true, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); - - targetFeedConfigs.Add( - new TargetFeedConfig( - TargetFeedContentType.Symbols, - StableSymbolsFeed, - FeedType.AzDoNugetFeed, - AzureDevOpsFeedsKey, - symbolTargetType: SymbolTargetType, - isolated: true, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); - - targetFeedConfigs.Add( - new TargetFeedConfig( - TargetFeedContentType.Package, - AzureDevOpsStaticTransportFeed, - FeedType.AzDoNugetFeed, - AzureDevOpsFeedsKey, - assetSelection: AssetSelection.NonShippingOnly, - symbolTargetType: SymbolTargetType, - isolated: false, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); + return null; + } - if (PublishInstallersAndChecksums) + private string GetFeedKey(string feed) + { + foreach (var prefix in FeedKeys.Keys) { - foreach (var contentType in Installers) - { - if (!string.IsNullOrEmpty(DotNetBuildsPublicUri) && string.Equals(InstallersTargetStaticFeed, PublishingConstants.FeedForInstallers, StringComparison.OrdinalIgnoreCase)) - { - targetFeedConfigs.Add( - new TargetFeedConfig( - contentType, - DotNetBuildsPublicUri, - FeedType.AzureStorageContainer, - null, - isolated: true, - symbolTargetType: SymbolTargetType, - latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, - @internal: false, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); - } - targetFeedConfigs.Add( - new TargetFeedConfig( - contentType, - InstallersTargetStaticFeed, - FeedType.AzureStorageFeed, - InstallersAzureAccountKey, - isolated: true, - symbolTargetType: SymbolTargetType, - latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, - @internal: false, - allowOverwrite: true, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); - } - - if (!string.IsNullOrEmpty(DotNetBuildsPublicChecksumsUri) && string.Equals(ChecksumsTargetStaticFeed, PublishingConstants.FeedForChecksums, StringComparison.OrdinalIgnoreCase)) + if (feed.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) { - targetFeedConfigs.Add( - new TargetFeedConfig( - TargetFeedContentType.Checksum, - DotNetBuildsPublicChecksumsUri, - FeedType.AzureStorageContainer, - null, - isolated: true, - symbolTargetType: SymbolTargetType, - latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, - @internal: false, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); + return FeedKeys[prefix]; } - - targetFeedConfigs.Add( - new TargetFeedConfig( - TargetFeedContentType.Checksum, - ChecksumsTargetStaticFeed, - FeedType.AzureStorageFeed, - ChecksumsAzureAccountKey, - isolated: true, - symbolTargetType: SymbolTargetType, - latestLinkShortUrlPrefix: LatestLinkShortUrlPrefix, - @internal: false, - allowOverwrite: true, - filenamesToExclude: FilesToExclude, - flatten: Flatten)); } - return targetFeedConfigs; + return null; } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetChannelConfig.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetChannelConfig.cs index b13fc7608fa..dced737bc3f 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetChannelConfig.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetChannelConfig.cs @@ -1,10 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.VersionTools.BuildManifest.Model; using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; +using Microsoft.DotNet.VersionTools.BuildManifest.Model; namespace Microsoft.DotNet.Build.Tasks.Feed.Model { @@ -32,30 +33,7 @@ public struct TargetChannelConfig /// public string AkaMSChannelName { get; } - /// - /// The URL (including the index.json suffix) of the *shipping* feed to be used for this channel. - /// - public string ShippingFeed { get; } - - /// - /// The URL (including the index.json suffix) of the *transport* feed to be used for this channel. - /// - public string TransportFeed { get; } - - /// - /// The URL (including the index.json suffix) of the *symbols* feed to be used for this channel. - /// - public string SymbolsFeed { get; } - - /// - /// The URL (including the index.json suffix) where *checksums* should be published to. - /// - public string ChecksumsFeed { get; } - - /// - /// The URL (including the index.json suffix) where *installers* should be published to. - /// - public string InstallersFeed { get; } + public ImmutableList TargetFeeds { get; } /// /// Should publish to Msdl @@ -64,7 +42,7 @@ public struct TargetChannelConfig public bool IsInternal { get; } - public List FilenamesToExclude { get; } + public ImmutableList FilenamesToExclude { get; } public bool Flatten { get; } @@ -73,11 +51,7 @@ public TargetChannelConfig( bool isInternal, PublishingInfraVersion publishingInfraVersion, string akaMSChannelName, - string shippingFeed, - string transportFeed, - string symbolsFeed, - string checksumsFeed, - string installersFeed, + IEnumerable targetFeeds, SymbolTargetType symbolTargetType, List filenamesToExclude = null, bool flatten = true) @@ -86,13 +60,9 @@ public TargetChannelConfig( IsInternal = isInternal; PublishingInfraVersion = publishingInfraVersion; AkaMSChannelName = akaMSChannelName; - ShippingFeed = shippingFeed; - TransportFeed = transportFeed; - SymbolsFeed = symbolsFeed; - ChecksumsFeed = checksumsFeed; - InstallersFeed = installersFeed; + TargetFeeds = targetFeeds.ToImmutableList(); SymbolTargetType = symbolTargetType; - FilenamesToExclude = filenamesToExclude ?? new List(); + FilenamesToExclude = filenamesToExclude?.ToImmutableList() ?? ImmutableList.Empty; Flatten = flatten; } @@ -102,11 +72,8 @@ public override string ToString() $"\n Channel ID: '{Id}' " + $"\n Infra-version: '{PublishingInfraVersion}' " + $"\n AkaMSChannelName: '{AkaMSChannelName}' " + - $"\n Shipping-feed: '{ShippingFeed}' " + - $"\n Transport-feed: '{TransportFeed}' " + - $"\n Symbols-feed: '{SymbolsFeed}' " + - $"\n Installers-feed: '{InstallersFeed}' " + - $"\n Checksums-feed: '{ChecksumsFeed}' " + + "\n Target Feeds:" + + $"\n {string.Join("\n ", TargetFeeds.Select(f => $"{string.Join(", ", f.ContentTypes)} -> {f.FeedUrl}"))}" + $"\n SymbolTargetType: '{SymbolTargetType}' " + $"\n IsInternal: '{IsInternal}'" + $"\n FilenamesToExclude: \n\t{string.Join("\n\t", FilenamesToExclude)}" + @@ -118,12 +85,9 @@ public override bool Equals(object other) if (other is TargetChannelConfig config && PublishingInfraVersion == config.PublishingInfraVersion && Id == config.Id && - String.Equals(AkaMSChannelName, config.AkaMSChannelName, StringComparison.OrdinalIgnoreCase) && - String.Equals(ShippingFeed, config.ShippingFeed, StringComparison.OrdinalIgnoreCase) && - String.Equals(TransportFeed, config.TransportFeed, StringComparison.OrdinalIgnoreCase) && - String.Equals(SymbolsFeed, config.SymbolsFeed, StringComparison.OrdinalIgnoreCase) && - String.Equals(ChecksumsFeed, config.ChecksumsFeed, StringComparison.OrdinalIgnoreCase) && - String.Equals(InstallersFeed, config.InstallersFeed, StringComparison.OrdinalIgnoreCase) && + string.Equals(AkaMSChannelName, config.AkaMSChannelName, StringComparison.OrdinalIgnoreCase) && + TargetFeeds.Count == config.TargetFeeds.Count && + TargetFeeds.Zip(config.TargetFeeds, (l, r) => l.Equals(r)).All(b => b) && IsInternal == config.IsInternal && Flatten == config.Flatten) { @@ -141,18 +105,77 @@ public override bool Equals(object other) public override int GetHashCode() { - return (PublishingInfraVersion, - Id, - IsInternal, - AkaMSChannelName, - ShippingFeed, - TransportFeed, - SymbolsFeed, - ChecksumsFeed, - InstallersFeed, - SymbolTargetType, - Flatten, - string.Join(" ", FilenamesToExclude)).GetHashCode(); + var hash = new HashCode(); + hash.Add(PublishingInfraVersion); + hash.Add(Id); + hash.Add(IsInternal); + hash.Add(AkaMSChannelName); + foreach (var feedSpec in TargetFeeds) + { + hash.Add(feedSpec); + } + hash.Add(SymbolTargetType); + hash.Add(Flatten); + foreach (string fileName in FilenamesToExclude) + { + hash.Add(fileName); + } + + return hash.ToHashCode(); + } + } + + public struct TargetFeedSpecification + { + public ImmutableList ContentTypes { get; } + public string FeedUrl { get; } + public AssetSelection Assets { get; } + + public static implicit operator TargetFeedSpecification((TargetFeedContentType[] types, string feed) tuple) + { + return new TargetFeedSpecification(tuple.types, tuple.feed, AssetSelection.All); + } + + public static implicit operator TargetFeedSpecification((TargetFeedContentType[] types, string feed, AssetSelection assets) tuple) + { + return new TargetFeedSpecification(tuple.types, tuple.feed, tuple.assets); + } + + public static implicit operator TargetFeedSpecification((TargetFeedContentType type, string feed) tuple) + { + return new TargetFeedSpecification(ImmutableList.Create(tuple.type), tuple.feed, AssetSelection.All); + } + + public static implicit operator TargetFeedSpecification((TargetFeedContentType type, string feed, AssetSelection assets) tuple) + { + return new TargetFeedSpecification(ImmutableList.Create(tuple.type), tuple.feed, tuple.assets); + } + + public TargetFeedSpecification(IEnumerable contentTypes, string feedUrl, AssetSelection assets) + { + ContentTypes = contentTypes.ToImmutableList(); + FeedUrl = feedUrl; + Assets = assets; + } + + public override bool Equals(object obj) + { + return obj is TargetFeedSpecification other && Equals(other); + } + + public bool Equals(TargetFeedSpecification other) => ContentTypes.Count == other.ContentTypes.Count && + ContentTypes.Zip(other.ContentTypes, (l, r) => l.Equals(r)).All(b => b) && + FeedUrl == other.FeedUrl; + + public override int GetHashCode() + { + var hash = new HashCode(); + foreach (var t in ContentTypes) + { + hash.Add(t); + } + hash.Add(FeedUrl); + return hash.ToHashCode(); } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs index f4a8a9aa961..3a61dcb8a5c 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/TargetFeedConfig.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; using Microsoft.Extensions.Azure; @@ -57,7 +58,7 @@ public class TargetFeedConfig public SymbolTargetType SymbolTargetType { get; } - public List FilenamesToExclude { get; } + public ImmutableList FilenamesToExclude { get; } public bool Flatten { get; } @@ -71,7 +72,7 @@ public TargetFeedConfig(TargetFeedContentType contentType, bool @internal = false, bool allowOverwrite = false, SymbolTargetType symbolTargetType = SymbolTargetType.None, - List filenamesToExclude = null, + IEnumerable filenamesToExclude = null, bool flatten = true) { ContentType = contentType; @@ -84,7 +85,7 @@ public TargetFeedConfig(TargetFeedContentType contentType, AllowOverwrite = allowOverwrite; LatestLinkShortUrlPrefix = latestLinkShortUrlPrefix ?? string.Empty; SymbolTargetType = symbolTargetType; - FilenamesToExclude = filenamesToExclude ?? new List(); + FilenamesToExclude = filenamesToExclude?.ToImmutableList() ?? ImmutableList.Empty; Flatten = flatten; } From 060c3d240a2a6bf75138ab7e59e0920827775c79 Mon Sep 17 00:00:00 2001 From: Alex Perovich Date: Fri, 24 Sep 2021 19:31:33 -0700 Subject: [PATCH 5/5] Put the base64 back in the right place --- .../tools/SdkTasks/PublishArtifactsInManifest.proj | 8 ++++---- .../src/PublishArtifactsInManifest.cs | 9 --------- .../src/model/SetupTargetFeedConfigV3.cs | 12 +++++++++++- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index 784933a84f8..73f4f779fef 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -119,10 +119,10 @@ - - - - + + + + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs index 4d58ddd2917..5fac2130aeb 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs @@ -353,14 +353,5 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui NonStreamingPublishingMaxClients = this.NonStreamingPublishingMaxClients, }; } - - private string ConvertFromBase64(string value) - { - if (value == null) - { - return null; - } - return Encoding.UTF8.GetString(Convert.FromBase64String(value)); - } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs index 1c5e677ae61..a0dcacbbb64 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Text; using Microsoft.Build.Framework; using Microsoft.DotNet.Build.Tasks.Feed.Model; @@ -53,11 +54,20 @@ public SetupTargetFeedConfigV3( FilesToExclude = filesToExclude ?? ImmutableList.Empty; Flatten = flatten; FeedKeys = feedKeys.ToImmutableDictionary(i => i.ItemSpec, i => i.GetMetadata("Key")); - FeedSasUris = feedSasUris.ToImmutableDictionary(i => i.ItemSpec, i => i.GetMetadata("Uri")); + FeedSasUris = feedSasUris.ToImmutableDictionary(i => i.ItemSpec, i => ConvertFromBase64(i.GetMetadata("Base64Uri"))); FeedOverrides = feedOverrides.ToImmutableDictionary(i => i.ItemSpec, i => i.GetMetadata("Replacement")); AzureDevOpsFeedsKey = FeedKeys.TryGetValue("https://pkgs.dev.azure.com/dnceng", out string key) ? key : null; } + private static string ConvertFromBase64(string value) + { + if (value == null) + { + return null; + } + return Encoding.UTF8.GetString(Convert.FromBase64String(value)); + } + public ImmutableDictionary FeedOverrides { get; set; } public ImmutableDictionary FeedSasUris { get; set; }