Add publishing to the dotnetbuilds storage account alongside dotnetcli

.vault-config/dotnetbuildskeys.yaml

@@ -0,0 +1,18 @@
+storageLocation:
+  type: azure-key-vault
+  parameters:
+    subscription: 11c6037b-227b-4d63-bee1-18c7b68c3a40
+    name: dotnetbuildskeys
+
+secrets:
+  dotnetbuilds-access-key:
+    type: azure-storage-key
+    parameters:
+      subscription: 11c6037b-227b-4d63-bee1-18c7b68c3a40
+      account: dotnetbuilds
+
+  dotnetbuilds-connection-string:
+    type: azure-storage-connection-string
+    parameters:
+      storageKeySecret: dotnetbuilds-access-key
+      account: dotnetbuilds

.vault-config/dotnetbuildstokens.yaml

@@ -0,0 +1,49 @@
+storageLocation:
+  type: azure-key-vault
+  parameters:
+    subscription: 11c6037b-227b-4d63-bee1-18c7b68c3a40
+    name: dotnetbuildstokens
+
+references:
+  dotnetbuildskeys:
+    type: azure-key-vault
+    parameters:
+      subscription: 11c6037b-227b-4d63-bee1-18c7b68c3a40
+      name: dotnetbuildskeys
+
+secrets:
+  dotnetbuilds-internal-container-uri:
+    type: azure-storage-container-sas-uri
+    parameters:
+      connectionString:
+        name: dotnetbuilds-connection-string
+        location: dotnetbuildskeys
+      permissions: rlwc
+      container: internal
+
+  dotnetbuilds-internal-container-checksum-uri:
+    type: azure-storage-container-sas-uri
+    parameters:
+      connectionString:
+        name: dotnetbuilds-connection-string
+        location: dotnetbuildskeys
+      permissions: rlwc
+      container: internal-checksums
+
+  dotnetbuilds-public-container-uri:
+    type: azure-storage-container-sas-uri
+    parameters:
+      connectionString:
+        name: dotnetbuilds-connection-string
+        location: dotnetbuildskeys
+      permissions: rlwc
+      container: public
+
+  dotnetbuilds-public-container-checksum-uri:
+    type: azure-storage-container-sas-uri
+    parameters:
+      connectionString:
+        name: dotnetbuilds-connection-string
+        location: dotnetbuildskeys
+      permissions: rlwc
+      container: public-checksums

eng/publishing/v3/publish-assets.yml

@@ -12,6 +12,7 @@ jobs:
   timeoutInMinutes: 120
   variables:
     - group: DotNet-Symbol-Server-Pats
+    - group: DotNetBuilds storage account tokens
     - name: BARBuildId
       value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
     - name: IsStableBuild
@@ -98,6 +99,10 @@ jobs:
           /p:UseStreamingPublishing='true'
           /p:StreamingPublishingMaxClients=16
           /p:NonStreamingPublishingMaxClients=12
+          /p:DotNetBuildsPublicUri='$(dotnetbuilds-public-container-uri)'
+          /p:DotNetBuildsPublicChecksumsUri='$(dotnetbuilds-public-container-checksum-uri)'
+          /p:DotNetBuildsInternalUri='$(dotnetbuilds-internal-container-uri)'
+          /p:DotNetBuildsInternalChecksumsUri='$(dotnetbuilds-internal-container-checksum-uri)'
     - template: /eng/common/templates/steps/publish-logs.yml
       parameters:
         StageLabel: '${{ parameters.stageName }}'

src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj

@@ -164,7 +164,12 @@
       AzureProject="$(AzureProject)"
       UseStreamingPublishing="$(UseStreamingPublishing)"
       StreamingPublishingMaxClients="$(StreamingPublishingMaxClients)"
-      NonStreamingPublishingMaxClients="$(NonStreamingPublishingMaxClients)"/>
+      NonStreamingPublishingMaxClients="$(NonStreamingPublishingMaxClients)"
+      DotNetBuildsPublicUri="$(DotNetBuildsPublicUri)"
+      DotNetBuildsPublicChecksumsUri="$(DotNetBuildsPublicChecksumsUri)"
+      DotNetBuildsInternalUri="$(DotNetBuildsInternalUri)"
+      DotNetBuildsInternalChecksumsUri="$(DotNetBuildsInternalChecksumsUri)"
+      />
   </Target>
 
   <ItemGroup>

src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishArtifactsInManifestTests.cs

@@ -136,7 +136,7 @@ public async Task FeedConfigParserTests2Async()
             await task.ParseTargetFeedConfigAsync();
             task.Log.HasLoggedErrors.Should().BeTrue();
             buildEngine.BuildErrorEvents.Should().Contain(e =>
-                e.Message.Equals("Invalid feed config type 'MyUnknownFeedType'. Possible values are: AzDoNugetFeed, AzureStorageFeed"));
+                e.Message.Equals("Invalid feed config type 'MyUnknownFeedType'. Possible values are: AzDoNugetFeed, AzureStorageFeed, AzureStorageContainer"));
         }
 
         [Fact]

src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs

@@ -82,7 +82,7 @@ public void PublishToBothSymbolServerTest()
         }
 
         [Fact]
-        public void TemporarySymbolDirectoryDoesNotExists()
+        public async Task TemporarySymbolDirectoryDoesNotExists()
         {
             var buildEngine = new MockBuildEngine();
             var task = new PublishArtifactsInManifestV3()
@@ -91,7 +91,7 @@ public void TemporarySymbolDirectoryDoesNotExists()
             };
             var path = TestInputs.GetFullPath("Symbol");
             var buildAsset = new Dictionary<string, HashSet<Asset>>();
-            var publish = task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, buildAsset, null, path);
+            await task.HandleSymbolPublishingAsync(path, MsdlToken, SymWebToken, "", false, buildAsset, null, path);
             Assert.True(task.Log.HasLoggedErrors);
         }
 
@@ -113,7 +113,6 @@ public void TemporarySymbolsDirectoryTest()
         [Fact]
         public void PublishSymbolApiIsCalledTest()
         {
-            var publishTask = new PublishArtifactsInManifestV3();
             var path = TestInputs.GetFullPath("Symbols");
             string[] fileEntries = Directory.GetFiles(path);
             var feedConfigsForSymbols = new HashSet<TargetFeedConfig>();
@@ -128,8 +127,6 @@ public void PublishSymbolApiIsCalledTest()
                 @internal: false,
                 allowOverwrite: true,
                 SymbolTargetType.SymWeb));
-            Dictionary<string, string> test =
-                publishTask.GetTargetSymbolServers(feedConfigsForSymbols, MsdlToken, SymWebToken);
             Assert.True(PublishSymbolsHelper.PublishAsync(null,
                 path,
                 SymWebToken,

src/Microsoft.DotNet.Build.Tasks.Feed.Tests/SetupTargetFeedConfigV3Tests.cs

@@ -162,6 +162,10 @@ public void StableFeeds(bool publishInstallersAndChecksums, bool isInternalBuild
                     AzureDevOpsFeedsKey,
                     buildEngine,
                     symbolTargetType,
+                    "public",
+                    "publicchecksums",
+                    "internal",
+                    "internalchecksums",
                     StablePackageFeed,
                     StableSymbolsFeed,
                     filesToExclude: FilesToExclude
@@ -275,6 +279,10 @@ public void NonStableAndInternal(bool publishInstallersAndChecksums)
                     AzureDevOpsFeedsKey,
                     buildEngine: buildEngine,
                     symbolTargetType,
+                    "public",
+                    "publicchecksums",
+                    "internal",
+                    "internalchecksums",
                     filesToExclude: FilesToExclude
                 );
 
@@ -385,6 +393,10 @@ public void NonStableAndPublic(bool publishInstallersAndChecksums)
                     AzureDevOpsFeedsKey,
                     buildEngine: buildEngine,
                     symbolTargetType,
+                    "public",
+                    "publicchecksums",
+                    "internal",
+                    "internalchecksums",
                     filesToExclude: FilesToExclude
                 );
 

src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisher.cs

@@ -0,0 +1,34 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Threading;
+using Microsoft.Build.Utilities;
+using Microsoft.DotNet.Maestro.Client.Models;
+using Task = System.Threading.Tasks.Task;
+
+namespace Microsoft.DotNet.Build.Tasks.Feed
+{
+    public abstract class AssetPublisher : IDisposable
+    {
+        public TaskLoggingHelper Log { get; }
+        public abstract AddAssetLocationToAssetAssetLocationType LocationType { get; }
+
+        protected AssetPublisher(TaskLoggingHelper log)
+        {
+            Log = log;
+        }
+
+        public abstract Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null);
+
+        protected virtual void Dispose(bool disposing)
+        {
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+    }
+}

src/Microsoft.DotNet.Build.Tasks.Feed/src/AssetPublisherFactory.cs

@@ -0,0 +1,35 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using Microsoft.Build.Utilities;
+using Microsoft.DotNet.Build.Tasks.Feed.Model;
+
+namespace Microsoft.DotNet.Build.Tasks.Feed
+{
+    public class AssetPublisherFactory
+    {
+        private readonly TaskLoggingHelper _log;
+
+        public AssetPublisherFactory(TaskLoggingHelper log)
+        {
+            _log = log;
+        }
+
+        public virtual AssetPublisher CreateAssetPublisher(TargetFeedConfig feedConfig, PublishArtifactsInManifestBase task)
+        {
+            switch (feedConfig.Type)
+            {
+                case FeedType.AzDoNugetFeed:
+                    return new AzureDevOpsNugetFeedAssetPublisher(_log, feedConfig.TargetURL, feedConfig.Token, task);
+                case FeedType.AzureStorageFeed:
+                    var action = new BlobFeedAction(feedConfig.TargetURL, feedConfig.Token, _log);
+                    return new AzureStorageFeedAssetPublisher(action.AccountName, action.AccountKey, action.ContainerName, _log);
+                case FeedType.AzureStorageContainer:
+                    return new AzureStorageContainerAssetPublisher(new Uri(feedConfig.TargetURL), _log);
+                default:
+                    throw new NotImplementedException();
+            }
+        }
+    }
+}

src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureDevOpsNugetFeedAssetPublisher.cs

@@ -0,0 +1,90 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+using Microsoft.Build.Utilities;
+using Microsoft.DotNet.Build.Tasks.Feed.Model;
+using Microsoft.DotNet.Maestro.Client.Models;
+using NuGet.Packaging;
+using NuGet.Packaging.Core;
+using Task = System.Threading.Tasks.Task;
+
+namespace Microsoft.DotNet.Build.Tasks.Feed
+{
+    public class AzureDevOpsNugetFeedAssetPublisher : AssetPublisher
+    {
+        private readonly string _targetUrl;
+        private readonly string _accessToken;
+        private readonly PublishArtifactsInManifestBase _task;
+        private readonly string _feedAccount;
+        private readonly string _feedVisibility;
+        private readonly string _feedName;
+        private readonly HttpClient _httpClient;
+
+        public AzureDevOpsNugetFeedAssetPublisher(TaskLoggingHelper log, string targetUrl, string accessToken, PublishArtifactsInManifestBase task) : base(log)
+        {
+            _targetUrl = targetUrl;
+            _accessToken = accessToken;
+            _task = task;
+
+            var parsedUri = Regex.Match(_targetUrl, PublishingConstants.AzDoNuGetFeedPattern);
+            if (!parsedUri.Success)
+            {
+                Log.LogError(
+                    $"Azure DevOps NuGetFeed was not in the expected format '{PublishingConstants.AzDoNuGetFeedPattern}'");
+                return;
+            }
+            _feedAccount = parsedUri.Groups["account"].Value;
+            _feedVisibility = parsedUri.Groups["visibility"].Value;
+            _feedName = parsedUri.Groups["feed"].Value;
+
+            _httpClient = new HttpClient(new HttpClientHandler {CheckCertificateRevocationList = true})
+            {
+                Timeout = TimeSpan.FromSeconds(300),
+                DefaultRequestHeaders =
+                {
+                    Authorization = new AuthenticationHeaderValue(
+                        "Basic",
+                        Convert.ToBase64String(Encoding.ASCII.GetBytes($":{_accessToken}")))
+                },
+            };
+        }
+
+        protected override void Dispose(bool disposing)
+        {
+            _httpClient?.Dispose();
+        }
+
+        public override AddAssetLocationToAssetAssetLocationType LocationType => AddAssetLocationToAssetAssetLocationType.NugetFeed;
+
+        public override async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null)
+        {
+            if (!file.EndsWith(GeneralUtils.PackageSuffix, StringComparison.OrdinalIgnoreCase))
+            {
+                Log.LogWarning(
+                    $"AzDO feed publishing not available for blobs. Blob '{file}' was not published.");
+                return;
+            }
+
+            using var packageReader = new PackageArchiveReader(file);
+            PackageIdentity packageIdentity = packageReader.GetIdentity();
+            string id = packageIdentity.Id;
+            string version = packageIdentity.Version.ToString();
+
+            try
+            {
+                var config = new TargetFeedConfig(default, _targetUrl, default, default);
+                await _task.PushNugetPackageAsync(config, _httpClient, file, id, version, _feedAccount, _feedVisibility, _feedName);
+            }
+            catch (Exception e)
+            {
+                Log.LogErrorFromException(e);
+            }
+        }
+    }
+}

src/Microsoft.DotNet.Build.Tasks.Feed/src/AzureStorageAssetPublisher.cs

@@ -0,0 +1,50 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Threading;
+using Azure.Storage.Blobs;
+using Microsoft.Build.Utilities;
+using Microsoft.DotNet.Maestro.Client.Models;
+using Task = System.Threading.Tasks.Task;
+
+namespace Microsoft.DotNet.Build.Tasks.Feed
+{
+    public abstract class AzureStorageAssetPublisher : AssetPublisher
+    {
+        protected AzureStorageAssetPublisher(TaskLoggingHelper log)
+            : base(log)
+        {
+        }
+
+        public override AddAssetLocationToAssetAssetLocationType LocationType => AddAssetLocationToAssetAssetLocationType.Container;
+
+        public abstract BlobClient CreateBlobClient(string blobPath);
+
+        public override async Task PublishAssetAsync(string file, string blobPath, PushOptions options, SemaphoreSlim clientThrottle = null)
+        {
+            using (await SemaphoreLock.LockAsync(clientThrottle))
+            {
+                blobPath = blobPath.Replace("\\", "/");
+                var blobClient = CreateBlobClient(blobPath);
+                if (!options.AllowOverwrite && await blobClient.ExistsAsync())
+                {
+                    if (options.PassIfExistingItemIdentical)
+                    {
+                        if (!await blobClient.IsFileIdenticalToBlobAsync(file))
+                        {
+                            Log.LogError($"Asset '{file}' already exists with different contents at '{blobPath}'");
+                        }
+
+                        return;
+                    }
+
+                    Log.LogError($"Asset '{file}' already exists at '{blobPath}'");
+                    return;
+                }
+
+                Log.LogMessage($"Uploading '{file}' to '{blobPath}'");
+                await blobClient.UploadAsync(file);
+            }
+        }
+    }
+}