Tests for publishing (#7346)

* tests * remove unwanted using * addressed feedback * change response * comments * Revert some changes * changed it cos, the test were timing out * few nits * review comment
dotnet · Jul 1, 2021 · c206f6f · c206f6f
1 parent 812a324
commit c206f6f
Show file tree

Hide file tree

Showing 4 changed files with 277 additions and 10 deletions.
diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj
@@ -38,4 +38,5 @@
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
+
 </Project>
diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs
@@ -1,6 +1,12 @@
+using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.Arcade.Common;
 using Microsoft.Arcade.Test.Common;
+using Microsoft.DotNet.Arcade.Test.Common;
 using Microsoft.DotNet.Build.Tasks.Feed.Model;
 using Microsoft.DotNet.Maestro.Client.Models;
 using Xunit;
@@ -40,7 +46,6 @@ public void PublishToSymbolServersTest(SymbolTargetType symbolTargetType , strin
             Assert.True(test.Count == 1);
         }
 
-
         [Fact]
         public void PublishToBothSymbolServerTest()
         {
@@ -139,5 +144,227 @@ public void PublishSymbolApiIsCalledTest()
                 false,
                 false).IsCompleted);
         }
+
+        [Fact]
+        public void DownloadFileAsyncSucceedsForValidUrl()
+        {
+            var buildEngine = new MockBuildEngine();
+            var publishTask = new PublishArtifactsInManifestV3
+            {
+                BuildEngine = buildEngine,
+            };
+
+            var testFile = Path.Combine("Symbols", "test.txt");
+            var responseContent = TestInputs.ReadAllBytes(testFile);
+            var response = new HttpResponseMessage(HttpStatusCode.OK)
+            {
+                Content = new ByteArrayContent(responseContent)
+            };
+
+            using HttpClient client = FakeHttpClient.WithResponses(response);
+            var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());
+
+            var test = publishTask.DownloadFileAsync(
+                client,
+                PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
+                "1234",
+                "test.txt",
+                path);
+
+            Assert.True(File.Exists(path));
+            publishTask.DeleteTemporaryFiles(path);
+            publishTask.DeleteTemporaryDirectory(path);
+        }
+
+        [Theory]
+        [InlineData(HttpStatusCode.BadRequest)]
+        [InlineData(HttpStatusCode.NotFound)]
+        [InlineData(HttpStatusCode.GatewayTimeout)]
+        public async Task DownloadFileAsyncFailsForInValidUrlTest(HttpStatusCode httpStatus)
+        {
+            var buildEngine = new MockBuildEngine();
+            var publishTask = new PublishArtifactsInManifestV3
+            {
+                BuildEngine = buildEngine,
+            };
+            var testFile = Path.Combine("Symbols", "test.txt");
+            var responseContent = TestInputs.ReadAllBytes(testFile);
+            publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 3, DelayBase = 1 };
+
+            var responses = new[]
+            {
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                },
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                },
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                }
+            };
+            using HttpClient client = FakeHttpClient.WithResponses(responses);
+            var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());
+
+            var actualError = await Assert.ThrowsAsync<Exception>(() =>
+                publishTask.DownloadFileAsync(
+                    client,
+                    PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
+                    "1234",
+                    "test.txt",
+                    path));
+            Assert.Contains($"Failed to download local file '{path}' after {publishTask.RetryHandler.MaxAttempts} attempts.  See inner exception for details,", actualError.Message);
+        }
+
+        [Theory]
+        [InlineData(HttpStatusCode.BadRequest)]
+        [InlineData(HttpStatusCode.NotFound)]
+        [InlineData(HttpStatusCode.GatewayTimeout)]
+        public async Task DownloadFailureWhenStatusCodeIsInvalid(HttpStatusCode httpStatus)
+        {
+            var buildEngine = new MockBuildEngine();
+            var publishTask = new PublishArtifactsInManifestV3
+            {
+                BuildEngine = buildEngine,
+            };
+            var testFile = Path.Combine("Symbols", "test.txt");
+            var responseContent = TestInputs.ReadAllBytes(testFile);
+            publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 3, DelayBase = 1 };
+
+            var responses = new[]
+            {
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                },
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                },
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                }
+            };
+            using HttpClient client = FakeHttpClient.WithResponses(responses);
+            var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());
+
+            var actualError = await Assert.ThrowsAsync<Exception>(() =>
+                publishTask.DownloadFileAsync(
+                    client,
+                    PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
+                    "1234",
+                    "test.txt",
+                    path));
+            Assert.Contains($"Failed to download local file '{path}' after {publishTask.RetryHandler.MaxAttempts} attempts.  See inner exception for details,", actualError.Message);
+        }
+
+        [Theory]
+        [InlineData(HttpStatusCode.BadRequest)]
+        [InlineData(HttpStatusCode.NotFound)]
+        [InlineData(HttpStatusCode.GatewayTimeout)]
+        public async Task DownloadFileSuccessfulAfterRetryTest(HttpStatusCode httpStatus)
+        {
+            var buildEngine = new MockBuildEngine();
+            var publishTask = new PublishArtifactsInManifestV3
+            {
+                BuildEngine = buildEngine,
+            };
+            var testFile = Path.Combine("Symbols", "test.txt");
+            var responseContent = TestInputs.ReadAllBytes(testFile);
+            publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 2, DelayBase = 1 };
+
+            var responses = new[]
+            {
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                },
+                new HttpResponseMessage(HttpStatusCode.OK)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                }
+            };
+            using HttpClient client = FakeHttpClient.WithResponses(responses);
+            var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());
+
+            await publishTask.DownloadFileAsync(
+                client,
+                PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
+                "1234",
+                "test.txt",
+                path);
+            Assert.True(File.Exists(path));
+            publishTask.DeleteTemporaryFiles(path);
+            publishTask.DeleteTemporaryDirectory(path);
+        }
+
+        [Theory]
+        [InlineData(PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts, "1")]
+        [InlineData(PublishArtifactsInManifestBase.ArtifactName.PackageArtifacts, "1234")]
+        public async Task GetContainerIdToDownloadArtifactAsync(PublishArtifactsInManifestBase.ArtifactName artifactName, string containerId)
+        {
+            var buildEngine = new MockBuildEngine();
+            var publishTask = new PublishArtifactsInManifestV3
+            {
+                BuildEngine = buildEngine,
+            };
+            publishTask.BuildId = "1243456";
+            var testPackageName = Path.Combine("Symbols", "test.txt");
+            var responseContent = TestInputs.ReadAllBytes(testPackageName);
+            var responses = new HttpResponseMessage(HttpStatusCode.OK)
+            {
+                Content = new ByteArrayContent(responseContent)
+            };
+
+            using HttpClient client = FakeHttpClient.WithResponses(responses);
+            var test = await publishTask.GetContainerIdAsync(
+                client,
+                artifactName);
+            Assert.Equal(containerId, test);
+        }
+
+        [Theory]
+        [InlineData(HttpStatusCode.BadRequest)]
+        [InlineData(HttpStatusCode.NotFound)]
+        public async Task ErrorAfterMaxRetriesToGetContainerId(HttpStatusCode httpStatus)
+        {
+            var buildEngine = new MockBuildEngine();
+            var publishTask = new PublishArtifactsInManifestV3
+            {
+                BuildEngine = buildEngine,
+            };
+            publishTask.BuildId = "1243456";
+            publishTask.RetryHandler = new ExponentialRetry() {MaxAttempts = 3, DelayBase = 1};
+
+            var testPackageName = Path.Combine("Symbols", "test.txt");
+            var responseContent = TestInputs.ReadAllBytes(testPackageName);
+            var responses = new[]
+            {
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                },
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                },
+                new HttpResponseMessage(httpStatus)
+                {
+                    Content = new ByteArrayContent(responseContent)
+                }
+            };
+
+            using HttpClient client = FakeHttpClient.WithResponses(responses);
+
+            var actualError = await Assert.ThrowsAsync<Exception>(() =>
+                publishTask.GetContainerIdAsync(
+                    client,
+                    PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts));
+            Assert.Contains($"Failed to get container id after {publishTask.RetryHandler.MaxAttempts} attempts.  See inner exception for details,", actualError.Message);
+        }
     }
 }
diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt
@@ -0,0 +1,35 @@
+{
+   "count":1,
+   "value":[
+      {
+         "id":2124,
+         "name":"PackageArtifacts",
+         "source":"testSource",
+         "resource":{
+            "type":"Container",
+            "data":"#/1234/PackageArtifacts",
+            "properties":{
+               "localpath":"D:\\workspace\\_work\\1\\s\\artifacts\\output\\packages",
+               "artifactsize":"7644905"
+            },
+            "url":"https://dev.azure.com/dnceng/url",
+            "downloadUrl":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=PackageArtifacts&api-version=6.0&%24format=zip"
+         }
+      },
+      {
+         "id":2123,
+         "name":"BlobArtifacts",
+         "source":"testSource",
+         "resource":{
+            "type":"Container",
+            "data":"#/1/BlobArtifacts",
+            "properties":{
+               "localpath":"D:\\workspace\\_work\\1\\s\\artifacts\\output\\packages",
+               "artifactsize":"7644905"
+            },
+            "url":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=BlobArtifacts&api-version=6.0",
+            "downloadUrl":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=PackageArtifacts&api-version=6.0&%24format=zip"
+         }
+      }
+   ]
+}
diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs
@@ -185,13 +185,13 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities
         /// </summary>
         public int RetryDelayMilliseconds { get; set; } = 5000;
 
-        public readonly ExponentialRetry RetryHandler = new ExponentialRetry
+        public ExponentialRetry RetryHandler = new ExponentialRetry
         {
             MaxAttempts = 5,
             DelayBase = 2.5 // 2.5 ^ 5 = ~1.5 minutes max wait between retries
         };
 
-        private enum ArtifactName
+        public enum ArtifactName
         {
             [Description("PackageArtifacts")]
             PackageArtifacts,
@@ -401,7 +401,9 @@ public async Task PublishSymbolsUsingStreamingAsync(
             Log.LogMessage(MessageImportance.High, 
                 $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false ");
             var symbolCategory = TargetFeedContentType.Symbols;
-            string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts);
+
+            using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
+            string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts);
 
             if (Log.HasLoggedErrors)
             {
@@ -830,7 +832,7 @@ public HttpClient CreateAzdoClient(
         /// </summary>
         /// <param name="artifactName">If it is PackageArtifacts or BlobArtifacts</param>
         /// <returns>ContainerId</returns>
-        private async Task<string> GetContainerIdAsync(ArtifactName artifactName)
+        public async Task<string> GetContainerIdAsync(HttpClient client, ArtifactName artifactName)
         {
             string uri =
                  $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}";
@@ -843,7 +845,6 @@ private async Task<string> GetContainerIdAsync(ArtifactName artifactName)
                     CancellationTokenSource timeoutTokenSource =
                         new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes));
 
-                    using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
                     using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri);
                     using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token);
 
@@ -892,7 +893,7 @@ private async Task<string> GetContainerIdAsync(ArtifactName artifactName)
         /// <param name="containerId">ContainerId where the packageArtifact and BlobArtifacts are stored</param>
         /// <param name="fileName">Name the file we are trying to download</param>
         /// <param name="path">Path where the file is being downloaded</param>
-        private async Task DownloadFileAsync(
+        public async Task DownloadFileAsync(
             HttpClient client,
             ArtifactName artifactName,
             string containerId,
@@ -1136,7 +1137,8 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync(
             TargetFeedConfig feedConfig,
             SemaphoreSlim clientThrottle)
         {
-            string containerId = await GetContainerIdAsync(ArtifactName.PackageArtifacts);
+            using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
+            string containerId = await GetContainerIdAsync(httpClient, ArtifactName.PackageArtifacts);
 
             if (Log.HasLoggedErrors)
             {
@@ -1491,7 +1493,8 @@ private async Task PublishBlobsUsingStreamingToAzDoNugetAsync(
             TargetFeedConfig feedConfig,
             SemaphoreSlim clientThrottle)
         {
-            string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts);
+            using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
+            string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts);
 
             if (Log.HasLoggedErrors)
             {
@@ -1654,7 +1657,8 @@ private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(
             TargetFeedConfig feedConfig,
             SemaphoreSlim clientThrottle)
         {
-            string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts);
+            using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
+            string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts);
 
             if (Log.HasLoggedErrors)
             {