From c206f6f604fa6369604000d3df409f7136040877 Mon Sep 17 00:00:00 2001 From: Epsitha Ananth <47157394+epananth@users.noreply.github.com> Date: Wed, 30 Jun 2021 20:40:08 -0700 Subject: [PATCH] Tests for publishing (#7346) * tests * remove unwanted using * addressed feedback * change response * comments * Revert some changes * changed it cos, the test were timing out * few nits * review comment --- ...osoft.DotNet.Build.Tasks.Feed.Tests.csproj | 1 + .../PublishToSymbolServerTest.cs | 229 +++++++++++++++++- .../TestInputs/Symbols/test.txt | 35 +++ .../src/PublishArtifactsInManifestBase.cs | 22 +- 4 files changed, 277 insertions(+), 10 deletions(-) create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj index d100e0d8321..548fac7ecdb 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj @@ -38,4 +38,5 @@ Always + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs index 4b2784921bb..9e08f09b226 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs @@ -1,6 +1,12 @@ +using System; using System.Collections.Generic; using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Arcade.Common; using Microsoft.Arcade.Test.Common; +using Microsoft.DotNet.Arcade.Test.Common; using Microsoft.DotNet.Build.Tasks.Feed.Model; using Microsoft.DotNet.Maestro.Client.Models; using Xunit; @@ -40,7 +46,6 @@ public void PublishToSymbolServersTest(SymbolTargetType symbolTargetType , strin Assert.True(test.Count == 1); } - [Fact] public void PublishToBothSymbolServerTest() { @@ -139,5 +144,227 @@ public void PublishSymbolApiIsCalledTest() false, false).IsCompleted); } + + [Fact] + public void DownloadFileAsyncSucceedsForValidUrl() + { + var buildEngine = new MockBuildEngine(); + var publishTask = new PublishArtifactsInManifestV3 + { + BuildEngine = buildEngine, + }; + + var testFile = Path.Combine("Symbols", "test.txt"); + var responseContent = TestInputs.ReadAllBytes(testFile); + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(responseContent) + }; + + using HttpClient client = FakeHttpClient.WithResponses(response); + var path = TestInputs.GetFullPath(Guid.NewGuid().ToString()); + + var test = publishTask.DownloadFileAsync( + client, + PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts, + "1234", + "test.txt", + path); + + Assert.True(File.Exists(path)); + publishTask.DeleteTemporaryFiles(path); + publishTask.DeleteTemporaryDirectory(path); + } + + [Theory] + [InlineData(HttpStatusCode.BadRequest)] + [InlineData(HttpStatusCode.NotFound)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task DownloadFileAsyncFailsForInValidUrlTest(HttpStatusCode httpStatus) + { + var buildEngine = new MockBuildEngine(); + var publishTask = new PublishArtifactsInManifestV3 + { + BuildEngine = buildEngine, + }; + var testFile = Path.Combine("Symbols", "test.txt"); + var responseContent = TestInputs.ReadAllBytes(testFile); + publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 3, DelayBase = 1 }; + + var responses = new[] + { + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + }, + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + }, + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + } + }; + using HttpClient client = FakeHttpClient.WithResponses(responses); + var path = TestInputs.GetFullPath(Guid.NewGuid().ToString()); + + var actualError = await Assert.ThrowsAsync(() => + publishTask.DownloadFileAsync( + client, + PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts, + "1234", + "test.txt", + path)); + Assert.Contains($"Failed to download local file '{path}' after {publishTask.RetryHandler.MaxAttempts} attempts. See inner exception for details,", actualError.Message); + } + + [Theory] + [InlineData(HttpStatusCode.BadRequest)] + [InlineData(HttpStatusCode.NotFound)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task DownloadFailureWhenStatusCodeIsInvalid(HttpStatusCode httpStatus) + { + var buildEngine = new MockBuildEngine(); + var publishTask = new PublishArtifactsInManifestV3 + { + BuildEngine = buildEngine, + }; + var testFile = Path.Combine("Symbols", "test.txt"); + var responseContent = TestInputs.ReadAllBytes(testFile); + publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 3, DelayBase = 1 }; + + var responses = new[] + { + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + }, + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + }, + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + } + }; + using HttpClient client = FakeHttpClient.WithResponses(responses); + var path = TestInputs.GetFullPath(Guid.NewGuid().ToString()); + + var actualError = await Assert.ThrowsAsync(() => + publishTask.DownloadFileAsync( + client, + PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts, + "1234", + "test.txt", + path)); + Assert.Contains($"Failed to download local file '{path}' after {publishTask.RetryHandler.MaxAttempts} attempts. See inner exception for details,", actualError.Message); + } + + [Theory] + [InlineData(HttpStatusCode.BadRequest)] + [InlineData(HttpStatusCode.NotFound)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public async Task DownloadFileSuccessfulAfterRetryTest(HttpStatusCode httpStatus) + { + var buildEngine = new MockBuildEngine(); + var publishTask = new PublishArtifactsInManifestV3 + { + BuildEngine = buildEngine, + }; + var testFile = Path.Combine("Symbols", "test.txt"); + var responseContent = TestInputs.ReadAllBytes(testFile); + publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 2, DelayBase = 1 }; + + var responses = new[] + { + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + }, + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(responseContent) + } + }; + using HttpClient client = FakeHttpClient.WithResponses(responses); + var path = TestInputs.GetFullPath(Guid.NewGuid().ToString()); + + await publishTask.DownloadFileAsync( + client, + PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts, + "1234", + "test.txt", + path); + Assert.True(File.Exists(path)); + publishTask.DeleteTemporaryFiles(path); + publishTask.DeleteTemporaryDirectory(path); + } + + [Theory] + [InlineData(PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts, "1")] + [InlineData(PublishArtifactsInManifestBase.ArtifactName.PackageArtifacts, "1234")] + public async Task GetContainerIdToDownloadArtifactAsync(PublishArtifactsInManifestBase.ArtifactName artifactName, string containerId) + { + var buildEngine = new MockBuildEngine(); + var publishTask = new PublishArtifactsInManifestV3 + { + BuildEngine = buildEngine, + }; + publishTask.BuildId = "1243456"; + var testPackageName = Path.Combine("Symbols", "test.txt"); + var responseContent = TestInputs.ReadAllBytes(testPackageName); + var responses = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(responseContent) + }; + + using HttpClient client = FakeHttpClient.WithResponses(responses); + var test = await publishTask.GetContainerIdAsync( + client, + artifactName); + Assert.Equal(containerId, test); + } + + [Theory] + [InlineData(HttpStatusCode.BadRequest)] + [InlineData(HttpStatusCode.NotFound)] + public async Task ErrorAfterMaxRetriesToGetContainerId(HttpStatusCode httpStatus) + { + var buildEngine = new MockBuildEngine(); + var publishTask = new PublishArtifactsInManifestV3 + { + BuildEngine = buildEngine, + }; + publishTask.BuildId = "1243456"; + publishTask.RetryHandler = new ExponentialRetry() {MaxAttempts = 3, DelayBase = 1}; + + var testPackageName = Path.Combine("Symbols", "test.txt"); + var responseContent = TestInputs.ReadAllBytes(testPackageName); + var responses = new[] + { + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + }, + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + }, + new HttpResponseMessage(httpStatus) + { + Content = new ByteArrayContent(responseContent) + } + }; + + using HttpClient client = FakeHttpClient.WithResponses(responses); + + var actualError = await Assert.ThrowsAsync(() => + publishTask.GetContainerIdAsync( + client, + PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts)); + Assert.Contains($"Failed to get container id after {publishTask.RetryHandler.MaxAttempts} attempts. See inner exception for details,", actualError.Message); + } } } diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt new file mode 100644 index 00000000000..5a604136aeb --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt @@ -0,0 +1,35 @@ +{ + "count":1, + "value":[ + { + "id":2124, + "name":"PackageArtifacts", + "source":"testSource", + "resource":{ + "type":"Container", + "data":"#/1234/PackageArtifacts", + "properties":{ + "localpath":"D:\\workspace\\_work\\1\\s\\artifacts\\output\\packages", + "artifactsize":"7644905" + }, + "url":"https://dev.azure.com/dnceng/url", + "downloadUrl":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=PackageArtifacts&api-version=6.0&%24format=zip" + } + }, + { + "id":2123, + "name":"BlobArtifacts", + "source":"testSource", + "resource":{ + "type":"Container", + "data":"#/1/BlobArtifacts", + "properties":{ + "localpath":"D:\\workspace\\_work\\1\\s\\artifacts\\output\\packages", + "artifactsize":"7644905" + }, + "url":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=BlobArtifacts&api-version=6.0", + "downloadUrl":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=PackageArtifacts&api-version=6.0&%24format=zip" + } + } + ] +} diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs index ddbbadb383a..8670e955c83 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs @@ -185,13 +185,13 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities /// public int RetryDelayMilliseconds { get; set; } = 5000; - public readonly ExponentialRetry RetryHandler = new ExponentialRetry + public ExponentialRetry RetryHandler = new ExponentialRetry { MaxAttempts = 5, DelayBase = 2.5 // 2.5 ^ 5 = ~1.5 minutes max wait between retries }; - private enum ArtifactName + public enum ArtifactName { [Description("PackageArtifacts")] PackageArtifacts, @@ -401,7 +401,9 @@ public async Task PublishSymbolsUsingStreamingAsync( Log.LogMessage(MessageImportance.High, $"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false "); var symbolCategory = TargetFeedContentType.Symbols; - string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); + + using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject); + string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts); if (Log.HasLoggedErrors) { @@ -830,7 +832,7 @@ public HttpClient CreateAzdoClient( /// /// If it is PackageArtifacts or BlobArtifacts /// ContainerId - private async Task GetContainerIdAsync(ArtifactName artifactName) + public async Task GetContainerIdAsync(HttpClient client, ArtifactName artifactName) { string uri = $"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}"; @@ -843,7 +845,6 @@ private async Task GetContainerIdAsync(ArtifactName artifactName) CancellationTokenSource timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes)); - using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject); using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri); using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token); @@ -892,7 +893,7 @@ private async Task GetContainerIdAsync(ArtifactName artifactName) /// ContainerId where the packageArtifact and BlobArtifacts are stored /// Name the file we are trying to download /// Path where the file is being downloaded - private async Task DownloadFileAsync( + public async Task DownloadFileAsync( HttpClient client, ArtifactName artifactName, string containerId, @@ -1136,7 +1137,8 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync( TargetFeedConfig feedConfig, SemaphoreSlim clientThrottle) { - string containerId = await GetContainerIdAsync(ArtifactName.PackageArtifacts); + using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject); + string containerId = await GetContainerIdAsync(httpClient, ArtifactName.PackageArtifacts); if (Log.HasLoggedErrors) { @@ -1491,7 +1493,8 @@ private async Task PublishBlobsUsingStreamingToAzDoNugetAsync( TargetFeedConfig feedConfig, SemaphoreSlim clientThrottle) { - string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); + using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject); + string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts); if (Log.HasLoggedErrors) { @@ -1654,7 +1657,8 @@ private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync( TargetFeedConfig feedConfig, SemaphoreSlim clientThrottle) { - string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts); + using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject); + string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts); if (Log.HasLoggedErrors) {