From c206f6f604fa6369604000d3df409f7136040877 Mon Sep 17 00:00:00 2001
From: Epsitha Ananth <47157394+epananth@users.noreply.github.com>
Date: Wed, 30 Jun 2021 20:40:08 -0700
Subject: [PATCH] Tests for publishing (#7346)
* tests
* remove unwanted using
* addressed feedback
* change response
* comments
* Revert some changes
* changed it cos, the test were timing out
* few nits
* review comment
---
...osoft.DotNet.Build.Tasks.Feed.Tests.csproj | 1 +
.../PublishToSymbolServerTest.cs | 229 +++++++++++++++++-
.../TestInputs/Symbols/test.txt | 35 +++
.../src/PublishArtifactsInManifestBase.cs | 22 +-
4 files changed, 277 insertions(+), 10 deletions(-)
create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt
diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj
index d100e0d8321..548fac7ecdb 100644
--- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj
+++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj
@@ -38,4 +38,5 @@
Always
+
diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs
index 4b2784921bb..9e08f09b226 100644
--- a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs
+++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs
@@ -1,6 +1,12 @@
+using System;
using System.Collections.Generic;
using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.Arcade.Common;
using Microsoft.Arcade.Test.Common;
+using Microsoft.DotNet.Arcade.Test.Common;
using Microsoft.DotNet.Build.Tasks.Feed.Model;
using Microsoft.DotNet.Maestro.Client.Models;
using Xunit;
@@ -40,7 +46,6 @@ public void PublishToSymbolServersTest(SymbolTargetType symbolTargetType , strin
Assert.True(test.Count == 1);
}
-
[Fact]
public void PublishToBothSymbolServerTest()
{
@@ -139,5 +144,227 @@ public void PublishSymbolApiIsCalledTest()
false,
false).IsCompleted);
}
+
+ [Fact]
+ public void DownloadFileAsyncSucceedsForValidUrl()
+ {
+ var buildEngine = new MockBuildEngine();
+ var publishTask = new PublishArtifactsInManifestV3
+ {
+ BuildEngine = buildEngine,
+ };
+
+ var testFile = Path.Combine("Symbols", "test.txt");
+ var responseContent = TestInputs.ReadAllBytes(testFile);
+ var response = new HttpResponseMessage(HttpStatusCode.OK)
+ {
+ Content = new ByteArrayContent(responseContent)
+ };
+
+ using HttpClient client = FakeHttpClient.WithResponses(response);
+ var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());
+
+ var test = publishTask.DownloadFileAsync(
+ client,
+ PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
+ "1234",
+ "test.txt",
+ path);
+
+ Assert.True(File.Exists(path));
+ publishTask.DeleteTemporaryFiles(path);
+ publishTask.DeleteTemporaryDirectory(path);
+ }
+
+ [Theory]
+ [InlineData(HttpStatusCode.BadRequest)]
+ [InlineData(HttpStatusCode.NotFound)]
+ [InlineData(HttpStatusCode.GatewayTimeout)]
+ public async Task DownloadFileAsyncFailsForInValidUrlTest(HttpStatusCode httpStatus)
+ {
+ var buildEngine = new MockBuildEngine();
+ var publishTask = new PublishArtifactsInManifestV3
+ {
+ BuildEngine = buildEngine,
+ };
+ var testFile = Path.Combine("Symbols", "test.txt");
+ var responseContent = TestInputs.ReadAllBytes(testFile);
+ publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 3, DelayBase = 1 };
+
+ var responses = new[]
+ {
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ },
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ },
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ }
+ };
+ using HttpClient client = FakeHttpClient.WithResponses(responses);
+ var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());
+
+ var actualError = await Assert.ThrowsAsync(() =>
+ publishTask.DownloadFileAsync(
+ client,
+ PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
+ "1234",
+ "test.txt",
+ path));
+ Assert.Contains($"Failed to download local file '{path}' after {publishTask.RetryHandler.MaxAttempts} attempts. See inner exception for details,", actualError.Message);
+ }
+
+ [Theory]
+ [InlineData(HttpStatusCode.BadRequest)]
+ [InlineData(HttpStatusCode.NotFound)]
+ [InlineData(HttpStatusCode.GatewayTimeout)]
+ public async Task DownloadFailureWhenStatusCodeIsInvalid(HttpStatusCode httpStatus)
+ {
+ var buildEngine = new MockBuildEngine();
+ var publishTask = new PublishArtifactsInManifestV3
+ {
+ BuildEngine = buildEngine,
+ };
+ var testFile = Path.Combine("Symbols", "test.txt");
+ var responseContent = TestInputs.ReadAllBytes(testFile);
+ publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 3, DelayBase = 1 };
+
+ var responses = new[]
+ {
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ },
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ },
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ }
+ };
+ using HttpClient client = FakeHttpClient.WithResponses(responses);
+ var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());
+
+ var actualError = await Assert.ThrowsAsync(() =>
+ publishTask.DownloadFileAsync(
+ client,
+ PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
+ "1234",
+ "test.txt",
+ path));
+ Assert.Contains($"Failed to download local file '{path}' after {publishTask.RetryHandler.MaxAttempts} attempts. See inner exception for details,", actualError.Message);
+ }
+
+ [Theory]
+ [InlineData(HttpStatusCode.BadRequest)]
+ [InlineData(HttpStatusCode.NotFound)]
+ [InlineData(HttpStatusCode.GatewayTimeout)]
+ public async Task DownloadFileSuccessfulAfterRetryTest(HttpStatusCode httpStatus)
+ {
+ var buildEngine = new MockBuildEngine();
+ var publishTask = new PublishArtifactsInManifestV3
+ {
+ BuildEngine = buildEngine,
+ };
+ var testFile = Path.Combine("Symbols", "test.txt");
+ var responseContent = TestInputs.ReadAllBytes(testFile);
+ publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 2, DelayBase = 1 };
+
+ var responses = new[]
+ {
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ },
+ new HttpResponseMessage(HttpStatusCode.OK)
+ {
+ Content = new ByteArrayContent(responseContent)
+ }
+ };
+ using HttpClient client = FakeHttpClient.WithResponses(responses);
+ var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());
+
+ await publishTask.DownloadFileAsync(
+ client,
+ PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
+ "1234",
+ "test.txt",
+ path);
+ Assert.True(File.Exists(path));
+ publishTask.DeleteTemporaryFiles(path);
+ publishTask.DeleteTemporaryDirectory(path);
+ }
+
+ [Theory]
+ [InlineData(PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts, "1")]
+ [InlineData(PublishArtifactsInManifestBase.ArtifactName.PackageArtifacts, "1234")]
+ public async Task GetContainerIdToDownloadArtifactAsync(PublishArtifactsInManifestBase.ArtifactName artifactName, string containerId)
+ {
+ var buildEngine = new MockBuildEngine();
+ var publishTask = new PublishArtifactsInManifestV3
+ {
+ BuildEngine = buildEngine,
+ };
+ publishTask.BuildId = "1243456";
+ var testPackageName = Path.Combine("Symbols", "test.txt");
+ var responseContent = TestInputs.ReadAllBytes(testPackageName);
+ var responses = new HttpResponseMessage(HttpStatusCode.OK)
+ {
+ Content = new ByteArrayContent(responseContent)
+ };
+
+ using HttpClient client = FakeHttpClient.WithResponses(responses);
+ var test = await publishTask.GetContainerIdAsync(
+ client,
+ artifactName);
+ Assert.Equal(containerId, test);
+ }
+
+ [Theory]
+ [InlineData(HttpStatusCode.BadRequest)]
+ [InlineData(HttpStatusCode.NotFound)]
+ public async Task ErrorAfterMaxRetriesToGetContainerId(HttpStatusCode httpStatus)
+ {
+ var buildEngine = new MockBuildEngine();
+ var publishTask = new PublishArtifactsInManifestV3
+ {
+ BuildEngine = buildEngine,
+ };
+ publishTask.BuildId = "1243456";
+ publishTask.RetryHandler = new ExponentialRetry() {MaxAttempts = 3, DelayBase = 1};
+
+ var testPackageName = Path.Combine("Symbols", "test.txt");
+ var responseContent = TestInputs.ReadAllBytes(testPackageName);
+ var responses = new[]
+ {
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ },
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ },
+ new HttpResponseMessage(httpStatus)
+ {
+ Content = new ByteArrayContent(responseContent)
+ }
+ };
+
+ using HttpClient client = FakeHttpClient.WithResponses(responses);
+
+ var actualError = await Assert.ThrowsAsync(() =>
+ publishTask.GetContainerIdAsync(
+ client,
+ PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts));
+ Assert.Contains($"Failed to get container id after {publishTask.RetryHandler.MaxAttempts} attempts. See inner exception for details,", actualError.Message);
+ }
}
}
diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt
new file mode 100644
index 00000000000..5a604136aeb
--- /dev/null
+++ b/src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt
@@ -0,0 +1,35 @@
+{
+ "count":1,
+ "value":[
+ {
+ "id":2124,
+ "name":"PackageArtifacts",
+ "source":"testSource",
+ "resource":{
+ "type":"Container",
+ "data":"#/1234/PackageArtifacts",
+ "properties":{
+ "localpath":"D:\\workspace\\_work\\1\\s\\artifacts\\output\\packages",
+ "artifactsize":"7644905"
+ },
+ "url":"https://dev.azure.com/dnceng/url",
+ "downloadUrl":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=PackageArtifacts&api-version=6.0&%24format=zip"
+ }
+ },
+ {
+ "id":2123,
+ "name":"BlobArtifacts",
+ "source":"testSource",
+ "resource":{
+ "type":"Container",
+ "data":"#/1/BlobArtifacts",
+ "properties":{
+ "localpath":"D:\\workspace\\_work\\1\\s\\artifacts\\output\\packages",
+ "artifactsize":"7644905"
+ },
+ "url":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=BlobArtifacts&api-version=6.0",
+ "downloadUrl":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=PackageArtifacts&api-version=6.0&%24format=zip"
+ }
+ }
+ ]
+}
diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs
index ddbbadb383a..8670e955c83 100644
--- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs
+++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs
@@ -185,13 +185,13 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities
///
public int RetryDelayMilliseconds { get; set; } = 5000;
- public readonly ExponentialRetry RetryHandler = new ExponentialRetry
+ public ExponentialRetry RetryHandler = new ExponentialRetry
{
MaxAttempts = 5,
DelayBase = 2.5 // 2.5 ^ 5 = ~1.5 minutes max wait between retries
};
- private enum ArtifactName
+ public enum ArtifactName
{
[Description("PackageArtifacts")]
PackageArtifacts,
@@ -401,7 +401,9 @@ public async Task PublishSymbolsUsingStreamingAsync(
Log.LogMessage(MessageImportance.High,
$"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false ");
var symbolCategory = TargetFeedContentType.Symbols;
- string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts);
+
+ using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
+ string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts);
if (Log.HasLoggedErrors)
{
@@ -830,7 +832,7 @@ public HttpClient CreateAzdoClient(
///
/// If it is PackageArtifacts or BlobArtifacts
/// ContainerId
- private async Task GetContainerIdAsync(ArtifactName artifactName)
+ public async Task GetContainerIdAsync(HttpClient client, ArtifactName artifactName)
{
string uri =
$"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}";
@@ -843,7 +845,6 @@ private async Task GetContainerIdAsync(ArtifactName artifactName)
CancellationTokenSource timeoutTokenSource =
new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes));
- using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri);
using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token);
@@ -892,7 +893,7 @@ private async Task GetContainerIdAsync(ArtifactName artifactName)
/// ContainerId where the packageArtifact and BlobArtifacts are stored
/// Name the file we are trying to download
/// Path where the file is being downloaded
- private async Task DownloadFileAsync(
+ public async Task DownloadFileAsync(
HttpClient client,
ArtifactName artifactName,
string containerId,
@@ -1136,7 +1137,8 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync(
TargetFeedConfig feedConfig,
SemaphoreSlim clientThrottle)
{
- string containerId = await GetContainerIdAsync(ArtifactName.PackageArtifacts);
+ using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
+ string containerId = await GetContainerIdAsync(httpClient, ArtifactName.PackageArtifacts);
if (Log.HasLoggedErrors)
{
@@ -1491,7 +1493,8 @@ private async Task PublishBlobsUsingStreamingToAzDoNugetAsync(
TargetFeedConfig feedConfig,
SemaphoreSlim clientThrottle)
{
- string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts);
+ using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
+ string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts);
if (Log.HasLoggedErrors)
{
@@ -1654,7 +1657,8 @@ private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(
TargetFeedConfig feedConfig,
SemaphoreSlim clientThrottle)
{
- string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts);
+ using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
+ string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts);
if (Log.HasLoggedErrors)
{