Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tests for publishing #7346

Merged
13 commits merged into from
Jul 1, 2021
Merged

Tests for publishing #7346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
5c886b7
tests
epananth May 5, 2021
83f4a34
Merge branch 'main' of https://github.com/dotnet/arcade into tests-fo…
epananth May 5, 2021
4b162a3
remove unwanted using
epananth May 5, 2021
67cf778
Merge branch 'main' of https://github.com/dotnet/arcade into tests-fo…
epananth Jun 19, 2021
fbd126c
addressed feedback
epananth Jun 19, 2021
3cb3735
change response
epananth Jun 19, 2021
1d026d1
comments
epananth Jun 22, 2021
26b6641
Revert some changes
epananth Jun 22, 2021
00ad66d
Merge branch 'main' of https://github.com/dotnet/arcade into tests-fo…
epananth Jun 23, 2021
bf3d491
Merge branch 'main' of https://github.com/dotnet/arcade into tests-fo…
epananth Jun 29, 2021
fafc680
changed it cos, the test were timing out
epananth Jun 30, 2021
b1296c6
few nits
epananth Jun 30, 2021
8fd9025
review comment
epananth Jun 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/Microsoft.DotNet.Build.Tasks.Feed.Tests/Microsoft.DotNet.Build.Tasks.Feed.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,5 @@
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
</ItemGroup>

</Project>
229 changes: 228 additions & 1 deletion src/Microsoft.DotNet.Build.Tasks.Feed.Tests/PublishToSymbolServerTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Arcade.Common;
using Microsoft.Arcade.Test.Common;
using Microsoft.DotNet.Arcade.Test.Common;
using Microsoft.DotNet.Build.Tasks.Feed.Model;
using Microsoft.DotNet.Maestro.Client.Models;
using Xunit;
Expand Down Expand Up @@ -40,7 +46,6 @@ public void PublishToSymbolServersTest(SymbolTargetType symbolTargetType , strin
Assert.True(test.Count == 1);
}


[Fact]
public void PublishToBothSymbolServerTest()
{
Expand Down Expand Up @@ -139,5 +144,227 @@ public void PublishSymbolApiIsCalledTest()
false,
false).IsCompleted);
}

[Fact]
public void DownloadFileAsyncSucceedsForValidUrl()
{
var buildEngine = new MockBuildEngine();
var publishTask = new PublishArtifactsInManifestV3
{
BuildEngine = buildEngine,
};

var testFile = Path.Combine("Symbols", "test.txt");
var responseContent = TestInputs.ReadAllBytes(testFile);
var response = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ByteArrayContent(responseContent)
};

using HttpClient client = FakeHttpClient.WithResponses(response);
var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());

var test = publishTask.DownloadFileAsync(
client,
PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
"1234",
"test.txt",
path);

Assert.True(File.Exists(path));
publishTask.DeleteTemporaryFiles(path);
publishTask.DeleteTemporaryDirectory(path);
}

[Theory]
[InlineData(HttpStatusCode.BadRequest)]
[InlineData(HttpStatusCode.NotFound)]
[InlineData(HttpStatusCode.GatewayTimeout)]
public async Task DownloadFileAsyncFailsForInValidUrlTest(HttpStatusCode httpStatus)
{
var buildEngine = new MockBuildEngine();
var publishTask = new PublishArtifactsInManifestV3
{
BuildEngine = buildEngine,
};
var testFile = Path.Combine("Symbols", "test.txt");
var responseContent = TestInputs.ReadAllBytes(testFile);
publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 3, DelayBase = 1 };

var responses = new[]
{
new HttpResponseMessage(httpStatus)
{
Content = new ByteArrayContent(responseContent)
},
new HttpResponseMessage(httpStatus)
{
Content = new ByteArrayContent(responseContent)
},
new HttpResponseMessage(httpStatus)
{
Content = new ByteArrayContent(responseContent)
}
};
using HttpClient client = FakeHttpClient.WithResponses(responses);
var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());

var actualError = await Assert.ThrowsAsync<Exception>(() =>
publishTask.DownloadFileAsync(
client,
PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
"1234",
"test.txt",
path));
Assert.Contains($"Failed to download local file '{path}' after {publishTask.RetryHandler.MaxAttempts} attempts. See inner exception for details,", actualError.Message);
}

[Theory]
[InlineData(HttpStatusCode.BadRequest)]
[InlineData(HttpStatusCode.NotFound)]
[InlineData(HttpStatusCode.GatewayTimeout)]
public async Task DownloadFailureWhenStatusCodeIsInvalid(HttpStatusCode httpStatus)
{
var buildEngine = new MockBuildEngine();
var publishTask = new PublishArtifactsInManifestV3
{
BuildEngine = buildEngine,
};
var testFile = Path.Combine("Symbols", "test.txt");
var responseContent = TestInputs.ReadAllBytes(testFile);
publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 3, DelayBase = 1 };

var responses = new[]
{
new HttpResponseMessage(httpStatus)
epananth marked this conversation as resolved.
Show resolved Hide resolved
{
Content = new ByteArrayContent(responseContent)
},
new HttpResponseMessage(httpStatus)
{
Content = new ByteArrayContent(responseContent)
},
new HttpResponseMessage(httpStatus)
{
Content = new ByteArrayContent(responseContent)
}
};
using HttpClient client = FakeHttpClient.WithResponses(responses);
var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());

var actualError = await Assert.ThrowsAsync<Exception>(() =>
publishTask.DownloadFileAsync(
client,
PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
"1234",
"test.txt",
path));
Assert.Contains($"Failed to download local file '{path}' after {publishTask.RetryHandler.MaxAttempts} attempts. See inner exception for details,", actualError.Message);
}

[Theory]
[InlineData(HttpStatusCode.BadRequest)]
[InlineData(HttpStatusCode.NotFound)]
[InlineData(HttpStatusCode.GatewayTimeout)]
public async Task DownloadFileSuccessfulAfterRetryTest(HttpStatusCode httpStatus)
{
var buildEngine = new MockBuildEngine();
var publishTask = new PublishArtifactsInManifestV3
{
BuildEngine = buildEngine,
};
var testFile = Path.Combine("Symbols", "test.txt");
var responseContent = TestInputs.ReadAllBytes(testFile);
publishTask.RetryHandler = new ExponentialRetry() { MaxAttempts = 2, DelayBase = 1 };

var responses = new[]
{
new HttpResponseMessage(httpStatus)
{
Content = new ByteArrayContent(responseContent)
},
new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ByteArrayContent(responseContent)
}
};
using HttpClient client = FakeHttpClient.WithResponses(responses);
var path = TestInputs.GetFullPath(Guid.NewGuid().ToString());

await publishTask.DownloadFileAsync(
client,
PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts,
"1234",
"test.txt",
path);
Assert.True(File.Exists(path));
publishTask.DeleteTemporaryFiles(path);
publishTask.DeleteTemporaryDirectory(path);
}

[Theory]
[InlineData(PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts, "1")]
[InlineData(PublishArtifactsInManifestBase.ArtifactName.PackageArtifacts, "1234")]
public async Task GetContainerIdToDownloadArtifactAsync(PublishArtifactsInManifestBase.ArtifactName artifactName, string containerId)
{
var buildEngine = new MockBuildEngine();
var publishTask = new PublishArtifactsInManifestV3
{
BuildEngine = buildEngine,
};
publishTask.BuildId = "1243456";
var testPackageName = Path.Combine("Symbols", "test.txt");
var responseContent = TestInputs.ReadAllBytes(testPackageName);
var responses = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ByteArrayContent(responseContent)
};

using HttpClient client = FakeHttpClient.WithResponses(responses);
var test = await publishTask.GetContainerIdAsync(
client,
artifactName);
Assert.Equal(containerId, test);
}

[Theory]
[InlineData(HttpStatusCode.BadRequest)]
[InlineData(HttpStatusCode.NotFound)]
public async Task ErrorAfterMaxRetriesToGetContainerId(HttpStatusCode httpStatus)
{
var buildEngine = new MockBuildEngine();
var publishTask = new PublishArtifactsInManifestV3
{
BuildEngine = buildEngine,
};
publishTask.BuildId = "1243456";
publishTask.RetryHandler = new ExponentialRetry() {MaxAttempts = 3, DelayBase = 1};

var testPackageName = Path.Combine("Symbols", "test.txt");
var responseContent = TestInputs.ReadAllBytes(testPackageName);
var responses = new[]
{
new HttpResponseMessage(httpStatus)
{
Content = new ByteArrayContent(responseContent)
},
new HttpResponseMessage(httpStatus)
{
Content = new ByteArrayContent(responseContent)
},
new HttpResponseMessage(httpStatus)
{
Content = new ByteArrayContent(responseContent)
}
};

using HttpClient client = FakeHttpClient.WithResponses(responses);

var actualError = await Assert.ThrowsAsync<Exception>(() =>
publishTask.GetContainerIdAsync(
client,
PublishArtifactsInManifestBase.ArtifactName.BlobArtifacts));
Assert.Contains($"Failed to get container id after {publishTask.RetryHandler.MaxAttempts} attempts. See inner exception for details,", actualError.Message);
}
}
}
35 changes: 35 additions & 0 deletions src/Microsoft.DotNet.Build.Tasks.Feed.Tests/TestInputs/Symbols/test.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
epananth marked this conversation as resolved.
Show resolved Hide resolved
"count":1,
"value":[
{
"id":2124,
"name":"PackageArtifacts",
"source":"testSource",
"resource":{
"type":"Container",
"data":"#/1234/PackageArtifacts",
"properties":{
"localpath":"D:\\workspace\\_work\\1\\s\\artifacts\\output\\packages",
"artifactsize":"7644905"
},
"url":"https://dev.azure.com/dnceng/url",
"downloadUrl":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=PackageArtifacts&api-version=6.0&%24format=zip"
}
},
{
"id":2123,
"name":"BlobArtifacts",
"source":"testSource",
"resource":{
"type":"Container",
"data":"#/1/BlobArtifacts",
"properties":{
"localpath":"D:\\workspace\\_work\\1\\s\\artifacts\\output\\packages",
"artifactsize":"7644905"
},
"url":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=BlobArtifacts&api-version=6.0",
"downloadUrl":"https://dev.azure.com/dnceng/_apis/build/builds/buildId/artifacts?artifactName=PackageArtifacts&api-version=6.0&%24format=zip"
}
}
]
}
22 changes: 13 additions & 9 deletions src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,13 +185,13 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities
/// </summary>
public int RetryDelayMilliseconds { get; set; } = 5000;

public readonly ExponentialRetry RetryHandler = new ExponentialRetry
public ExponentialRetry RetryHandler = new ExponentialRetry
epananth marked this conversation as resolved.
Show resolved Hide resolved
{
MaxAttempts = 5,
DelayBase = 2.5 // 2.5 ^ 5 = ~1.5 minutes max wait between retries
};

private enum ArtifactName
public enum ArtifactName
{
[Description("PackageArtifacts")]
PackageArtifacts,
Expand Down Expand Up @@ -401,7 +401,9 @@ public async Task PublishSymbolsUsingStreamingAsync(
Log.LogMessage(MessageImportance.High,
$"Performing symbol publishing... \nExpirationInDays : {ExpirationInDays} \nConvertPortablePdbsToWindowsPdb : false \ndryRun: false ");
var symbolCategory = TargetFeedContentType.Symbols;
string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts);

using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts);

if (Log.HasLoggedErrors)
{
Expand Down Expand Up @@ -830,7 +832,7 @@ public HttpClient CreateAzdoClient(
/// </summary>
/// <param name="artifactName">If it is PackageArtifacts or BlobArtifacts</param>
/// <returns>ContainerId</returns>
private async Task<string> GetContainerIdAsync(ArtifactName artifactName)
public async Task<string> GetContainerIdAsync(HttpClient client, ArtifactName artifactName)
epananth marked this conversation as resolved.
Show resolved Hide resolved
{
string uri =
$"{AzureDevOpsBaseUrl}/{AzureDevOpsOrg}/{AzureProject}/_apis/build/builds/{BuildId}/artifacts?api-version={AzureDevOpsFeedsApiVersion}";
Expand All @@ -843,7 +845,6 @@ private async Task<string> GetContainerIdAsync(ArtifactName artifactName)
CancellationTokenSource timeoutTokenSource =
new CancellationTokenSource(TimeSpan.FromMinutes(TimeoutInMinutes));

using HttpClient client = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
using HttpRequestMessage getMessage = new HttpRequestMessage(HttpMethod.Get, uri);
using HttpResponseMessage response = await client.GetAsync(uri, timeoutTokenSource.Token);

Expand Down Expand Up @@ -892,7 +893,7 @@ private async Task<string> GetContainerIdAsync(ArtifactName artifactName)
/// <param name="containerId">ContainerId where the packageArtifact and BlobArtifacts are stored</param>
/// <param name="fileName">Name the file we are trying to download</param>
/// <param name="path">Path where the file is being downloaded</param>
private async Task DownloadFileAsync(
public async Task DownloadFileAsync(
HttpClient client,
ArtifactName artifactName,
string containerId,
Expand Down Expand Up @@ -1136,7 +1137,8 @@ private async Task PublishPackagesUsingStreamingToAzdoNugetAsync(
TargetFeedConfig feedConfig,
SemaphoreSlim clientThrottle)
{
string containerId = await GetContainerIdAsync(ArtifactName.PackageArtifacts);
using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
string containerId = await GetContainerIdAsync(httpClient, ArtifactName.PackageArtifacts);

if (Log.HasLoggedErrors)
{
Expand Down Expand Up @@ -1491,7 +1493,8 @@ private async Task PublishBlobsUsingStreamingToAzDoNugetAsync(
TargetFeedConfig feedConfig,
SemaphoreSlim clientThrottle)
{
string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts);
using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts);

if (Log.HasLoggedErrors)
{
Expand Down Expand Up @@ -1654,7 +1657,8 @@ private async Task PublishBlobsToAzureStorageNugetUsingStreamingPublishingAsync(
TargetFeedConfig feedConfig,
SemaphoreSlim clientThrottle)
{
string containerId = await GetContainerIdAsync(ArtifactName.BlobArtifacts);
using HttpClient httpClient = CreateAzdoClient(AzureDevOpsOrg, false, AzureProject);
string containerId = await GetContainerIdAsync(httpClient, ArtifactName.BlobArtifacts);

if (Log.HasLoggedErrors)
{
Expand Down