Skip to content

Commit

Permalink
Upped timeout and added logs for downloading and publishing operation (…
Browse files Browse the repository at this point in the history 
…#7351)

* increased http timeout and added more logging around download and publishing operations
  • Loading branch information
@epananth
epananth authored May 6, 2021
1 parent 3094203 commit f1e67a4
Showing 1 changed file with 34 additions and 3 deletions.
37 changes: 34 additions & 3 deletions src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
Expand Down Expand Up @@ -191,7 +192,7 @@ private enum ArtifactName
BlobArtifacts
}

private int TimeoutInSeconds = 180;
private int TimeoutInSeconds = 300;

public override bool Execute()
{
Expand Down Expand Up @@ -418,6 +419,7 @@ public async Task PublishSymbolsUsingStreamingAsync(
{
using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true))
{
client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds);
await Task.WhenAll(symbolsToPublish.Select(async symbol =>
{
try
Expand All @@ -427,13 +429,18 @@ await Task.WhenAll(symbolsToPublish.Select(async symbol =>
string localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol);
symbolLog.AppendLine($"Downloading symbol : {symbol} to {localSymbolPath}");

Stopwatch gatherDownloadTime = Stopwatch.StartNew();
await DownloadFileAsync(
client,
ArtifactName.BlobArtifacts,
containerId,
symbol,
localSymbolPath);

gatherDownloadTime.Stop();
symbolLog.AppendLine($"Time taken to download file to '{localSymbolPath}' is {gatherDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)");
symbolLog.AppendLine($"Successfully downloaded symbol : {symbol} to {localSymbolPath}");

List<string> symbolFiles = new List<string>();
symbolFiles.Add(localSymbolPath);
symbolLog.AppendLine($"Uploading symbol file '{string.Join(",", symbolFiles)}'");
Expand All @@ -443,6 +450,7 @@ await DownloadFileAsync(
var serverPath = server.Key;
var token = server.Value;
symbolLog.AppendLine($"Publishing symbol file {symbol} to {serverPath}:");
Stopwatch gatherSymbolPublishingTime = Stopwatch.StartNew();

try
{
Expand All @@ -465,6 +473,10 @@ await PublishSymbolsHelper.PublishAsync(
{
Log.LogError(ex.Message);
}

gatherSymbolPublishingTime.Stop();
symbolLog.AppendLine(
$"Symbol publishing for {symbol} took {gatherSymbolPublishingTime.ElapsedMilliseconds / 1000.0} (seconds)");
}

DeleteTemporaryDirectory(temporarySymbolsDirectory);
Expand Down Expand Up @@ -1148,6 +1160,7 @@ await Task.WhenAll(packagesToPublish.Select(async package =>
Log.LogMessage(MessageImportance.Low,
$"Downloading package : {packageFilename} to {localPackagePath}");

Stopwatch gatherPackageDownloadTime = Stopwatch.StartNew();
await DownloadFileAsync(
client,
ArtifactName.PackageArtifacts,
Expand All @@ -1162,6 +1175,8 @@ await DownloadFileAsync(
return;
}

gatherPackageDownloadTime.Stop();
Log.LogMessage(MessageImportance.Low, $"Time taken to download file to '{localPackagePath}' is {gatherPackageDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)");
Log.LogMessage(MessageImportance.Low,
$"Successfully downloaded package : {packageFilename} to {localPackagePath}");

Expand All @@ -1183,7 +1198,10 @@ await DownloadFileAsync(
Convert.ToBase64String(
Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token))));

Stopwatch gatherPackagePublishingTime = Stopwatch.StartNew();
await PushPackageToNugetFeed(httpClient, feedConfig, localPackagePath, package.Id, package.Version);
gatherPackagePublishingTime.Stop();
Log.LogMessage(MessageImportance.Low,$"Publishing package {localPackagePath} took {gatherPackagePublishingTime.ElapsedMilliseconds / 1000.0} (seconds)");

DeleteTemporaryDirectory(localPackagePath);
}
Expand Down Expand Up @@ -1243,7 +1261,7 @@ public async Task PushNugetPackagesAsync<T>(
using (HttpClient httpClient = new HttpClient(new HttpClientHandler
{CheckCertificateRevocationList = true}))
{
httpClient.Timeout = TimeSpan.FromSeconds(180);
httpClient.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds);
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(
"Basic",
Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token))));
Expand Down Expand Up @@ -1500,6 +1518,7 @@ await Task.WhenAll(blobsToPublish.Select(async blob =>
string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName);
Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}");

Stopwatch gatherBlobDownloadTime = Stopwatch.StartNew();
await DownloadFileAsync(
client,
ArtifactName.BlobArtifacts,
Expand All @@ -1511,6 +1530,8 @@ await DownloadFileAsync(
{
Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'");
}
gatherBlobDownloadTime.Stop();
Log.LogMessage(MessageImportance.Low, $"Time taken to download file to '{localBlobPath}' is {gatherBlobDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)");

Log.LogMessage(MessageImportance.Low,
$"Successfully downloaded blob : {fileName} to {localBlobPath}");
Expand All @@ -1524,11 +1545,14 @@ await DownloadFileAsync(
version = packageIdentity.Version.ToString();
}

Stopwatch gatherBlobPublishingTime = Stopwatch.StartNew();
await PushBlobToNugetFeed(
feedConfig,
localBlobPath,
id,
version);
gatherBlobPublishingTime.Stop();
Log.LogMessage(MessageImportance.Low, $"Time taken to publish blob {localBlobPath} is {gatherBlobPublishingTime.ElapsedMilliseconds / 1000.0} (seconds)");

DeleteTemporaryDirectory(temporaryBlobDirectory);
}
Expand Down Expand Up @@ -1556,7 +1580,7 @@ private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string local

using HttpClient httpClient = new HttpClient(new HttpClientHandler
{CheckCertificateRevocationList = true});
httpClient.Timeout = TimeSpan.FromSeconds(180);
httpClient.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds);
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(
"Basic",
Convert.ToBase64String(
Expand Down Expand Up @@ -1656,6 +1680,7 @@ await Task.WhenAll(blobsToPublish.Select(async blob =>
var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName);
Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}");

Stopwatch gatherBlobDownloadTime = Stopwatch.StartNew();
await DownloadFileAsync(
client,
ArtifactName.BlobArtifacts,
Expand All @@ -1667,6 +1692,8 @@ await DownloadFileAsync(
{
Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'");
}
gatherBlobDownloadTime.Stop();
Log.LogMessage(MessageImportance.Low, $"Time taken to download file to '{localBlobPath}' is {gatherBlobDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)");

Log.LogMessage(MessageImportance.Low,
$"Successfully downloaded blob : {fileName} to {localBlobPath}");
Expand All @@ -1684,7 +1711,11 @@ await DownloadFileAsync(
feedConfig,
AddAssetLocationToAssetAssetLocationType.Container);

Stopwatch gatherBlobPublishingTime = Stopwatch.StartNew();
await blobFeedAction.UploadAssetAsync(item, pushOptions, null);
gatherBlobPublishingTime.Stop();
Log.LogMessage(MessageImportance.Low,$"Publishing {item.ItemSpec} completed in {gatherBlobPublishingTime.ElapsedMilliseconds / 1000.0} (seconds)");

DeleteTemporaryDirectory(temporaryBlobDirectory);
}
finally
Expand Down

0 comments on commit f1e67a4

Please sign in to comment.