Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upped timeout and added logs for downloading and publishing operation #7351

Merged
merged 8 commits into from
May 6, 2021
Merged

Upped timeout and added logs for downloading and publishing operation #7351

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8883772
increased timeout and added more logging
epananth May 6, 2021
117d1f0
test sdk changes
epananth May 6, 2021
030561e
package push timeout to 300
epananth May 6, 2021
53d4525
test sdk
epananth May 6, 2021
cbdc39f
revert test sdk
epananth May 6, 2021
f62f440
unwanted code
epananth May 6, 2021
551ad33
added stop watch
epananth May 6, 2021
0588cab
get decimals
epananth May 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 34 additions & 3 deletions src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
Expand Down Expand Up @@ -191,7 +192,7 @@ private enum ArtifactName
BlobArtifacts
}

private int TimeoutInSeconds = 180;
private int TimeoutInSeconds = 300;

public override bool Execute()
{
Expand Down Expand Up @@ -418,6 +419,7 @@ public async Task PublishSymbolsUsingStreamingAsync(
{
using (HttpClient client = CreateAzdoClient(AzureDevOpsOrg, true))
{
client.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds);
await Task.WhenAll(symbolsToPublish.Select(async symbol =>
{
try
Expand All @@ -427,13 +429,18 @@ await Task.WhenAll(symbolsToPublish.Select(async symbol =>
string localSymbolPath = Path.Combine(temporarySymbolsDirectory, symbol);
symbolLog.AppendLine($"Downloading symbol : {symbol} to {localSymbolPath}");

Stopwatch gatherDownloadTime = Stopwatch.StartNew();
await DownloadFileAsync(
client,
ArtifactName.BlobArtifacts,
containerId,
symbol,
localSymbolPath);

gatherDownloadTime.Stop();
symbolLog.AppendLine($"Time taken to download file to '{localSymbolPath}' is {gatherDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)");
symbolLog.AppendLine($"Successfully downloaded symbol : {symbol} to {localSymbolPath}");

List<string> symbolFiles = new List<string>();
symbolFiles.Add(localSymbolPath);
symbolLog.AppendLine($"Uploading symbol file '{string.Join(",", symbolFiles)}'");
Expand All @@ -443,6 +450,7 @@ await DownloadFileAsync(
var serverPath = server.Key;
var token = server.Value;
symbolLog.AppendLine($"Publishing symbol file {symbol} to {serverPath}:");
Stopwatch gatherSymbolPublishingTime = Stopwatch.StartNew();

try
{
Expand All @@ -465,6 +473,10 @@ await PublishSymbolsHelper.PublishAsync(
{
Log.LogError(ex.Message);
}

gatherSymbolPublishingTime.Stop();
symbolLog.AppendLine(
$"Symbol publishing for {symbol} took {gatherSymbolPublishingTime.ElapsedMilliseconds / 1000.0} (seconds)");
}

DeleteTemporaryDirectory(temporarySymbolsDirectory);
Expand Down Expand Up @@ -1148,6 +1160,7 @@ await Task.WhenAll(packagesToPublish.Select(async package =>
Log.LogMessage(MessageImportance.Low,
$"Downloading package : {packageFilename} to {localPackagePath}");

Stopwatch gatherPackageDownloadTime = Stopwatch.StartNew();
await DownloadFileAsync(
client,
ArtifactName.PackageArtifacts,
Expand All @@ -1162,6 +1175,8 @@ await DownloadFileAsync(
return;
}

gatherPackageDownloadTime.Stop();
Log.LogMessage(MessageImportance.Low, $"Time taken to download file to '{localPackagePath}' is {gatherPackageDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)");
Log.LogMessage(MessageImportance.Low,
$"Successfully downloaded package : {packageFilename} to {localPackagePath}");

Expand All @@ -1183,7 +1198,10 @@ await DownloadFileAsync(
Convert.ToBase64String(
Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token))));

Stopwatch gatherPackagePublishingTime = Stopwatch.StartNew();
await PushPackageToNugetFeed(httpClient, feedConfig, localPackagePath, package.Id, package.Version);
gatherPackagePublishingTime.Stop();
Log.LogMessage(MessageImportance.Low,$"Publishing package {localPackagePath} took {gatherPackagePublishingTime.ElapsedMilliseconds / 1000.0} (seconds)");

DeleteTemporaryDirectory(localPackagePath);
}
Expand Down Expand Up @@ -1243,7 +1261,7 @@ public async Task PushNugetPackagesAsync<T>(
using (HttpClient httpClient = new HttpClient(new HttpClientHandler
{CheckCertificateRevocationList = true}))
{
httpClient.Timeout = TimeSpan.FromSeconds(180);
httpClient.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds);
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(
"Basic",
Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.Token))));
Expand Down Expand Up @@ -1500,6 +1518,7 @@ await Task.WhenAll(blobsToPublish.Select(async blob =>
string localBlobPath = Path.Combine(temporaryBlobDirectory, fileName);
Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}");

Stopwatch gatherBlobDownloadTime = Stopwatch.StartNew();
await DownloadFileAsync(
client,
ArtifactName.BlobArtifacts,
Expand All @@ -1511,6 +1530,8 @@ await DownloadFileAsync(
{
Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'");
}
gatherBlobDownloadTime.Stop();
Log.LogMessage(MessageImportance.Low, $"Time taken to download file to '{localBlobPath}' is {gatherBlobDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)");

Log.LogMessage(MessageImportance.Low,
$"Successfully downloaded blob : {fileName} to {localBlobPath}");
Expand All @@ -1524,11 +1545,14 @@ await DownloadFileAsync(
version = packageIdentity.Version.ToString();
}

Stopwatch gatherBlobPublishingTime = Stopwatch.StartNew();
await PushBlobToNugetFeed(
feedConfig,
localBlobPath,
id,
version);
gatherBlobPublishingTime.Stop();
Log.LogMessage(MessageImportance.Low, $"Time taken to publish blob {localBlobPath} is {gatherBlobPublishingTime.ElapsedMilliseconds / 1000.0} (seconds)");

DeleteTemporaryDirectory(temporaryBlobDirectory);
}
Expand Down Expand Up @@ -1556,7 +1580,7 @@ private async Task PushBlobToNugetFeed(TargetFeedConfig feedConfig, string local

using HttpClient httpClient = new HttpClient(new HttpClientHandler
{CheckCertificateRevocationList = true});
httpClient.Timeout = TimeSpan.FromSeconds(180);
httpClient.Timeout = TimeSpan.FromSeconds(TimeoutInSeconds);
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(
"Basic",
Convert.ToBase64String(
Expand Down Expand Up @@ -1656,6 +1680,7 @@ await Task.WhenAll(blobsToPublish.Select(async blob =>
var localBlobPath = Path.Combine(temporaryBlobDirectory, fileName);
Log.LogMessage(MessageImportance.Low, $"Downloading blob : {fileName} to {localBlobPath}");

Stopwatch gatherBlobDownloadTime = Stopwatch.StartNew();
await DownloadFileAsync(
client,
ArtifactName.BlobArtifacts,
Expand All @@ -1667,6 +1692,8 @@ await DownloadFileAsync(
{
Log.LogError($"Could not locate '{blob.Id} at '{localBlobPath}'");
}
gatherBlobDownloadTime.Stop();
Log.LogMessage(MessageImportance.Low, $"Time taken to download file to '{localBlobPath}' is {gatherBlobDownloadTime.ElapsedMilliseconds / 1000.0} (seconds)");

Log.LogMessage(MessageImportance.Low,
$"Successfully downloaded blob : {fileName} to {localBlobPath}");
Expand All @@ -1684,7 +1711,11 @@ await DownloadFileAsync(
feedConfig,
AddAssetLocationToAssetAssetLocationType.Container);

Stopwatch gatherBlobPublishingTime = Stopwatch.StartNew();
await blobFeedAction.UploadAssetAsync(item, pushOptions, null);
gatherBlobPublishingTime.Stop();
Log.LogMessage(MessageImportance.Low,$"Publishing {item.ItemSpec} completed in {gatherBlobPublishingTime.ElapsedMilliseconds / 1000.0} (seconds)");

DeleteTemporaryDirectory(temporaryBlobDirectory);
}
finally
Expand Down