Skip to content

Commit

Permalink
Cargo project manager retry 500 errors. (#471)
Browse files Browse the repository at this point in the history 
* Cargo project manager should retry 500 errors.

* Switch to polly retry
  • Loading branch information
@gregorymacharia
gregorymacharia authored Oct 11, 2024
1 parent 7d9fada commit ed0e2cf
Show file tree
Hide file tree
Showing 2 changed files with 194 additions and 43 deletions.
115 changes: 75 additions & 40 deletions src/Shared/PackageManagers/CargoProjectManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ namespace Microsoft.CST.OpenSource.PackageManagers
using Model.Enums;
using Octokit;
using PackageUrl;
using Polly.Retry;
using Polly;
using System;
using System.Collections.Generic;
using System.IO;
Expand All @@ -35,10 +37,19 @@ public class CargoProjectManager : TypedManager<IManagerPackageVersionMetadata,

[System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE0044:Add readonly modifier", Justification = "Modified through reflection.")]
public string ENV_CARGO_ENDPOINT_STATIC { get; set; } = "https://static.crates.io";

[System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE0044:Add readonly modifier", Justification = "Modified through reflection.")]
public string ENV_CARGO_INDEX_ENDPOINT { get; set; } = "https://raw.githubusercontent.com/rust-lang/crates.io-index/master";

private static int InternalServerErrorMaxRetries = 3;

private AsyncRetryPolicy retryPolicy = Policy
.Handle<HttpRequestException>(ex => ex.StatusCode == HttpStatusCode.InternalServerError)
.RetryAsync(InternalServerErrorMaxRetries, (exception, retryCount) =>
{
Logger.Debug(exception, "Internal Server error 500, retrying... {0}/{1} tries", retryCount, InternalServerErrorMaxRetries);
});

public CargoProjectManager(
string directory,
IManagerPackageActions<IManagerPackageVersionMetadata>? actions = null,
Expand Down Expand Up @@ -86,33 +97,41 @@ public override async Task<IEnumerable<string>> DownloadVersionAsync(PackageURL
}

Uri url = (await GetArtifactDownloadUrisAsync(purl, cached).ToListAsync()).Single().Uri;

try
{
string targetName = $"cargo-{fileName}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
// if the cache is already present, no need to extract
if (doExtract && cached && Directory.Exists(extractionPath))
await retryPolicy.ExecuteAsync(async () =>
{
downloadedPaths.Add(extractionPath);
return downloadedPaths;
}
Logger.Debug("Downloading {0}", url);
string targetName = $"cargo-{fileName}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
// if the cache is already present, no need to extract
if (doExtract && cached && Directory.Exists(extractionPath))
{
downloadedPaths.Add(extractionPath);
return;
}
Logger.Debug("Downloading {0}", url);

HttpClient httpClient = CreateHttpClient();
HttpClient httpClient = CreateHttpClient();

System.Net.Http.HttpResponseMessage result = await httpClient.GetAsync(url);
result.EnsureSuccessStatusCode();
System.Net.Http.HttpResponseMessage result = await httpClient.GetAsync(url);
result.EnsureSuccessStatusCode();

if (doExtract)
{
downloadedPaths.Add(await ArchiveHelper.ExtractArchiveAsync(TopLevelExtractionDirectory, targetName, await result.Content.ReadAsStreamAsync(), cached));
}
else
{
extractionPath += Path.GetExtension(url.ToString()) ?? "";
await File.WriteAllBytesAsync(extractionPath, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(extractionPath);
}
if (doExtract)
{
downloadedPaths.Add(await ArchiveHelper.ExtractArchiveAsync(TopLevelExtractionDirectory, targetName, await result.Content.ReadAsStreamAsync(), cached));
}
else
{
extractionPath += Path.GetExtension(url.ToString()) ?? "";
await File.WriteAllBytesAsync(extractionPath, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(extractionPath);
}
});
}
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.InternalServerError)
{
Logger.Debug($"Max retries reached. Unable to download the package: {purl}, error: {ex.Message}");
}
catch (Exception ex)
{
Expand Down Expand Up @@ -147,23 +166,31 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR

try
{
string? packageName = purl.Name;
HttpClient httpClient = CreateHttpClient();
// NOTE: The file isn't valid json, so use the custom rule.
JsonDocument doc = await GetJsonCache(httpClient, $"{ENV_CARGO_INDEX_ENDPOINT}/{CreatePath(packageName)}", jsonParsingOption: JsonParsingOption.NotInArrayNotCsv);
List<string> versionList = new();
foreach (JsonElement versionObject in doc.RootElement.EnumerateArray())
return await retryPolicy.ExecuteAsync(async () =>
{
if (versionObject.TryGetProperty("vers", out JsonElement version))
string? packageName = purl.Name;
HttpClient httpClient = CreateHttpClient();
// NOTE: The file isn't valid json, so use the custom rule.
JsonDocument doc = await GetJsonCache(httpClient, $"{ENV_CARGO_INDEX_ENDPOINT}/{CreatePath(packageName)}", jsonParsingOption: JsonParsingOption.NotInArrayNotCsv);
List<string> versionList = new();
foreach (JsonElement versionObject in doc.RootElement.EnumerateArray())
{
Logger.Debug("Identified {0} version {1}.", packageName, version.ToString());
if (version.ToString() is string s)
if (versionObject.TryGetProperty("vers", out JsonElement version))
{
versionList.Add(s);
Logger.Debug("Identified {0} version {1}.", packageName, version.ToString());
if (version.ToString() is string s)
{
versionList.Add(s);
}
}
}
}
return SortVersions(versionList.Distinct());
return SortVersions(versionList.Distinct());
});
}
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.InternalServerError)
{
Logger.Debug($"Max retries reached. Unable to enumerate versions for package: {purl}, error: {ex.Message}");
throw;
}
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
{
Expand All @@ -174,18 +201,26 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
{
Logger.Debug("Unable to enumerate versions: {0}", ex.Message);
throw;
}
}
}

/// <inheritdoc />
public override async Task<string?> GetMetadataAsync(PackageURL purl, bool useCache = true)
{
try
{
string? packageName = purl.Name;
HttpClient httpClient = CreateHttpClient();
string? content = await GetHttpStringCache(httpClient, $"{ENV_CARGO_ENDPOINT}/api/v1/crates/{packageName}", useCache);
return content;
return await retryPolicy.ExecuteAsync(async () =>
{
string? packageName = purl.Name;
HttpClient httpClient = CreateHttpClient();
string? content = await GetHttpStringCache(httpClient, $"{ENV_CARGO_ENDPOINT}/api/v1/crates/{packageName}", useCache);
return content;
});
}
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.InternalServerError)
{
Logger.Debug($"Max retries reached. Unable to get metadata for package: {purl}, error: {ex.Message}");
throw;
}
catch (Exception ex)
{
Expand Down Expand Up @@ -261,7 +296,7 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl)
}
return new Uri(url);
}

/// <summary>
/// Helper method to create the path for the crates.io index for this package name.
/// </summary>
Expand Down
122 changes: 119 additions & 3 deletions src/oss-tests/ProjectManagerTests/CargoProjectManagerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace Microsoft.CST.OpenSource.Tests.ProjectManagerTests
{
using Microsoft.CodeAnalysis.Sarif;
using Microsoft.CST.OpenSource.Extensions;
using Microsoft.CST.OpenSource.Model;
using Microsoft.CST.OpenSource.PackageActions;
Expand All @@ -15,6 +16,9 @@ namespace Microsoft.CST.OpenSource.Tests.ProjectManagerTests
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using VisualStudio.TestTools.UnitTesting;

Expand All @@ -27,15 +31,28 @@ public class CargoProjectManagerTests
{ "https://crates.io/api/v1/crates/rand", Resources.cargo_rand_json },
}.ToImmutableDictionary();

private readonly IDictionary<string, bool> _retryTestsPackages = new Dictionary<string, bool>()
{
{ "https://crates.io/api/v1/crates/a-mazed*", true },
{ "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/a-/ma/a-mazed", true},
{ "https://crates.io/api/v1/crates/A2VConverter*", false},
{ "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/A2/VC/A2VConverter", false},
}.ToImmutableDictionary();

private readonly CargoProjectManager _projectManager;
private readonly IHttpClientFactory _httpFactory;

public CargoProjectManagerTests()
{
Mock<IHttpClientFactory> mockFactory = new();

MockHttpMessageHandler mockHttp = new();

foreach ((string Url, bool ShouldSucceedAfterRetry) in _retryTestsPackages)
{
ConfigureMockHttpForRetryMechanismTests(mockHttp, Url, ShouldSucceedAfterRetry);
}

foreach ((string url, string json) in _packages)
{
MockHttpFetchResponse(HttpStatusCode.OK, url, json, mockHttp);
Expand All @@ -47,6 +64,77 @@ public CargoProjectManagerTests()
_projectManager = new CargoProjectManager(".", new NoOpPackageActions(), _httpFactory);
}

[DataTestMethod]
[DataRow("pkg:cargo/[email protected]")]
public async Task GetMetadataAsyncRetries500InternalServerError(string purlString)
{
PackageURL purl = new(purlString);
string? sampleResult = await new StringContent(JsonSerializer.Serialize(new { Name = "sampleName", Content = "sampleContent" }), Encoding.UTF8, "application/json").ReadAsStringAsync();

string? result = await _projectManager.GetMetadataAsync(purl, useCache: false);

Assert.IsNotNull(result);
Assert.AreEqual(sampleResult, result);
}

[DataTestMethod]
[DataRow("pkg:cargo/[email protected]")]
public async Task DownloadVersionAsyncRetries500InternalServerError(string purlString)
{
PackageURL purl = new(purlString);

IEnumerable<string> downloadedPath = await _projectManager.DownloadVersionAsync(purl, doExtract: true, cached: false);

Assert.IsFalse(downloadedPath.IsEmptyEnumerable());
}

[DataTestMethod]
[DataRow("pkg:cargo/[email protected]")]
public async Task EnumerateVersionsAsyncRetries500InternalServerError(string purlString)
{
PackageURL purl = new(purlString);

IEnumerable<string> versionsList = await _projectManager.EnumerateVersionsAsync(purl);

Assert.IsNotNull(versionsList);
}

[DataTestMethod]
[DataRow("pkg:cargo/[email protected]")]
public async Task GetMetadataAsyncThrowsExceptionAfterMaxRetries(string purlString)
{
PackageURL purl = new(purlString);

HttpRequestException exception = await Assert.ThrowsExceptionAsync<HttpRequestException>(() => _projectManager.GetPackageMetadataAsync(purl, useCache: false));

Assert.IsNotNull(exception);
Assert.AreEqual(exception.StatusCode, HttpStatusCode.InternalServerError);
}

[DataTestMethod]
[DataRow("pkg:cargo/[email protected]")]
public async Task DownloadVersionAsyncThrowsExceptionAfterMaxRetries(string purlString)
{
PackageURL purl = new(purlString);

IEnumerable<string> downloadedPath = await _projectManager.DownloadVersionAsync(purl, doExtract: true, cached: false);

Assert.IsTrue(downloadedPath.IsEmptyEnumerable());
}

[DataTestMethod]
[DataRow("pkg:cargo/[email protected]")]
public async Task EnumerateVersionsAsyncThrowsExceptionAfterMaxRetries(string purlString)
{
PackageURL purl = new(purlString);

HttpRequestException exception = await Assert.ThrowsExceptionAsync<HttpRequestException>(() => _projectManager.EnumerateVersionsAsync(purl));

Assert.IsNotNull(exception);
Assert.AreEqual(exception.StatusCode, HttpStatusCode.InternalServerError);
}


[DataTestMethod]
[DataRow("pkg:cargo/[email protected]", "https://crates.io/api/v1/crates/rand/0.8.5/download")]
[DataRow("pkg:cargo/[email protected]", "https://crates.io/api/v1/crates/quote/1.0.21/download")]
Expand Down Expand Up @@ -80,7 +168,7 @@ public async Task PackageVersionExistsAsyncSucceeds(string purlString)

Assert.IsTrue(await _projectManager.PackageVersionExistsAsync(purl, useCache: false));
}

[DataTestMethod]
[DataRow("pkg:cargo/[email protected]")]
public async Task PackageVersionDoesntExistsAsyncSucceeds(string purlString)
Expand Down Expand Up @@ -122,7 +210,35 @@ private static void MockHttpFetchResponse(
httpMock
.When(HttpMethod.Get, url)
.Respond(statusCode, "application/json", content);
}

private static void ConfigureMockHttpForRetryMechanismTests(MockHttpMessageHandler mockHttp, string url, bool shouldSucceedAfterRetry = true)
{
if (shouldSucceedAfterRetry)
{
int callCount = 0;
mockHttp
.When(HttpMethod.Get, url)
.Respond(_ =>
{
callCount++;
if (callCount == 1) // Fail and return 500 on 1st attempt
{
return new HttpResponseMessage(HttpStatusCode.InternalServerError);
}
return new HttpResponseMessage(HttpStatusCode.OK) // Succeed on subsequent attempts
{
Content = new StringContent(JsonSerializer.Serialize(new { Name = "sampleName", Content = "sampleContent" }), Encoding.UTF8, "application/json")

};
});
}
else
{
mockHttp
.When(HttpMethod.Get, url)
.Respond(HttpStatusCode.InternalServerError);
}
}
}
}

0 comments on commit ed0e2cf

Please sign in to comment.