From c593588bd63c90d91b84321540ba52f5d27b9298 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 17 Mar 2023 15:44:41 -0700 Subject: [PATCH 1/8] first cut at retry mechanism --- .../Store/GitProcessHandler.cs | 188 +++++++++++------- 1 file changed, 113 insertions(+), 75 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs index b77604f26fe..7bb7bf844c6 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs @@ -8,6 +8,8 @@ using System.Threading.Tasks; using Azure.Sdk.Tools.TestProxy.Common; using Azure.Sdk.Tools.TestProxy.Common.Exceptions; +using Microsoft.Extensions.Internal; +using NuGet.Protocol.Core.Types; namespace Azure.Sdk.Tools.TestProxy.Store { @@ -26,6 +28,7 @@ public class CommandResult /// public class GitProcessHandler { + public const int RETRY_INTERMITTENT_FAILURE_COUNT = 3; /// /// Internal class to hold the minimum supported version of git. If that /// version changes we only need to change it here. @@ -127,54 +130,67 @@ public virtual CommandResult Run(string arguments, string workingDirectory) { try { - DebugLogger.LogInformation($"git {arguments}"); - - var output = new List(); - var error = new List(); - - using (var process = new Process()) + int attempts = 1; + bool continueToAttempt = true; + while (attempts <= RETRY_INTERMITTENT_FAILURE_COUNT) { - process.StartInfo = processStartInfo; + DebugLogger.LogInformation($"git {arguments}"); - process.OutputDataReceived += (s, e) => - { - lock (output) - { - output.Add(e.Data); - } - }; + var output = new List(); + var error = new List(); - process.ErrorDataReceived += (s, e) => + using (var process = new Process()) { - lock (error) - { - error.Add(e.Data); - } - }; - - process.Start(); - process.BeginErrorReadLine(); - process.BeginOutputReadLine(); - process.WaitForExit(); - - int returnCode = process.ExitCode; - var stdOut = string.Join(Environment.NewLine, output); - var stdError = string.Join(Environment.NewLine, error); + process.StartInfo = processStartInfo; - DebugLogger.LogDebug($"StdOut: {stdOut}"); - DebugLogger.LogDebug($"StdErr: {stdError}"); - DebugLogger.LogDebug($"ExitCode: {process.ExitCode}"); + process.OutputDataReceived += (s, e) => + { + lock (output) + { + output.Add(e.Data); + } + }; - result.ExitCode = process.ExitCode; - result.StdErr = string.Join(Environment.NewLine, stdError); - result.StdOut = string.Join(Environment.NewLine, stdOut); + process.ErrorDataReceived += (s, e) => + { + lock (error) + { + error.Add(e.Data); + } + }; + + process.Start(); + process.BeginErrorReadLine(); + process.BeginOutputReadLine(); + process.WaitForExit(); + + int returnCode = process.ExitCode; + var stdOut = string.Join(Environment.NewLine, output); + var stdError = string.Join(Environment.NewLine, error); + + DebugLogger.LogDebug($"StdOut: {stdOut}"); + DebugLogger.LogDebug($"StdErr: {stdError}"); + DebugLogger.LogDebug($"ExitCode: {process.ExitCode}"); + + result.ExitCode = process.ExitCode; + result.StdErr = string.Join(Environment.NewLine, stdError); + result.StdOut = string.Join(Environment.NewLine, stdOut); + + if (result.ExitCode != 0) + { + continueToAttempt = IsRetriableGitError(result); - if (result.ExitCode != 0) - { - throw new GitProcessException(result); + if(!continueToAttempt) + { + throw new GitProcessException(result); + } + } + attempts++; } } } + // exceptions caught here will be to do with inability to start the git process + // otherwise all "error" states should be handled by the output to stdErr and non-zero exitcode. catch (Exception e) { DebugLogger.LogDebug(e.Message); @@ -189,6 +205,15 @@ public virtual CommandResult Run(string arguments, string workingDirectory) return result; } + public bool IsRetriableGitError(CommandResult result) + { + if (result.ExitCode != 0) { + + } + + return false; + } + /// /// Invokes git binary against a GitAssetsConfiguration. /// @@ -214,52 +239,65 @@ public virtual bool TryRun(string arguments, string workingDirectory, out Comman { try { - DebugLogger.LogInformation($"git {arguments}"); - var output = new List(); - var error = new List(); - - using (var process = new Process()) + int attempts = 1; + bool continueToAttempt = true; + while (attempts <= RETRY_INTERMITTENT_FAILURE_COUNT) { - process.StartInfo = processStartInfo; + DebugLogger.LogInformation($"git {arguments}"); + var output = new List(); + var error = new List(); - process.OutputDataReceived += (s, e) => + using (var process = new Process()) { - lock (output) - { - output.Add(e.Data); - } - }; + process.StartInfo = processStartInfo; - process.ErrorDataReceived += (s, e) => - { - lock (error) + process.OutputDataReceived += (s, e) => { - error.Add(e.Data); - } - }; + lock (output) + { + output.Add(e.Data); + } + }; - process.Start(); - process.BeginErrorReadLine(); - process.BeginOutputReadLine(); - process.WaitForExit(); - - int returnCode = process.ExitCode; - var stdOut = string.Join(Environment.NewLine, output); - var stdError = string.Join(Environment.NewLine, error); - - DebugLogger.LogDebug($"StdOut: {stdOut}"); - DebugLogger.LogDebug($"StdErr: {stdError}"); - DebugLogger.LogDebug($"ExitCode: {process.ExitCode}"); + process.ErrorDataReceived += (s, e) => + { + lock (error) + { + error.Add(e.Data); + } + }; + + process.Start(); + process.BeginErrorReadLine(); + process.BeginOutputReadLine(); + process.WaitForExit(); + + int returnCode = process.ExitCode; + var stdOut = string.Join(Environment.NewLine, output); + var stdError = string.Join(Environment.NewLine, error); + + DebugLogger.LogDebug($"StdOut: {stdOut}"); + DebugLogger.LogDebug($"StdErr: {stdError}"); + DebugLogger.LogDebug($"ExitCode: {process.ExitCode}"); + + commandResult = new CommandResult() + { + ExitCode = process.ExitCode, + StdErr = stdError, + StdOut = stdOut, + Arguments = arguments + }; - commandResult = new CommandResult() - { - ExitCode = process.ExitCode, - StdErr = stdError, - StdOut = stdOut, - Arguments = arguments - }; + if (commandResult.ExitCode != 0) + { + continueToAttempt = IsRetriableGitError(commandResult); + } + attempts++; + } } } + // exceptions caught here will be to do with inability to start the git process + // otherwise all "error" states should be handled by the output to stdErr and non-zero exitcode. catch (Exception e) { DebugLogger.LogDebug(e.Message); From 5dc343d7611fdfbd5d284cc8d39ef51a1f48c2e9 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 17 Mar 2023 17:05:35 -0700 Subject: [PATCH 2/8] add retry based on git output --- .../GitProcessHandlerTests.cs | 56 +++++++++++++++++++ .../Store/GitProcessHandler.cs | 51 ++++++++++++++++- 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/GitProcessHandlerTests.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/GitProcessHandlerTests.cs index de32d03f64b..fa962ad8a45 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/GitProcessHandlerTests.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/GitProcessHandlerTests.cs @@ -15,6 +15,62 @@ public GitProcessHandlerTests() DebugLogger.ConfigureLogger(loggerFactory); } + const string longTimeOutError = @" + Cloning into '.'... + fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': Failed to connect to github.com port 443: Operation timed out + + + at Azure.Sdk.Tools.TestProxy.Store.GitStore.InitializeAssetsRepo(GitAssetsConfiguration config, Boolean forceInit) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs:line 456 + at Azure.Sdk.Tools.TestProxy.Store.GitStore.Restore(String pathToAssetsJson) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs:line 152 + at Azure.Sdk.Tools.TestProxy.Startup.Run(Object commandObj) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Startup.cs:line 154 + at CommandLine.ParserResultExtensions.WithParsedAsync[T](ParserResult`1 result, Func`2 action) + at Azure.Sdk.Tools.TestProxy.Startup.Main(String[] args) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Startup.cs:line 67 + at Azure.Sdk.Tools.TestProxy.Startup.
(String[] args)"; + + const string tooManyRequestsError = @"fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': The requested URL returned error: 429 + fatal: could not fetch 26bfd3d8ada54a78573cb64f6fa79c8d4a300c8c from promisor remote + + at Azure.Sdk.Tools.TestProxy.Store.GitStore.CheckoutRepoAtConfig(GitAssetsConfiguration config) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs:line 330 + at Azure.Sdk.Tools.TestProxy.Store.GitStore.InitializeAssetsRepo(GitAssetsConfiguration config, Boolean forceInit) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs:line 459 + at Azure.Sdk.Tools.TestProxy.Store.GitStore.Restore(String pathToAssetsJson) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs:line 152 + at Azure.Sdk.Tools.TestProxy.RecordingHandler.RestoreAssetsJson(String assetsJson, Boolean forceCheckout) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/RecordingHandler.cs:line 164 + at Azure.Sdk.Tools.TestProxy.RecordingHandler.StartPlaybackAsync(String sessionId, HttpResponse outgoingResponse, RecordingType mode, String assetsPath) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/RecordingHandler.cs:line 373 + at Azure.Sdk.Tools.TestProxy.Playback.Start() in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Playback.cs:line 46 + at lambda_method37(Closure , Object )"; + + const string longTimeOutWithValue = @"fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': Failed to connect to github.com port 443 after 21019 ms: Couldn't connect to server + fatal: could not fetch 87d05b71fcffcf3c7fa8e611bda09b505ff586a4 from promisor remote + + + at Azure.Sdk.Tools.TestProxy.Store.GitStore.CheckoutRepoAtConfig(GitAssetsConfiguration config) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs:line 330 + at Azure.Sdk.Tools.TestProxy.Store.GitStore.InitializeAssetsRepo(GitAssetsConfiguration config, Boolean forceInit) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs:line 459 + at Azure.Sdk.Tools.TestProxy.Store.GitStore.Restore(String pathToAssetsJson) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs:line 152 + at Azure.Sdk.Tools.TestProxy.RecordingHandler.RestoreAssetsJson(String assetsJson, Boolean forceCheckout) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/RecordingHandler.cs:line 164 + at Azure.Sdk.Tools.TestProxy.RecordingHandler.StartPlaybackAsync(String sessionId, HttpResponse outgoingResponse, RecordingType mode, String assetsPath) in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/RecordingHandler.cs:line 373 + at Azure.Sdk.Tools.TestProxy.Playback.Start() in /mnt/vss/_work/1/s/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Playback.cs:line 46 + at lambda_method37(Closure , Object )"; + + [Theory] + [InlineData("", longTimeOutError, 1, true)] + [InlineData("", tooManyRequestsError, 1, true)] + [InlineData("", longTimeOutWithValue, 1, true)] + [InlineData("", "fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': Failed to connect to github.com port 443 after 1 ms: Couldn't connect to server", 1, true)] + [InlineData("", "fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': Failed to connect to github.com port 443 after 0 ms: Couldn't connect to server", 1, true)] + [InlineData("", "fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': Failed to connect to github.com port 443 after ms: Couldn't connect to server", 1, false)] + [InlineData("", "", 0, false)] + public void VerifyGitExceptionParser(string inputStdOut, string inputStdErr, int exitCode, bool expectedResult) + { + var processHandler = new GitProcessHandler(); + var commandResult = new CommandResult(); + commandResult.StdOut = inputStdOut; + commandResult.StdErr = inputStdErr; + commandResult.ExitCode = exitCode; + + var checkResult = processHandler.IsRetriableGitError(commandResult); + + Assert.Equal(expectedResult, checkResult); + } + [Theory] // 2.37.0 is the min version of git required by the TestProxy // Windows git version strings diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs index 7bb7bf844c6..640f9ec3abf 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs @@ -23,6 +23,7 @@ public class CommandResult public Exception CommandException; } + /// /// This class offers an easy wrapper abstraction for shelling out to git. /// @@ -176,7 +177,11 @@ public virtual CommandResult Run(string arguments, string workingDirectory) result.StdErr = string.Join(Environment.NewLine, stdError); result.StdOut = string.Join(Environment.NewLine, stdOut); - if (result.ExitCode != 0) + if (result.ExitCode == 0) + { + break; + } + else if (result.ExitCode != 0) { continueToAttempt = IsRetriableGitError(result); @@ -205,10 +210,48 @@ public virtual CommandResult Run(string arguments, string workingDirectory) return result; } + /// + /// This function evaluates a git command invocation result. The result of "yes you should retry this" only occurs + /// when the necessary data is available. Otherwise we default to NOT retry. + /// + /// Check Azure/azure-sdk-tools#5660 for additional detail on occurrence. + /// + /// + /// The id of the git repo being retrieved from. Used within format string to assist pattern matching errors. + /// public bool IsRetriableGitError(CommandResult result) { if (result.ExitCode != 0) { + // we cannot evaluate an empty stderr to see if it is retriable + if (string.IsNullOrEmpty(result.StdErr)) + { + return false; + } + + // fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': The requested URL returned error: 429 + if (result.StdErr.Contains("The requested URL returned error: 429")) + { + return true; + } + + // fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': Failed to connect to github.com port 443: Connection timed out + if (result.StdErr.Contains("Failed to connect to github.com port 443: Connection timed out")) + { + return true; + } + // fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': Failed to connect to github.com port 443: Operation timed out + if (result.StdErr.Contains("Failed to connect to github.com port 443: Operation timed out")) + { + return true; + } + + // fatal: unable to access 'https://github.com/Azure/azure-sdk-assets/': Failed to connect to github.com port 443 after 21019 ms: Couldn't connect to server + var regex = new Regex(@"Failed to connect to github.com port 443 after [\d]+ ms: Couldn't connect to server"); + if (regex.IsMatch(result.StdErr)) + { + return true; + } } return false; @@ -288,7 +331,11 @@ public virtual bool TryRun(string arguments, string workingDirectory, out Comman Arguments = arguments }; - if (commandResult.ExitCode != 0) + if (commandResult.ExitCode == 0) + { + break; + } + else if (commandResult.ExitCode != 0) { continueToAttempt = IsRetriableGitError(commandResult); } From e3ae96beee866c71c27d5515f781497ae3811e3a Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 17 Mar 2023 17:10:41 -0700 Subject: [PATCH 3/8] clean up usings --- .../Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs index 640f9ec3abf..803bcae0cb6 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs @@ -1,15 +1,11 @@ using System; -using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Text.RegularExpressions; -using System.Threading.Tasks; using Azure.Sdk.Tools.TestProxy.Common; using Azure.Sdk.Tools.TestProxy.Common.Exceptions; -using Microsoft.Extensions.Internal; -using NuGet.Protocol.Core.Types; namespace Azure.Sdk.Tools.TestProxy.Store { From ddbf93467499927a13774a534927c216491c7e4e Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 17 Mar 2023 17:34:13 -0700 Subject: [PATCH 4/8] remove extraneous param in code summary --- .../Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs index 803bcae0cb6..85afb001429 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs @@ -213,7 +213,6 @@ public virtual CommandResult Run(string arguments, string workingDirectory) /// Check Azure/azure-sdk-tools#5660 for additional detail on occurrence. ///
/// - /// The id of the git repo being retrieved from. Used within format string to assist pattern matching errors. /// public bool IsRetriableGitError(CommandResult result) { From 45a4e1304f2e10c34cb922fa8c61f3f210bc3b3e Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 17 Mar 2023 17:51:50 -0700 Subject: [PATCH 5/8] ensure we bomb out early --- .../Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs index 85afb001429..75911378978 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs @@ -129,7 +129,7 @@ public virtual CommandResult Run(string arguments, string workingDirectory) { int attempts = 1; bool continueToAttempt = true; - while (attempts <= RETRY_INTERMITTENT_FAILURE_COUNT) + while (continueToAttempt && attempts <= RETRY_INTERMITTENT_FAILURE_COUNT) { DebugLogger.LogInformation($"git {arguments}"); @@ -279,7 +279,7 @@ public virtual bool TryRun(string arguments, string workingDirectory, out Comman { int attempts = 1; bool continueToAttempt = true; - while (attempts <= RETRY_INTERMITTENT_FAILURE_COUNT) + while (continueToAttempt && attempts <= RETRY_INTERMITTENT_FAILURE_COUNT) { DebugLogger.LogInformation($"git {arguments}"); var output = new List(); From 7e339e2c40a92b003f164eaca39146cfb4b8d3f5 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 21 Mar 2023 18:10:18 -0700 Subject: [PATCH 6/8] simplify the error handling. not certain why I complicated it so much when IsRetriableGitError is definitely the abstraction I needed to keep it simple --- .../Store/GitProcessHandler.cs | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs index 75911378978..d9b6f1cef3f 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs @@ -128,8 +128,7 @@ public virtual CommandResult Run(string arguments, string workingDirectory) try { int attempts = 1; - bool continueToAttempt = true; - while (continueToAttempt && attempts <= RETRY_INTERMITTENT_FAILURE_COUNT) + while (attempts <= RETRY_INTERMITTENT_FAILURE_COUNT) { DebugLogger.LogInformation($"git {arguments}"); @@ -177,14 +176,9 @@ public virtual CommandResult Run(string arguments, string workingDirectory) { break; } - else if (result.ExitCode != 0) + if (!IsRetriableGitError(result)) { - continueToAttempt = IsRetriableGitError(result); - - if(!continueToAttempt) - { - throw new GitProcessException(result); - } + throw new GitProcessException(result); } attempts++; } @@ -330,10 +324,7 @@ public virtual bool TryRun(string arguments, string workingDirectory, out Comman { break; } - else if (commandResult.ExitCode != 0) - { - continueToAttempt = IsRetriableGitError(commandResult); - } + continueToAttempt = IsRetriableGitError(commandResult); attempts++; } } From 0d771d1f1d69872957206f44b8aef28b56855afc Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 22 Mar 2023 11:14:41 -0700 Subject: [PATCH 7/8] add delay --- .../Store/GitProcessHandler.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs index d9b6f1cef3f..a57cd83344d 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs @@ -4,6 +4,8 @@ using System.Diagnostics; using System.IO; using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; using Azure.Sdk.Tools.TestProxy.Common; using Azure.Sdk.Tools.TestProxy.Common.Exceptions; @@ -176,10 +178,17 @@ public virtual CommandResult Run(string arguments, string workingDirectory) { break; } - if (!IsRetriableGitError(result)) + var continueToAttempt = IsRetriableGitError(result); + + if (!continueToAttempt) { throw new GitProcessException(result); } + + if (continueToAttempt) + { + Task.Delay(attempts * 2 * 1000).Wait(); + } attempts++; } } @@ -325,6 +334,11 @@ public virtual bool TryRun(string arguments, string workingDirectory, out Comman break; } continueToAttempt = IsRetriableGitError(commandResult); + + if (continueToAttempt) + { + Task.Delay(attempts * 2 * 1000).Wait(); + } attempts++; } } From 33951c264fe64f35f1b6f5e18c2e000df24e357e Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 22 Mar 2023 11:40:24 -0700 Subject: [PATCH 8/8] update retry count logic --- .../Store/GitProcessHandler.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs index a57cd83344d..fd4581b0d3c 100644 --- a/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs +++ b/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitProcessHandler.cs @@ -4,7 +4,6 @@ using System.Diagnostics; using System.IO; using System.Text.RegularExpressions; -using System.Threading; using System.Threading.Tasks; using Azure.Sdk.Tools.TestProxy.Common; using Azure.Sdk.Tools.TestProxy.Common.Exceptions; @@ -185,11 +184,12 @@ public virtual CommandResult Run(string arguments, string workingDirectory) throw new GitProcessException(result); } - if (continueToAttempt) + attempts++; + + if (continueToAttempt && attempts < RETRY_INTERMITTENT_FAILURE_COUNT) { Task.Delay(attempts * 2 * 1000).Wait(); } - attempts++; } } } @@ -335,11 +335,11 @@ public virtual bool TryRun(string arguments, string workingDirectory, out Comman } continueToAttempt = IsRetriableGitError(commandResult); - if (continueToAttempt) + attempts++; + if (continueToAttempt && attempts < RETRY_INTERMITTENT_FAILURE_COUNT) { Task.Delay(attempts * 2 * 1000).Wait(); } - attempts++; } } }