From 4a1e38095b53ebdd9eed320756794de2fd8d1a0e Mon Sep 17 00:00:00 2001 From: Aiqiao Yan <55104035+aiqiaoy@users.noreply.github.com> Date: Tue, 9 Apr 2024 14:13:07 -0400 Subject: [PATCH] =?UTF-8?q?backoff=20if=20we=20retried=20polling=20for=20m?= =?UTF-8?q?ore=20than=2050=20times=20in=20less=20than=2030m=E2=80=A6=20(#3?= =?UTF-8?q?232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * backoff if we retried polling for more than 50 times in less than 30minutes * run dotnet format * move delay to after no message trace * run dotnet format --- src/Runner.Listener/MessageListener.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 403c2bfc720..7b51423e6f5 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -236,6 +236,7 @@ public async Task GetNextMessageAsync(CancellationToken token) ArgUtil.NotNull(_settings, nameof(_settings)); bool encounteringError = false; int continuousError = 0; + int continuousEmptyMessage = 0; string errorMessage = string.Empty; Stopwatch heartbeat = new(); heartbeat.Restart(); @@ -359,16 +360,27 @@ public async Task GetNextMessageAsync(CancellationToken token) if (message == null) { + continuousEmptyMessage++; if (heartbeat.Elapsed > TimeSpan.FromMinutes(30)) { Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes."); heartbeat.Restart(); + continuousEmptyMessage = 0; } else { Trace.Verbose($"No message retrieved from session '{_session.SessionId}'."); } + if (continuousEmptyMessage > 50) + { + // retried more than 50 times in less than 30mins and still getting empty message + // something is not right on the service side, backoff for 15-30s before retry + _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval); + Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds); + await HostContext.Delay(_getNextMessageRetryInterval, token); + } + continue; }