From 5a946114d517183eb85bd32e513f99a5e8edeb0b Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 16:42:47 +0200
Subject: [PATCH 01/11] wip

---
 src/utilities/downloads_backend.jl | 17 +++++++++++++
 src/utilities/request.jl           | 38 ++++++++++++++++++++++++++----
 2 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index bbce7ebf92..74e33b3810 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -110,6 +110,23 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
         return nothing
     end
 
+    check = (s, e) -> begin
+            if isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500
+                @debug "AWS.jl Downloads inner retry for status >= 500" retry = true reason = "status >= 500" status = AWS._http_status(
+                    e
+                ) exception = e
+                return true
+            elseif isa(e, Downloads.RequestError)
+                @debug "AWS.jl Downloads inner retry for Downloads.RequestError" retry =
+                    true reason = "Downloads.RequestError" exception = e
+                return true
+            else
+                @debug "AWS.jl Downloads inner retry declined" retry = false reason = "Exception passed onwards" exception =
+                    e
+                return false
+            end
+        end
+
     try
         retry(get_response; check=check, delays=delays)()
     finally
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index a214eec436..4760ab2d6f 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -147,6 +147,10 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             if e isa HTTP.StatusError
                 e = AWSException(e, stream)
                 rethrow(e)
+            elseif !(e isa AWSException)
+                @debug "AWS.jl declined to retry non-AWSException" retry = false reason = "Non-AWSException" exception =
+                    e
+                return false
             end
             rethrow()
         end
@@ -158,11 +162,17 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             return false
         end
 
-        occursin("Signature expired", e.message) && return true
+        if occursin("Signature expired", e.message)
+            @debug "AWS.jl retry for signature expired" retry = true reason = "Signature expired" exception =
+                e
+            return true
+        end
 
         # Handle ExpiredToken...
         # https://github.com/aws/aws-sdk-go/blob/v1.31.5/aws/request/retryer.go#L98
         if e isa AWSException && e.code in EXPIRED_ERROR_CODES
+            @debug "AWS.jl retry for expired credentials" retry = true reason = "Credentials expired" exception =
+                e
             check_credentials(credentials(aws); force_refresh=true)
             return true
         end
@@ -170,24 +180,42 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
         # Throttle handling
         # https://github.com/boto/botocore/blob/1.16.17/botocore/data/_retry.json
         # https://docs.aws.amazon.com/general/latest/gr/api-retries.html
-        if _http_status(e.cause) == TOO_MANY_REQUESTS || e.code in THROTTLING_ERROR_CODES
+        if _http_status(e.cause) == TOO_MANY_REQUESTS
+            @debug "AWS.jl retry too many requests" retry = true reason = "too many requests" exception =
+                e
+            return true
+        elseif e.code in THROTTLING_ERROR_CODES
+            @debug "AWS.jl retry for throttling" retry = true reason = "throttled" exception =
+                e
             return true
         end
 
         # Handle BadDigest error and CRC32 check sum failure
-        if _header(e.cause, "crc32body") == "x-amz-crc32" ||
-            e.code in ("BadDigest", "RequestTimeout", "RequestTimeoutException")
+        if _header(e.cause, "crc32body") == "x-amz-crc32"
+            @debug "AWS.jl retry for check sum failure" retry = true reason = "Check sum failure" exception =
+                e
+            return true
+        end
+
+        if e.code in ("BadDigest", "RequestTimeout", "RequestTimeoutException")
+            @debug "AWS.jl retry for $(e.code)" retry = true reason = "$(e.code)" exception =
+                e
             return true
         end
 
         if occursin("Missing Authentication Token", e.message) &&
             aws.credentials === nothing
+            @debug "AWS.jl declined to retry request without credentials" retry = false reason = "No credentials" exception = e
+
             return throw(
                 NoCredentials(
-                    "You're attempting to perform a request without credentials set."
+                    "You're attempting to perform a request without credentials set.",
                 ),
             )
         end
+
+        @debug "AWS.jl declined to retry uncaught error" retry = false reason = "Error unhandled" exception =
+            e
         return false
     end
 

From 5d6916085d4830458c917860230e8b521510e0a9 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 16:54:27 +0200
Subject: [PATCH 02/11] add more logging

---
 src/utilities/request.jl | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 4760ab2d6f..d30ca2ba15 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -262,14 +262,32 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
         return nothing
     end
 
-    check = function (s, e)
-        return isa(e, HTTP.ConnectError) ||
-               isa(e, HTTP.RequestError) ||
-               (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
-    end
-
     delays = AWSExponentialBackoff(; max_attempts=4)
 
+    check = (s, e) -> begin
+            # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
+            # caught and wrapped in an `HTTP.IOError`
+            # https://github.com/JuliaWeb/HTTP.jl/issues/382
+            errors = (Sockets.DNSError, HTTP.ParseError, HTTP.IOError, Base.IOError)
+            for error in errors
+                if isa(e, error)
+                    @debug "AWS.jl HTTP inner retry for $error" retry = true reason = "$error" exception =
+                        e
+                    return true
+                end
+            end
+            if (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
+                @debug "AWS.jl HTTP inner retry for status >= 500" retry = true reason = "status >= 500" status = AWS._http_status(
+                    e
+                ) exception = e
+                return true
+            end
+            @debug "AWS.jl HTTP inner retry declined" retry = false reason = "Exception passed onwards" exception =
+                e
+            return false
+        end
+>>>>>>> fe6382e31 (add more logging)
+
     try
         retry(get_response; check=check, delays=delays)()
     finally

From ba7a65afac1a14ae635673db652c6e6a2de24aef Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 19:23:07 +0200
Subject: [PATCH 03/11] wip tests

---
 Project.toml             |  5 +++++
 src/utilities/request.jl | 33 ++++++++++++++++-----------------
 test/issues.jl           | 14 ++++++++++----
 test/runtests.jl         |  5 +++--
 4 files changed, 34 insertions(+), 23 deletions(-)

diff --git a/Project.toml b/Project.toml
index bebfc3f51f..cd81434d76 100644
--- a/Project.toml
+++ b/Project.toml
@@ -37,6 +37,7 @@ XMLDict = "0.3, 0.4"
 julia = "1.6"
 
 [extras]
+Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
@@ -44,4 +45,8 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [targets]
+<<<<<<< HEAD
 test = ["Pkg", "Suppressor", "StableRNGs", "Test", "UUIDs"]
+=======
+test = ["Pkg", "Logging", "Random", "Suppressor", "Test", "UUIDs"]
+>>>>>>> 387b8d31f (wip tests)
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index d30ca2ba15..78beb72632 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -265,28 +265,27 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
     delays = AWSExponentialBackoff(; max_attempts=4)
 
     check = (s, e) -> begin
-            # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
-            # caught and wrapped in an `HTTP.IOError`
-            # https://github.com/JuliaWeb/HTTP.jl/issues/382
-            errors = (Sockets.DNSError, HTTP.ParseError, HTTP.IOError, Base.IOError)
-            for error in errors
-                if isa(e, error)
-                    @debug "AWS.jl HTTP inner retry for $error" retry = true reason = "$error" exception =
-                        e
-                    return true
-                end
-            end
-            if (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
-                @debug "AWS.jl HTTP inner retry for status >= 500" retry = true reason = "status >= 500" status = AWS._http_status(
+        # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
+        # caught and wrapped in an `HTTP.IOError`
+        # https://github.com/JuliaWeb/HTTP.jl/issues/382
+        errors = (Sockets.DNSError, HTTP.ParseError, HTTP.IOError, Base.IOError)
+        for error in errors
+            if isa(e, error)
+                @debug "AWS.jl HTTP inner retry for $error" retry = true reason = "$error" exception =
                     e
-                ) exception = e
                 return true
             end
-            @debug "AWS.jl HTTP inner retry declined" retry = false reason = "Exception passed onwards" exception =
+        end
+        if (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
+            @debug "AWS.jl HTTP inner retry for status >= 500" retry = true reason = "status >= 500" status = AWS._http_status(
                 e
-            return false
+            ) exception = e
+            return true
         end
->>>>>>> fe6382e31 (add more logging)
+        @debug "AWS.jl HTTP inner retry declined" retry = false reason = "Exception passed onwards" exception =
+            e
+        return false
+    end
 
     try
         retry(get_response; check=check, delays=delays)()
diff --git a/test/issues.jl b/test/issues.jl
index 9c67e7f437..09da3a7c5b 100644
--- a/test/issues.jl
+++ b/test/issues.jl
@@ -202,12 +202,18 @@ try
         config = AWSConfig(; creds=nothing)
 
         @testset "Fail 2 attempts then succeed" begin
-            apply(_incomplete_patch(; data=data, num_attempts_to_fail=2)) do
-                retrieved = S3.get_object(bucket, key; aws_config=config)
+            logger = Test.TestLogger()
+            with_logger(logger) do
+                apply(_incomplete_patch(; data=data, num_attempts_to_fail=2)) do
+                    retrieved = S3.get_object(bucket, key; aws_config=config)
 
-                @test length(retrieved) == n
-                @test retrieved == data
+                    @test length(retrieved) == n
+                    @test retrieved == data
+                end
             end
+            logs = logger.logs
+            @test logs[1].level == Logging.Debug
+            @test logs[1].kwargs.retry
         end
 
         @testset "Fail all 4 attempts then throw" begin
diff --git a/test/runtests.jl b/test/runtests.jl
index 3df8afe121..3731e4b8af 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -25,16 +25,17 @@ using GitHub
 using HTTP
 using IniFile: Inifile
 using JSON
-using OrderedCollections: LittleDict, OrderedDict
+using Logging
 using MbedTLS: digest, MD_SHA256, MD_MD5
 using Mocking
+using OrderedCollections: LittleDict, OrderedDict
 using Pkg
 using Random
+using StableRNGs
 using Suppressor
 using Test
 using UUIDs
 using XMLDict
-using StableRNGs
 
 Mocking.activate()
 

From c4741ba8a9c53050ddb8cfaf0b7c0efbeb47aeba Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 20:14:03 +0200
Subject: [PATCH 04/11] add tests

---
 test/issues.jl | 41 ++++++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/test/issues.jl b/test/issues.jl
index 09da3a7c5b..a1dc956522 100644
--- a/test/issues.jl
+++ b/test/issues.jl
@@ -2,6 +2,15 @@
 
 BUCKET_NAME = "aws-jl-test-issues---" * _now_formatted()
 
+# We try to be insensitive to the existence of other logging messages, so we test
+# by counting the number of messages that meet some criteria
+function log_is_retry(successful)
+    return record ->
+        record.level == Logging.Debug &&
+            haskey(record.kwargs, :retry) &&
+            record.kwargs[:retry] == successful
+end
+
 try
     S3.create_bucket(BUCKET_NAME)
 
@@ -202,7 +211,7 @@ try
         config = AWSConfig(; creds=nothing)
 
         @testset "Fail 2 attempts then succeed" begin
-            logger = Test.TestLogger()
+            logger = Test.TestLogger(; min_level=Logging.Debug)
             with_logger(logger) do
                 apply(_incomplete_patch(; data=data, num_attempts_to_fail=2)) do
                     retrieved = S3.get_object(bucket, key; aws_config=config)
@@ -212,8 +221,10 @@ try
                 end
             end
             logs = logger.logs
-            @test logs[1].level == Logging.Debug
-            @test logs[1].kwargs.retry
+            # Two successful retries
+            @test count(log_is_retry(true), logs) == 2
+            # No unsuccessful ones
+            @test count(log_is_retry(false), logs) == 0
         end
 
         @testset "Fail all 4 attempts then throw" begin
@@ -224,15 +235,23 @@ try
             end
             io = IOBuffer()
 
-            apply(_incomplete_patch(; data=data, num_attempts_to_fail=4)) do
-                params = Dict("response_stream" => io)
-                @test_throws err_t S3.get_object(bucket, key, params; aws_config=config)
-
-                seekstart(io)
-                retrieved = read(io)
-                @test length(retrieved) == n - 1
-                @test retrieved == data[1:(n - 1)]
+            logger = Test.TestLogger(; min_level=Logging.Debug)
+            with_logger(logger) do
+                apply(_incomplete_patch(; data=data, num_attempts_to_fail=4)) do
+                    params = Dict("response_stream" => io)
+                    @test_throws err_t S3.get_object(bucket, key, params; aws_config=config)
+
+                    seekstart(io)
+                    retrieved = read(io)
+                    @test length(retrieved) == n - 1
+                    @test retrieved == data[1:(n - 1)]
+                end
             end
+            logs = logger.logs
+            # Three successful retries - from the inner retry loop
+            @test count(log_is_retry(true), logs) == 3
+            # One unsuccessful one - from the outer loop where we pass it on
+            @test count(log_is_retry(false), logs) == 1
         end
     end
 

From f96081715e530a62b895fea16a75f814c12f7501 Mon Sep 17 00:00:00 2001
From: Eric Hanson <5846501+ericphanson@users.noreply.github.com>
Date: Wed, 11 May 2022 20:39:51 +0200
Subject: [PATCH 05/11] Update src/utilities/request.jl

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
---
 src/utilities/request.jl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 78beb72632..5c9d98dfa7 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -156,9 +156,11 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
         end
     end
 
-    check = function (s, e)
+    check = (s, e) -> begin
         # Pass on non-AWS exceptions.
         if !(e isa AWSException)
+            @debug "AWS.jl declined to retry non-AWSException" retry = false reason = "Non-AWSException" exception =
+                e
             return false
         end
 
@@ -220,7 +222,6 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
     end
 
     delays = AWSExponentialBackoff(; max_attempts=max_attempts(aws))
-
     retry(upgrade_error(get_response); check=check, delays=delays)()
 
     if request.use_response_type

From f5c8cc74a1dd8965d05dc822406b9e89dbeb8f37 Mon Sep 17 00:00:00 2001
From: Matt Brzezinski <matt.brzezinski@beacon.bio>
Date: Mon, 20 Mar 2023 15:32:32 -0500
Subject: [PATCH 06/11] Longer line lengths

---
 src/utilities/downloads_backend.jl | 10 +++-------
 src/utilities/request.jl           | 26 +++++++++-----------------
 test/issues.jl                     |  3 +--
 3 files changed, 13 insertions(+), 26 deletions(-)

diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index 74e33b3810..e2c0b44b15 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -112,17 +112,13 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
 
     check = (s, e) -> begin
             if isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500
-                @debug "AWS.jl Downloads inner retry for status >= 500" retry = true reason = "status >= 500" status = AWS._http_status(
-                    e
-                ) exception = e
+                @debug "AWS.jl Downloads inner retry for status >= 500" retry=true reason="status >= 500" status=._http_status(e) exception=e
                 return true
             elseif isa(e, Downloads.RequestError)
-                @debug "AWS.jl Downloads inner retry for Downloads.RequestError" retry =
-                    true reason = "Downloads.RequestError" exception = e
+                @debug "AWS.jl Downloads inner retry for Downloads.RequestError" retry=true reason="Downloads.RequestError" exception=e
                 return true
             else
-                @debug "AWS.jl Downloads inner retry declined" retry = false reason = "Exception passed onwards" exception =
-                    e
+                @debug "AWS.jl Downloads inner retry declined" retry=false reason="Exception passed onwards" exception=e
                 return false
             end
         end
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index 5c9d98dfa7..e43229158c 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -159,22 +159,19 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
     check = (s, e) -> begin
         # Pass on non-AWS exceptions.
         if !(e isa AWSException)
-            @debug "AWS.jl declined to retry non-AWSException" retry = false reason = "Non-AWSException" exception =
-                e
+            @debug "AWS.jl declined to retry non-AWSException" retry=false reason="Non-AWSException" exception=e
             return false
         end
 
         if occursin("Signature expired", e.message)
-            @debug "AWS.jl retry for signature expired" retry = true reason = "Signature expired" exception =
-                e
+            @debug "AWS.jl retry for signature expired" retry=true reason="Signature expired" exception=e
             return true
         end
 
         # Handle ExpiredToken...
         # https://github.com/aws/aws-sdk-go/blob/v1.31.5/aws/request/retryer.go#L98
         if e isa AWSException && e.code in EXPIRED_ERROR_CODES
-            @debug "AWS.jl retry for expired credentials" retry = true reason = "Credentials expired" exception =
-                e
+            @debug "AWS.jl retry for expired credentials" retry=true reason="Credentials expired" exception=e
             check_credentials(credentials(aws); force_refresh=true)
             return true
         end
@@ -183,31 +180,27 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
         # https://github.com/boto/botocore/blob/1.16.17/botocore/data/_retry.json
         # https://docs.aws.amazon.com/general/latest/gr/api-retries.html
         if _http_status(e.cause) == TOO_MANY_REQUESTS
-            @debug "AWS.jl retry too many requests" retry = true reason = "too many requests" exception =
-                e
+            @debug "AWS.jl retry too many requests" retry=true reason="too many requests" exception=e
             return true
         elseif e.code in THROTTLING_ERROR_CODES
-            @debug "AWS.jl retry for throttling" retry = true reason = "throttled" exception =
-                e
+            @debug "AWS.jl retry for throttling" retry=true reason="throttled" exception=e
             return true
         end
 
         # Handle BadDigest error and CRC32 check sum failure
         if _header(e.cause, "crc32body") == "x-amz-crc32"
-            @debug "AWS.jl retry for check sum failure" retry = true reason = "Check sum failure" exception =
-                e
+            @debug "AWS.jl retry for check sum failure" retry=true reason="Check sum failure" exception=e
             return true
         end
 
         if e.code in ("BadDigest", "RequestTimeout", "RequestTimeoutException")
-            @debug "AWS.jl retry for $(e.code)" retry = true reason = "$(e.code)" exception =
-                e
+            @debug "AWS.jl retry for $(e.code)" retry=true reason="$(e.code)" exception=e
             return true
         end
 
         if occursin("Missing Authentication Token", e.message) &&
             aws.credentials === nothing
-            @debug "AWS.jl declined to retry request without credentials" retry = false reason = "No credentials" exception = e
+            @debug "AWS.jl declined to retry request without credentials" retry=false reason="No credentials" exception=e
 
             return throw(
                 NoCredentials(
@@ -216,8 +209,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             )
         end
 
-        @debug "AWS.jl declined to retry uncaught error" retry = false reason = "Error unhandled" exception =
-            e
+        @debug "AWS.jl declined to retry uncaught error" retry=false reason="Error unhandled" exception=e
         return false
     end
 
diff --git a/test/issues.jl b/test/issues.jl
index a1dc956522..b82e904b18 100644
--- a/test/issues.jl
+++ b/test/issues.jl
@@ -7,8 +7,7 @@ BUCKET_NAME = "aws-jl-test-issues---" * _now_formatted()
 function log_is_retry(successful)
     return record ->
         record.level == Logging.Debug &&
-            haskey(record.kwargs, :retry) &&
-            record.kwargs[:retry] == successful
+        get(record.kwargs, :retry, nothing) == successful
 end
 
 try

From 95ccc75a47d1b5ce0e272401b5508d175e0b6423 Mon Sep 17 00:00:00 2001
From: Matt Brzezinski <matt.brzezinski@beacon.bio>
Date: Mon, 20 Mar 2023 15:47:16 -0500
Subject: [PATCH 07/11] Rebase fix

---
 Project.toml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/Project.toml b/Project.toml
index cd81434d76..a0435f1f05 100644
--- a/Project.toml
+++ b/Project.toml
@@ -45,8 +45,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [targets]
-<<<<<<< HEAD
-test = ["Pkg", "Suppressor", "StableRNGs", "Test", "UUIDs"]
-=======
 test = ["Pkg", "Logging", "Random", "Suppressor", "Test", "UUIDs"]
->>>>>>> 387b8d31f (wip tests)

From d8a00ddd63c12dde38f2eefb77301476c1787e2c Mon Sep 17 00:00:00 2001
From: Matt Brzezinski <matt.brzezinski@beacon.bio>
Date: Mon, 20 Mar 2023 15:57:09 -0500
Subject: [PATCH 08/11] Rebase fixes x2

---
 Project.toml                       | 2 +-
 src/utilities/downloads_backend.jl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Project.toml b/Project.toml
index a0435f1f05..0339e617f2 100644
--- a/Project.toml
+++ b/Project.toml
@@ -45,4 +45,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [targets]
-test = ["Pkg", "Logging", "Random", "Suppressor", "Test", "UUIDs"]
+test = ["Pkg", "Logging", "StableRNGs", "Suppressor", "Test", "UUIDs"]
diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index e2c0b44b15..a54fdbcc68 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -112,7 +112,7 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
 
     check = (s, e) -> begin
             if isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500
-                @debug "AWS.jl Downloads inner retry for status >= 500" retry=true reason="status >= 500" status=._http_status(e) exception=e
+                @debug "AWS.jl Downloads inner retry for status >= 500" retry=true reason="status >= 500" status=_http_status(e) exception=e
                 return true
             elseif isa(e, Downloads.RequestError)
                 @debug "AWS.jl Downloads inner retry for Downloads.RequestError" retry=true reason="Downloads.RequestError" exception=e

From 5b625673c4dff7ce1c45ec1c51239e6959d1d6b0 Mon Sep 17 00:00:00 2001
From: Matt Brzezinski <matt.brzezinski@beacon.bio>
Date: Mon, 20 Mar 2023 16:24:23 -0500
Subject: [PATCH 09/11] Fix?

---
 src/utilities/request.jl | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index e43229158c..b3660afd57 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -148,9 +148,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
                 e = AWSException(e, stream)
                 rethrow(e)
             elseif !(e isa AWSException)
-                @debug "AWS.jl declined to retry non-AWSException" retry = false reason = "Non-AWSException" exception =
-                    e
-                return false
+                @debug "AWS.jl declined to retry non-AWSException" retry=false reason="Non-AWSException" exception=e
             end
             rethrow()
         end

From 5b6c71668924e123abb176ae281d1a1fbd8ea273 Mon Sep 17 00:00:00 2001
From: Matt Brzezinski <matt.brzezinski@beacon.bio>
Date: Mon, 20 Mar 2023 16:39:50 -0500
Subject: [PATCH 10/11] one more pass

---
 src/utilities/downloads_backend.jl |  2 +-
 src/utilities/request.jl           | 15 ++++-----------
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl
index a54fdbcc68..ca95cd5a6c 100644
--- a/src/utilities/downloads_backend.jl
+++ b/src/utilities/downloads_backend.jl
@@ -112,7 +112,7 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
 
     check = (s, e) -> begin
             if isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500
-                @debug "AWS.jl Downloads inner retry for status >= 500" retry=true reason="status >= 500" status=_http_status(e) exception=e
+                @debug "AWS.jl Downloads inner retry for status >= 500" retry=true reason="status >= 500" status=AWS._http_status(e) exception=e
                 return true
             elseif isa(e, Downloads.RequestError)
                 @debug "AWS.jl Downloads inner retry for Downloads.RequestError" retry=true reason="Downloads.RequestError" exception=e
diff --git a/src/utilities/request.jl b/src/utilities/request.jl
index b3660afd57..a0d1e6d118 100644
--- a/src/utilities/request.jl
+++ b/src/utilities/request.jl
@@ -256,25 +256,18 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
     delays = AWSExponentialBackoff(; max_attempts=4)
 
     check = (s, e) -> begin
-        # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
-        # caught and wrapped in an `HTTP.IOError`
-        # https://github.com/JuliaWeb/HTTP.jl/issues/382
-        errors = (Sockets.DNSError, HTTP.ParseError, HTTP.IOError, Base.IOError)
+        errors = (Sockets.DNSError, HTTP.ParseError, Base.IOError)
         for error in errors
             if isa(e, error)
-                @debug "AWS.jl HTTP inner retry for $error" retry = true reason = "$error" exception =
-                    e
+                @debug "AWS.jl HTTP inner retry for $error" retry=true reason="$error" exception=e
                 return true
             end
         end
         if (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
-            @debug "AWS.jl HTTP inner retry for status >= 500" retry = true reason = "status >= 500" status = AWS._http_status(
-                e
-            ) exception = e
+            @debug "AWS.jl HTTP inner retry for status >= 500" retry=true reason="status >= 500" status=_http_status(e) exception=e
             return true
         end
-        @debug "AWS.jl HTTP inner retry declined" retry = false reason = "Exception passed onwards" exception =
-            e
+        @debug "AWS.jl HTTP inner retry declined" retry=false reason="Exception passed onwards" exception=e
         return false
     end
 

From fcc394ba6f968560cdb952bc54bfe5d3dc84e52e Mon Sep 17 00:00:00 2001
From: Matt Brzezinski <matt.brzezinski@beacon.bio>
Date: Tue, 21 Mar 2023 09:29:41 -0500
Subject: [PATCH 11/11] debug statements

---
 test/issues.jl | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/issues.jl b/test/issues.jl
index b82e904b18..c4d17e2d1e 100644
--- a/test/issues.jl
+++ b/test/issues.jl
@@ -220,6 +220,10 @@ try
                 end
             end
             logs = logger.logs
+
+            println("Fail two ")
+            println(logs)
+
             # Two successful retries
             @test count(log_is_retry(true), logs) == 2
             # No unsuccessful ones
@@ -247,6 +251,10 @@ try
                 end
             end
             logs = logger.logs
+
+            println("Fail 4")
+            println(logs)
+
             # Three successful retries - from the inner retry loop
             @test count(log_is_retry(true), logs) == 3
             # One unsuccessful one - from the outer loop where we pass it on