From cd3ad530ef0ec13171f80b99c9161d512f929e8f Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 20 Jul 2021 01:19:05 +0200 Subject: [PATCH 1/2] add downloads backend --- Project.toml | 3 +- src/AWS.jl | 2 + src/utilities/downloads_backend.jl | 87 ++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 src/utilities/downloads_backend.jl diff --git a/Project.toml b/Project.toml index b775616e0a..1a8299c891 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ version = "1.52.0" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" IniFile = "83e8ac13-25f8-5344-8a64-a9f2b223428f" @@ -32,7 +33,7 @@ OrderedCollections = "1" Retry = "0.3, 0.4" URIs = "1" XMLDict = "0.3, 0.4" -julia = "1" +julia = "1.3" [extras] Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" diff --git a/src/AWS.jl b/src/AWS.jl index e205dff8ad..a856959d4e 100644 --- a/src/AWS.jl +++ b/src/AWS.jl @@ -3,6 +3,7 @@ module AWS using Compat: Compat, @something using Base64 using Dates +using Downloads using HTTP using MbedTLS using Mocking @@ -33,6 +34,7 @@ include("AWSMetadata.jl") include(joinpath("utilities", "request.jl")) include(joinpath("utilities", "sign.jl")) +include(joinpath("utilities", "downloads_backend.jl")) using ..AWSExceptions diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl new file mode 100644 index 0000000000..c37681e6b5 --- /dev/null +++ b/src/utilities/downloads_backend.jl @@ -0,0 +1,87 @@ +struct DownloadsBackend <: AWS.AbstractBackend + downloader::Union{Nothing, Downloads.Downloader} +end + +DownloadsBackend() = DownloadsBackend(nothing) + +const AWS_DOWNLOADER = Ref{Union{Nothing, Downloader}}(nothing) +const AWS_DOWNLOAD_LOCK = ReentrantLock() + +# Here we mimic Download.jl's own setup for using a global downloader. +# We do this to have our own downloader (separate from Downloads.jl's global downloader) +# because we add a hook to avoid redirects in order to try to match the HTTPBackend's +# implementation, and we don't want to mutate the global downloader from Downloads.jl. +# https://github.com/JuliaLang/Downloads.jl/blob/84e948c02b8a0625552a764bf90f7d2ee97c949c/src/Downloads.jl#L293-L301 +function get_downloader(downloader=nothing) + lock(AWS_DOWNLOAD_LOCK) do + yield() # let other downloads finish + downloader isa Downloader && return + while true + downloader = AWS_DOWNLOADER[] + downloader isa Downloader && return + D = Downloader() + D.easy_hook = (easy, info) -> Curl.setopt(easy, Curl.CURLOPT_FOLLOWLOCATION, false) + AWS_DOWNLOADER[] = D + end + end + return downloader +end + + +function AWS._http_request(backend::DownloadsBackend, request) + # If we pass `output`, Downloads.jl will expect a message + # body in the response. Specifically, it sets + # + # only when we do not pass the `output` argument. + # + # When the method is `HEAD`, the response may have a Content-Length + # but not send any content back (which appears to be correct, + # ). + # + # Thus, if we did not set `CURLOPT_NOBODY`, and it gets a Content-Length + # back, it will hang waiting for that body. + # + # Therefore, we do not pass an `output` when the `request_method` is `HEAD`. + if request.request_method != "HEAD" + output = IOBuffer() + output_arg = (; output=output) + + # We set a callback so later on we know how to get the `body` back. + body_arg = () -> (; body = take!(output)) + else + output_arg = NamedTuple() + body_arg = () -> NamedTuple() + end + + # We pass an `input` only when we have content we wish to send. + if !isempty(request.content) + input = IOBuffer() + write(input, request.content) + input_arg = (; input=input) + else + input_arg = NamedTuple() + end + + @repeat 4 try + downloader = @something(backend.downloader, get_downloader()) + # set the hook so that we don't follow redirects. Only + # need to do this on per-request downloaders, because we + # set our global one with this hook already. + if backend.downloader !== nothing + downloader.easy_hook = (easy, info) -> Curl.setopt(easy, Curl.CURLOPT_FOLLOWLOCATION, false) + end + response = Downloads.request(request.url; input_arg..., output_arg..., + method = request.request_method, + request.headers, verbose=false, throw=true, + downloader) + http_response = HTTP.Response(response.status, response.headers; body_arg()..., request=nothing) + + if HTTP.iserror(http_response) + target = HTTP.resource(HTTP.URI(request.url)) + throw(HTTP.StatusError(http_response.status, request.request_method, target, http_response)) + end + return http_response + catch e + @delay_retry if ((isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500) || isa(e, Downloads.RequestError)) end + end +end From 65fe8ce77f5e1496fa5307f31971953f6db53819 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 20 Jul 2021 01:19:11 +0200 Subject: [PATCH 2/2] test with it --- test/runtests.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index c79b758356..66cf990d68 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -34,6 +34,8 @@ function _now_formatted() return lowercase(Dates.format(now(Dates.UTC), dateformat"yyyymmdd\THHMMSSsss\Z")) end +AWS.DEFAULT_BACKEND[] = AWS.DownloadsBackend() + @testset "AWS.jl" begin include("AWS.jl") include("AWSCredentials.jl")