diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 72831a61cc..4b2873c4a3 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,25 +1,29 @@ # This file is machine-generated - editing it directly is not advised [[AWS]] -deps = ["Base64", "Compat", "Dates", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Retry", "Sockets", "UUIDs", "XMLDict"] +deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Retry", "Sockets", "URIs", "UUIDs", "XMLDict"] path = ".." uuid = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" -version = "1.25.2" +version = "1.61.0" + +[[ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" [[Artifacts]] -deps = ["Pkg"] -git-tree-sha1 = "c30985d8821e0cd73870b17b0ed0ce6dc44cb744" uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -version = "1.3.0" [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "919c7f3151e79ff196add81d7f4e45d91bbf420b" +git-tree-sha1 = "96b0bc6c52df76506efc8a441c6cf1adcb1babc4" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.25.0" +version = "3.42.0" + +[[CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" [[Dates]] deps = ["Printf"] @@ -34,10 +38,10 @@ deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[DocStringExtensions]] -deps = ["LibGit2", "Markdown", "Pkg", "Test"] -git-tree-sha1 = "50ddf44c53698f5e784bbebb3f4b21c5807401b1" +deps = ["LibGit2"] +git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.8.3" +version = "0.8.6" [[Documenter]] deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] @@ -45,10 +49,14 @@ git-tree-sha1 = "21fb992ef1b28ff8f315354d3808ebf4a8fa6e45" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" version = "0.26.2" +[[Downloads]] +deps = ["ArgTools", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" + [[ExprTools]] -git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e" +git-tree-sha1 = "56559bbef6ca5ea0c0818fa5c90320398a6fbf8d" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" -version = "0.1.3" +version = "0.1.8" [[EzXML]] deps = ["Printf", "XML2_jll"] @@ -57,16 +65,16 @@ uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" version = "1.1.0" [[GitHub]] -deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal"] -git-tree-sha1 = "a4f61fc1b1724e6eec1d9333eac2d4b01d8fcc8f" +deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal", "URIs"] +git-tree-sha1 = "056781ae7b953289778408b136f8708a46837979" uuid = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" -version = "5.4.0" +version = "5.7.2" [[HTTP]] -deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets", "URIs"] -git-tree-sha1 = "942c1a9c750bbe79912b7bd060a420932afd35b8" +deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"] +git-tree-sha1 = "0fa77022fe4b511826b39c894c90daf5fce3334a" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "0.9.3" +version = "0.9.17" [[IOCapture]] deps = ["Logging"] @@ -75,46 +83,58 @@ uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" version = "0.1.1" [[IniFile]] -deps = ["Test"] -git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8" +git-tree-sha1 = "f550e6e32074c939295eb5ea6de31849ac2c9625" uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" -version = "0.5.0" +version = "0.5.1" [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[IterTools]] -git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18" +git-tree-sha1 = "fa6287a4469f5e048d763df38279ee729fbd44e5" uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" -version = "1.3.0" +version = "1.4.0" [[JLLWrappers]] -git-tree-sha1 = "a431f5f2ca3f4feef3bd7a5e94b8b8d4f2f647a0" +deps = ["Preferences"] +git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.2.0" +version = "1.4.1" [[JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4" +git-tree-sha1 = "3c837543ddb02250ef42f4738347454f95079d4e" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.1" +version = "0.21.3" + +[[LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" + +[[LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" [[LibGit2]] -deps = ["Printf"] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +[[LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" + [[Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [[Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "8e924324b2e9275a51407a4e06deb3455b1e359f" +git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" -version = "1.16.0+7" +version = "1.16.1+1" [[LinearAlgebra]] -deps = ["Libdl"] +deps = ["Libdl", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[Logging]] @@ -131,45 +151,59 @@ uuid = "739be429-bea8-5141-9913-cc70e7f3736d" version = "1.0.3" [[MbedTLS_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "0eef589dd1c26a3ac9d753fe1a8bcad63f956fa6" +deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.16.8+1" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[Mocking]] -deps = ["ExprTools"] -git-tree-sha1 = "916b850daad0d46b8c71f65f719c49957e9513ed" +deps = ["Compat", "ExprTools"] +git-tree-sha1 = "29714d0a7a8083bba8427a4fbfb00a540c681ce7" uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" -version = "0.7.1" +version = "0.7.3" + +[[MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" + +[[NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" + +[[OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" [[OrderedCollections]] -git-tree-sha1 = "4fa2ba51070ec13fcc7517db714445b4ab986bdf" +git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.4.0" +version = "1.4.1" [[Parsers]] deps = ["Dates"] -git-tree-sha1 = "50c9a9ed8c714945e01cd53a21007ed3865ed714" +git-tree-sha1 = "85b5da0fa43588c75bb1ff986493443f821c70b7" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "1.0.15" +version = "2.2.3" [[Pkg]] -deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +[[Preferences]] +deps = ["TOML"] +git-tree-sha1 = "d3538e7f8a790dc8903519090857ef8e1283eecd" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.2.5" + [[Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" [[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets"] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[Random]] -deps = ["Serialization"] +deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[Retry]] @@ -204,14 +238,22 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +[[TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + +[[Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" + [[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[URIs]] -git-tree-sha1 = "7855809b88d7b16e9b029afd17880930626f54a2" +git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -version = "1.2.0" +version = "1.3.0" [[UUIDs]] deps = ["Random", "SHA"] @@ -222,9 +264,9 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[XML2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "be0db24f70aae7e2b89f2f3092e93b8606d659a6" +git-tree-sha1 = "1acf5bdf07aa0907e0a37d3718bb88d4b687b74a" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" -version = "2.9.10+3" +version = "2.9.12+0" [[XMLDict]] deps = ["EzXML", "IterTools", "OrderedCollections"] @@ -233,13 +275,23 @@ uuid = "228000da-037f-5747-90a9-8195ccbf91a5" version = "0.4.1" [[Zlib_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "320228915c8debb12cb434c59057290f0834dbf6" +deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.11+18" + +[[libblastrampoline_jll]] +deps = ["Artifacts", "Libdl", "OpenBLAS_jll"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" [[libsodium_jll]] -deps = ["Libdl", "Pkg"] -git-tree-sha1 = "7127f5f40332ccfa43ee07dcd0c4d81a27d9bb23" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "848ab3d00fe39d6fbc2a8641048f8f272af1c51e" uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" -version = "1.0.18+1" +version = "1.0.20+0" + +[[nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" + +[[p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/docs/make.jl b/docs/make.jl index 868b067ec4..df0aa934c3 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -69,6 +69,7 @@ makedocs(; ), pages=[ "Home" => "index.md", + "Backends" => "backends.md", "AWS" => "aws.md", "Services" => _generate_high_level_services_docs(), ], diff --git a/docs/src/backends.md b/docs/src/backends.md new file mode 100644 index 0000000000..e0edc9d6ef --- /dev/null +++ b/docs/src/backends.md @@ -0,0 +1,20 @@ +# Backends + +AWS.jl supports two "backends" which serve as HTTP clients to reach the AWS REST API. + +The backend can be specified in two ways: by setting the global [`AWS.DEFAULT_BACKEND`](@ref), or by setting the backend on a per-request basis by setting the `"backend"` key in `params` dictionaries: + +```julia +using AWS +@service S3 use_response_type = true +result = S3.get_object(bucket, key, Dict("backend" => AWS.DownloadsBackend())) + +``` +Note: `use_response_type=true` is not needed here for the backend selection to work; it is just a recommended option in general. See [`@service`](@ref) for more. + +```@docs +AWS.AbstractBackend +AWS.DEFAULT_BACKEND +AWS.HTTPBackend +AWS.DownloadsBackend +``` diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl index ba663f239d..ab31e4756b 100644 --- a/src/utilities/downloads_backend.jl +++ b/src/utilities/downloads_backend.jl @@ -1,5 +1,22 @@ using HTTP.MessageRequest: body_was_streamed +""" + DownloadsBackend <: AWS.AbstractBackend + +This backend uses the Downloads.jl stdlib to use libcurl +as an HTTP client to connect to the AWS REST API. + +It has one field, + +- `downloader::Union{Nothing,Downloads.Downloader}` + +which is the `Downloads.Downloader` to use. If set to `nothing`, the default, +then a global downloader object will be used. + +Downloads.jl tends to perform better under concurrent operation than HTTP.jl, +particularly with `@async` / `asyncmap`. As of March 2022, threading (e.g. `@spawn` or `@threads`) with Downloads.jl is broken on all releases of Julia ([Downloads.jl#110](https://github.com/JuliaLang/Downloads.jl/issues/110)), and there are still reported issues on the upcoming +1.7.3 and 1.8 releases ([Downloads.jl#182](https://github.com/JuliaLang/Downloads.jl/issues/182])). +""" struct DownloadsBackend <: AWS.AbstractBackend downloader::Union{Nothing,Downloads.Downloader} end diff --git a/src/utilities/request.jl b/src/utilities/request.jl index 39f8c463b2..fe1a48c1fc 100644 --- a/src/utilities/request.jl +++ b/src/utilities/request.jl @@ -1,12 +1,28 @@ # Used to allow custom dispatches to `_http_request` +""" + AWS.AbstractBackend + +An abstract type representing a "backend" to use as an HTTP client +to connect to the AWS REST API. +""" abstract type AbstractBackend end """ AWS.HTTPBackend <: AWS.AbstractBackend -An `HTTPBackend` can hold default `http_options::AbstractDict{Symbol,<:Any}` -to pass to HTTP.jl, which can be overwritten per-request by any `http_options` -supplied there. +This backend uses HTTP.jl as an HTTP client to connect to the AWS +REST API, and has one field: + +- `http_options::AbstractDict{Symbol,<:Any}` + +which defaults to an empty dictionary. This field provides +default options to use, which can be any of the keyword +arguments to [`HTTP.request`](https://juliaweb.github.io/HTTP.jl/stable/public_interface/#HTTP.request). These options are overwritten by any per-request options. + +This is the default backend, and the only option until AWS.jl v1.57.0. Therefore, it has +been used more often in practice, and may be more mature. Note, however, HTTP.jl currently +(March 2022) has issues with concurrency (see [HTTP.jl#517](https://github.com/JuliaWeb/HTTP.jl/issues/517)). Therefore, it may be advisable to switch to the [`DownloadsBackend`](@ref) if you +are using concurrency. """ struct HTTPBackend <: AbstractBackend http_options::AbstractDict{Symbol,<:Any} @@ -20,6 +36,24 @@ function HTTPBackend(; kwargs...) end end # populated in `__init__` +""" + AWS.DEFAULT_BACKEND = Ref{AbstractBackend}() + +This specifies the default backend to use. This can be modified +to change the default backend used by AWS.jl: + +```julia +using AWS +AWS.DEFAULT_BACKEND[] = AWS.DownloadsBackend() +``` + +As an alternative, the `backend` can be specified on a per-request basis, by +adding a pair `"backend" => AWS.DownloadsBackend()` to the `params` argument of +AWS.jl functions. + +!!! warning + Setting the `AWS.DEFAULT_BACKEND` is a global change that affects all packages in your Julia session using AWS.jl. Therefore, it is not recommended for library code to change the default backend, and instead set the backend on a per-request basis if needed (or ask the user to set a default backend). If you do wish to change the default backend inside package code which is precompiled, then it must be changed from within the `__init__` method. See the [Julia manual](https://docs.julialang.org/en/v1/manual/modules/#Module-initialization-and-precompilation) for more on module initialization. +""" const DEFAULT_BACKEND = Ref{AbstractBackend}() Base.@kwdef mutable struct Request