got IOError during POST with Form email

[deahhh]

bug report with:

• Julia 1.6
• HTTP.jl v0.9.17

using HTTP
function test3()
	url = "http://post.oreilly.com/client/o/oreilly/forms/quicksignup.cgi"
	r = HTTP.request("POST", url, [], HTTP.Form(Dict(["email_addr"=>"[email protected]"])), timeout=5)
end
test3()

output as

IOError(EOFError() during request(https://get.oreilly.com/email-signup.html?utm_medium=email&utm_source=oreilly.com&utm_campaign=na&utm_content=20180518+ipost+redirect))

    read_to_buffer(::HTTP.ConnectionPool.Transaction{MbedTLS.SSLContext}, ::Int64)@ConnectionPool.jl:275
    readuntil(::HTTP.ConnectionPool.Transaction{MbedTLS.SSLContext}, ::Function, ::Int64)@ConnectionPool.jl:294
    readuntil@ConnectionPool.jl:292[inlined]
    readheaders@Messages.jl:537[inlined]
    startread(::HTTP.Streams.Stream{HTTP.Messages.Response, HTTP.ConnectionPool.Transaction{MbedTLS.SSLContext}})@Streams.jl:170
    macro expansion@StreamRequest.jl:67[inlined]
    macro expansion@task.jl:387[inlined]
    var"#request#1"(::Bool, ::Nothing, ::Nothing, ::Int64, ::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:timeout, :redirect), Tuple{Int64, Bool}}}, ::typeof(HTTP.request), ::Type{HTTP.StreamRequest.StreamLayer{Union{}}}, ::HTTP.ConnectionPool.Transaction{MbedTLS.SSLContext}, ::HTTP.Messages.Request, ::HTTP.Form)@StreamRequest.jl:57
    var"#request#1"(::Nothing, ::Type, ::Int64, ::Base.Iterators.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :reached_redirect_limit, :timeout, :redirect), Tuple{Nothing, Bool, Int64, Bool}}}, ::typeof(HTTP.request), ::Type{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}, ::URIs.URI, ::HTTP.Messages.Request, ::HTTP.Form)@ConnectionRequest.jl:108
    var"#request#1"(::Base.Iterators.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :reached_redirect_limit, :timeout, :redirect), Tuple{Nothing, Bool, Int64, Bool}}}, ::typeof(HTTP.request), ::Type{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}, ::URIs.URI, ::Vararg{Any, N} where N)@ExceptionRequest.jl:19
    (::Base.var"#70#71#73"{ExponentialBackOff, HTTP.RetryRequest.var"#2#3"{Bool, HTTP.Messages.Request}, typeof(HTTP.request)})(::Base.Iterators.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :reached_redirect_limit, :timeout, :redirect), Tuple{Nothing, Bool, Int64, Bool}}}, ::Base.var"#70#72"{Base.var"#70#71#73"{ExponentialBackOff, HTTP.RetryRequest.var"#2#3"{Bool, HTTP.Messages.Request}, typeof(HTTP.request)}}, ::Type, ::Vararg{Any, N} where N)@error.jl:288
    #request#[email protected]:44[inlined]
    var"#request#1"(::VersionNumber, ::String, ::HTTP.Messages.Response, ::Nothing, ::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:reached_redirect_limit, :timeout, :redirect), Tuple{Bool, Int64, Bool}}}, ::typeof(HTTP.request), ::Type{HTTP.MessageRequest.MessageLayer{HTTP.RetryRequest.RetryLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}}, ::String, ::URIs.URI, ::Vector{Pair{SubString{String}, SubString{String}}}, ::HTTP.Form)@MessageRequest.jl:66
    var"#request#1"(::Base.Iterators.Pairs{Symbol, Any, NTuple{4, Symbol}, NamedTuple{(:reached_redirect_limit, :timeout, :redirect, :parent), Tuple{Bool, Int64, Bool, HTTP.Messages.Response}}}, ::typeof(HTTP.request), ::Type{HTTP.BasicAuthRequest.BasicAuthLayer{HTTP.MessageRequest.MessageLayer{HTTP.RetryRequest.RetryLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}}}, ::String, ::URIs.URI, ::Vector{Pair{SubString{String}, SubString{String}}}, ::HTTP.Form)@BasicAuthRequest.jl:28
    var"#request#1"(::Int64, ::Bool, ::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:timeout, :redirect), Tuple{Int64, Bool}}}, ::typeof(HTTP.request), ::Type{HTTP.RedirectRequest.RedirectLayer{HTTP.BasicAuthRequest.BasicAuthLayer{HTTP.MessageRequest.MessageLayer{HTTP.RetryRequest.RetryLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}}}}, ::String, ::URIs.URI, ::Vector{Pair{SubString{String}, SubString{String}}}, ::HTTP.Form)@RedirectRequest.jl:28
    #request#[email protected]:15[inlined]
    var"#request#9"(::Vector{Any}, ::HTTP.Form, ::Nothing, ::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:timeout, :redirect), Tuple{Int64, Bool}}}, ::typeof(HTTP.request), ::String, ::String, ::Vector{Any}, ::HTTP.Form)@HTTP.jl:330
    test3()@[Other: 3](http://0.0.0.0:1234/edit?id=eda36532-d4ff-11ec-26c4-1565cdb7f07a#)
    top-level scope@[Local: 1](http://0.0.0.0:1234/edit?id=eda36532-d4ff-11ec-26c4-1565cdb7f07a#)[inlined]

[fonsp]

I can reproduce on latest Julia & HTTP. The same request seems to go through on JS:

➜  ~ deno
Deno 1.19.2
exit using ctrl+d or close()
> let url = "http://post.oreilly.com/client/o/oreilly/forms/quicksignup.cgi"
undefined
> let data = new FormData()
undefined
> data.append("email_addr", "[email protected]")
undefined
> let r = await fetch(url, {
	method: "post",
	body: data,
})
undefined

> await r.text()
`\n<!DOCTYPE html>\n<html lang="en"><head>\n  <meta charset="utf-8">\n  <meta http-equiv="X-UA-Compatible" content="IE=edge">\n  <meta name="viewport" content="width=device-width, initial-scale=1">\n  <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->\n  <meta name="description" content="Keep ahead of what's next with the latest ideas, and knowledge you can put into action, in all the subjects you care about, from O'Reilly's community of experts.">\n  <meta name="author" content><meta name="robots" content="index, nofollow">\n\n  <link rel="shortcut icon" href="https://www.oreilly.com/favicon.ico" type="image/vnd.microsoft.icon">\n\n  <title>O'Reilly - Subscription Center</title>\n\n  <link rel="stylesheet" type="text/css" href="https://cdn.oreillystatic.com/assets/css/2017_font_face.css">\n  <link rel="stylesheet" type="text/css" href="https://cdn.oreillystatic.com/assets/css/subscription-center-topics.css">\n  \n  <noscript>\n    <style>\n      .mktoForm ...... etc

[quinnj]

Thanks for the report @deahhh and for confirming @fonsp! I have a PR up that should fix this case: #829, and clean up our "which method should be used in redirect requests" behavior more generally. The core issue here is we were using/sending the original request method (POST) in the redirect request, which the server did not accept/support and hence the error. It turns out that should be legal in this case since the server is returning a 301 response status, but in practice, most http clients will switch POST -> GET, so most servers now expect that behavior (hence why other language implementations were working here and we were erroring). I am proposing a new redirect_method keyword arg that will allow users precise control over this behavior if needed, similar to how curl provides the ability to override this for redirected POST requests.

[deahhh]

Thanks a lot! @quinnj @fonsp