From 1226c4b29d6833bb93bc753b8683d9a848cefcaf Mon Sep 17 00:00:00 2001 From: quinnj Date: Wed, 30 Nov 2016 20:06:00 -0700 Subject: [PATCH] Initial commit including url parsing, http common types, http request/response parsing and a new Server type --- src/HTTP.jl | 20 +- src/parser.jl | 311 ++++++++++++++++++++++++++++++++ src/serve.jl | 213 ++++++++++++++++++++++ src/statuscodes.jl | 74 ++++++++ src/types.jl | 124 +++++++++++++ src/uri.jl | 441 +++++++++++++++++++++++++++++++++++++++++++++ src/utils.jl | 34 ++++ test/cert.pem | 18 ++ test/key.pem | 28 +++ test/parser.jl | 195 ++++++++++++++++++++ test/runtests.jl | 73 +++++++- test/uri.jl | 66 +++++++ 12 files changed, 1592 insertions(+), 5 deletions(-) create mode 100644 src/parser.jl create mode 100644 src/serve.jl create mode 100644 src/statuscodes.jl create mode 100644 src/types.jl create mode 100644 src/uri.jl create mode 100644 src/utils.jl create mode 100644 test/cert.pem create mode 100644 test/key.pem create mode 100644 test/parser.jl create mode 100644 test/uri.jl diff --git a/src/HTTP.jl b/src/HTTP.jl index 1cacfa12b..ebc0096ea 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -1,5 +1,23 @@ module HTTP -# package code goes here +export Request, Response, URI + +using MbedTLS + +const TLS = MbedTLS + +import Base.== + +include("statuscodes.jl") +include("utils.jl") +include("uri.jl") +include("types.jl") +include("parser.jl") +include("serve.jl") + +# package-wide inits +function __init__() + __init__parser() +end end # module diff --git a/src/parser.jl b/src/parser.jl new file mode 100644 index 000000000..0dcd29c85 --- /dev/null +++ b/src/parser.jl @@ -0,0 +1,311 @@ +const libhttp_parser = "/Users/jacobquinn/.julia/v0.5/HttpParser/deps/usr/lib/libhttp_parser.dylib" + +parsertype(::Type{Request}) = 0 # HTTP_REQUEST +parsertype(::Type{Response}) = 1 # HTTP_RESPONSE + +# A composite type that matches bit-for-bit a C struct. +type Parser + # parser + flag = Single byte + type_and_flags::Cuchar + + state::Cuchar + header_state::Cuchar + index::Cuchar + + nread::UInt32 + content_length::UInt64 + + http_major::Cushort + http_minor::Cushort + status_code::Cushort + method::Cuchar + + # http_errno + upgrade = Single byte + errno_and_upgrade::Cuchar + + data::Any +end + +parserwrapper(::Type{Request}) = ParserRequest() +parserwrapper(::Type{Response}) = ParserResponse() + +function Parser{T}(::Type{T}=Request) + parser = Parser(convert(Cuchar, 0), convert(Cuchar, 0), convert(Cuchar, 0), convert(Cuchar, 0), + convert(UInt32, 0), convert(UInt64, 0), + convert(Cushort, 0), convert(Cushort, 0), convert(Cushort, 0), convert(Cuchar, 0), + convert(Cuchar, 0), + convert(Ptr{UInt8}, C_NULL)) + parser.data = parserwrapper(T) + http_parser_init(parser, T) + return parser +end + +# Intializes the Parser object with the correct memory +function http_parser_init{T}(parser, ::Type{T}) + ccall((:http_parser_init, libhttp_parser), Void, (Ptr{Parser}, Cint), &parser, parsertype(T)) +end + +# A composite type that is expecting C functions to be run as callbacks. +type ParserSettings + on_message_begin_cb::Ptr{Void} + on_url_cb::Ptr{Void} + on_status_complete_cb::Ptr{Void} + on_header_field_cb::Ptr{Void} + on_header_value_cb::Ptr{Void} + on_headers_complete_cb::Ptr{Void} + on_body_cb::Ptr{Void} + on_message_complete_cb::Ptr{Void} + on_chunk_header::Ptr{Void} + on_chunk_complete::Ptr{Void} +end + +ParserSettings(on_message_begin_cb, on_url_cb, on_status_complete_cb, on_header_field_cb, on_header_value_cb, on_headers_complete_cb, on_body_cb, on_message_complete_cb) = + ParserSettings(on_message_begin_cb, on_url_cb, on_status_complete_cb, on_header_field_cb, on_header_value_cb, on_headers_complete_cb, on_body_cb, on_message_complete_cb, C_NULL, C_NULL) + +function Base.show(io::IO, p::Parser) + print(io,"libhttp-parser: v$(version()), ") + print(io,"HTTP/$(p.http_major).$(p.http_minor), ") + print(io,"Content-Length: $(p.content_length)") +end + +function version() + ver = ccall((:http_parser_version, libhttp_parser), Culong, ()) + major = (ver >> 16) & 255 + minor = (ver >> 8) & 255 + patch = ver & 255 + return VersionNumber(major, minor, patch) +end + +# Run a request through a parser with specific callbacks on the settings instance +function http_parser_execute(parser::Parser, settings::ParserSettings, request, len=sizeof(request)) + return ccall((:http_parser_execute, libhttp_parser), Csize_t, + (Ptr{Parser}, Ptr{ParserSettings}, Cstring, Csize_t,), + Ref(parser), Ref(settings), convert(Cstring, pointer(request)), len) +end + +"Returns a string version of the HTTP method." +function http_method_str(method) + val = ccall((:http_method_str, libhttp_parser), Cstring, (Int,), method) + return unsafe_string(val) +end + +# Is the request a keep-alive request? +http_should_keep_alive(parser::Ptr{Parser}) = ccall((:http_should_keep_alive, libhttp_parser), Int, (Ptr{Parser},), Ref(parser)) != 0 + +"Pauses the parser." +pause(parser::Parser) = ccall((:http_parser_pause, libhttp_parser), Void, (Ptr{Parser}, Cint), Ref(parser), one(Cint)) +"Resumes the parser." +resume(parser::Parser) = ccall((:http_parser_pause, libhttp_parser), Void,(Ptr{Parser}, Cint), Ref(parser), zero(Cint)) +"Checks if this is the final chunk of the body." +isfinalchunk(parser::Parser) = ccall((:http_parser_pause, libhttp_parser), Cint, (Ptr{Parser},), Ref(parser)) == 1 + +upgrade(parser::Parser) = (parser.errno_and_upgrade & 0b10000000) > 0 +errno(parser::Parser) = parser.errno_and_upgrade & 0b01111111 +errno_name(errno::Integer) = unsafe_string(ccall((:http_errno_name, libhttp_parser), Cstring, (Int32,), errno)) +errno_description(errno::Integer) = unsafe_string(ccall((:http_errno_description, libhttp_parser), Cstring, (Int32,), errno)) + +immutable ParserError <: Exception + errno::Int32 + ParserError(errno::Integer) = new(Int32(errno)) +end + +Base.show(io::IO, err::ParserError) = print(io, "HTTP.ParserError: ", errno_name(err.errno), " (", err.errno, "): ", errno_description(err.errno)) + +# Dedicated types for parsing Request/Response types +type ParserRequest + val::Request + parsedfield::Bool + fieldbuffer::Vector{UInt8} + valuebuffer::Vector{UInt8} + messagecomplete::Bool +end + +ParserRequest() = ParserRequest(Request(), true, UInt8[], UInt8[], false) + +type ParserResponse + val::Response + parsedfield::Bool + fieldbuffer::Vector{UInt8} + valuebuffer::Vector{UInt8} + messagecomplete::Bool +end + +ParserResponse() = ParserResponse(Response(), true, UInt8[], UInt8[], false) + +# Default callbacks for requests and responses +getrequest(p::Ptr{Parser}) = (unsafe_load(p).data)::ParserRequest +getresponse(p::Ptr{Parser}) = (unsafe_load(p).data)::ParserResponse + +# on_message_begin +function request_on_message_begin(parser) + r = getrequest(parser) + r.messagecomplete = false + return 0 +end + +function response_on_message_begin(parser) + r = getresponse(parser) + r.messagecomplete = false + return 0 +end + +# on_url (requests only) +function request_on_url(parser, at, len) + r = getrequest(parser) + r.val.resource = string(r.val.resource, unsafe_string(convert(Ptr{UInt8}, at), len)) + r.val.uri = URI(r.val.resource) + return 0 +end +response_on_url(parser, at, len) = 0 + +# on_status_complete (responses only) +function response_on_status_complete(parser) + r = getresponse(parser) + r.val.status = unsafe_load(parser).status_code + return 0 +end +request_on_status_complete(parser) = 0 + +# on_header_field, on_header_value +function request_on_header_field(parser, at, len) + r = getrequest(parser) + if r.parsedfield + append!(r.fieldbuffer, unsafe_wrap(Array, convert(Ptr{UInt8}, at), len)) + else + r.val.headers[String(r.fieldbuffer)] = String(r.valuebuffer) + r.fieldbuffer = unsafe_wrap(Array, convert(Ptr{UInt8}, at), len) + end + r.parsedfield = true + return 0 +end + +function request_on_header_value(parser, at, len) + r = getrequest(parser) + if r.parsedfield + r.valuebuffer = unsafe_wrap(Array, convert(Ptr{UInt8}, at), len) + else + append!(r.valuebuffer, unsafe_wrap(Array, convert(Ptr{UInt8}, at), len)) + end + r.parsedfield = false + return 0 +end + +function response_on_header_field(parser, at, len) + r = getresponse(parser) + if r.parsedfield + append!(r.fieldbuffer, unsafe_wrap(Array, convert(Ptr{UInt8}, at), len)) + else + r.val.headers[String(r.fieldbuffer)] = String(r.valuebuffer) + r.fieldbuffer = unsafe_wrap(Array, convert(Ptr{UInt8}, at), len) + end + r.parsedfield = true + return 0 +end + +function response_on_header_value(parser, at, len) + r = getresponse(parser) + if r.parsedfield + r.valuebuffer = unsafe_wrap(Array, convert(Ptr{UInt8}, at), len) + else + append!(r.valuebuffer, unsafe_wrap(Array, convert(Ptr{UInt8}, at), len)) + end + r.parsedfield = false + return 0 +end + +# on_headers_complete +function request_on_headers_complete(parser) + r = getrequest(parser) + p = unsafe_load(parser) + if length(r.fieldbuffer) > 0 + r.val.headers[String(r.fieldbuffer)] = String(r.valuebuffer) + end + r.val.method = http_method_str(p.method) + r.val.major = p.http_major + r.val.minor = p.http_minor + r.val.keepalive = http_should_keep_alive(parser) != 0 + return 0 +end + +function response_on_headers_complete(parser) + r = getresponse(parser) + p = unsafe_load(parser) + if length(r.fieldbuffer) > 0 + r.val.headers[String(r.fieldbuffer)] = String(r.valuebuffer) + end + r.val.status = p.status_code + r.val.major = p.http_major + r.val.minor = p.http_minor + return 0 +end + +# on_body +function on_body(parser, at, len) + append!(unsafe_load(parser).data.val.data, unsafe_wrap(Array, convert(Ptr{UInt8}, at), len)) + return 0 +end + +# on_message_complete +function request_on_message_complete(parser) + r = getrequest(parser) + r.messagecomplete = true + return 0 +end + +function response_on_message_complete(parser) + r = getresponse(parser) + r.messagecomplete = true + return 0 +end + +# Main user-facing functions +function parse(::Type{Request}, str) + http_parser_init(DEFAULT_REQUEST_PARSER, Request) + http_parser_execute(DEFAULT_REQUEST_PARSER, DEFAULT_REQUEST_PARSER_SETTINGS, str, sizeof(str)) + if errno(DEFAULT_REQUEST_PARSER) != 0 + throw(ParserError(errno(DEFAULT_REQUEST_PARSER))) + end + return (DEFAULT_REQUEST_PARSER.data.val)::Request +end + +function parse(::Type{Response}, str) + http_parser_init(DEFAULT_REQUEST_PARSER, Response) + http_parser_execute(DEFAULT_RESPONSE_PARSER, DEFAULT_RESPONSE_PARSER_SETTINGS, str, sizeof(str)) + if errno(DEFAULT_RESPONSE_PARSER) != 0 + throw(ParserError(errno(DEFAULT_RESPONSE_PARSER))) + end + return (DEFAULT_RESPONSE_PARSER.data.val)::Response +end + +function __init__parser() + HTTP_CB = (Int, (Ptr{Parser},)) + HTTP_DATA_CB = (Int, (Ptr{Parser}, Ptr{Cchar}, Csize_t,)) + # Turn all the callbacks into C callable functions. + global const request_on_message_begin_cb = cfunction(request_on_message_begin, HTTP_CB...) + global const request_on_url_cb = cfunction(request_on_url, HTTP_DATA_CB...) + global const request_on_status_complete_cb = cfunction(request_on_status_complete, HTTP_CB...) + global const request_on_header_field_cb = cfunction(request_on_header_field, HTTP_DATA_CB...) + global const request_on_header_value_cb = cfunction(request_on_header_value, HTTP_DATA_CB...) + global const request_on_headers_complete_cb = cfunction(request_on_headers_complete, HTTP_CB...) + global const on_body_cb = cfunction(on_body, HTTP_DATA_CB...) + global const request_on_message_complete_cb = cfunction(request_on_message_complete, HTTP_CB...) + global const DEFAULT_REQUEST_PARSER_SETTINGS = ParserSettings(request_on_message_begin_cb, request_on_url_cb, + request_on_status_complete_cb, request_on_header_field_cb, + request_on_header_value_cb, request_on_headers_complete_cb, + on_body_cb, request_on_message_complete_cb) + global const response_on_message_begin_cb = cfunction(response_on_message_begin, HTTP_CB...) + global const response_on_url_cb = cfunction(response_on_url, HTTP_DATA_CB...) + global const response_on_status_complete_cb = cfunction(response_on_status_complete, HTTP_CB...) + global const response_on_header_field_cb = cfunction(response_on_header_field, HTTP_DATA_CB...) + global const response_on_header_value_cb = cfunction(response_on_header_value, HTTP_DATA_CB...) + global const response_on_headers_complete_cb = cfunction(response_on_headers_complete, HTTP_CB...) + global const response_on_message_complete_cb = cfunction(response_on_message_complete, HTTP_CB...) + global const DEFAULT_RESPONSE_PARSER_SETTINGS = ParserSettings(response_on_message_begin_cb, response_on_url_cb, + response_on_status_complete_cb, response_on_header_field_cb, + response_on_header_value_cb, response_on_headers_complete_cb, + on_body_cb, response_on_message_complete_cb) + # + global const DEFAULT_REQUEST_PARSER = Parser(Request) + global const DEFAULT_RESPONSE_PARSER = Parser(Response) + return +end diff --git a/src/serve.jl b/src/serve.jl new file mode 100644 index 000000000..46063e7e8 --- /dev/null +++ b/src/serve.jl @@ -0,0 +1,213 @@ +# High-level + # Bind TCPServer and listen to host:port + # Outer event loop: + # accept TCPSocket connections + # @async the connection off into it's own green thread + # initialize a new HTTP Parser for this new TCPSocket connection + # continue reading data off connection and passing it thru to our parser + # as parser detects "checkpoints", callback functions will be called + # on_url: choose appropriate resource handler from those "registered" + # on_message_complete: + +#TODO: + # request/response timeout abilities + # appropriate size limits/timeouts for different parts of request messages + # handle Expect 100 Continue + # special case OPTIONS method like go? + # handle redirects + # read through RFCs, writing tests + # support auto-decompress, deflate, gunzip + +abstract ServerType + +immutable http <: ServerType end +immutable https <: ServerType end +# immutable ws <: ServerType end +# immutable wss <: ServerType end + +type Server{T <: ServerType} # {I <: IPAddr, H <: ServerHandler, T <: ServerType} + host::IPAddr + port::Int + handler::Function + logger::IO + count::Int + tlsconfig::TLS.SSLConfig + tcp::Base.TCPServer + + function Server(host::IPAddr, port::Integer, handler::Function, logger::IO) + new{T}(host, port, handler, logger, 0) + end +end + +Base.listen(s::Server) = (s.tcp = listen(s.host, s.port); return nothing) + +const DEFAULT_BUFFER_SIZE = 1024 + +type ServerClient{T <: ServerType, I <: IO} + id::Int + server::Server + parser::Parser + keepalive::Bool + replied::Bool + buffer::Vector{UInt8} + tcp::TCPSocket + socket::I + request::Request + response::Response + + function ServerClient(id, server, parser, keepalive, replied, buffer, tcp, socket) + sc = new(id, server, parser, keepalive, replied, buffer, tcp, socket) + sc.request = parser.data.val + sc.response = Response() + return sc + end +end + +function ServerClient{T}(server::Server{T}) + if T == https + tls = TLS.SSLContext() + TLS.setup!(tls, server.tlsconfig) + c = ServerClient{https, TLS.SSLContext}(server.count += 1, server, Parser(Request), false, false, Vector{UInt8}(DEFAULT_BUFFER_SIZE), TCPSocket(), tls) + else + tcp = TCPSocket() + c = ServerClient{http, TCPSocket}(server.count += 1, server, Parser(Request), false, false, Vector{UInt8}(DEFAULT_BUFFER_SIZE), tcp, tcp) + end + finalizer(c, x->close(x.tcp)) + return c +end + +# if haskey(r.headers, "Expect") +# if r.headers["Expect"] == "100-continue" +# write(c.socket, "HTTP/1.1 100 Continue\r\n\r\n") +# else +# # expectation failed +# write(c.socket, "HTTP/1.1 417 Expectation Failed\r\n\r\n") +# c.replied = true +# end +# end + + +# gets called when a full request has been parsed by our HTTP.Parser +function handle!(client::ServerClient) + local response + println(client.server.logger, "Received request on client: $(client.id) \n"); flush(STDOUT) + show(client.server.logger, client.request) + println(client.server.logger); flush(STDOUT) + if !client.replied + try + response = client.server.handler(client.request, client.response) + catch err + response = Response(500) + Base.display_error(err, catch_backtrace()) + end + + response.headers["Connection"] = client.keepalive ? "keep-alive" : "close" + println(client.server.logger, "Responding with response on client: $(client.id) \n"); flush(STDOUT) + show(client.server.logger, response) + println(client.server.logger); flush(STDOUT) + write(client.socket, response) + end + client.replied = false + client.keepalive || close(client.socket) + return +end + +initTLS!(client::ServerClient{http}) = return +function initTLS!(client::ServerClient{https}) + try + TLS.associate!(client.socket, client.tcp) + TLS.handshake!(client.socket) + catch e + println("Error establishing SSL connection: ", e); flush(STDOUT) + close(client.tcp) + end + return +end + +function process!{T}(client::ServerClient{T}) + println(client.server.logger, "`process!`: client connection: ", client.id); flush(STDOUT) + initTLS!(client) + retry = 1 + while !eof(client.socket) + # if no data after 30 seconds, break out + nb = @timeout 10 readbytes!(client.socket, client.buffer, nb_available(client.socket)) break + if nb < 1 + # `retry` tracks how many times we've unsuccessfully read data from the client + # give up after 10 seconds of no data received + sleep(1) + retry += 1 + retry == 10 && break + continue + else + http_parser_execute(client.parser, DEFAULT_REQUEST_PARSER_SETTINGS, client.buffer, nb) + if errno(client.parser) != 0 + # error in parsing the http request + break + elseif client.parser.data.messagecomplete + retry = 0 + handle!(client) + end + end + end + println(client.server.logger, "`process!`: finished processing client: ", client.id); flush(STDOUT) +end + +#behavior I'm seeing + # setup TCPServer listening on localhost:port + # create TCPSocket() + # call `accept(server, client)` + # call @async process!(client) + # initialize another TCPSocket() client + # the @async process! seems to be getting the 2nd TCPSocket client + +function serve{T}(server::Server{T}) + println(server.logger, "Starting server to listen on: $(server.host):$(server.port)"); flush(STDOUT) + listen(server) + + while true + # initialize a ServerClient w/ Parser + client = ServerClient(server) + println(server.logger, "New client initialized: ", client.id); flush(STDOUT) + try + # accept blocks until a new connection is detected + accept(server.tcp, client.tcp) + println(server.logger, "New client connection accepted: ", client.id); flush(STDOUT) + + let client = client + @async process!(client) + end + catch e + if typeof(e) <: InterruptException + println(server.logger, "Interrupt detectd, shutting down..."); flush(STDOUT) + break + else + if !isopen(server.tcp) + println(server.logger, "Server TCPServer is closed, shutting down..."); flush(STDOUT) + # Server was closed while waiting to accept client. Exit gracefully. + break + end + println(server.logger, "Error encountered: $e"); flush(STDOUT) + println(server.logger, "Resuming serving..."); flush(STDOUT) + end + end + end + close(server.tcp) + return +end + +function serve(host::IPAddr,port::Int, + handler=(req, rep) -> Response("Hello World!"), + logger=STDOUT; + cert::String="", + key::String="") + if cert != "" && key != "" + server = Server{https}(host, port, handler, logger) + server.tlsconfig = TLS.SSLConfig(cert, key) + else + server = Server{http}(host, port, handler, logger) + end + + return serve(server) +end +serve(; host::IPAddr=IPv4(127,0,0,1), port::Int=8081, handler::Function=(req, rep) -> Response("Hello World!"), + logger::IO=STDOUT, cert::String="", key::String="") = serve(host, port, handler, logger; cert=cert, key=key) diff --git a/src/statuscodes.jl b/src/statuscodes.jl new file mode 100644 index 000000000..5d44fb211 --- /dev/null +++ b/src/statuscodes.jl @@ -0,0 +1,74 @@ +# HTTP status code => description +const STATUS_CODES = Dict( + 100 => "Continue", + 101 => "Switching Protocols", + 102 => "Processing", # RFC 2518 => obsoleted by RFC 4918 + 200 => "OK", + 201 => "Created", + 202 => "Accepted", + 203 => "Non-Authoritative Information", + 204 => "No Content", + 205 => "Reset Content", + 206 => "Partial Content", + 207 => "Multi-Status", # RFC 4918 + 300 => "Multiple Choices", + 301 => "Moved Permanently", + 302 => "Moved Temporarily", + 303 => "See Other", + 304 => "Not Modified", + 305 => "Use Proxy", + 307 => "Temporary Redirect", + 400 => "Bad Request", + 401 => "Unauthorized", + 402 => "Payment Required", + 403 => "Forbidden", + 404 => "Not Found", + 405 => "Method Not Allowed", + 406 => "Not Acceptable", + 407 => "Proxy Authentication Required", + 408 => "Request Time-out", + 409 => "Conflict", + 410 => "Gone", + 411 => "Length Required", + 412 => "Precondition Failed", + 413 => "Request Entity Too Large", + 414 => "Request-URI Too Large", + 415 => "Unsupported Media Type", + 416 => "Requested Range Not Satisfiable", + 417 => "Expectation Failed", + 418 => "I'm a teapot", # RFC 2324 + 422 => "Unprocessable Entity", # RFC 4918 + 423 => "Locked", # RFC 4918 + 424 => "Failed Dependency", # RFC 4918 + 425 => "Unordered Collection", # RFC 4918 + 426 => "Upgrade Required", # RFC 2817 + 428 => "Precondition Required", # RFC 6585 + 429 => "Too Many Requests", # RFC 6585 + 431 => "Request Header Fields Too Large", # RFC 6585 + 440 => "Login Timeout", + 444 => "nginx error: No Response", + 495 => "nginx error: SSL Certificate Error", + 496 => "nginx error: SSL Certificate Required", + 497 => "nginx error: HTTP -> HTTPS", + 499 => "nginx error or Antivirus intercepted request or ArcGIS error", + 500 => "Internal Server Error", + 501 => "Not Implemented", + 502 => "Bad Gateway", + 503 => "Service Unavailable", + 504 => "Gateway Time-out", + 505 => "HTTP Version Not Supported", + 506 => "Variant Also Negotiates", # RFC 2295 + 507 => "Insufficient Storage", # RFC 4918 + 509 => "Bandwidth Limit Exceeded", + 510 => "Not Extended", # RFC 2774 + 511 => "Network Authentication Required", # RFC 6585 + 520 => "CloudFlare Server Error: Unknown", + 521 => "CloudFlare Server Error: Connection Refused", + 522 => "CloudFlare Server Error: Connection Timeout", + 523 => "CloudFlare Server Error: Origin Server Unreachable", + 524 => "CloudFlare Server Error: Connection Timeout", + 525 => "CloudFlare Server Error: Connection Failed", + 526 => "CloudFlare Server Error: Invalid SSL Ceritificate", + 527 => "CloudFlare Server Error: Railgun Error", + 530 => "Site Frozen" +) diff --git a/src/types.jl b/src/types.jl new file mode 100644 index 000000000..55937e201 --- /dev/null +++ b/src/types.jl @@ -0,0 +1,124 @@ +typealias Headers Dict{String,String} +headers() = Headers( + "Server" => "Julia/$VERSION", + "Content-Type" => "text/html; charset=utf-8", + "Content-Language" => "en", + "Date" => Dates.format(now(Dates.UTC), Dates.RFC1123Format) ) + +type Request + method::String # HTTP method string (e.g. "GET") + major::Int8 + minor::Int8 + resource::String # Resource requested (e.g. "/hello/world") + uri::URI + headers::Headers + keepalive::Bool + data::Vector{UInt8} +end +Request() = Request("", 1, 1, "", URI(""), Headers(), true, UInt8[]) +Request(method, resource, headers, data) = Request(method, 1, 1, resource, URI(resource), headers, true, data) + +==(a::Request,b::Request) = (a.method == b.method) && + (a.major == b.major) && + (a.minor == b.minor) && + (a.resource == b.resource) && + (a.uri == b.uri) && + (a.headers == b.headers) && + (a.keepalive == b.keepalive) && + (a.data == b.data) + +function Base.show(io::IO, r::Request) + println(io, "$(r.method) $(r.resource) HTTP/1.1") + for (k, v) in r.headers + println(io, "$k: $v") + end + if length(r.data) > 100 + println(io, "[Request body of $(length(r.data)) bytes]") + elseif length(r.data) > 0 + println(io, String(r.data)) + end +end +Base.showcompact(io::IO, r::Request) = print(io, "Request(", r.resource, ", ", + length(r.headers), " headers, ", + sizeof(r.data), " bytes in body)") + +type Cookie + name::String + value::String + attrs::Dict{String, String} +end +Cookie(name, value) = Cookie(name, value, Dict{String, String}()) +Base.show(io::IO, c::Cookie) = print(io, "Cookie(", c.name, ", ", c.value, + ", ", length(c.attrs), " attributes)") + +type Response + status::Int + major::Int8 + minor::Int8 + headers::Headers + cookies::Dict{String, Cookie} + data::Vector{UInt8} + request::Nullable{Request} + history::Vector{Response} +end + +typealias HttpData Union{Vector{UInt8}, AbstractString} +Response(s::Int, h::Headers, d::HttpData) = + Response(s, 1, 1, h, Dict{String, Cookie}(), d, Nullable(), Response[]) +Response(status, major, minor, headers, cookies, data) = + Response(status, major, minor, headers, cookies, data, Nullable(), Response[]) + +Response(s::Int, h::Headers) = Response(s, h, UInt8[]) +Response(s::Int, d::HttpData) = Response(s, headers(), d) +Response(d::HttpData, h::Headers) = Response(200, h, d) +Response(d::HttpData) = Response(d, headers()) +Response(s::Int) = Response(s, headers(), UInt8[]) +Response() = Response(200) + +==(a::Response,b::Response) = (a.status == b.status) && + (a.major == b.major) && + (a.minor == b.minor) && + (a.headers == b.headers) && + (a.cookies == b.cookies) && + (a.data == b.data) + +function Base.show(io::IO, r::Response) + println(io, "HTTP/1.1 $(r.status) ", get(STATUS_CODES, r.status, "Unknown Code")) + for (k, v) in r.headers + println(io, "$k: $v") + end + if length(r.data) > 100 + println(io, "[Response body of $(length(r.data)) bytes]") + elseif length(r.data) > 0 + println(io, String(r.data)) + end +end + +function Base.showcompact(io::IO, r::Response) + print(io, "Response(", r.status, " ", get(STATUS_CODES, r.status, "Unknown Code"), ", ", + length(r.headers)," headers, ", + sizeof(r.data)," bytes in body)") +end + +"Converts a `Response` to an HTTP response string" +function Base.write(io::IO, response::Response) + write(io, join(["HTTP/1.1", response.status, STATUS_CODES[response.status], "\r\n"], " ")) + + response.headers["Content-Length"] = string(sizeof(response.data)) + for (header,value) in response.headers + write(io, string(join([ header, ": ", value ]), "\r\n")) + end + for (cookie_name, cookie) in response.cookies + write(io, "Set-Cookie: ", cookie_name, "=", cookie.value) + for (attr_name, attr_val) in cookie.attrs + write(io, "; ", attr_name) + if !isempty(attr_val) + write(io, "=", attr_val) + end + end + write(io, "\r\n") + end + + write(io, "\r\n") + write(io, response.data) +end diff --git a/src/uri.jl b/src/uri.jl new file mode 100644 index 000000000..9a684db18 --- /dev/null +++ b/src/uri.jl @@ -0,0 +1,441 @@ +is_url_char(c) = ((@assert UInt32(c) < 0x80); 'A' <= c <= '~' || '$' <= c <= '>' || c == '\f' || c == '\t') +is_mark(c) = (c == '-') || (c == '_') || (c == '.') || (c == '!') || (c == '~') || + (c == '*') || (c == '\'') || (c == '(') || (c == ')') +is_userinfo_char(c) = isalnum(c) || is_mark(c) || (c == '%') || (c == ';') || + (c == ':') || (c == '&') || (c == '+') || (c == '$' || c == ',') +isnum(c) = ('0' <= c <= '9') +ishex(c) = (isnum(c) || 'a' <= lowercase(c) <= 'f') +is_host_char(c) = isalnum(c) || (c == '.') || (c == '-') || (c == '_') || (c == "~") + + +immutable URI + scheme::String + host::String + port::UInt16 + path::String + query::String + fragment::String + userinfo::String + specifies_authority::Bool + URI(scheme,host,port,path,query="",fragment="",userinfo="",specifies_authority=false) = + new(scheme,host,UInt16(port),path,query,fragment,userinfo,specifies_authority) +end + +==(a::URI,b::URI) = (a.scheme == b.scheme) && + (a.host == b.host) && + (a.port == b.port) && + (a.path == b.path) && + (a.query == b.query) && + (a.fragment == b.fragment) && + (a.userinfo == b.userinfo) + +URI(host, path) = URI("http", host, UInt16(80), path, "", "", "", true) + +# URL parser based on the http-parser package by Joyent +# Licensed under the BSD license + +# Parse authority (user@host:port) +# return (host,port,user) +function parse_authority(authority,seen_at) + host="" + port="" + user="" + last_state = state = seen_at ? :http_userinfo_start : :http_host_start + i = start(authority) + li = s = 0 + while true + if done(authority,li) + last_state = state + state = :done + end + + if s == 0 + s = li + end + + if state != last_state + r = s:prevind(authority,li) + s = li + if last_state == :http_userinfo + user = authority[r] + elseif last_state == :http_host || last_state == :http_host_v6 + host = authority[r] + elseif last_state == :http_host_port + port = authority[r] + end + end + + if state == :done + break + end + + if done(authority,i) + li = i + continue + end + + li = i + (ch,i) = next(authority,i) + + last_state = state + if state == :http_userinfo || state == :http_userinfo_start + if ch == '@' + state = :http_host_start + elseif is_userinfo_char(ch) + state = :http_userinfo + else + error("Unexpected character '$ch' in userinfo") + end + elseif state == :http_host_start + if ch == '[' + state = :http_host_v6_start + elseif is_host_char(ch) + state = :http_host + else + error("Unexpected character '$ch' at the beginning of the host string") + end + elseif state == :http_host + if ch == ':' + state = :http_host_port_start + elseif !is_host_char(ch) + error("Unexpected character '$ch' in host") + end + elseif state == :http_host_v6_end + if ch != ':' + error("Only port allowed in authority after IPv6 address") + end + state = :http_host_port_start + elseif state == :http_host_v6 || state == :http_host_v6_start + if ch == ']' && state == :http_host_v6 + state = :http_host_v6_end + elseif ishex(ch) || ch == ':' || ch == '.' + state = :http_host_v6 + else + error("Unrecognized character in IPv6 address") + end + elseif state == :http_host_port || state == :http_host_port_start + if !isnum(ch) + error("Port must be numeric (decimal)") + end + state = :http_host_port + else + error("Unexpected state $state") + end + end + (host, UInt16(port == "" ? 0 : Base.parse(Int,port,10)), user) +end + +function parse_url(url) + scheme = "" + host = "" + server = "" + port = 80 + query = "" + fragment = "" + username = "" + pass = "" + path = "/" + last_state = state = :req_spaces_before_url + seen_at = false + specifies_authority = false + i = start(url) + li = s = 0 + while true + if done(url,li) + last_state = state + state = :done + end + + if s == 0 + s = li + end + + if state != last_state + r = s:prevind(url,li) + s = li + if last_state == :req_scheme + scheme = url[r] + elseif last_state == :req_server_start + specifies_authority = true + elseif last_state == :req_server + server = url[r] + elseif last_state == :req_query_string + query = url[r] + elseif last_state == :req_path + path = url[r] + elseif last_state == :req_fragment + fragment = url[r] + end + end + if state == :done + break + end + + if done(url,i) + li = i + continue + end + + li = i + (ch,i) = next(url,i) + if !isascii(ch) + error("Non-ASCII characters not supported in URIs. Encode the URL and try again.") + end + + last_state = state + + if state == :req_spaces_before_url + if ch == '/' || ch == '*' + state = :req_path + elseif isalpha(ch) + state = :req_scheme + else + error("Unexpected start of URL") + end + elseif state == :req_scheme + if ch == ':' + state = :req_scheme_slash + elseif !(isalpha(ch) || isdigit(ch) || ch == '+' || ch == '-' || ch == '.') + error("Unexpected character $ch after scheme") + end + elseif state == :req_scheme_slash + if ch == '/' + state = :req_scheme_slash_slash + elseif is_url_char(ch) + state = :req_path + else + error("Expecting scheme:path scheme:/path format not scheme:$ch") + end + elseif state == :req_scheme_slash_slash + if ch == '/' + state = :req_server_start + elseif is_url_char(ch) + s -= 1 + state = :req_path + else + error("Expecting scheme:// or scheme: format not scheme:/$ch") + end + elseif state == :req_server_start || state == :req_server + # In accordence with RFC3986: + # 'The authority component is preceded by a double slash ("//") and isterminated by the next slash ("/")' + # This is different from the joyent http-parser, which considers empty hosts to be invalid. c.f. also the + # following part of RFC 3986: + # "If the URI scheme defines a default for host, then that default + # applies when the host subcomponent is undefined or when the + # registered name is empty (zero length). For example, the "file" URI + # scheme is defined so that no authority, an empty host, and + # "localhost" all mean the end-user's machine, whereas the "http" + # scheme considers a missing authority or empty host invalid." + if ch == '/' + state = :req_path + elseif ch == '?' + state = :req_query_string_start + elseif ch == '@' + seen_at = true + state = :req_server + elseif is_userinfo_char(ch) || ch == '[' || ch == ']' + state = :req_server + else + error("Unexpected character $ch in server") + end + elseif state == :req_path + if ch == '?' + state = :req_query_string_start + elseif ch == '#' + state = :req_fragment_start + elseif !is_url_char(ch) && ch != '@' + error("Path contained unexpected character") + end + elseif state == :req_query_string_start || state == :req_query_string + if ch == '?' + state = :req_query_string + elseif ch == '#' + state = :req_fragment_start + elseif !is_url_char(ch) + error("Query string contained unexpected character") + else + state = :req_query_string + end + elseif state == :req_fragment_start + if ch == '?' + state = :req_fragment + elseif ch == '#' + state = :req_fragment_start + elseif ch != '#' && !is_url_char(ch) + error("Start of fragment contained unexpected character") + else + state = :req_fragment + end + elseif state == :req_fragment + if !is_url_char(ch) && ch != '?' && ch != '#' + error("Fragment contained unexpected character") + end + else + error("Unrecognized state") + end + end + host, port, user = parse_authority(server,seen_at) + return URI(lowercase(scheme),host,port,path,query,fragment,user,specifies_authority) +end + +URI(url) = parse_url(url) + +Base.show(io::IO, uri::URI) = print(io, "HTTP.URI(\"", uri, "\")") + +function Base.print(io::IO, uri::URI) + if uri.specifies_authority || !isempty(uri.host) + print(io,uri.scheme,"://") + if !isempty(uri.userinfo) + print(io,uri.userinfo,'@') + end + if ':' in uri.host #is IPv6 + print(io,'[',uri.host,']') + else + print(io,uri.host) + end + if uri.port != 0 + print(io,':',Int(uri.port)) + end + else + print(io,uri.scheme,":") + end + print(io,uri.path) + if !isempty(uri.query) + print(io,"?",uri.query) + end + if !isempty(uri.fragment) + print(io,"#",uri.fragment) + end +end + +function Base.show(io::IO, ::MIME"text/html", uri::URI) + print(io, "") + print(io, uri) + print(io, "") +end + +const escaped_regex = r"%([0-9a-fA-F]{2})" + +# Escaping +const control_array = vcat(map(UInt8, 0:Base.parse(Int,"1f",16))) +const control = String(control_array)*"\x7f" +const space = String(" ") +const delims = String("%<>\"") +const unwise = String("(){}|\\^`") + +const reserved = String(",;/?:@&=+\$![]'*#") +# Strings to be escaped +# (Delims goes first so '%' gets escaped first.) +const unescaped = delims * reserved * control * space * unwise +const unescaped_form = delims * reserved * control * unwise + +function unescape(str) + r = UInt8[] + l = length(str) + i = 1 + while i <= l + c = str[i] + i += 1 + if c == '%' + c = Base.parse(UInt8, str[i:i+1], 16) + i += 2 + end + push!(r, c) + end + return String(r) +end +unescape_form(str) = unescape(replace(str, "+", " ")) + +hex_string(x) = string('%', uppercase(hex(x,2))) + +# Escapes chars (in second string); also escapes all non-ASCII chars. +function escape_with(str, use) + str = String(str) + out = IOBuffer() + chars = Set(use) + i = start(str) + e = endof(str) + while i <= e + i_next = nextind(str, i) + if i_next == i + 1 + _char = str[i] + if _char in chars + write(out, hex_string(Int(_char))) + else + write(out, _char) + end + else + while i < i_next + write(out, hex_string(str.data[i])) + i += 1 + end + end + i = i_next + end + takebuf_string(out) +end + +escape(str) = escape_with(str, unescaped) +escape_form(str) = replace(escape_with(str, unescaped_form), " ", "+") + +## +# Splits the userinfo portion of an URI in the format user:password and +# returns the components as tuple. +# +# Note: This is just a convenience method, and this form of usage is +# deprecated as of rfc3986. +# See: http://tools.ietf.org/html/rfc3986#section-3.2.1 +function userinfo(uri::URI) + Base.warn_once("Use of the format user:password is deprecated (rfc3986)") + uinfo = uri.userinfo + sep = search(uinfo, ':') + l = length(uinfo) + username = uinfo[1:(sep-1)] + password = ((sep == l) || (sep == 0)) ? "" : uinfo[(sep+1):l] + (username, password) +end + +## +# Splits the path into components and parameters +# See: http://tools.ietf.org/html/rfc3986#section-3.3 +function splitpath(uri::URI, starting=2) + elems = String[] + p = uri.path + len = length(p) + len > 1 || return elems + start_ind = i = starting # p[1] == '/' + while true + c = p[i] + if c == '/' || i == len + push!(elems, p[start_ind:i-1]) + start_ind = i + 1 + end + i += 1 + (i > len || c in ('?', '#')) && break + end + return elems +end + +# Create equivalent URI without the fragment +defrag(uri::URI) = URI(uri.scheme, uri.host, uri.port, uri.path, uri.query, "", uri.userinfo, uri.specifies_authority) + +# Validate known URI formats +const uses_authority = ["hdfs", "ftp", "http", "gopher", "nntp", "telnet", "imap", "wais", "file", "mms", "https", "shttp", "snews", "prospero", "rtsp", "rtspu", "rsync", "svn", "svn+ssh", "sftp" ,"nfs", "git", "git+ssh", "ldap"] +const uses_params = ["ftp", "hdl", "prospero", "http", "imap", "https", "shttp", "rtsp", "rtspu", "sip", "sips", "mms", "sftp", "tel"] +const non_hierarchical = ["gopher", "hdl", "mailto", "news", "telnet", "wais", "imap", "snews", "sip", "sips"] +const uses_query = ["http", "wais", "imap", "https", "shttp", "mms", "gopher", "rtsp", "rtspu", "sip", "sips", "ldap"] +const uses_fragment = ["hdfs", "ftp", "hdl", "http", "gopher", "news", "nntp", "wais", "https", "shttp", "snews", "file", "prospero"] + +function isvalid(uri::URI) + scheme = uri.scheme + isempty(scheme) && error("Can not validate relative URI") + if ((scheme in non_hierarchical) && (search(uri.path, '/') > 1)) || # path hierarchy not allowed + (!(scheme in uses_query) && !isempty(uri.query)) || # query component not allowed + (!(scheme in uses_fragment) && !isempty(uri.fragment)) || # fragment identifier component not allowed + (!(scheme in uses_authority) && (!isempty(uri.host) || (0 != uri.port) || !isempty(uri.userinfo))) # authority component not allowed + return false + end + true +end diff --git a/src/utils.jl b/src/utils.jl new file mode 100644 index 000000000..56f8f5158 --- /dev/null +++ b/src/utils.jl @@ -0,0 +1,34 @@ +""" +escapeHTML(i::String) + +Returns a string with special HTML characters escaped: &, <, >, ", ' +""" +function escapeHTML(i::String) + # Refer to http://stackoverflow.com/a/7382028/3822752 for spec. links + o = replace(i, "&", "&") + o = replace(o, "\"", """) + o = replace(o, "'", "'") + o = replace(o, "<", "<") + o = replace(o, ">", ">") + return o +end + +""" +@timeout secs expr then pollint + +Start executing `expr`; if it doesn't finish executing in `secs` seconds, +then execute `then`. `pollint` controls the amount of time to wait in between +checking if `expr` has finished executing (short for polling interval). +""" +macro timeout(t, expr, then, pollint=0.01) + return quote + tm = Float64($t) + start = time() + tsk = @async $expr + while !istaskdone(tsk) && (time() - start < tm) + sleep($pollint) + end + istaskdone(tsk) || $then + tsk.result + end +end diff --git a/test/cert.pem b/test/cert.pem new file mode 100644 index 000000000..85029c67b --- /dev/null +++ b/test/cert.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC+zCCAeOgAwIBAgIJAJjLegbzdgUPMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAeFw0xNTA5MjUyMjAyMzlaFw00MzAyMTAyMjAyMzlaMBQx +EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBALxC4OgxUv2v4zJCxChb6c9Qr+kgx5Y3XMsJ5pBcrPyMAyJ8aHAy8HvkUoWi +EmGsghamZujpovTctwUP4CHqDW0NPv5ewmpZNlLJZlpUufeAQ0FIVrWzUL0qKYyB +lilkCoCCpNyTKI9D1QtLOk/EvqD/P8ZoFvDFvgQW/LZSxNj5TWa81rjOT6Ral2b7 +Xdsc6eaJq/aCnt6T9z20NXderh9nFCmkAN98LD12QtV2CP8RCSD8vYqN1KnOLjUr +xtLe3K0bjQF8xTKajsT3ppRV00uWG+6glJVfsqCw+LsCsKWpDmLxooo6b+zC6V61 +c0XLNYYkB+gRKCV9zjXASP+g9hMCAwEAAaNQME4wHQYDVR0OBBYEFJ2Qo9xfnsyo +rbl/Q/28kOsd0D8wMB8GA1UdIwQYMBaAFJ2Qo9xfnsyorbl/Q/28kOsd0D8wMAwG +A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAD7jtxExdnynxDrPqYHdozAP +WgD3WYgNvygEU7WySz67UECPPjhyH/q/3hMFqntDANYCwFBrQ59zXIrYHDuZ1G5b +BlyZsmKdWaOokpkmAEYZUglj1AHwF5C8XPCuAk7/oP0VYx27T5jEsqTTfpsKVImP +c4HBbdXoy07LRzDRzo/zE8KxXW9Z7divV87AxXbTA7VwBV75uhFfQ58gpuTA4grk +kvF00zTT5P2/hCVKTDhiiLtIZz9c9W0KLT8cJN5zm4S8yPNZu8K5TI1XUTjcrNQM +bcSW0BksGgwB8kH+S5W+eyPsr0y+3a7Omg9S8sMza0Pc6P9GduqN8Q0S+LYRZO4= +-----END CERTIFICATE----- diff --git a/test/key.pem b/test/key.pem new file mode 100644 index 000000000..8387ce8bf --- /dev/null +++ b/test/key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8QuDoMVL9r+My +QsQoW+nPUK/pIMeWN1zLCeaQXKz8jAMifGhwMvB75FKFohJhrIIWpmbo6aL03LcF +D+Ah6g1tDT7+XsJqWTZSyWZaVLn3gENBSFa1s1C9KimMgZYpZAqAgqTckyiPQ9UL +SzpPxL6g/z/GaBbwxb4EFvy2UsTY+U1mvNa4zk+kWpdm+13bHOnmiav2gp7ek/c9 +tDV3Xq4fZxQppADffCw9dkLVdgj/EQkg/L2KjdSpzi41K8bS3tytG40BfMUymo7E +96aUVdNLlhvuoJSVX7KgsPi7ArClqQ5i8aKKOm/swuletXNFyzWGJAfoESglfc41 +wEj/oPYTAgMBAAECggEBAIB6y+7qqo7DWLRWaHR6tchscoERg+R6h/NxIE7pUI1S +KFmCuevId+K1YbQddZn/FxDKI3VU7YdakfT8bqP2jY8c+R60IM5fb/lzxUxkgj3s +5PlKmxKJ+9H9Ujm3vnkk8x3dCxIVxBpx2pVIk9UYmlhZmnaXVwCekx1LatArEHha +D8FGhxHfFKnIVuL4A579+lyGNsHXJl/B7yJEudwFcdT1ZnwkRLidWvLRHkoxnH7m +PsghdjHLNTYaOgYHCVqntr/LInGgZVE7+jxqWJGqTCmTg0MXvabg37QxXSoDgPWa +J8u+64qE+GIKaQ2sOeuNMGR2eB3Hw2h6fsJ33HhzI4ECgYEA4RFP2aJzj1J8jst9 +evSfZcfM7mXev8DReA+wmlcnUjWn0Bz1b7pPUYHDZrTPb9ONTPaB+EUGCDhuPI2b +W0Mk0iPqkfZAssXQBDQIHZnWim/hRoqdkzPe4KWA8hjgKATLrqfcc1MAvhaHOjqe +gAmBykN7WaFRMA/j+cr8FVFo+aECgYEA1iKV+uYIqIbowY6NChc8aBIkSXy7xcz5 +HntR2UgI2UiYCdppZXfQayRlng1CYhTvmFJcd528mZ64sg/1hyD0bT0oZ7Xgn8Tl +IBdEvpZFAafW9xO/akshvt6HSEPrFoZnOlr3MjTr52TRg5JGmM06evfj9sWWsbFD +qFDbckMbWzMCgYATLviRYklbQ/qd6TZOzp7ve/I5t7Eewv6Xry6sWRVe6nfdQzqg +RU8RcXAIRw0PSQbYMoKteKSk+rpaqu88/iIbTzhlLIojMr0iPpUagMxKjHK1Iod/ +zoIGv9SXzgr9HjuGLYSax85eZWktS2XLIARSCyJuZ1OWNySFXAnUf1XlQQKBgERw +VWMVNls2kxmZx/YbqxDQC4z5MsJrWoulemlpnnpju0Qa7GijvJch0OCM+FSEwHb8 +i9UnMuoeUoWGmECSBc0MKOfMt3gY4+o3xZ7sRC3dSNU7GIiObsCkOrScEHzohAGg +pTUEuQkBrfzROYMIxNIcfF2YlStBrpATF7ATRqEFAoGBAI/C2xjcr58U4lyp8Uuq +IYvC0dyz2nPe30wTBtWI0nKEuTGvB0A0xXfkW+OeyDQ2Wmt3OV6hq89EI4kmFxUn +k1LW8w4qOW9lwXnWBEFF0H38EZ8L5Y9Gv9DO3kJPN32ypSfQL7kl0mX7gUaoHskY +jQp1r0nwNJ6zfYXOaj1jr5lz +-----END PRIVATE KEY----- diff --git a/test/parser.jl b/test/parser.jl new file mode 100644 index 000000000..7ce15f19e --- /dev/null +++ b/test/parser.jl @@ -0,0 +1,195 @@ +reqstr = "GET http://www.techcrunch.com/ HTTP/1.1\r\n" * + "Host: www.techcrunch.com\r\n" * + "User-Agent: Fake\r\n" * + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" * + "Accept-Language: en-us,en;q=0.5\r\n" * + "Accept-Encoding: gzip,deflate\r\n" * + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" * + "Keep-Alive: 300\r\n" * + "Content-Length: 7\r\n" * + "Proxy-Connection: keep-alive\r\n\r\n" + +req = HTTP.Request("GET", + 1, 1, + "http://www.techcrunch.com/", + Dict("Content-Length"=>"7","Host"=>"www.techcrunch.com","Accept"=>"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Accept-Charset"=>"ISO-8859-1,utf-8;q=0.7,*;q=0.7","Proxy-Connection"=>"keep-alive","Accept-Language"=>"en-us,en;q=0.5","Keep-Alive"=>"300","User-Agent"=>"Fake","Accept-Encoding"=>"gzip,deflate"), + true, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "GET / HTTP/1.1\r\n" * + "Host: foo.com\r\n\r\n" + +req = HTTP.Request("GET", + 1, 1, + "/", HTTP.Headers("Host"=>"foo.com"), + true, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "GET //user@host/is/actually/a/path/ HTTP/1.1\r\n" * + "Host: test\r\n\r\n" + +req = HTTP.Request("GET", + 1, 1, + "//user@host/is/actually/a/path/", + HTTP.Headers("Host"=>"test"), + true, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "GET ../../../../etc/passwd HTTP/1.1\r\n" * + "Host: test\r\n\r\n" + +@test_throws HTTP.ParserError HTTP.parse(HTTP.Request, reqstr) + +reqstr = "GET HTTP/1.1\r\n" * + "Host: test\r\n\r\n" + +@test_throws HTTP.ParserError HTTP.parse(HTTP.Request, reqstr) + +reqstr = "POST / HTTP/1.1\r\n" * + "Host: foo.com\r\n" * + "Transfer-Encoding: chunked\r\n\r\n" * + "3\r\nfoo\r\n" * + "3\r\nbar\r\n" * + "0\r\n" * + "Trailer-Key: Trailer-Value\r\n" * + "\r\n" + +req = HTTP.Request("POST", + 1, 1, + "/", + HTTP.Headers("Transfer-Encoding"=>"chunked", "Host"=>"foo.com"), + true, + "foobar".data +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "POST / HTTP/1.1\r\n" * + "Host: foo.com\r\n" * + "Transfer-Encoding: chunked\r\n" * + "Content-Length: 9999\r\n\r\n" * # to be removed. + "3\r\nfoo\r\n" * + "3\r\nbar\r\n" * + "0\r\n" * + "\r\n" + +@test_throws HTTP.ParserError HTTP.parse(HTTP.Request, reqstr) + +reqstr = "CONNECT www.google.com:443 HTTP/1.1\r\n\r\n" + +req = HTTP.Request("CONNECT", + 1, 1, + "www.google.com:443", + HTTP.Headers(), + true, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "CONNECT 127.0.0.1:6060 HTTP/1.1\r\n\r\n" + +req = HTTP.Request("CONNECT", + 1, 1, + "127.0.0.1:6060", + HTTP.Headers(), + true, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "CONNECT /_goRPC_ HTTP/1.1\r\n\r\n" + +req = HTTP.Request("CONNECT", + 1, 1, + "/_goRPC_", + HTTP.Headers(), + true, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "NOTIFY * HTTP/1.1\r\nServer: foo\r\n\r\n" + +req = HTTP.Request("NOTIFY", + 1, 1, + "*", + HTTP.Headers("Server"=>"foo"), + true, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "OPTIONS * HTTP/1.1\r\nServer: foo\r\n\r\n" + +req = HTTP.Request("OPTIONS", + 1, 1, + "*", + HTTP.Headers("Server"=>"foo"), + true, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "GET / HTTP/1.1\r\nHost: issue8261.com\r\nConnection: close\r\n\r\n" + +req = HTTP.Request("GET", + 1, 1, + "/", + HTTP.Headers("Host"=>"issue8261.com", "Connection"=>"close"), + false, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "HEAD / HTTP/1.1\r\nHost: issue8261.com\r\nConnection: close\r\nContent-Length: 0\r\n\r\n" + +req = HTTP.Request("HEAD", + 1, 1, + "/", + HTTP.Headers("Host"=>"issue8261.com", "Connection"=>"close", "Content-Length"=>"0"), + false, + UInt8[] +) + +@test HTTP.parse(HTTP.Request, reqstr) == req + +reqstr = "POST /cgi-bin/process.cgi HTTP/1.1\r\n" * + "User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r\n" * + "Host: www.tutorialspoint.com\r\n" * + "Content-Type: text/xml; charset=utf-8\r\n" * + "Content-Length: 19\r\n" * + "Accept-Language: en-us\r\n" * + "Accept-Encoding: gzip, deflate\r\n" * + "Connection: Keep-Alive\r\n\r\n" * + "first=Zara&last=Ali\r\n\r\n" + +req = HTTP.Request("POST", + 1, 1, + "/cgi-bin/process.cgi", + HTTP.Headers("Host"=>"www.tutorialspoint.com", + "Connection"=>"Keep-Alive", + "Content-Length"=>"19", + "User-Agent"=>"Mozilla/4.0 (compatible; MSIE5.01; Windows NT)", + "Content-Type"=>"text/xml; charset=utf-8", + "Accept-Language"=>"en-us", + "Accept-Encoding"=>"gzip, deflate"), + true, + "first=Zara&last=Ali".data +) + +@test HTTP.parse(HTTP.Request, reqstr) == req diff --git a/test/runtests.jl b/test/runtests.jl index a1b29087f..acce0546f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,70 @@ -using HTTP -using Base.Test +using HTTP, Base.Test -# write your own tests here -@test 1 == 2 +include("uri.jl") +include("parser.jl") + +server_task = @async HTTP.serve() + +client = connect("127.0.0.1:8081") +write(client, "OPTIONS * HTTP/1.1\r\n\r\n") +sleep(1) +resp = String(readavailable(client)) + +# reject invalid HTTP versions + +# test a variety of invalid requests (see http_parser error codes?) + # invalid METHOD (501 not implemented status) + # URI too long (414 status) + # invalid HTTP versions + # whitespace where there shouldn't be + # non-encoded target resource w/ spaces (return 400) + # duplicate headers + # no space between header field name and colon + # reject obs-fold multi-line header field values (400 bad request) + + +# limit on overall header size? body size? + +# no response body + # HEAD requests + # 1xx and 2xx responses + # CONNECT requests + # 204, 304 + +# no transfer-encoding header in response to: + # 1xx or 204 response statuses + # CONNECT request + +# unsupported transfer-endcoding => 501 not implemented + +# bad request 400 on multiple Content-Length headers + +# timeout and close connection when bytes received don't match Content-Length from client + +# https://tools.ietf.org/html/rfc7230#section-3.5 +# ignore preceeding CRLF to request-line + +# https://tools.ietf.org/html/rfc7230#section-4.1.1 +# ignore unsupported chunk-extensions + +# https://tools.ietf.org/html/rfc7230#section-4.1.2 +# chunk trailer headers are handled appropriately + +# https://tools.ietf.org/html/rfc7230#section-4.2 +# transfer-encodings supported: compress, deflate, gzip, x-gzip + +# https://tools.ietf.org/html/rfc7230#section-5.4 +# 400 for no Host or multiple Host headers + +# https://tools.ietf.org/html/rfc7230#section-6.1 +# test keep-alive support + +# https://tools.ietf.org/html/rfc7230#section-6.5 +# test an inactive client (i.e. sent keep-alive, but didn't send anything else) + + + + + + +# diff --git a/test/uri.jl b/test/uri.jl new file mode 100644 index 000000000..2b539c60b --- /dev/null +++ b/test/uri.jl @@ -0,0 +1,66 @@ +urls = ["hdfs://user:password@hdfshost:9000/root/folder/file.csv#frag", + "https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag", + "https://user:password@httphost:9000/path1/path2?q=a&p=r#frag", + "https://user:password@httphost:9000/path1/path2;paramstring#frag", + "https://user:password@httphost:9000/path1/path2#frag", + "file:///path/to/file/with%3fshould%3dwork%23fine", + "ftp://ftp.is.co.za/rfc/rfc1808.txt", "http://www.ietf.org/rfc/rfc2396.txt", + "ldap://[2001:db8::7]/c=GB?objectClass?one", "mailto:John.Doe@example.com", + "news:comp.infosystems.www.servers.unix", "tel:+1-816-555-1212", "telnet://192.0.2.16:80/", + "urn:oasis:names:specification:docbook:dtd:xml:4.1.2"] + +failed = 0 +for url in urls + u = HTTP.URI(url) + if !(string(u) == url) || !isvalid(u) + failed += 1 + println("Test failed for ",url) + end +end +if failed != 0 + exit(failed) +end + +@test HTTP.URI("hdfs://user:password@hdfshost:9000/root/folder/file.csv") == HTTP.URI("hdfs","hdfshost",9000,"/root/folder/file.csv","","","user:password") +@test HTTP.URI("google.com","/some/path") == HTTP.URI("http://google.com:80/some/path") +g = HTTP.URI("google.com","/some/path") +@test HTTP.URI(g,port=160) == HTTP.URI("http://google.com:160/some/path") + +@test HTTP.escape("abcdef αβ 1234-=~!@#\$()_+{}|[]a;") == "abcdef%20%CE%B1%CE%B2%201234-%3D~%21%40%23%24%28%29_%2B%7B%7D%7C%5B%5Da%3B" +@test HTTP.unescape(HTTP.escape("abcdef 1234-=~!@#\$()_+{}|[]a;")) == "abcdef 1234-=~!@#\$()_+{}|[]a;" +@test HTTP.unescape(HTTP.escape("👽")) == "👽" + +@test HTTP.escape_form("abcdef 1234-=~!@#\$()_+{}|[]a;") == "abcdef+1234-%3D~%21%40%23%24%28%29_%2B%7B%7D%7C%5B%5Da%3B" +@test HTTP.unescape_form(HTTP.escape_form("abcdef 1234-=~!@#\$()_+{}|[]a;")) == "abcdef 1234-=~!@#\$()_+{}|[]a;" + +@test ("user", "password") == HTTP.userinfo(HTTP.URI("https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag")) +@test HTTP.URI("https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r") == HTTP.defrag(HTTP.URI("https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag")) + +@test ["dc","example","dc","com"] == HTTP.path_params(HTTP.URI("ldap://ldap.example.com/dc=example,dc=com"))[1] +@test ["servlet","jsessionid","OI24B9ASD7BSSD"] == HTTP.path_params(HTTP.URI("http://www.mysite.com/servlet;jsessionid=OI24B9ASD7BSSD"))[1] + +@test Dict("q"=>"a","p"=>"r") == HTTP.query_params(HTTP.URI("https://httphost/path1/path2;paramstring?q=a&p=r#frag")) +@test Dict("q"=>"a","malformed"=>"") == HTTP.query_params(HTTP.URI("https://foo.net/?q=a&malformed")) + +@test false == isvalid(HTTP.URI("file:///path/to/file/with?should=work#fine")) +@test true == isvalid(HTTP.URI("file:///path/to/file/with%3fshould%3dwork%23fine")) + +@test HTTP.URI("s3://bucket/key") == HTTP.URI("s3","bucket",0,"/key") + +@test sprint(show, HTTP.URI("http://google.com")) == "HTTP.URI(http://google.com/)" + +# Error paths +# Non-ASCII characters +@test_throws ErrorException HTTP.URI("http://🍕.com") +# Unexpected start of URL +@test_throws ErrorException HTTP.URI(".google.com") +# Unexpected character after scheme +@test_throws ErrorException HTTP.URI("ht!tp://google.com") + +# Issue #27 +@test HTTP.escape("t est\n") == "t%20est%0A" + +# Issue #2 +@test sprint((io, mime, obj)->show(io, mime, obj), + MIME("text/html"), HTTP.URI("http://google.com")) == + """http://google.com/"""