diff --git a/.travis.yml b/.travis.yml index 3433889..57ba28d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ os: - linux - osx julia: - - 0.6 + - 0.7 - nightly notifications: email: false @@ -14,8 +14,8 @@ notifications: # - julia -e 'Pkg.clone(pwd()); Pkg.build("GoogleCloud"); Pkg.test("GoogleCloud"; coverage=true)' after_success: # push coverage results to Coveralls - - julia -e 'cd(Pkg.dir("GoogleCloud")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' + - julia -e 'import Pkg; cd(Pkg.dir("GoogleCloud")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' # push coverage results to Codecov - - julia -e 'cd(Pkg.dir("GoogleCloud")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' - - julia -e 'Pkg.add("Documenter")' - - julia -e 'cd(Pkg.dir("GoogleCloud")); include(joinpath("docs", "make.jl"))' + - julia -e 'import Pkg; cd(Pkg.dir("GoogleCloud")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' + - julia -e 'import Pkg; Pkg.add("Documenter")' + - julia -e 'import Pkg; cd(Pkg.dir("GoogleCloud")); include(joinpath("docs", "make.jl"))' diff --git a/README.md b/README.md index 0c150e4..7f5d1b8 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # GoogleCloud.jl -[![Build Status](https://travis-ci.org/joshbode/GoogleCloud.jl.svg?branch=master)](https://travis-ci.org/joshbode/GoogleCloud.jl) +[![Build Status](https://travis-ci.org/juliacloud/GoogleCloud.jl.svg?branch=master)](https://travis-ci.org/juliacloud/GoogleCloud.jl) [![Build status](https://ci.appveyor.com/api/projects/status/itmgxkcc75m9ulqd?svg=true)](https://ci.appveyor.com/project/JoshBode/googlecloud-jl) -[![Coverage Status](https://coveralls.io/repos/github/joshbode/GoogleCloud.jl/badge.svg?branch=master)](https://coveralls.io/github/joshbode/GoogleCloud.jl?branch=master) -[![codecov.io](http://codecov.io/github/joshbode/GoogleCloud.jl/coverage.svg?branch=master)](http://codecov.io/github/joshbode/GoogleCloud.jl?branch=master) +[![Coverage Status](https://coveralls.io/repos/github/juliacloud/GoogleCloud.jl/badge.svg?branch=master)](https://coveralls.io/github/juliacloud/GoogleCloud.jl?branch=master) +[![codecov.io](http://codecov.io/github/juliacloud/GoogleCloud.jl/coverage.svg?branch=master)](http://codecov.io/github/juliacloud/GoogleCloud.jl?branch=master) -[![](https://img.shields.io/badge/docs-latest-blue.svg)](https://joshbode.github.io/GoogleCloud.jl/latest) +[![](https://img.shields.io/badge/docs-latest-blue.svg)](https://juliacloud.github.io/GoogleCloud.jl/latest) Google Cloud APIs for Julia @@ -15,8 +15,5 @@ Google Cloud APIs for Julia This package thinly wraps the Google Cloud JSON APIs for Julia. -Currently the package wraps the Google Cloud Storage API. Other APIs will be added in the future. See [this tutorial](https://joshbode.github.io/GoogleCloud.jl/latest) for a detailed walk through the Google Storage API. +See [this tutorial](https://juliacloud.github.io/GoogleCloud.jl/latest) for a detailed walk through the Google Storage API. -See [here](https://joshbode.github.io/GoogleCloud.jl/latest) for the remaining docs. - -**Note**: This package includes OAuth 2.0 functionality, which may be extracted into a separate package in the future. diff --git a/REQUIRE b/REQUIRE index 6d42f1b..557e091 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,8 +1,11 @@ -julia 0.6 +julia 0.7 -JSON 0.7.0 -MbedTLS 0.3.1 -Requests +Dates +Base64 +Printf +Markdown +JSON +MbedTLS +HTTP Libz MsgPack -URIParser diff --git a/appveyor.yml b/appveyor.yml index 46383cc..ad918ab 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,7 @@ environment: matrix: - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.7/julia-0.7-latest-win32.exe" + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7-latest-win64.exe" - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" @@ -28,8 +28,8 @@ install: build_script: # Need to convert from shallow to complete for Pkg.clone to work - IF EXIST .git\shallow (git fetch --unshallow) - - C:\projects\julia\bin\julia -e "versioninfo(); + - C:\projects\julia\bin\julia -e "versioninfo(); import Pkg; Pkg.clone(pwd(), \"GoogleCloud\"); Pkg.build(\"GoogleCloud\")" test_script: - - C:\projects\julia\bin\julia -e "Pkg.test(\"GoogleCloud\")" + - C:\projects\julia\bin\julia -e "import Pkg; Pkg.test(\"GoogleCloud\")" diff --git a/docs/src/index.md b/docs/src/index.md index f4eb618..b13b08f 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -113,7 +113,7 @@ Upload an object to the _a12345foo_ bucket: ```julia # String containing the contents of test_image.jpg. The semi-colon avoids an error caused by printing the returned value. -file_contents = readstring(open("test_image.jpg", "r")); +file_contents = read(open("test_image.jpg", "r"), String); # Upload storage(:Object, :insert, "a12345foo"; # Returns metadata about the object diff --git a/src/GoogleCloud.jl b/src/GoogleCloud.jl index 5f3fb68..0b77083 100644 --- a/src/GoogleCloud.jl +++ b/src/GoogleCloud.jl @@ -1,5 +1,3 @@ -__precompile__(true) - """ Google Cloud APIs """ diff --git a/src/api/api.jl b/src/api/api.jl index 342468c..2035d50 100644 --- a/src/api/api.jl +++ b/src/api/api.jl @@ -5,13 +5,13 @@ module api export APIRoot, APIResource, APIMethod, set_session!, get_session, iserror -using Base.Dates +using Dates, Base64 -using Requests +using HTTP import MbedTLS -import URIParser import Libz import JSON +using Markdown using ..session using ..error @@ -33,7 +33,7 @@ path_tokens("/{foo}/{bar}/x/{baz}") "{baz}" ``` """ -path_tokens(path::AbstractString) = matchall(r"{\w+}", path) +path_tokens(path::AbstractString) = collect((m.match for m = eachmatch(r"{\w+}", path))) """ path_replace(path, values) @@ -48,10 +48,20 @@ path_replace("/{foo}/{bar}/{baz}", ["this", "is", "it"]) "/this/is/it" ``` """ -path_replace(path::AbstractString, values) = reduce((x, y) -> replace(x, y[1], URIParser.escape(y[2]), 1), path, zip(path_tokens(path), values)) +path_replace(path::AbstractString, values) = reduce( + (x, y) -> replace(x, y[1]=>HTTP.URIs.escapeuri(y[2]), count=1), + zip(path_tokens(path), values); init=path) +#function path_replace(path::AbstractString, values) +# for value in values +# path = replace(path, r"{\w+}"=>value, count=1) +# end +# path +# #reduce((x, y) -> replace(x, y[1], HTTP.URIs.escapeuri(y[2]), 1), path, +# # zip(path_tokens(path), values)) +#end """Check if response is/contains an error""" -iserror(x::Associative{Symbol}) = haskey(x, :error) +iserror(x::AbstractDict{Symbol}) = haskey(x, :error) iserror(::Any) = false """ @@ -66,14 +76,14 @@ struct APIMethod default_params::Dict{Symbol, Any} transform::Function function APIMethod(verb::Symbol, path::AbstractString, description::AbstractString, - default_params::Associative{Symbol}=Dict{Symbol, Any}(); + default_params::AbstractDict{Symbol}=Dict{Symbol, Any}(); transform=(x, t) -> x) new(verb, path, description, default_params, transform) end end function Base.print(io::IO, x::APIMethod) println(io, "$(x.verb): $(x.path)") - Base.Markdown.print_wrapped(io, x.description) + Markdown.print_wrapped(io, x.description) end Base.show(io::IO, x::APIMethod) = print(io, x) @@ -134,7 +144,7 @@ struct APIRoot An API rooted at `path` with specified OAuth 2.0 access scopes and resources. """ - function APIRoot(path::AbstractString, scopes::Associative{<: AbstractString, <: AbstractString}; resources...) + function APIRoot(path::AbstractString, scopes::AbstractDict{<: AbstractString, <: AbstractString}; resources...) if !isurl(path) throw(APIError("API root must be a valid URL.")) end @@ -156,6 +166,7 @@ struct APIRoot new(path, scopes, resources) end end + function Base.print(io::IO, x::APIRoot) println(io, x.path, "\n") for (name, resource) in sort(collect(x.resources), by=(x) -> x[1]) @@ -174,7 +185,7 @@ Base.show(io::IO, x::APIRoot) = print(io, x) Set the default session for a specific API. Set session to `nothing` to forget session. """ -function set_session!(api::APIRoot, session::Union{GoogleSession, Void}) +function set_session!(api::APIRoot, session::Union{GoogleSession, Nothing}) _default_session[api] = session nothing end @@ -208,20 +219,21 @@ function (api::APIRoot)(resource_name::Symbol, method_name::Symbol, args...; kwa end """ - execute(session::GoogleSession, resource::APIResource, method::APIMethod, path_args::AbstractString...[; ...]) + execute(session::GoogleSession, resource::APIResource, method::APIMethod, + path_args::AbstractString...[; ...]) Execute a method against the provided path arguments. Optionally provide parameters and data (with optional MIME content-type). """ -function execute(session::GoogleSession, resource::APIResource, method::APIMethod, path_args::AbstractString...; - data::Union{AbstractString, Associative, Vector{UInt8}, Void}=nothing, - gzip::Bool=false, content_type::AbstractString="application/json", - debug::Bool=false, raw::Bool=false, - max_backoff::TimePeriod=Second(64), max_attempts::Int64=10, - params... -) - if length(path_args) != length(path_tokens(method.path)) +function execute(session::GoogleSession, resource::APIResource, method::APIMethod, + path_args::AbstractString...; + data::Union{AbstractString, AbstractDict, Vector{UInt8}}="", + gzip::Bool=false, content_type::AbstractString="application/json", + debug::Bool=false, raw::Bool=false, + max_backoff::TimePeriod=Second(64), max_attempts::Int64=10, + params...) + if length(path_args) != path_tokens(method.path) |> length throw(APIError("Number of path arguments do not match")) end @@ -233,18 +245,19 @@ function execute(session::GoogleSession, resource::APIResource, method::APIMetho params = Dict(params) # check if data provided when not expected - if xor((data !== nothing), in(method.verb, (:POST, :UPDATE, :PATCH, :PUT))) - data = nothing + if xor((!isempty(data)), in(method.verb, (:POST, :UPDATE, :PATCH, :PUT))) + data = "" content_type = "" headers["Content-Length"] = "0" end + if !isempty(content_type) + headers["Content-Type"] = content_type + end + # serialise data to JSON if necessary - if data !== nothing - if !isempty(content_type) - headers["Content-Type"] = content_type - end - if isa(data, Associative) || content_type == "application/json" + if !isempty(data) + if isa(data, AbstractDict) || content_type == "application/json" data = JSON.json(data) elseif isempty(data) headers["Content-Length"] = "0" @@ -272,32 +285,36 @@ function execute(session::GoogleSession, resource::APIResource, method::APIMetho max_backoff = Millisecond(max_backoff) for attempt = 1:max(max_attempts, 1) if debug - info("Attempt: $attempt") + @info("Attempt: $attempt") end res = try - Requests.do_request( - URIParser.URI(path_replace(method.path, path_args)), string(method.verb); - query=params, data=data, headers=headers, compressed=true - ) + HTTP.request(string(method.verb), + path_replace(method.path, path_args), headers, data; + query=params ) catch e - if isa(e, Base.UVError) && e.code in (Base.UV_ECONNRESET, Base.UV_ECONNREFUSED, Base.UV_ECONNABORTED, Base.UV_EPIPE, Base.UV_ETIMEDOUT) - elseif isa(e, MbedTLS.MbedException) && e.ret in (MbedTLS.MBEDTLS_ERR_SSL_TIMEOUT, MbedTLS.MBEDTLS_ERR_SSL_CONN_EOF) - else - rethrow(e) - end + # if isa(e, Base.IOError) && + # e.code in (Base.UV_ECONNRESET, Base.UV_ECONNREFUSED, Base.UV_ECONNABORTED, + # Base.UV_EPIPE, Base.UV_ETIMEDOUT) + # elseif isa(e, MbedTLS.MbedException) && + # e.ret in (MbedTLS.MBEDTLS_ERR_SSL_TIMEOUT, MbedTLS.MBEDTLS_ERR_SSL_CONN_EOF) + # else + # println("get a HTTP request error: ", e) + # rethrow(e) + # end + println("get a HTTP request error: ", e) + rethrow(e) end if debug && (res !== nothing) - info("Request URL: $(get(res.request).uri)") - info("Request Headers:\n" * join((" $name: $value" for (name, value) in sort(collect(get(res.request).headers))), "\n")) - info("Request Data:\n " * base64encode(get(res.request).data)) - info("Response Headers:\n" * join((" $name: $value" for (name, value) in sort(collect(res.headers))), "\n")) - info("Response Data:\n " * base64encode(res.data)) - info("Status: ", statuscode(res)) + @info("Request URL: $(res.request.target)") + @info("Response Headers:\n" * join((" $name: $value" for (name, value) in + sort(collect(res.headers))), "\n")) + @info("Response Data:\n " * base64encode(res.body)) + @info("Status: ", res.status) end # https://cloud.google.com/storage/docs/exponential-backoff - if (res === nothing) || (div(statuscode(res), 100) == 5) || (statuscode(res) == 429) + if (res === nothing) || (div(res.status, 100) == 5) || (res.status == 429) if attempt < max_attempts backoff = min(Millisecond(floor(Int, 1000 * (2 ^ (attempt - 1) + rand()))), max_backoff) warn("Unable to complete request: Retrying ($attempt/$max_attempts) in $backoff") @@ -311,16 +328,19 @@ function execute(session::GoogleSession, resource::APIResource, method::APIMetho end # if response is JSON, parse and return. otherwise, just dump data - if contains(res.headers["Content-Type"], "application/json") - if get(res.headers, "Content-Length", "") == "0" - nothing + # HTTP response header type is Vector{Pair{String,String}} + # https://github.com/JuliaWeb/HTTP.jl/blob/master/src/Messages.jl#L166 + headers = Dict(res.headers) + if occursin("application/json", headers["Content-Type"]) + if get(headers, "Content-Length", "") == "0" + return nothing else - result = Requests.json(res; dicttype=Dict{Symbol, Any}) - raw || (statuscode(res) >= 400) ? result : method.transform(result, resource.transform) + result = JSON.parse(read(IOBuffer(res.body), String); dicttype=Dict{Symbol, Any}) + return raw || (res.status >= 400) ? result : method.transform(result, resource.transform) end else - result, status = res.data, statuscode(res) - status == 200 ? result : Dict{Symbol, Any}(:error => Dict{Symbol, Any}(:message => result, :code => status)) + result, status = res.body, res.status + return status == 200 ? result : Dict{Symbol, Any}(:error => Dict{Symbol, Any}(:message => result, :code => status)) end end diff --git a/src/api/datastore.jl b/src/api/datastore.jl index 5a3d73b..33b53f7 100644 --- a/src/api/datastore.jl +++ b/src/api/datastore.jl @@ -3,6 +3,8 @@ Google Cloud Datastore API """ module _datastore +using Base64 + export datastore using ..api @@ -12,7 +14,7 @@ using ...root module types export ValueType, OperatorType, wrap, unwrap - using Base.Dates + using Dates import JSON @@ -47,7 +49,7 @@ module types AbstractString => stringValue, Char => stringValue, Enum => stringValue, - Void => nullValue, + Nothing => nullValue, ) function wrap(x) T = typeof(x) diff --git a/src/collection.jl b/src/collection.jl index 4c824b7..b5d2b78 100644 --- a/src/collection.jl +++ b/src/collection.jl @@ -3,7 +3,8 @@ module collection export KeyStore, connect!, destroy!, watch, unwatch -using Base.Dates +using Dates +using Printf import JSON import MsgPack @@ -20,13 +21,13 @@ end _deserialize_bytes(x) = deserialize(IOBuffer(x)) # key serialiser/deserialiser pairs -key_format_map = Dict{Symbol, Tuple{Function, Function}}( +const KEY_FORMAT_MAP = Dict{Symbol, Tuple{Function, Function}}( :json => (JSON.json, JSON.parse), :string => (string, identity) ) # value serialiser/deserialiser pairs -val_format_map = Dict{Symbol, Tuple{Function, Function}}( +const VAL_FORMAT_MAP = Dict{Symbol, Tuple{Function, Function}}( :json => (JSON.json, JSON.parse), :string => (string, identity), :data => (identity, identity), @@ -37,7 +38,7 @@ val_format_map = Dict{Symbol, Tuple{Function, Function}}( """ High-level container wrapping a Google Storage bucket """ -immutable KeyStore{K, V} <: Associative{K, V} +struct KeyStore{K, V} <: AbstractDict{K, V} bucket_name::String session::GoogleSession key_decoder::Function @@ -46,39 +47,47 @@ immutable KeyStore{K, V} <: Associative{K, V} writer::Function gzip::Bool channel::Dict{Symbol, Any} - function KeyStore{K, V}(bucket_name::AbstractString, session::GoogleSession=get_session(storage); - location::AbstractString="US", empty::Bool=false, gzip::Bool=true, - key_format::Union{Symbol, AbstractString}=K <: String ? :string : :json, - val_format::Union{Symbol, AbstractString}=V <: String ? :string : :json - ) where {K, V} - key_encoder, key_decoder = try key_format_map[Symbol(key_format)] catch - error("Unknown key format: $key_format") - end - writer, reader = try val_format_map[Symbol(val_format)] catch - error("Unknown value format: $val_format") - end - store = new{K, V}(bucket_name, session, - (x) -> convert(K, key_decoder(x)), key_encoder, - reader, writer, - gzip, Dict{Symbol, Any}() - ) - # establish availability of bucket - connect!(store; location=location, empty=empty) - store +end + +function KeyStore{K,V}(bucket_name::AbstractString; session::GoogleSession=get_session(storage), + location::AbstractString="US", empty::Bool=false, gzip::Bool=true, + key_format::Union{Symbol, AbstractString}=K <: String ? :string : :json, + val_format::Union{Symbol, AbstractString}=V <: String ? :string : :json, + debug=false) where {K,V} + + key_encoder, key_decoder = try KEY_FORMAT_MAP[Symbol(key_format)] catch + error("Unknown key format: $key_format") end + writer, reader = try VAL_FORMAT_MAP[Symbol(val_format)] catch + error("Unknown value format: $val_format") + end + + store = KeyStore{K,V}(bucket_name, session, + (x) -> convert(K, key_decoder(x)), key_encoder, + reader, writer, + gzip, Dict{Symbol, Any}() + ) + # establish availability of bucket + connect!(store; location=location, empty=empty, debug=debug) + store end -function connect!(store::KeyStore; location::AbstractString="US", empty::Bool=false) - response = storage(:Bucket, :get, store.bucket_name; session=store.session, fields="") +function connect!(store::KeyStore; location::AbstractString="US", + empty::Bool=false, debug=false) + response = storage(:Bucket, :get, store.bucket_name; + session=store.session, fields="", debug=debug) if iserror(response) code = response[:error][:code] if code == 404 # not found (available) - response = storage(:Bucket, :insert; session=store.session, data=Dict(:name => store.bucket_name, :location => location), fields="") + response = storage(:Bucket, :insert; session=store.session, + data=Dict(:name => store.bucket_name, :location => location), + fields="", debug=debug) if iserror(response) error("Unable to create bucket: $(response[:error][:message])") end elseif code == 403 # forbidden (not available) - error("Authorization failure or bucket name already taken: $(response[:error][:message])") + error("Authorization failure or bucket name already taken: ", + response[:error][:message]) else error("Error checking bucket: $(response[:error][:message])") end @@ -88,7 +97,7 @@ function connect!(store::KeyStore; location::AbstractString="US", empty::Bool=fa store end -function destroy!{K, V}(store::KeyStore{K, V}) +function destroy!(store::KeyStore{K, V}) where {K, V} response = storage(:Bucket, :delete, store.bucket_name; session=store.session, fields="") if iserror(response) error("Unable to delete bucket: $(response[:error][:message])") @@ -96,13 +105,13 @@ function destroy!{K, V}(store::KeyStore{K, V}) nothing end -function Base.print{K, V}(io::IO, store::KeyStore{K, V}) +function Base.print(io::IO, store::KeyStore{K, V}) where {K, V} print(io, @sprintf("""KeyStore{%s, %s}("%s")""", K, V, store.bucket_name)) end Base.show(io::IO, store::KeyStore) = print(io, store) Base.display(store::KeyStore) = print(store) -function Base.setindex!{K, V}(store::KeyStore{K, V}, val::V, key::K) +function Base.setindex!(store::KeyStore{K, V}, val::V, key::K) where {K, V} name = store.key_encoder(key) data = store.writer(val) response = storage(:Object, :insert, store.bucket_name; session=store.session, @@ -114,7 +123,7 @@ function Base.setindex!{K, V}(store::KeyStore{K, V}, val::V, key::K) store end -function Base.getindex{K, V}(store::KeyStore{K, V}, key::K) +function Base.getindex(store::KeyStore{K, V}, key::K) where {K, V} name = store.key_encoder(key) data = storage(:Object, :get, store.bucket_name, name; session=store.session) if iserror(data) @@ -124,7 +133,7 @@ function Base.getindex{K, V}(store::KeyStore{K, V}, key::K) convert(V, val) end -function Base.get{K, V}(store::KeyStore{K, V}, key::K, default) +function Base.get(store::KeyStore{K, V}, key::K, default) where {K, V} try return getindex(store, key) catch e @@ -136,14 +145,14 @@ function Base.get{K, V}(store::KeyStore{K, V}, key::K, default) end end -function Base.haskey{K, V}(store::KeyStore{K, V}, key::K) +function Base.haskey(store::KeyStore{K, V}, key::K) where {K, V} name = store.key_encoder(key) response = storage(:Object, :get, store.bucket_name, name; session=store.session, alt="", fields="") !iserror(response) end # WARNING: potential for race condition. don't zip keys with values... use collect instead -function Base.keys{K, V}(store::KeyStore{K, V}) +function Base.keys(store::KeyStore{K, V}) where {K, V} result = K[] for response in storage(:Object, :list, store.bucket_name; session=store.session, fields="items(name)") name = response[:name] @@ -157,9 +166,11 @@ function Base.keys{K, V}(store::KeyStore{K, V}) end # avoiding race condition where values might have been deleted after keys were generated -Base.values{K, V}(store::KeyStore{K, V}) = (x for x in (get(store, key, Void) for key in keys(store)) if x !== Void) +@inline function Base.values(store::KeyStore{K, V}) where {K, V} + (x for x in (get(store, key, Nothing) for key in keys(store)) if x !== Voide) +end -function Base.delete!{K, V}(store::KeyStore{K, V}, key::K) +function Base.delete!(store::KeyStore{K, V}, key::K) where {K, V} name = store.key_encoder(key) response = storage(:Object, :delete, store.bucket_name, name; session=store.session, fields="") if iserror(response) @@ -168,19 +179,19 @@ function Base.delete!{K, V}(store::KeyStore{K, V}, key::K) store end -function Base.pop!{K, V}(store::KeyStore{K, V}, key::K) +function Base.pop!(store::KeyStore{K, V}, key::K) where {K, V} val = getindex(store, key) delete!(store, key) val end -function Base.pop!{K, V}(store::KeyStore{K, V}, key::K, default) +function Base.pop!(store::KeyStore{K, V}, key::K, default) where {K, V} val = get(store, key, default) delete!(store, key) val end -function Base.merge!{K, V}(store::KeyStore{K, V}, d::Associative{K, V}) +function Base.merge!(store::KeyStore{K, V}, d::AbstractDict{K, V}) where {K, V} for (k, v) in d store[k] = v end @@ -190,7 +201,7 @@ Base.length(store::KeyStore) = length(collect(keys(store))) Base.isempty(store::KeyStore) = length(store) == 0 -function Base.empty!{K, V}(store::KeyStore{K, V}) +function Base.empty!(store::KeyStore{K, V}) where {K, V} for object in storage(:Object, :list, store.bucket_name; session=store.session, fields="items(name)") response = storage(:Object, :delete, store.bucket_name, object[:name]; session=store.session, fields="") if iserror(response) @@ -201,36 +212,39 @@ function Base.empty!{K, V}(store::KeyStore{K, V}) end """Skip over missing keys if any deleted since key list was geenrated""" -function fast_forward{K, V}(store::KeyStore{K, V}, key_list) +function fast_forward(store::KeyStore{K, V}, key_list) where {K, V} while !isempty(key_list) key = pop!(key_list) - val = get(store, key, Void) - if val !== Void + val = get(store, key, Nothing) + if val !== Nothing return Pair{K, V}(key, val) end end nothing end -function Base.start{K, V}(store::KeyStore{K, V}) +function Base.start(store::KeyStore{K, V}) where {K, V} key_list = collect(keys(store)) return (fast_forward(store, key_list), key_list) end -function Base.next{K, V}(store::KeyStore{K, V}, state) +function Base.next(store::KeyStore{K, V}, state) where {K, V} pair, key_list = state return pair, (fast_forward(store, key_list), key_list) end -function Base.done{K, V}(store::KeyStore{K, V}, state) +function Base.done(store::KeyStore{K, V}, state) where {K, V} pair, key_list = state pair === nothing end -Base.iteratorsize{K, V}(::Type{KeyStore{K, V}}) = Base.SizeUnknown() +@inline function Base.IteratorSize(::Type{KeyStore{K, V}}) where {K, V} + Base.SizeUnknown() +end # notifications -function watch{K, V}(store::KeyStore{K, V}, channel_id::AbstractString, address::AbstractString) +function watch(store::KeyStore{K, V}, channel_id::AbstractString, + address::AbstractString) where {K, V} if !isempty(store.channel) error("Already watching: $store.channel") end @@ -244,7 +258,7 @@ function watch{K, V}(store::KeyStore{K, V}, channel_id::AbstractString, address: store.channel = channel end -function unwatch{K, V}(store::KeyStore{K, V}) +function unwatch(store::KeyStore{K, V}) where {K, V} response = storage(:Channel, :stop; data=store.channel, session=store.session) if iserror(response) error("Unable to unwatch bucket: $(response[:error][:message])") diff --git a/src/credentials.jl b/src/credentials.jl index 6f4cd47..5ddebe8 100644 --- a/src/credentials.jl +++ b/src/credentials.jl @@ -8,8 +8,9 @@ export Credentials, JSONCredentials, MetadataCredentials import Base: show, print import JSON import MbedTLS -import Requests -using URIParser +#import Requests +import HTTP +using HTTP.URIs using ..error using ..root @@ -53,8 +54,8 @@ function Base.get(credentials::MetadataCredentials, path::AbstractString; contex else throw(CredentialError("Unknown metadata context: $context")) end - res = Requests.get(joinpath(url, path), headers=headers) - if Requests.statuscode(res) != 200 + res = HTTP.get(joinpath(url, path), headers=headers) + if HTTP.Messages.status(res) != 200 throw(CredentialError("Unable to obtain credentials from metadata server")) end String(res.data) @@ -66,7 +67,6 @@ end Parse JSON credentials created for a service-account at [Google Cloud Platform Console](https://console.cloud.google.com/apis/credentials) """ - struct JSONCredentials <: Credentials account_type::String project_id::String @@ -98,9 +98,10 @@ end Initialise credentials from dictionary containing values. """ -function JSONCredentials(data::Associative{Symbol, <: AbstractString}) +function JSONCredentials(data::AbstractDict{Symbol, <: AbstractString}) fields = fieldnames(JSONCredentials) - fields[findfirst(fields, :account_type)] = :type # type is a keyword! + fields = [fields...,] + fields[findfirst(x->x==:account_type, fields)] = :type # type is a keyword! missing = setdiff(fields, keys(data)) if !isempty(missing) info(missing) diff --git a/src/root.jl b/src/root.jl index b2fef92..07d0e73 100644 --- a/src/root.jl +++ b/src/root.jl @@ -10,7 +10,7 @@ export API_ROOT, SCOPE_ROOT, AUD_ROOT, METADATA_ROOT, isurl Return true if `path` is a URL and false a path fragment. """ -isurl(path::AbstractString) = ismatch(r"^https?://", path) +isurl(path::AbstractString) = occursin(r"^https?://", path) const API_ROOT = "https://www.googleapis.com" const SCOPE_ROOT = "$API_ROOT/auth" diff --git a/src/session.jl b/src/session.jl index 867cdf8..669ea7b 100644 --- a/src/session.jl +++ b/src/session.jl @@ -6,9 +6,11 @@ module session export GoogleSession, authorize import Base: string, print, show -using Base.Dates +using Dates, Base64 + +#using Requests +using HTTP, HTTP.Messages -using Requests import JSON import MbedTLS @@ -48,10 +50,11 @@ mutable struct GoogleSession{T <: Credentials} """ function GoogleSession(credentials::T, scopes::AbstractVector{<: AbstractString}) where {T <: Credentials} scopes = [isurl(scope) ? scope : "$SCOPE_ROOT/$scope" for scope in scopes] - new{T}(credentials, scopes, Dict{String, String}(), DateTime()) + new{T}(credentials, scopes, Dict{String, String}(), DateTime(1)) end end -function GoogleSession(credentials::Union{AbstractString, Void}, + +function GoogleSession(credentials::Union{AbstractString, Nothing}, scopes::AbstractVector{<: AbstractString}=String[]) if credentials === nothing credentials = "" @@ -70,6 +73,7 @@ function GoogleSession(credentials::Union{AbstractString, Void}, end GoogleSession(credentials, scopes) end + function GoogleSession(scopes::AbstractVector{<: AbstractString}) GoogleSession(get(ENV, "GOOGLE_APPLICATION_CREDENTIALS", ""), scopes) end @@ -137,19 +141,21 @@ function JWS(credentials::JSONCredentials, claimset::JWTClaimSet, header::JWTHea "$payload.$signature" end -function token(credentials::JSONCredentials, scopes::AbstractVector{<: AbstractString}) +function token(credentials::JSONCredentials, + scopes::AbstractVector{<: AbstractString}) # construct claim-set from service account email and requested scopes claimset = JWTClaimSet(credentials.client_email, scopes) - data = Requests.format_query_str(Dict{Symbol, String}( + data = HTTP.URIs.escapeuri(Dict{Symbol, String}( :grant_type => "urn:ietf:params:oauth:grant-type:jwt-bearer", :assertion => JWS(credentials, claimset) )) - headers = Dict{String, String}("Content-Type" => "application/x-www-form-urlencoded") - res = Requests.post("$AUD_ROOT"; data=data, headers=headers) - if statuscode(res) != 200 - throw(SessionError("Unable to obtain authorization: $(readstring(res))")) + headers = Dict{String, String}( + "Content-Type" => "application/x-www-form-urlencoded") + res = HTTP.post("$AUD_ROOT", headers, data) + if res.status != 200 + throw(SessionError("Unable to obtain authorization: $(read(res, String))")) end - authorization = Requests.json(res; dicttype=Dict{Symbol, Any}) + authorization = JSON.parse(payload(res, String); dicttype=Dict{Symbol, Any}) authorization, claimset.assertion end @@ -173,7 +179,7 @@ function authorize(session::GoogleSession; cache::Bool=true) end authorization, assertion = try token(session.credentials, session.scopes) catch e - session.expiry = DateTime() + session.expiry = DateTime(1) empty!(session.authorization) rethrow(e) end @@ -183,7 +189,7 @@ function authorize(session::GoogleSession; cache::Bool=true) session.expiry = assertion + Second(authorization[:expires_in] - 30) session.authorization = authorization else - session.expiry = DateTime() + session.expiry = DateTime(1) empty!(session.authorization) end authorization diff --git a/test/api.jl b/test/api.jl new file mode 100644 index 0000000..6292f52 --- /dev/null +++ b/test/api.jl @@ -0,0 +1,11 @@ +using Test +using GoogleCloud +import GoogleCloud.api + +@testset "test api functions" begin + @test "/this/is/it" == GoogleCloud.api.path_replace( + "/{foo}/{bar}/{baz}", + ["this", "is", "it"]) + + @test api.path_tokens("/{foo}/{bar}/x/{baz}") == ["{foo}", "{bar}", "{baz}"] +end diff --git a/test/runtests.jl b/test/runtests.jl index 3bf0ddd..e2f88af 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,3 +1,5 @@ using GoogleCloud -using Base.Test -@test 1 == 1 +using Test + +include("api.jl") +#include("storage.jl") diff --git a/test/storage.jl b/test/storage.jl new file mode 100644 index 0000000..299bf44 --- /dev/null +++ b/test/storage.jl @@ -0,0 +1,10 @@ +using Test +using GoogleCloud + +creds = JSONCredentials("/secrets/google-secret.json") + +session = GoogleSession(creds, ["devstorage.full_control"]) + +set_session!(storage, session) + +bkts = storage(:Bucket, :list)