diff --git a/.travis.yml b/.travis.yml index 92f60ef..261c61d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: julia julia: - - 0.4 - 0.5 - nightly script: diff --git a/REQUIRE b/REQUIRE index 945ecfb..d28a369 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,3 +1,3 @@ -julia 0.4 -Compat 0.9.4 +julia 0.5 +Compat 0.13.0 Reexport diff --git a/appveyor.yml b/appveyor.yml index d0870af..98a37c4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,5 @@ environment: matrix: - - JULIAVERSION: "julialang/bin/winnt/x86/0.4/julia-0.4-latest-win32.exe" - - JULIAVERSION: "julialang/bin/winnt/x64/0.4/julia-0.4-latest-win64.exe" - JULIAVERSION: "julialang/bin/winnt/x86/0.5/julia-0.5-latest-win32.exe" - JULIAVERSION: "julialang/bin/winnt/x64/0.5/julia-0.5-latest-win64.exe" - JULIAVERSION: "julianightlies/bin/winnt/x86/julia-latest-win32.exe" diff --git a/perf/map.jl b/perf/map.jl index 6740220..cd5d853 100644 --- a/perf/map.jl +++ b/perf/map.jl @@ -15,33 +15,33 @@ f(x::Float64) = 5 * x function profile_map() - println("Method: map!(f, dest, src) (0 missing entries, lift=true)") + println("Method: map!(f, dest, src) (0 missing entries)") print(" for Array{Float64}: ") map!(f, C, A); @time map!(f, C, A); print(" for NullableArray{Float64}: ") - map!(f, Z, X; lift=true); - @time map!(f, Z, X; lift=true); + map!(f, Z, X); + @time map!(f, Z, X); println() - println("Method: map!(f, dest, src) (~half missing entries, lift=true)") + println("Method: map!(f, dest, src) (~half missing entries)") print(" for NullableArray{Float64}: ") - map!(f, Z, Y; lift=true); - @time map!(f, Z, Y; lift=true); + map!(f, Z, Y); + @time map!(f, Z, Y); println() - println("Method: map(f, src) (0 missing entries, lift=true)") + println("Method: map(f, src) (0 missing entries)") print(" for Array{Float64}: ") map(f, A); @time map(f, A); print(" for NullableArray{Float64}: ") - map(f, X; lift=true); - @time map(f, X; lift=true); + map(f, X); + @time map(f, X); println() - println("Method: map(f, src) (~half missing entries, lift=true)") + println("Method: map(f, src) (~half missing entries)") print(" for NullableArray{Float64}: ") - map(f, Y; lift=true); - @time map(f, Y; lift=true); + map(f, Y); + @time map(f, Y); println() end diff --git a/src/NullableArrays.jl b/src/NullableArrays.jl index 4e32e26..b267467 100644 --- a/src/NullableArrays.jl +++ b/src/NullableArrays.jl @@ -25,6 +25,7 @@ include("typedefs.jl") include("constructors.jl") include("primitives.jl") include("indexing.jl") +include("lift.jl") include("map.jl") include("nullablevector.jl") include("operators.jl") diff --git a/src/broadcast.jl b/src/broadcast.jl index 7684f82..0931740 100644 --- a/src/broadcast.jl +++ b/src/broadcast.jl @@ -1,5 +1,6 @@ -using Base: promote_eltype -using Base.Cartesian +using Base: _default_eltype +using Compat + if VERSION >= v"0.6.0-dev.693" using Base.Broadcast: check_broadcast_indices, broadcast_indices else @@ -8,199 +9,96 @@ else const broadcast_indices = broadcast_shape end -if VERSION >= v"0.5.0-dev+5189" - _to_shape(dims::Base.DimsOrInds) = map(_to_shape, dims) - _to_shape(r::Base.OneTo) = Int(last(r)) -else - _to_shape(x) = x -end +if VERSION < v"0.6.0-dev" # Old approach needed for inference to work + ftype(f, A) = typeof(f) + ftype(f, A...) = typeof(a -> f(a...)) + ftype(T::DataType, A) = Type{T} + ftype(T::DataType, A...) = Type{T} -if VERSION < v"0.5.0-dev+5434" - function gen_nullcheck(narrays::Int, nd::Int) - e_nullcheck = macroexpand(:( @nref $nd isnull_1 d->j_d_1 )) - for k = 2:narrays - isnull = Symbol("isnull_$k") - j_d_k = Symbol("j_d_$k") - e_isnull_k = macroexpand(:( @nref $nd $(isnull) d->$(j_d_k) )) - e_nullcheck = Expr(:||, e_nullcheck, e_isnull_k) - end - return e_nullcheck + if isdefined(Base, :Iterators) + using Base.Iterators: Zip2 + else + using Base: Zip2 end + ziptype(A) = Tuple{eltype(A)} + ziptype(A, B) = Zip2{Tuple{eltype(A)}, Tuple{eltype(B)}} + @inline ziptype(A, B, C, D...) = Zip{Tuple{eltype(A)}, ziptype(B, C, D...)} - function gen_broadcast_body(nd::Int, narrays::Int, f, lift::Bool) - F = Expr(:quote, f) - e_nullcheck = gen_nullcheck(narrays, nd) - if lift - return quote - # set up aliases to facilitate subsequent Base.Cartesian magic - B_isnull = B.isnull - @nexprs $narrays k->(values_k = A_k.values) - @nexprs $narrays k->(isnull_k = A_k.isnull) - # check size - @assert ndims(B) == $nd - @ncall $narrays check_broadcast_shape size(B) k->A_k - # main loops - @nloops($nd, i, B, - d->(@nexprs $narrays k->(j_d_k = size(A_k, d) == 1 ? 1 : i_d)), # pre - begin # body - if $e_nullcheck - @inbounds (@nref $nd B_isnull i) = true - else - @nexprs $narrays k->(@inbounds v_k = @nref $nd values_k d->j_d_k) - @inbounds (@nref $nd B i) = (@ncall $narrays $F v) - end - end - ) - end - else - return Base.Broadcast.gen_broadcast_body_cartesian(nd, narrays, f) - end - end - - function gen_broadcast_function(nd::Int, narrays::Int, f, lift::Bool) - As = [Symbol("A_"*string(i)) for i = 1:narrays] - body = gen_broadcast_body(nd, narrays, f, lift) - @eval let - local _F_ - function _F_(B, $(As...)) - $body - end - _F_ - end + nullable_broadcast_eltype(f, As...) = + eltype(_default_eltype(Base.Generator{ziptype(As...), ftype(f, As...)})) +else + Base.@pure nullable_eltypestuple(a) = Tuple{eltype(eltype(a))} + Base.@pure nullable_eltypestuple(T::Type) = Tuple{Type{eltype(T)}} + Base.@pure nullable_eltypestuple(a, b...) = + Tuple{nullable_eltypestuple(a).types..., nullable_eltypestuple(b...).types...} + + Base.@pure function nullable_broadcast_eltype(f, As...) + T = Core.Inference.return_type(f, nullable_eltypestuple(As...)) + T === Union{} ? Any : T end +end - function Base.broadcast!(f, X::NullableArray; lift::Bool=false) - broadcast!(f, X, X; lift=lift) - end +invoke_broadcast!{F, N}(f::F, dest, As::Vararg{NullableArray, N}) = + invoke(broadcast!, Tuple{F, AbstractArray, Vararg{AbstractArray, N}}, f, dest, As...) - @eval let cache = Dict{Any, Dict{Bool, Dict{Int, Dict{Int, Any}}}}() - """ - broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false) - - This method implements the same behavior as that of `broadcast!` when called on - regular `Array` arguments. It also includes the `lift` keyword argument, which - when set to true will lift `f` over the entries of the `As`. - - Lifting is disabled by default. Note that this method's signature specifies - the destination `B` array as well as the source `As` arrays as all - `NullableArray`s. Thus, calling `broadcast!` on a arguments consisting - of both `Array`s and `NullableArray`s will fall back to the implementation - of `broadcast!` in `base/broadcast.jl`. - """ - function Base.broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false) - nd = ndims(B) - narrays = length(As) - - cache_f = Base.@get! cache f Dict{Bool, Dict{Int, Dict{Int, Any}}}() - cache_lift = Base.@get! cache_f lift Dict{Int, Dict{Int, Any}}() - cache_f_na = Base.@get! cache_lift narrays Dict{Int, Any}() - func = Base.@get! cache_f_na nd gen_broadcast_function(nd, narrays, f, lift) - - func(B, As...) - return B - end - end # let cache -else - using Base.Broadcast: newindexer, map_newindexer, newindex - - function _nullcheck(nargs) - nullcheck = :(isnull_1[I_1]) - for i in 2:nargs - sym_isnull = Symbol("isnull_$i") - sym_idx = Symbol("I_$i") - nullcheck = Expr(:||, :($sym_isnull[$sym_idx]), nullcheck) - end - # if 0 argument arrays, treat nullcheck as though it returns false - nargs >= 1 ? nullcheck : :(false) - end +""" + broadcast(f, As::NullableArray...) - @generated function Base.Broadcast._broadcast!{K,ID,XT,nargs}(f, - Z::NullableArray, keeps::K, Idefaults::ID, Xs::XT, ::Type{Val{nargs}}; lift=false) - nullcheck = _nullcheck(nargs) - quote - T = eltype(Z) - $(Expr(:meta, :noinline)) - # destructure keeps and Xs tuples (common to both lifted and non-lifted broadcast) - @nexprs $nargs i->(keep_i = keeps[i]) - @nexprs $nargs i->(Idefault_i = Idefaults[i]) - if !lift - # destructure the keeps and As tuples - @nexprs $nargs i->(X_i = Xs[i]) - @simd for I in CartesianRange(indices(Z)) - # reverse-broadcast the indices - @nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i)) - # extract array values - @nexprs $nargs i->(@inbounds val_i = X_i[I_i]) - # call the function and store the result - @inbounds Z[I] = @ncall $nargs f val - end - else - # destructure the indexmaps and Xs tuples - @nexprs $nargs i->(values_i = Xs[i].values) - @nexprs $nargs i->(isnull_i = Xs[i].isnull) - @simd for I in CartesianRange(indices(Z)) - # reverse-broadcast the indices - @nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i)) - if $nullcheck - # if any args are null, store null - @inbounds Z.isnull[I] = true - else - # extract array values - @nexprs $nargs i->(@inbounds val_i = values_i[I_i]) - # call the function and store the result - @inbounds Z[I] = @ncall $nargs f val - end - end - end - end - end +Call `broadcast` with nullable lifting semantics and return a `NullableArray`. +Lifting means calling function `f` on the the values wrapped inside `Nullable` entries +of the input arrays, and returning null if any entry is missing. - """ - broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false) - - This method implements the same behavior as that of `broadcast!` when called - on regular `Array` arguments. It also includes the `lift` keyword argument, - which when set to true will lift `f` over the entries of the `As`. - - Lifting is disabled by default. Note that this method's signature specifies - the destination `B` array as well as the source `As` arrays as all - `NullableArray`s. Thus, calling `broadcast!` on a arguments consisting of - both `Array`s and `NullableArray`s will fall back to the implementation of - `broadcast!` in `base/broadcast.jl`. - """ - # Required to solve dispatch ambiguity between - # broadcast!(f, X::AbstractArray, x::Number...) - # broadcast!(f, Z::NullableArrays.NullableArray, Xs::NullableArrays.NullableArray...) - @inline Base.broadcast!(f, Z::NullableArray; lift=false) = - broadcast!(f, Z, Z; lift=lift) - - @inline function Base.broadcast!(f, Z::NullableArray, Xs::NullableArray...; - lift=false) - nargs = length(Xs) - shape = indices(Z) - check_broadcast_indices(shape, Xs...) - keeps, Idefaults = map_newindexer(shape, Xs) - Base.Broadcast._broadcast!(f, Z, keeps, Idefaults, Xs, Val{nargs}; lift=lift) - return Z - end +Note that this method's signature specifies the source `As` arrays as all +`NullableArray`s. Thus, calling `broadcast` on arguments consisting +of both `Array`s and `NullableArray`s will fall back to the standard implementation +of `broadcast` (i.e. without lifting). +""" +function Base.broadcast{F}(f::F, As::NullableArray...) + # These definitions are needed to avoid allocation due to splatting + @inline f2(x1) = lift(f, (x1,)) + @inline f2(x1, x2) = lift(f, (x1, x2)) + @inline f2(x1, x2, x3) = lift(f, (x1, x2, x3)) + @inline f2(x1, x2, x3, x4) = lift(f, (x1, x2, x3, x4)) + @inline f2(x1, x2, x3, x4, x5) = lift(f, (x1, x2, x3, x4, x5)) + @inline f2(x1, x2, x3, x4, x5, x6) = lift(f, (x1, x2, x3, x4, x5, x6)) + @inline f2(x1, x2, x3, x4, x5, x6, x7) = lift(f, (x1, x2, x3, x4, x5, x6, x7)) + @inline f2(x...) = lift(f, x) + + T = nullable_broadcast_eltype(f, As...) + dest = similar(NullableArray{T}, broadcast_indices(As...)) + invoke_broadcast!(f2, dest, As...) end """ - broadcast(f, As::NullableArray...;lift::Bool=false) + broadcast!(f, dest::NullableArray, As::NullableArray...) -This method implements the same behavior as that of `broadcast` when called on -regular `Array` arguments. It also includes the `lift` keyword argument, which -when set to true will lift `f` over the entries of the `As`. +Call `broadcast!` with nullable lifting semantics. +Lifting means calling function `f` on the the values wrapped inside `Nullable` entries +of the input arrays, and returning null if any entry is missing. -Lifting is disabled by default. Note that this method's signature specifies the -source `As` arrays as all `NullableArray`s. Thus, calling `broadcast!` on -arguments consisting of both `Array`s and `NullableArray`s will fall back to the -implementation of `broadcast` in `base/broadcast.jl`. +Note that this method's signature specifies the destination `dest` array as well as the +source `As` arrays as all `NullableArray`s. Thus, calling `broadcast!` on a arguments +consisting of both `Array`s and `NullableArray`s will fall back to the standard implementation +of `broadcast!` (i.e. without lifting). """ -@inline function Base.broadcast(f, Xs::NullableArray...;lift::Bool=false) - return broadcast!(f, NullableArray(eltype(promote_eltype(Xs...)), - _to_shape(broadcast_indices(Xs...))), - Xs...; lift=lift) +function Base.broadcast!{F}(f::F, dest::NullableArray, As::NullableArray...) + # These definitions are needed to avoid allocation due to splatting + @inline f2(x1) = lift(f, (x1,)) + @inline f2(x1, x2) = lift(f, (x1, x2)) + @inline f2(x1, x2, x3) = lift(f, (x1, x2, x3)) + @inline f2(x1, x2, x3, x4) = lift(f, (x1, x2, x3, x4)) + @inline f2(x1, x2, x3, x4, x5) = lift(f, (x1, x2, x3, x4, x5)) + @inline f2(x1, x2, x3, x4, x5, x6) = lift(f, (x1, x2, x3, x4, x5, x6)) + @inline f2(x1, x2, x3, x4, x5, x6, x7) = lift(f, (x1, x2, x3, x4, x5, x6, x7)) + @inline f2(x...) = lift(f, x) + + invoke_broadcast!(f2, dest, As...) +end + +# To fix ambiguity +function Base.broadcast!{F}(f::F, dest::NullableArray) + f2() = lift(f) + invoke_broadcast!(f2, dest) end # broadcasted ops diff --git a/src/lift.jl b/src/lift.jl new file mode 100644 index 0000000..685a700 --- /dev/null +++ b/src/lift.jl @@ -0,0 +1,37 @@ +eltype_nullable(x::Nullable) = eltype(x) +eltype_nullable(x) = typeof(x) +eltype_nullable{T<:Nullable}(::Type{T}) = eltype(T) +eltype_nullable{T}(::Type{T}) = T + +eltypes(x) = Tuple{eltype_nullable(x)} +eltypes(x, xs...) = Tuple{eltype_nullable(x), eltypes(xs...).parameters...} + +""" + lift(f, xs...) + +Lift function `f`, passing it arguments `xs...`, using standard lifting semantics: +for a function call `f(xs...)`, return null if any `x` in `xs` is null; otherwise, +return `f` applied to values of `xs`. +""" +@inline @generated function lift{F, N, T}(f::F, xs::NTuple{N, T}) + args = (:(unsafe_get(xs[$i])) for i in 1:N) + checknull = (:(!isnull(xs[$i])) for i in 1:N) + if null_safe_op(f.instance, map(eltype_nullable, xs.parameters)...) + return quote + val = f($(args...)) + nonull = (&)($(checknull...)) + @compat Nullable(val, nonull) + end + else + return quote + U = Core.Inference.return_type(f, eltypes(xs...)) + if (&)($(checknull...)) + return Nullable(f($(args...))) + else + return isleaftype(U) ? Nullable{U}() : Nullable() + end + end + end +end + +lift(f) = Nullable(f()) diff --git a/src/map.jl b/src/map.jl index bc0cc11..940ee5a 100644 --- a/src/map.jl +++ b/src/map.jl @@ -1,248 +1,62 @@ -# if VERSION < v"0.5.0-dev+3294" -# include("map0_4.jl") -# else -using Base: ith_all -if VERSION < v"0.5.0-dev+3294" - include("map0_4.jl") -else - using Base: collect_similar, Generator -end - -function _return_type(f, Xs...) - rtypes = Base.return_types(f, tuple([ inner_eltype(X) for X in Xs ]...)) - T = isempty(rtypes) ? Union{} : rtypes[1] -end - -macro nullcheck(Xs, nargs) - res = :($(Xs)[1].isnull[i]) - for i = 2:nargs - e = :($(Xs)[$i].isnull[i]) - res = Expr(:||, res, e) - end - return res -end - -macro fcall(Xs, nargs) - res = Expr(:call, :f) - for i in 1:nargs - push!(res.args, :($(Xs)[$i].values[i])) - end - return res -end - -inner_eltype{T}(X::NullableArray{T}) = T - -# Base.map! - -Base.map!{F}(f::F, X::NullableArray; lift=false) = map!(f, X, X; lift=lift) -function Base.map!{F}(f::F, dest::NullableArray, X::NullableArray; lift=false) - if lift - for (i, j) in zip(eachindex(dest), eachindex(X)) - if X.isnull[j] - dest.isnull[i] = true - else - dest.isnull[i] = false - dest.values[i] = f(X.values[j]) - end - end - else - for (i, j) in zip(eachindex(dest), eachindex(X)) - dest[i] = f(X[j]) - end - end - return dest -end - -function Base.map!{F}(f::F, dest::NullableArray, X1::NullableArray, - X2::NullableArray; lift=false) - if lift - for (i, j, k) in zip(eachindex(dest), eachindex(X1), eachindex(X2)) - if X1.isnull[j] | X2.isnull[k] - dest.isnull[i] = true - else - dest.isnull[i] = false - dest.values[i] = f(X1.values[j], X2.values[k]) - end - end - else - for (i, j, k) in zip(eachindex(dest), eachindex(X1), eachindex(X2)) - dest[i] = f(X1[j], X2[k]) - end - end - return dest -end - -function Base.map!{F}(f::F, dest::NullableArray, Xs::NullableArray...; lift=false) - _mapn!(f, dest, Xs, lift) -end - -@generated function _mapn!{F, N}(f::F, dest::NullableArray, Xs::NTuple{N, NullableArray}, lift) - return quote - if lift - for i in eachindex(dest) - if @nullcheck Xs $N - dest.isnull[i] = true - else - dest.isnull[i] = false - dest.values[i] = @fcall Xs $N - end - end - else - for i in eachindex(dest) - dest[i] = f(ith_all(i, Xs)...) - end - end - return dest - end -end - -# Base.map - -if VERSION < v"0.5.0-dev+3294" - function Base.map(f, X::NullableArray; lift=false) - lift ? _liftedmap(f, X) : _map(f, X) - end - function Base.map(f, X1::NullableArray, X2::NullableArray; lift=false) - lift ? _liftedmap(f, X1, X2) : _map(f, X1, X2) - end - function Base.map(f, Xs::NullableArray...; lift=false) - lift ? _liftedmap(f, Xs) : _map(f, Xs...) - end -else - function Base.map(f, X::NullableArray; lift=false) - lift ? _liftedmap(f, X) : collect_similar(X, Generator(f, X)) - end - function Base.map(f, X1::NullableArray, X2::NullableArray; lift=false) - lift ? _liftedmap(f, X1, X2) : collect(Generator(f, X1, X2)) - end - function Base.map(f, Xs::NullableArray...; lift=false) - lift ? _liftedmap(f, Xs) : collect(Generator(f, Xs...)) - end -end - -function _liftedmap(f, X::NullableArray) - len = length(X) - # if X is empty, fall back on type inference - len > 0 || return NullableArray(_return_type(f, X), 0) - i = 1 - while X.isnull[i] & (i < len) - i += 1 - end - # if X is all null, fall back on type inference - if X.isnull[i] - T = _return_type(f, X) - return similar(X, T) - end - # otherwise, initialize and map to destination array - v = f(X.values[i]) - dest = similar(X, typeof(v)) - dest[i] = v - _liftedmap_to!(f, dest, X, i+1, len) -end - -function _liftedmap(f, X1::NullableArray, X2::NullableArray) - len = prod(promote_shape(size(X1), size(X2))) - len > 0 || return NullableArray(_return_type(f, X1, X2), 0) - i = 1 - while (X1.isnull[i] | X2.isnull[i]) & (i < len) - i += 1 - end - if X1.isnull[i] | X2.isnull[i] - T = _return_type(f, X1, X2) - return similar(X1, T) - end - v = f(X1.values[i], X2.values[i]) - dest = similar(X1, typeof(v)) - dest[i] = v - _liftedmap_to!(f, dest, X1, X2, i+1, len) -end - -@generated function _liftedmap{N}(f, Xs::NTuple{N, NullableArray}) - return quote - shp = mapreduce(size, promote_shape, Xs) - len = prod(shp) - len > 0 || return NullableArray(_return_type(f, Xs...), 0) - i = 1 - while (@nullcheck Xs $N) & (i < len) - i += 1 - end - if @nullcheck Xs $N - T = _return_type(f, Xs...) - return similar(Xs[1], T) - end - v = @fcall Xs $N - dest = similar(Xs[1], typeof(v)) - dest[i] = v - _liftedmap_to!(f, dest, Xs, i+1, len) - end -end - -function _liftedmap_to!{T}(f, dest::NullableArray{T}, X, offs, len) - # map to dest array, checking the type of each result. if a result does not - # match, widen the result type and re-dispatch. - i = offs - while i <= len - @inbounds if X.isnull[i] - i += 1; continue - end - @inbounds el = f(X.values[i]) - S = typeof(el) - if S === T || S <: T - @inbounds dest[i] = el::T - i += 1 - else - R = typejoin(T, S) - new = similar(dest, R) - copy!(new, 1, dest, 1, i-1) - @inbounds new[i] = el - return map_to!(f, new, X, i+1, len) - end - end - return dest -end - -function _liftedmap_to!{T}(f, dest::NullableArray{T}, X1, X2, offs, len) - i = offs - while i <= len - @inbounds if X1.isnull[i] | X2.isnull[i] - i += 1; continue - end - @inbounds el = f(X1.values[i], X2.values[i]) - S = typeof(el) - if S === T || S <: T - @inbounds dest[i] = el::T - i += 1 - else - R = typejoin(T, S) - new = similar(dest, R) - copy!(new, 1, dest, 1, i-1) - @inbounds new[i] = el - return map_to!(f, new, X1, X2, i+1, len) - end - end - return dest -end - -@generated function _liftedmap_to!{T, N}(f, dest::NullableArray{T}, Xs::NTuple{N,NullableArray}, offs, len) - return quote - i = offs - while i <= len - @inbounds if @nullcheck Xs $N - i += 1; continue - end - @inbounds el = @fcall Xs $N - S = typeof(el) - if S === T || S <: T - @inbounds dest[i] = el::T - i += 1 - else - R = typejoin(T, S) - new = similar(dest, R) - copy!(new, 1, dest, 1, i-1) - @inbounds new[i] = el - return map_to!(f, new, Xs, i+1, len) - end - end - return dest - end +invoke_map!{F, N}(f::F, dest, As::Vararg{NullableArray, N}) = + invoke(map!, Tuple{F, AbstractArray, Vararg{AbstractArray, N}}, f, dest, As...) + +""" + map(f, As::NullableArray...) + +Call `map` with nullable lifting semantics and return a `NullableArray`. +Lifting means calling function `f` on the the values wrapped inside `Nullable` entries +of the input arrays, and returning null if any entry is missing. + +Note that this method's signature specifies the source `As` arrays as all +`NullableArray`s. Thus, calling `map` on arguments consisting +of both `Array`s and `NullableArray`s will fall back to the standard implementation +of `map` (i.e. without lifting). +""" +function Base.map{F}(f::F, As::NullableArray...) + # These definitions are needed to avoid allocation due to splatting + @inline f2(x1) = lift(f, (x1,)) + @inline f2(x1, x2) = lift(f, (x1, x2)) + @inline f2(x1, x2, x3) = lift(f, (x1, x2, x3)) + @inline f2(x1, x2, x3, x4) = lift(f, (x1, x2, x3, x4)) + @inline f2(x1, x2, x3, x4, x5) = lift(f, (x1, x2, x3, x4, x5)) + @inline f2(x1, x2, x3, x4, x5, x6) = lift(f, (x1, x2, x3, x4, x5, x6)) + @inline f2(x1, x2, x3, x4, x5, x6, x7) = lift(f, (x1, x2, x3, x4, x5, x6, x7)) + @inline f2(x...) = lift(f, x) + + T = nullable_broadcast_eltype(f, As...) + dest = similar(NullableArray{T}, size(As[1])) + invoke_map!(f2, dest, As...) +end + +""" + map!(f, dest::NullableArray, As::NullableArray...) + +Call `map!` with nullable lifting semantics. +Lifting means calling function `f` on the the values wrapped inside `Nullable` entries +of the input arrays, and returning null if any entry is missing. + +Note that this method's signature specifies the destination `dest` array as well as the +source `As` arrays as all `NullableArray`s. Thus, calling `map!` on a arguments +consisting of both `Array`s and `NullableArray`s will fall back to the standard implementation +of `map!` (i.e. without lifting). +""" +function Base.map!{F}(f::F, dest::NullableArray, As::NullableArray...) + # These definitions are needed to avoid allocation due to splatting + @inline f2(x1) = lift(f, (x1,)) + @inline f2(x1, x2) = lift(f, (x1, x2)) + @inline f2(x1, x2, x3) = lift(f, (x1, x2, x3)) + @inline f2(x1, x2, x3, x4) = lift(f, (x1, x2, x3, x4)) + @inline f2(x1, x2, x3, x4, x5) = lift(f, (x1, x2, x3, x4, x5)) + @inline f2(x1, x2, x3, x4, x5, x6) = lift(f, (x1, x2, x3, x4, x5, x6)) + @inline f2(x1, x2, x3, x4, x5, x6, x7) = lift(f, (x1, x2, x3, x4, x5, x6, x7)) + @inline f2(x...) = lift(f, x) + + invoke_map!(f2, dest, As...) +end + +# This definition is needed to avoid dispatch loops going back to the above one +function Base.map!{F}(f::F, dest::NullableArray) + f2(x1) = lift(f, (x1,)) + invoke_map!(f2, dest, dest) end diff --git a/src/map0_4.jl b/src/map0_4.jl deleted file mode 100644 index ce079dd..0000000 --- a/src/map0_4.jl +++ /dev/null @@ -1,91 +0,0 @@ -using Base: promote_eltype - -function _map(f, X) - if isempty(X) - return isa(f, Type) ? similar(X, f) : similar(X) - end - st = start(X) - x1, st = next(X, st) - first = f(x1) - dest = similar(X, typeof(first)) - dest[1] = first - return map_to!(f, 2, st, dest, X) -end - -function _map(f, X1, X2) - shp = promote_shape(size(X1), size(X2)) - if prod(shp) == 0 - return similar(X1, promote_type(eltype(X1), eltype(X2)), shp) - end - first = f(X1[1], X2[1]) - dest = similar(X1, typeof(first), shp) - dest[1] = first - return map_to!(f, 2, dest, X1, X2) -end - -function _map(f, Xs...) - shape = mapreduce(size, promote_shape, Xs) - if prod(shape) == 0 - return similar(Xs[1], promote_eltype(Xs...), shape) - end - first = f(ith_all(1, Xs)...) - dest = similar(Xs[1], typeof(first), shape) - dest[1] = first - return map_to_n!(f, 2, dest, Xs) -end - - -function map_to!{T,F}(f::F, offs, st, dest::NullableArray{T}, X) - # map to dest array, checking the type of each result. if a result does not - # match, widen the result type and re-dispatch. - i = offs - while !done(X, st) - @inbounds Xi, st = next(X, st) - el = f(Xi) - S = typeof(el) - if S === T || S <: T - @inbounds dest[i] = el::T - i += 1 - else - R = typejoin(T, S) - new = similar(dest, R) - copy!(new,1, dest,1, i-1) - @inbounds new[i] = el - return map_to!(f, i+1, st, new, X) - end - end - return dest -end - -function map_to!{T,F}(f::F, offs, dest::NullableArray{T}, X1::NullableArray, X2::NullableArray) - for i = offs:length(X1) - @inbounds X1i, X2i = X1[i], X2[i] - el = f(X1i, X2i) - S = typeof(el) - if (S !== T) && !(S <: T) - R = typejoin(T, S) - new = similar(dest, R) - copy!(new,1, dest,1, i-1) - @inbounds new[i] = el - return map_to!(f, i+1, new, X1, X2) - end - @inbounds dest[i] = el::T - end - return dest -end - -function map_to_n!{T,F}(f::F, offs, dest::NullableArray{T}, Xs) - for i = offs:length(Xs[1]) - el = f(ith_all(i, Xs)...) - S = typeof(el) - if (S !== T) && !(S <: T) - R = typejoin(T, S) - new = similar(dest, R) - copy!(new,1, dest,1, i-1) - @inbounds new[i] = el - return map_to_n!(f, i+1, new, Xs) - end - @inbounds dest[i] = el::T - end - return dest -end diff --git a/src/operators.jl b/src/operators.jl index f4d825a..8b430dd 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -20,8 +20,7 @@ if VERSION < v"0.5.0-dev+5096" end """ - null_safe_op(f::Any, ::Type)::Bool - null_safe_op(f::Any, ::Type, ::Type)::Bool + null_safe_op(f::Any, ::Type...)::Bool Returns whether an operation `f` can safely be applied to any value of the passed type(s). Returns `false` by default. @@ -37,8 +36,7 @@ Types declared as safe can benefit from higher performance for operations on nul always computing the result even for null values, a branch is avoided, which helps vectorization. """ -null_safe_op(f::Any, ::Type) = false -null_safe_op(f::Any, ::Type, ::Type) = false +null_safe_op(f::Any, ::Type...) = false typealias SafeSignedInts Union{Int128,Int16,Int32,Int64,Int8} typealias SafeUnsignedInts Union{Bool,UInt128,UInt16,UInt32,UInt64,UInt8} diff --git a/test/broadcast.jl b/test/broadcast.jl index f68c989..2825e40 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -1,6 +1,7 @@ module TestBroadcast using NullableArrays using Base.Test + using Compat A1 = rand(10) M1 = rand(Bool, 10) @@ -31,15 +32,10 @@ module TestBroadcast Z3 = NullableArray(Float64, 10, [dims; i]...) f() = 5 - f(x) = Nullable(5) * x - f(x, y) = x + y - f(x, y, z) = x + y + z - g() = 5 - g(x::Float64) = 5 * x - g(x::Float64, y::Float64) = x * y - g(x::Float64, y::Float64, z::Float64) = x * y * z + f(x::Float64) = 5 * x + f(x::Float64, y::Float64) = x * y + f(x::Float64, y::Float64, z::Float64) = x * y * z - i = 1 for (dests, arrays, nullablearrays, mask) in ( ((C2, Z2), (A1, A2), (U1, U2), ()), ((C3, Z3), (A2, A3), (U2, U3), ()), @@ -50,26 +46,18 @@ module TestBroadcast ((C3, Z3), (A1, A2, A3), (V1, V2, V3), (Q3,)), ) - # Base.broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false) + # Base.broadcast!(f, B::NullableArray, As::NullableArray...) broadcast!(f, dests[1], arrays...) broadcast!(f, dests[2], nullablearrays...) @test isequal(dests[2], NullableArray(dests[1], mask...)) - broadcast!(g, dests[1], arrays...) - broadcast!(g, dests[2], nullablearrays...; lift=true) - @test isequal(dests[2], NullableArray(dests[1], mask...)) - - # Base.broadcast(f, As::NullableArray...;lift::Bool=false) + # Base.broadcast(f, As::NullableArray...) D = broadcast(f, arrays...) X = broadcast(f, nullablearrays...) @test isequal(X, NullableArray(D, mask...)) - - D = broadcast(g, arrays...) - X = broadcast(g, nullablearrays...; lift=true) - @test isequal(X, NullableArray(D, mask...)) end - # Base.broadcast!(f, X::NullableArray; lift::Bool=false) + # Base.broadcast!(f, X::NullableArray) for (array, nullablearray, mask) in ( (A1, U1, ()), (A2, U2, ()), (A3, U3, ()), (A1, V1, (M1,)), (A2, V2, (M2,)), (A3, V3, (M3,)), @@ -77,10 +65,6 @@ module TestBroadcast broadcast!(f, array) broadcast!(f, nullablearray) @test isequal(nullablearray, NullableArray(array, mask...)) - - broadcast!(g, array) - broadcast!(g, nullablearray; lift=true) - @test isequal(nullablearray, NullableArray(array, mask...)) end # test broadcasted arithmetic operators @@ -93,23 +77,33 @@ module TestBroadcast M = rand(Bool, 10, dims...) Y = NullableArray(B, M) - for op in ( - (.+), - (.-), - (.*), - (./), - (.%), - (.^), - (.==), - (.!=), - (.<), - (.>), - (.<=), - (.>=), - ) - @test isequal(op(X1, X2), NullableArray(op(A, B))) - @test isequal(op(X1, Y), NullableArray(op(A, B), M)) + if VERSION < v"0.6.0-dev.1632" + for op in ( + (.+), + (.-), + (.*), + (./), + (.%), + (.^), + (.==), + (.!=), + (.<), + (.>), + (.<=), + (.>=), + ) + @test isequal(op(X1, X2), NullableArray(op(A, B))) + @test isequal(op(X1, Y), NullableArray(op(A, B), M)) + end end + A = rand(Bool, 100) + B = rand(Bool, 100) + M1 = rand(Bool, 100) + M2 = rand(Bool, 100) + X = NullableArray(A, M1) + Y = NullableArray(B, M2) + @test isequal(broadcast(&, X, Y), NullableArray(A .& B, M1 .| M2)) + @test isequal(broadcast(|, X, Y), NullableArray(A .| B, M1 .| M2)) end # module diff --git a/test/map.jl b/test/map.jl index a8a8cbe..9e53c5b 100644 --- a/test/map.jl +++ b/test/map.jl @@ -25,8 +25,6 @@ module TestMap R = map(|, Ms...) f(x...) = sum(x) - g(x::Float64...) = prod(x) - dests = (C, Z) # 1 arg @@ -35,21 +33,13 @@ module TestMap ) for i in 1:m # map! - map!(f, args[1][i]) # map!(f, As[i]) - map!(f, args[2][i]) # map!(f, Xs[i]) - @test isequal(args[2][i], NullableArray(args[1][i], masks[i]...)) - # lifted map! - map!(g, args[1][i]) - map!(g, args[2][i]; lift=true) + map!(f, args[1][i], args[1][i]) # map!(f, As[i], As[i]) + map!(f, args[2][i], args[2][i]) # map!(f, Xs[i], Xs[i]) @test isequal(args[2][i], NullableArray(args[1][i], masks[i]...)) # map A = map(f, args[1][i]) X = map(f, args[2][i]) @test isequal(X, NullableArray(A, masks[i]...)) - # lifted map - A = map(f, args[1][i]) - X = map(g, args[2][i]; lift=true) - @test isequal(X, NullableArray(A, masks[i]...)) end end # 2 arg @@ -62,18 +52,10 @@ module TestMap map!(f, dests[1], args[1][i], args[1][j]) map!(f, dests[2], args[2][i], args[2][j]) @test isequal(dests[2], NullableArray(dests[1], mask...)) - # lifted map! - map!(g, dests[1], args[1][i], args[1][j]) - map!(g, dests[2], args[2][i], args[2][j]; lift=true) - @test isequal(dests[2], NullableArray(dests[1], mask...)) # map map(f, args[1][i], args[1][j]) map(f, args[2][i], args[2][j]) @test isequal(dests[2], NullableArray(dests[1], mask...)) - # lifted map - map(g, args[1][i], args[1][j]) - map(g, args[2][i], args[2][j]; lift=true) - @test isequal(dests[2], NullableArray(dests[1], mask...)) end # n arg for (args, mask) in ( @@ -83,79 +65,59 @@ module TestMap map!(f, dests[1], args[1]...) map!(f, dests[2], args[2]...) @test isequal(dests[2], NullableArray(dests[1], mask...)) - # lifted map! - map!(g, dests[1], args[1]...) - map!(g, dests[2], args[2]...; lift=true) - @test isequal(dests[2], NullableArray(dests[1], mask...)) # map map(f, args[1]...) map(f, args[2]...) @test isequal(dests[2], NullableArray(dests[1], mask...)) - # lifted map - map(g, args[1]...) - map(g, args[2]...; lift=true) - @test isequal(dests[2], NullableArray(dests[1], mask...)) end # test map over empty NullableArrays X = NullableArray(Int[]) h1(x) = 5.0*x - h2(x::Nullable) = x - h2(x::Nullable...) = prod(x) + h2(x) = x + h2(x...) = prod(x) Z1 = map(h1, X) Z2 = map(h2, X) - if VERSION >= v"0.5.0-dev+3294" - @test isequal(Z1, NullableArray(Union{}[])) # no type inference + @test isempty(Z1) + if VERSION >= v"0.6.0-dev" + @test isa(Z1, NullableArray{Float64}) end - if VERSION >= v"0.5.0-dev+5297" - @test isequal(Z2, NullableArray(Float64[])) # use type inference + @test isempty(Z2) + if VERSION >= v"0.6.0-dev" + @test isa(Z2, NullableArray{Int}) end - Z1 = map(h1, X; lift=true) # use type inference - @test isequal(Z1, NullableArray(Int64[])) - - # if a function has no method for inner_eltype of empty NullableArray, - # result should be empty NullableArray{Union{}}() + # if a function has no method for inner eltype of empty NullableArray, + # result should be empty NullableArray{Any}() for consistency with generic map() h3(x::Float64...) = prod(x) Z3 = map(h3, X) - @test isequal(Z3, NullableArray(Union{}[])) + @test isempty(Z3) + @test isa(Z3, NullableArray{Any}) Z3 = map(h3, X, X) - @test isequal(Z3, NullableArray(Union{}[])) + @test isempty(Z3) + @test isa(Z3, NullableArray{Any}) Z3 = map(h3, X, X, X) - @test isequal(Z3, NullableArray(Union{}[])) - Z3 = map(h3, X; lift=true) - @test isequal(Z3, NullableArray(Union{}[])) - Z3 = map(h3, X, X; lift=true) - @test isequal(Z3, NullableArray(Union{}[])) - Z3 = map(h3, X, X, X; lift=true) - @test isequal(Z3, NullableArray(Union{}[])) + @test isempty(Z3) + @test isa(Z3, NullableArray{Any}) # test map over all null NullableArray n = rand(10:100) Ys = [ NullableArray(rand(Int, n), fill(true, n)) for i in 1:rand(3:5) ] - Z2 = map(h2, Ys[1]) # behavior should be determined solely by h2, i.e. no type inference + Z2 = map(h2, Ys[1]) @test isequal(Z2, NullableArray(Int, n)) + if VERSION >= v"0.6.0-dev" + @test isa(Z2, NullableArray{Int}) + end Z2 = map(h2, Ys[1], Ys[2]) @test isequal(Z2, NullableArray(Int, n)) + if VERSION >= v"0.6.0-dev" + @test isa(Z2, NullableArray{Int}) + end Z2 = map(h2, Ys...) @test isequal(Z2, NullableArray(Int, n)) - - Z1 = map(h1, Ys[1]; lift=true) # fall back on type inference - @test isequal(Z1, NullableArray(Float64, n)) - Z1 = map(h1, Ys[1], Ys[2]; lift=true) - @test isequal(Z1, NullableArray(Float64, n)) - Z1 = map(h1, Ys...; lift=true) - @test isequal(Z1, NullableArray(Float64, n)) - - # if a function has no method for inner_eltype of all null NullableArray, - # result of lifted map should be NullableArray{Union{}}() - h4(x::Int...) = prod(x) - Z4 = map(h4, Ys[1]; lift=true) - @test isequal(Z4, NullableArray(Union{}, n)) - Z4 = map(h4, Ys[1], Ys[2]; lift=true) - @test isequal(Z4, NullableArray(Union{}, n)) - Z4 = map(h4, Ys...; lift=true) - @test isequal(Z4, NullableArray(Union{}, n)) + if VERSION >= v"0.6.0-dev" + @test isa(Z2, NullableArray{Int}) + end end # module TestMap