diff --git a/src/NullableArrays.jl b/src/NullableArrays.jl index 4e32e26..723fa69 100644 --- a/src/NullableArrays.jl +++ b/src/NullableArrays.jl @@ -27,6 +27,7 @@ include("primitives.jl") include("indexing.jl") include("map.jl") include("nullablevector.jl") +include("lift.jl") include("operators.jl") include("broadcast.jl") include("reduce.jl") diff --git a/src/broadcast.jl b/src/broadcast.jl index 1e7168b..d067519 100644 --- a/src/broadcast.jl +++ b/src/broadcast.jl @@ -1,9 +1,11 @@ -using Base: promote_eltype -using Base.Cartesian if VERSION >= v"0.6.0-dev.693" using Base.Broadcast: check_broadcast_indices, broadcast_indices + using Base.Broadcast: check_broadcast_indices, broadcast_indices, + _default_eltype, ftype, ziptype else using Base.Broadcast: check_broadcast_shape, broadcast_shape + using Base.Broadcast: check_broadcast_shape, broadcast_shape, + _default_eltype, ftype, ziptype const check_broadcast_indices = check_broadcast_shape const broadcast_indices = broadcast_shape end @@ -15,177 +17,9 @@ else _to_shape(x) = x end -if VERSION < v"0.5.0-dev+5434" - function gen_nullcheck(narrays::Int, nd::Int) - e_nullcheck = macroexpand(:( @nref $nd isnull_1 d->j_d_1 )) - for k = 2:narrays - isnull = Symbol("isnull_$k") - j_d_k = Symbol("j_d_$k") - e_isnull_k = macroexpand(:( @nref $nd $(isnull) d->$(j_d_k) )) - e_nullcheck = Expr(:||, e_nullcheck, e_isnull_k) - end - return e_nullcheck - end - - function gen_broadcast_body(nd::Int, narrays::Int, f, lift::Bool) - F = Expr(:quote, f) - e_nullcheck = gen_nullcheck(narrays, nd) - if lift - return quote - # set up aliases to facilitate subsequent Base.Cartesian magic - B_isnull = B.isnull - @nexprs $narrays k->(values_k = A_k.values) - @nexprs $narrays k->(isnull_k = A_k.isnull) - # check size - @assert ndims(B) == $nd - @ncall $narrays check_broadcast_shape size(B) k->A_k - # main loops - @nloops($nd, i, B, - d->(@nexprs $narrays k->(j_d_k = size(A_k, d) == 1 ? 1 : i_d)), # pre - begin # body - if $e_nullcheck - @inbounds (@nref $nd B_isnull i) = true - else - @nexprs $narrays k->(@inbounds v_k = @nref $nd values_k d->j_d_k) - @inbounds (@nref $nd B i) = (@ncall $narrays $F v) - end - end - ) - end - else - return Base.Broadcast.gen_broadcast_body_cartesian(nd, narrays, f) - end - end - - function gen_broadcast_function(nd::Int, narrays::Int, f, lift::Bool) - As = [Symbol("A_"*string(i)) for i = 1:narrays] - body = gen_broadcast_body(nd, narrays, f, lift) - @eval let - local _F_ - function _F_(B, $(As...)) - $body - end - _F_ - end - end - - function Base.broadcast!(f, X::NullableArray; lift::Bool=false) - broadcast!(f, X, X; lift=lift) - end - - @eval let cache = Dict{Any, Dict{Bool, Dict{Int, Dict{Int, Any}}}}() - @doc """ - `broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)` - This method implements the same behavior as that of `broadcast!` when called on - regular `Array` arguments. It also includes the `lift` keyword argument, which - when set to true will lift `f` over the entries of the `As`. - - Lifting is disabled by default. Note that this method's signature specifies - the destination `B` array as well as the source `As` arrays as all - `NullableArray`s. Thus, calling `broadcast!` on a arguments consisting - of both `Array`s and `NullableArray`s will fall back to the implementation - of `broadcast!` in `base/broadcast.jl`. - """ -> - function Base.broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false) - nd = ndims(B) - narrays = length(As) - - cache_f = Base.@get! cache f Dict{Bool, Dict{Int, Dict{Int, Any}}}() - cache_lift = Base.@get! cache_f lift Dict{Int, Dict{Int, Any}}() - cache_f_na = Base.@get! cache_lift narrays Dict{Int, Any}() - func = Base.@get! cache_f_na nd gen_broadcast_function(nd, narrays, f, lift) - - func(B, As...) - return B - end - end # let cache -else - using Base.Broadcast: newindexer, map_newindexer, newindex - - function _nullcheck(nargs) - nullcheck = :(isnull_1[I_1]) - for i in 2:nargs - sym_isnull = Symbol("isnull_$i") - sym_idx = Symbol("I_$i") - nullcheck = Expr(:||, :($sym_isnull[$sym_idx]), nullcheck) - end - # if 0 argument arrays, treat nullcheck as though it returns false - nargs >= 1 ? nullcheck : :(false) - end - - @generated function Base.Broadcast._broadcast!{K,ID,XT,nargs}(f, - Z::NullableArray, keeps::K, Idefaults::ID, Xs::XT, ::Type{Val{nargs}}; lift=false) - nullcheck = _nullcheck(nargs) - quote - T = eltype(Z) - $(Expr(:meta, :noinline)) - # destructure keeps and Xs tuples (common to both lifted and non-lifted broadcast) - @nexprs $nargs i->(keep_i = keeps[i]) - @nexprs $nargs i->(Idefault_i = Idefaults[i]) - if !lift - # destructure the keeps and As tuples - @nexprs $nargs i->(X_i = Xs[i]) - @simd for I in CartesianRange(indices(Z)) - # reverse-broadcast the indices - @nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i)) - # extract array values - @nexprs $nargs i->(@inbounds val_i = X_i[I_i]) - # call the function and store the result - @inbounds Z[I] = @ncall $nargs f val - end - else - # destructure the indexmaps and Xs tuples - @nexprs $nargs i->(values_i = Xs[i].values) - @nexprs $nargs i->(isnull_i = Xs[i].isnull) - @simd for I in CartesianRange(indices(Z)) - # reverse-broadcast the indices - @nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i)) - if $nullcheck - # if any args are null, store null - @inbounds Z.isnull[I] = true - else - # extract array values - @nexprs $nargs i->(@inbounds val_i = values_i[I_i]) - # call the function and store the result - @inbounds Z[I] = @ncall $nargs f val - end - end - end - end - end - - @doc """ - `broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)` - - This method implements the same behavior as that of `broadcast!` when called - on regular `Array` arguments. It also includes the `lift` keyword argument, - which when set to true will lift `f` over the entries of the `As`. - - Lifting is disabled by default. Note that this method's signature specifies - the destination `B` array as well as the source `As` arrays as all - `NullableArray`s. Thus, calling `broadcast!` on a arguments consisting of - both `Array`s and `NullableArray`s will fall back to the implementation of - `broadcast!` in `base/broadcast.jl`. - """ -> - # Required to solve dispatch ambiguity between - # broadcast!(f, X::AbstractArray, x::Number...) - # broadcast!(f, Z::NullableArrays.NullableArray, Xs::NullableArrays.NullableArray...) - @inline Base.broadcast!(f, Z::NullableArray; lift=false) = - broadcast!(f, Z, Z; lift=lift) - - @inline function Base.broadcast!(f, Z::NullableArray, Xs::NullableArray...; - lift=false) - nargs = length(Xs) - shape = indices(Z) - check_broadcast_indices(shape, Xs...) - keeps, Idefaults = map_newindexer(shape, Xs) - Base.Broadcast._broadcast!(f, Z, keeps, Idefaults, Xs, Val{nargs}; lift=lift) - return Z - end -end @doc """ -`broadcast(f, As::NullableArray...;lift::Bool=false)` +`broadcast(f, As::NullableArray...)` This method implements the same behavior as that of `broadcast` when called on regular `Array` arguments. It also includes the `lift` keyword argument, which @@ -196,10 +30,20 @@ source `As` arrays as all `NullableArray`s. Thus, calling `broadcast!` on arguments consisting of both `Array`s and `NullableArray`s will fall back to the implementation of `broadcast` in `base/broadcast.jl`. """ -> -@inline function Base.broadcast(f, Xs::NullableArray...;lift::Bool=false) - return broadcast!(f, NullableArray(eltype(promote_eltype(Xs...)), - _to_shape(broadcast_indices(Xs...))), - Xs...; lift=lift) +function Base.broadcast{N}(f, As::Vararg{NullableArray, N}) + f2(x...) = lift(f, x...) + T = _default_eltype(Base.Generator{ziptype(As...), ftype(f2, As...)}) + if isleaftype(T) && !(T <: Nullable) + return invoke(broadcast, Tuple{Function, Vararg{AbstractArray, N}}, f2, As...) + else + dest = NullableArray(eltype(T), _to_shape(broadcast_indices(As...))) + return invoke(broadcast!, Tuple{Function, AbstractArray, Vararg{AbstractArray, N}}, f2, dest, As...) + end +end + +function Base.broadcast!{N}(f, dest::AbstractArray, As::Vararg{NullableArray, N}) + f2(x...) = lift(f, x...) + invoke(broadcast!, Tuple{Function, AbstractArray, Vararg{AbstractArray, N}}, f2, dest, As...) end # broadcasted ops diff --git a/src/lift.jl b/src/lift.jl new file mode 100644 index 0000000..342e3f9 --- /dev/null +++ b/src/lift.jl @@ -0,0 +1,139 @@ +import Base: null_safe_op + +############################################################################## +## +## Standard lifting semantics +## +## For a function call f(xs...), return null if any x in xs is null; +## otherwise, return f applied to values of xs. +## +############################################################################## + +@inline function lift(f, x) + if null_safe_op(f, typeof(x)) + return @compat Nullable(f(x.value), !isnull(x)) + else + U = Core.Inference.return_type(f, Tuple{eltype(typeof(x))}) + if isnull(x) + return Nullable{U}() + else + return Nullable(f(unsafe_get(x))) + end + end +end + +@inline function lift(f, x1, x2) + if null_safe_op(f, typeof(x1), typeof(x2)) + return @compat Nullable( + f(x1.value, x2.value), !(isnull(x1) | isnull(x2)) + ) + else + U = Core.Inference.return_type( + f, Tuple{eltype(typeof(x1)), eltype(typeof(x2))} + ) + if isnull(x1) | isnull(x2) + return Nullable{U}() + else + return Nullable(f(unsafe_get(x1), unsafe_get(x2))) + end + end +end + +hasnulls(xs...) = any(isnull.(xs)) + +@inline function lift(f, xs...) + if null_safe_op(f, map(typeof, xs)...) + return @compat Nullable( + f(map(unsafe_get, xs)...), !(mapreduce(isnull, |, xs)) + ) + else + U = Core.Inference.return_type( + f, Tuple{map(x->eltype(typeof(x)), xs)...} + ) + if hasnulls(xs...) + return Nullable{U}() + else + return Nullable(f(map(unsafe_get, xs)...)) + end + end +end + +############################################################################## +## +## Non-standard lifting semantics +## +############################################################################## + +# three-valued logic implementation +@inline function lift(::typeof(&), x, y)::Nullable{Bool} + return ifelse( isnull(x), + ifelse( isnull(y), + Nullable{Bool}(), # x, y null + ifelse( unsafe_get(y), + Nullable{Bool}(), # x null, y == true + Nullable(false) # x null, y == false + ) + ), + ifelse( isnull(y), + ifelse( unsafe_get(x), + Nullable{Bool}(), # x == true, y null + Nullable(false) # x == false, y null + ), + Nullable(unsafe_get(x) & unsafe_get(y)) # x, y not null + ) + ) +end + +# three-valued logic implementation +@inline function lift(::typeof(|), x, y)::Nullable{Bool} + return ifelse( isnull(x), + ifelse( isnull(y), + Nullable{Bool}(), # x, y null + ifelse( unsafe_get(y), + Nullable(true), # x null, y == true + Nullable{Bool}() # x null, y == false + ) + ), + ifelse( isnull(y), + ifelse( unsafe_get(x), + Nullable(true), # x == true, y null + Nullable{Bool}() # x == false, y null + ), + Nullable(unsafe_get(x) | unsafe_get(y)) # x, y not null + ) + ) +end + +# TODO: Decide on semantics for isequal and uncomment the following +# @inline function lift(::typeof(isequal), x, y) +# return ifelse( isnull(x), +# ifelse( isnull(y), +# true, # x, y null +# false # x null, y not null +# ), +# ifelse( isnull(y), +# false, # x not null, y null +# isequal(unsafe_get(x), unsafe_get(y)) # x, y not null +# ) +# ) +# end + +@inline function lift(::typeof(isless), x, y)::Bool + if null_safe_op(isless, typeof(x), typeof(y)) + return ifelse( isnull(x), + false, # x null + ifelse( isnull(y), + true, # x not null, y null + isless(unsafe_get(x), unsafe_get(y)) # x, y not null + ) + ) + else + return isnull(x) ? false : + isnull(y) ? true : isless(unsafe_get(x), unsafe_get(y)) + end +end + +@inline lift(::typeof(isnull), x) = isnull(x) +@inline lift(::typeof(get), x::Nullable) = get(x) +@inline lift(::typeof(get), x::Nullable, y) = get(x, y) + diff --git a/test/broadcast.jl b/test/broadcast.jl index f68c989..c7e8ecd 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -31,15 +31,10 @@ module TestBroadcast Z3 = NullableArray(Float64, 10, [dims; i]...) f() = 5 - f(x) = Nullable(5) * x - f(x, y) = x + y - f(x, y, z) = x + y + z - g() = 5 - g(x::Float64) = 5 * x - g(x::Float64, y::Float64) = x * y - g(x::Float64, y::Float64, z::Float64) = x * y * z + f(x::Float64) = 5 * x + f(x::Float64, y::Float64) = x * y + f(x::Float64, y::Float64, z::Float64) = x * y * z - i = 1 for (dests, arrays, nullablearrays, mask) in ( ((C2, Z2), (A1, A2), (U1, U2), ()), ((C3, Z3), (A2, A3), (U2, U3), ()), @@ -50,26 +45,18 @@ module TestBroadcast ((C3, Z3), (A1, A2, A3), (V1, V2, V3), (Q3,)), ) - # Base.broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false) + # Base.broadcast!(f, B::NullableArray, As::NullableArray...) broadcast!(f, dests[1], arrays...) broadcast!(f, dests[2], nullablearrays...) @test isequal(dests[2], NullableArray(dests[1], mask...)) - broadcast!(g, dests[1], arrays...) - broadcast!(g, dests[2], nullablearrays...; lift=true) - @test isequal(dests[2], NullableArray(dests[1], mask...)) - - # Base.broadcast(f, As::NullableArray...;lift::Bool=false) + # Base.broadcast(f, As::NullableArray...) D = broadcast(f, arrays...) X = broadcast(f, nullablearrays...) @test isequal(X, NullableArray(D, mask...)) - - D = broadcast(g, arrays...) - X = broadcast(g, nullablearrays...; lift=true) - @test isequal(X, NullableArray(D, mask...)) end - # Base.broadcast!(f, X::NullableArray; lift::Bool=false) + # Base.broadcast!(f, X::NullableArray) for (array, nullablearray, mask) in ( (A1, U1, ()), (A2, U2, ()), (A3, U3, ()), (A1, V1, (M1,)), (A2, V2, (M2,)), (A3, V3, (M3,)), @@ -77,10 +64,6 @@ module TestBroadcast broadcast!(f, array) broadcast!(f, nullablearray) @test isequal(nullablearray, NullableArray(array, mask...)) - - broadcast!(g, array) - broadcast!(g, nullablearray; lift=true) - @test isequal(nullablearray, NullableArray(array, mask...)) end # test broadcasted arithmetic operators