Skip to content
This repository has been archived by the owner on May 4, 2019. It is now read-only.

Commit

Permalink
Rewrite broadcast() based on lift()
Browse files Browse the repository at this point in the history
Remove the custom implementation of broadcast(), and just call
the base method on the lift()ed method. This implements the
blacklist approach: methods with a custom lifting behavior like
isnull() should override lift() to get passed Nullable values
directly; if they do not return a Nullable, the result is a standard
Array rather than a NullableArray.

Performance will probably be worse than before, but at least the
semantics will be correct. We can always re-implement the custom
and faster versions later when broadcast() has stabilized in Base
and Nullable support has settled.
  • Loading branch information
nalimilan committed Nov 11, 2016
1 parent f0d2e3d commit 3432db6
Show file tree
Hide file tree
Showing 4 changed files with 165 additions and 198 deletions.
1 change: 1 addition & 0 deletions src/NullableArrays.jl
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ include("primitives.jl")
include("indexing.jl")
include("map.jl")
include("nullablevector.jl")
include("lift.jl")
include("operators.jl")
include("broadcast.jl")
include("reduce.jl")
Expand Down
194 changes: 19 additions & 175 deletions src/broadcast.jl
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
using Base: promote_eltype
using Base.Cartesian
if VERSION >= v"0.6.0-dev.693"
using Base.Broadcast: check_broadcast_indices, broadcast_indices
using Base.Broadcast: check_broadcast_indices, broadcast_indices,
_default_eltype, ftype, ziptype
else
using Base.Broadcast: check_broadcast_shape, broadcast_shape
using Base.Broadcast: check_broadcast_shape, broadcast_shape,
_default_eltype, ftype, ziptype
const check_broadcast_indices = check_broadcast_shape
const broadcast_indices = broadcast_shape
end
Expand All @@ -15,177 +17,9 @@ else
_to_shape(x) = x
end

if VERSION < v"0.5.0-dev+5434"
function gen_nullcheck(narrays::Int, nd::Int)
e_nullcheck = macroexpand(:( @nref $nd isnull_1 d->j_d_1 ))
for k = 2:narrays
isnull = Symbol("isnull_$k")
j_d_k = Symbol("j_d_$k")
e_isnull_k = macroexpand(:( @nref $nd $(isnull) d->$(j_d_k) ))
e_nullcheck = Expr(:||, e_nullcheck, e_isnull_k)
end
return e_nullcheck
end

function gen_broadcast_body(nd::Int, narrays::Int, f, lift::Bool)
F = Expr(:quote, f)
e_nullcheck = gen_nullcheck(narrays, nd)
if lift
return quote
# set up aliases to facilitate subsequent Base.Cartesian magic
B_isnull = B.isnull
@nexprs $narrays k->(values_k = A_k.values)
@nexprs $narrays k->(isnull_k = A_k.isnull)
# check size
@assert ndims(B) == $nd
@ncall $narrays check_broadcast_shape size(B) k->A_k
# main loops
@nloops($nd, i, B,
d->(@nexprs $narrays k->(j_d_k = size(A_k, d) == 1 ? 1 : i_d)), # pre
begin # body
if $e_nullcheck
@inbounds (@nref $nd B_isnull i) = true
else
@nexprs $narrays k->(@inbounds v_k = @nref $nd values_k d->j_d_k)
@inbounds (@nref $nd B i) = (@ncall $narrays $F v)
end
end
)
end
else
return Base.Broadcast.gen_broadcast_body_cartesian(nd, narrays, f)
end
end

function gen_broadcast_function(nd::Int, narrays::Int, f, lift::Bool)
As = [Symbol("A_"*string(i)) for i = 1:narrays]
body = gen_broadcast_body(nd, narrays, f, lift)
@eval let
local _F_
function _F_(B, $(As...))
$body
end
_F_
end
end

function Base.broadcast!(f, X::NullableArray; lift::Bool=false)
broadcast!(f, X, X; lift=lift)
end

@eval let cache = Dict{Any, Dict{Bool, Dict{Int, Dict{Int, Any}}}}()
@doc """
`broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)`
This method implements the same behavior as that of `broadcast!` when called on
regular `Array` arguments. It also includes the `lift` keyword argument, which
when set to true will lift `f` over the entries of the `As`.
Lifting is disabled by default. Note that this method's signature specifies
the destination `B` array as well as the source `As` arrays as all
`NullableArray`s. Thus, calling `broadcast!` on a arguments consisting
of both `Array`s and `NullableArray`s will fall back to the implementation
of `broadcast!` in `base/broadcast.jl`.
""" ->
function Base.broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)
nd = ndims(B)
narrays = length(As)

cache_f = Base.@get! cache f Dict{Bool, Dict{Int, Dict{Int, Any}}}()
cache_lift = Base.@get! cache_f lift Dict{Int, Dict{Int, Any}}()
cache_f_na = Base.@get! cache_lift narrays Dict{Int, Any}()
func = Base.@get! cache_f_na nd gen_broadcast_function(nd, narrays, f, lift)

func(B, As...)
return B
end
end # let cache
else
using Base.Broadcast: newindexer, map_newindexer, newindex

function _nullcheck(nargs)
nullcheck = :(isnull_1[I_1])
for i in 2:nargs
sym_isnull = Symbol("isnull_$i")
sym_idx = Symbol("I_$i")
nullcheck = Expr(:||, :($sym_isnull[$sym_idx]), nullcheck)
end
# if 0 argument arrays, treat nullcheck as though it returns false
nargs >= 1 ? nullcheck : :(false)
end

@generated function Base.Broadcast._broadcast!{K,ID,XT,nargs}(f,
Z::NullableArray, keeps::K, Idefaults::ID, Xs::XT, ::Type{Val{nargs}}; lift=false)
nullcheck = _nullcheck(nargs)
quote
T = eltype(Z)
$(Expr(:meta, :noinline))
# destructure keeps and Xs tuples (common to both lifted and non-lifted broadcast)
@nexprs $nargs i->(keep_i = keeps[i])
@nexprs $nargs i->(Idefault_i = Idefaults[i])
if !lift
# destructure the keeps and As tuples
@nexprs $nargs i->(X_i = Xs[i])
@simd for I in CartesianRange(indices(Z))
# reverse-broadcast the indices
@nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i))
# extract array values
@nexprs $nargs i->(@inbounds val_i = X_i[I_i])
# call the function and store the result
@inbounds Z[I] = @ncall $nargs f val
end
else
# destructure the indexmaps and Xs tuples
@nexprs $nargs i->(values_i = Xs[i].values)
@nexprs $nargs i->(isnull_i = Xs[i].isnull)
@simd for I in CartesianRange(indices(Z))
# reverse-broadcast the indices
@nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i))
if $nullcheck
# if any args are null, store null
@inbounds Z.isnull[I] = true
else
# extract array values
@nexprs $nargs i->(@inbounds val_i = values_i[I_i])
# call the function and store the result
@inbounds Z[I] = @ncall $nargs f val
end
end
end
end
end

@doc """
`broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)`
This method implements the same behavior as that of `broadcast!` when called
on regular `Array` arguments. It also includes the `lift` keyword argument,
which when set to true will lift `f` over the entries of the `As`.
Lifting is disabled by default. Note that this method's signature specifies
the destination `B` array as well as the source `As` arrays as all
`NullableArray`s. Thus, calling `broadcast!` on a arguments consisting of
both `Array`s and `NullableArray`s will fall back to the implementation of
`broadcast!` in `base/broadcast.jl`.
""" ->
# Required to solve dispatch ambiguity between
# broadcast!(f, X::AbstractArray, x::Number...)
# broadcast!(f, Z::NullableArrays.NullableArray, Xs::NullableArrays.NullableArray...)
@inline Base.broadcast!(f, Z::NullableArray; lift=false) =
broadcast!(f, Z, Z; lift=lift)

@inline function Base.broadcast!(f, Z::NullableArray, Xs::NullableArray...;
lift=false)
nargs = length(Xs)
shape = indices(Z)
check_broadcast_indices(shape, Xs...)
keeps, Idefaults = map_newindexer(shape, Xs)
Base.Broadcast._broadcast!(f, Z, keeps, Idefaults, Xs, Val{nargs}; lift=lift)
return Z
end
end

@doc """
`broadcast(f, As::NullableArray...;lift::Bool=false)`
`broadcast(f, As::NullableArray...)`
This method implements the same behavior as that of `broadcast` when called on
regular `Array` arguments. It also includes the `lift` keyword argument, which
Expand All @@ -196,10 +30,20 @@ source `As` arrays as all `NullableArray`s. Thus, calling `broadcast!` on
arguments consisting of both `Array`s and `NullableArray`s will fall back to the
implementation of `broadcast` in `base/broadcast.jl`.
""" ->
@inline function Base.broadcast(f, Xs::NullableArray...;lift::Bool=false)
return broadcast!(f, NullableArray(eltype(promote_eltype(Xs...)),
_to_shape(broadcast_indices(Xs...))),
Xs...; lift=lift)
function Base.broadcast{N}(f, As::Vararg{NullableArray, N})
f2(x...) = lift(f, x...)
T = _default_eltype(Base.Generator{ziptype(As...), ftype(f2, As...)})
if isleaftype(T) && !(T <: Nullable)
return invoke(broadcast, Tuple{Function, Vararg{AbstractArray, N}}, f2, As...)
else
dest = NullableArray(eltype(T), _to_shape(broadcast_indices(As...)))
return invoke(broadcast!, Tuple{Function, AbstractArray, Vararg{AbstractArray, N}}, f2, dest, As...)
end
end

function Base.broadcast!{N}(f, dest::AbstractArray, As::Vararg{NullableArray, N})
f2(x...) = lift(f, x...)
invoke(broadcast!, Tuple{Function, AbstractArray, Vararg{AbstractArray, N}}, f2, dest, As...)
end

# broadcasted ops
Expand Down
139 changes: 139 additions & 0 deletions src/lift.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
import Base: null_safe_op

##############################################################################
##
## Standard lifting semantics
##
## For a function call f(xs...), return null if any x in xs is null;
## otherwise, return f applied to values of xs.
##
##############################################################################

@inline function lift(f, x)
if null_safe_op(f, typeof(x))
return @compat Nullable(f(x.value), !isnull(x))
else
U = Core.Inference.return_type(f, Tuple{eltype(typeof(x))})
if isnull(x)
return Nullable{U}()
else
return Nullable(f(unsafe_get(x)))
end
end
end

@inline function lift(f, x1, x2)
if null_safe_op(f, typeof(x1), typeof(x2))
return @compat Nullable(
f(x1.value, x2.value), !(isnull(x1) | isnull(x2))
)
else
U = Core.Inference.return_type(
f, Tuple{eltype(typeof(x1)), eltype(typeof(x2))}
)
if isnull(x1) | isnull(x2)
return Nullable{U}()
else
return Nullable(f(unsafe_get(x1), unsafe_get(x2)))
end
end
end

hasnulls(xs...) = any(isnull.(xs))

@inline function lift(f, xs...)
if null_safe_op(f, map(typeof, xs)...)
return @compat Nullable(
f(map(unsafe_get, xs)...), !(mapreduce(isnull, |, xs))
)
else
U = Core.Inference.return_type(
f, Tuple{map(x->eltype(typeof(x)), xs)...}
)
if hasnulls(xs...)
return Nullable{U}()
else
return Nullable(f(map(unsafe_get, xs)...))
end
end
end

##############################################################################
##
## Non-standard lifting semantics
##
##############################################################################

# three-valued logic implementation
@inline function lift(::typeof(&), x, y)::Nullable{Bool}
return ifelse( isnull(x),
ifelse( isnull(y),
Nullable{Bool}(), # x, y null
ifelse( unsafe_get(y),
Nullable{Bool}(), # x null, y == true
Nullable(false) # x null, y == false
)
),
ifelse( isnull(y),
ifelse( unsafe_get(x),
Nullable{Bool}(), # x == true, y null
Nullable(false) # x == false, y null
),
Nullable(unsafe_get(x) & unsafe_get(y)) # x, y not null
)
)
end

# three-valued logic implementation
@inline function lift(::typeof(|), x, y)::Nullable{Bool}
return ifelse( isnull(x),
ifelse( isnull(y),
Nullable{Bool}(), # x, y null
ifelse( unsafe_get(y),
Nullable(true), # x null, y == true
Nullable{Bool}() # x null, y == false
)
),
ifelse( isnull(y),
ifelse( unsafe_get(x),
Nullable(true), # x == true, y null
Nullable{Bool}() # x == false, y null
),
Nullable(unsafe_get(x) | unsafe_get(y)) # x, y not null
)
)
end

# TODO: Decide on semantics for isequal and uncomment the following
# @inline function lift(::typeof(isequal), x, y)
# return ifelse( isnull(x),
# ifelse( isnull(y),
# true, # x, y null
# false # x null, y not null
# ),
# ifelse( isnull(y),
# false, # x not null, y null
# isequal(unsafe_get(x), unsafe_get(y)) # x, y not null
# )
# )
# end

@inline function lift(::typeof(isless), x, y)::Bool
if null_safe_op(isless, typeof(x), typeof(y))
return ifelse( isnull(x),
false, # x null
ifelse( isnull(y),
true, # x not null, y null
isless(unsafe_get(x), unsafe_get(y)) # x, y not null
)
)
else
return isnull(x) ? false :
isnull(y) ? true : isless(unsafe_get(x), unsafe_get(y))
end
end

@inline lift(::typeof(isnull), x) = isnull(x)
@inline lift(::typeof(get), x::Nullable) = get(x)
@inline lift(::typeof(get), x::Nullable, y) = get(x, y)

Loading

0 comments on commit 3432db6

Please sign in to comment.