From cf31fa05f7b7b274be339f3591ecca682e5b1bcb Mon Sep 17 00:00:00 2001 From: piever Date: Sun, 18 Mar 2018 12:08:11 +0000 Subject: [PATCH 01/11] collect without inference --- src/IndexedTables.jl | 3 ++- src/collect.jl | 34 ++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/test_collect.jl | 27 +++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 src/collect.jl create mode 100644 test/test_collect.jl diff --git a/src/IndexedTables.jl b/src/IndexedTables.jl index 224a869a..6d632894 100644 --- a/src/IndexedTables.jl +++ b/src/IndexedTables.jl @@ -10,7 +10,7 @@ import Base: permutedims, reducedim, serialize, deserialize, sort, sort! export NDSparse, flush!, aggregate!, aggregate_vec, where, pairs, convertdim, columns, column, rows, - itable, update!, aggregate, reducedim_vec, dimlabels + itable, update!, aggregate, reducedim_vec, dimlabels, collectcolumns const Tup = Union{Tuple,NamedTuple} const DimName = Union{Int,Symbol} @@ -19,6 +19,7 @@ include("utils.jl") include("columns.jl") include("table.jl") include("ndsparse.jl") +include("collect.jl") #= # Poor man's traits diff --git a/src/collect.jl b/src/collect.jl new file mode 100644 index 00000000..06930c08 --- /dev/null +++ b/src/collect.jl @@ -0,0 +1,34 @@ +collectcolumns(itr) = collectcolumns(itr, Base.iteratorsize(itr)) + +function collectcolumns(itr, ::Union{Base.HasShape, Base.HasLength}) + st = start(itr) + el, st = next(itr, st) + dest = similar(arrayof(typeof(el)), length(itr)) + dest[1] = el + collect_to_columns!(dest, itr, 2, st) +end + +function collect_to_columns!(dest::Columns{T, U}, itr, offs, st) where {T, U} + # collect to dest array, checking the type of each result. if a result does not + # match, widen the result type and re-dispatch. + i = offs + while !done(itr, st) + el, st = next(itr, st) + S = typeof(el) + if all((s <: t) for (s, t) in zip(S.parameters, T.parameters)) + @inbounds dest[i] = el::T + i += 1 + else + Rparams = map(typejoin, T.parameters, S.parameters) + R = get_tuple_type_from_params(el, Rparams) + new = similar(arrayof(R), length(itr)) + @inbounds for l in 1:i-1; new[l] = dest[l]; end + @inbounds new[i] = el + return collect_to_columns!(new, itr, i+1, st) + end + end + return dest +end + +get_tuple_type_from_params(el::Tuple, params) = Tuple{params...} +get_tuple_type_from_params(el::NamedTuple, params) = eval(:(NamedTuples.@NT($(keys(el)...)))){params...} diff --git a/test/runtests.jl b/test/runtests.jl index 4c0d46f2..0ce7166b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,5 +10,6 @@ using Base.Test include("test_core.jl") include("test_utils.jl") include("test_tabletraits.jl") +include("test_collect.jl") end diff --git a/test/test_collect.jl b/test/test_collect.jl new file mode 100644 index 00000000..8077aceb --- /dev/null +++ b/test/test_collect.jl @@ -0,0 +1,27 @@ +@testset "collectnamedtuples" begin + v = [@NT(a = 1, b = 2), @NT(a = 1, b = 3)] + @test collectcolumns(v) == Columns(@NT(a = Int[1, 1], b = Int[2, 3])) + + v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = 3)] + @test collectcolumns(v) == Columns(@NT(a = Real[1, 1.2], b = Int[2, 3])) + + v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = "3")] + @test collectcolumns(v) == Columns(@NT(a = Real[1, 1.2], b = Any[2, "3"])) + + v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = 2), @NT(a = 1, b = "3")] + @test collectcolumns(v) == Columns(@NT(a = Real[1, 1.2, 1], b = Any[2, 2, "3"])) +end + +@testset "collecttuples" begin + v = [(1, 2), (1, 3)] + @test collectcolumns(v) == Columns((Int[1, 1], Int[2, 3])) + + v = [(1, 2), (1.2, 3)] + @test collectcolumns(v) == Columns((Real[1, 1.2], Int[2, 3])) + + v = [(1, 2), (1.2, "3")] + @test collectcolumns(v) == Columns((Real[1, 1.2], Any[2, "3"])) + + v = [(1, 2), (1.2, 2), (1, "3")] + @test collectcolumns(v) == Columns((Real[1, 1.2, 1], Any[2, 2, "3"])) +end From ef30634659ec6a7d8d915cf579486cb507cd7bfd Mon Sep 17 00:00:00 2001 From: piever Date: Sun, 18 Mar 2018 19:57:16 +0000 Subject: [PATCH 02/11] use generated function to optimize type checking step --- src/collect.jl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/collect.jl b/src/collect.jl index 06930c08..e8583a26 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -14,11 +14,11 @@ function collect_to_columns!(dest::Columns{T, U}, itr, offs, st) where {T, U} i = offs while !done(itr, st) el, st = next(itr, st) - S = typeof(el) - if all((s <: t) for (s, t) in zip(S.parameters, T.parameters)) + if fieldwise_isa(el, T) @inbounds dest[i] = el::T i += 1 else + S = typeof(el) Rparams = map(typejoin, T.parameters, S.parameters) R = get_tuple_type_from_params(el, Rparams) new = similar(arrayof(R), length(itr)) @@ -32,3 +32,11 @@ end get_tuple_type_from_params(el::Tuple, params) = Tuple{params...} get_tuple_type_from_params(el::NamedTuple, params) = eval(:(NamedTuples.@NT($(keys(el)...)))){params...} + +@generated function fieldwise_isa(el::S, ::Type{T}) where {S, T} + if all((s <: t) for (s, t) in zip(S.parameters, T.parameters)) + return :(true) + else + return :(false) + end +end From a74cbe78fb60d8631406db90be54193530946c33 Mon Sep 17 00:00:00 2001 From: piever Date: Sun, 18 Mar 2018 20:29:49 +0000 Subject: [PATCH 03/11] only change relevant columns --- src/collect.jl | 15 ++++++++------- test/test_collect.jl | 2 ++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/collect.jl b/src/collect.jl index e8583a26..96ad318e 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -19,10 +19,14 @@ function collect_to_columns!(dest::Columns{T, U}, itr, offs, st) where {T, U} i += 1 else S = typeof(el) - Rparams = map(typejoin, T.parameters, S.parameters) - R = get_tuple_type_from_params(el, Rparams) - new = similar(arrayof(R), length(itr)) - @inbounds for l in 1:i-1; new[l] = dest[l]; end + sp, tp = S.parameters, T.parameters + idx = find(!(s <: t) for (s, t) in zip(sp, tp)) + new = dest + for l in idx + newcol = Array{typejoin(sp[l], tp[l])}(length(dest)) + copy!(newcol, 1, column(dest, l), 1, i-1) + new = setcol(new, l, newcol) + end @inbounds new[i] = el return collect_to_columns!(new, itr, i+1, st) end @@ -30,9 +34,6 @@ function collect_to_columns!(dest::Columns{T, U}, itr, offs, st) where {T, U} return dest end -get_tuple_type_from_params(el::Tuple, params) = Tuple{params...} -get_tuple_type_from_params(el::NamedTuple, params) = eval(:(NamedTuples.@NT($(keys(el)...)))){params...} - @generated function fieldwise_isa(el::S, ::Type{T}) where {S, T} if all((s <: t) for (s, t) in zip(S.parameters, T.parameters)) return :(true) diff --git a/test/test_collect.jl b/test/test_collect.jl index 8077aceb..6e4f8cb2 100644 --- a/test/test_collect.jl +++ b/test/test_collect.jl @@ -1,6 +1,7 @@ @testset "collectnamedtuples" begin v = [@NT(a = 1, b = 2), @NT(a = 1, b = 3)] @test collectcolumns(v) == Columns(@NT(a = Int[1, 1], b = Int[2, 3])) + #@inferred collectcolumns(v) v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = 3)] @test collectcolumns(v) == Columns(@NT(a = Real[1, 1.2], b = Int[2, 3])) @@ -15,6 +16,7 @@ end @testset "collecttuples" begin v = [(1, 2), (1, 3)] @test collectcolumns(v) == Columns((Int[1, 1], Int[2, 3])) + @inferred collectcolumns(v) v = [(1, 2), (1.2, 3)] @test collectcolumns(v) == Columns((Real[1, 1.2], Int[2, 3])) From 08145977c503a35967ad125b17e6b25252f7e51b Mon Sep 17 00:00:00 2001 From: piever Date: Sun, 18 Mar 2018 20:52:23 +0000 Subject: [PATCH 04/11] remove incorrect typeassert --- src/collect.jl | 2 +- test/test_collect.jl | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/collect.jl b/src/collect.jl index 96ad318e..988dc3dd 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -15,7 +15,7 @@ function collect_to_columns!(dest::Columns{T, U}, itr, offs, st) where {T, U} while !done(itr, st) el, st = next(itr, st) if fieldwise_isa(el, T) - @inbounds dest[i] = el::T + @inbounds dest[i] = el i += 1 else S = typeof(el) diff --git a/test/test_collect.jl b/test/test_collect.jl index 6e4f8cb2..8addb01b 100644 --- a/test/test_collect.jl +++ b/test/test_collect.jl @@ -1,7 +1,14 @@ @testset "collectnamedtuples" begin v = [@NT(a = 1, b = 2), @NT(a = 1, b = 3)] @test collectcolumns(v) == Columns(@NT(a = Int[1, 1], b = Int[2, 3])) - #@inferred collectcolumns(v) + + # test inferrability with constant eltype + itr = [@NT(a = 1, b = 2), @NT(a = 1, b = 2), @NT(a = 1, b = 12)] + st = start(itr) + el, st = next(itr, st) + dest = IndexedTables.arrayof(eltype(el), 3) + dest[1] = el + @inferred IndexedTables.collect_to_columns!(dest, itr, 2, st) v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = 3)] @test collectcolumns(v) == Columns(@NT(a = Real[1, 1.2], b = Int[2, 3])) From 59347bbe2bc185165ed2b126d49fa4c0dfcb39c6 Mon Sep 17 00:00:00 2001 From: piever Date: Sun, 18 Mar 2018 21:19:31 +0000 Subject: [PATCH 05/11] fix test --- test/test_collect.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_collect.jl b/test/test_collect.jl index 8addb01b..97ea127a 100644 --- a/test/test_collect.jl +++ b/test/test_collect.jl @@ -6,7 +6,7 @@ itr = [@NT(a = 1, b = 2), @NT(a = 1, b = 2), @NT(a = 1, b = 12)] st = start(itr) el, st = next(itr, st) - dest = IndexedTables.arrayof(eltype(el), 3) + dest = similar(IndexedTables.arrayof(typeof(el)), 3) dest[1] = el @inferred IndexedTables.collect_to_columns!(dest, itr, 2, st) From 6c05925d561d515c6f74e6be7a69b4c193978bda Mon Sep 17 00:00:00 2001 From: piever Date: Mon, 19 Mar 2018 11:10:00 +0000 Subject: [PATCH 06/11] refactor and add scalar case --- src/collect.jl | 40 +++++++++++++++++++++++++++++----------- test/test_collect.jl | 9 +++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/collect.jl b/src/collect.jl index 988dc3dd..490768e3 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -8,7 +8,7 @@ function collectcolumns(itr, ::Union{Base.HasShape, Base.HasLength}) collect_to_columns!(dest, itr, 2, st) end -function collect_to_columns!(dest::Columns{T, U}, itr, offs, st) where {T, U} +function collect_to_columns!(dest::AbstractArray{T}, itr, offs, st) where {T} # collect to dest array, checking the type of each result. if a result does not # match, widen the result type and re-dispatch. i = offs @@ -18,15 +18,7 @@ function collect_to_columns!(dest::Columns{T, U}, itr, offs, st) where {T, U} @inbounds dest[i] = el i += 1 else - S = typeof(el) - sp, tp = S.parameters, T.parameters - idx = find(!(s <: t) for (s, t) in zip(sp, tp)) - new = dest - for l in idx - newcol = Array{typejoin(sp[l], tp[l])}(length(dest)) - copy!(newcol, 1, column(dest, l), 1, i-1) - new = setcol(new, l, newcol) - end + new = widencolumns(dest, i, el, T) @inbounds new[i] = el return collect_to_columns!(new, itr, i+1, st) end @@ -34,10 +26,36 @@ function collect_to_columns!(dest::Columns{T, U}, itr, offs, st) where {T, U} return dest end -@generated function fieldwise_isa(el::S, ::Type{T}) where {S, T} +@generated function fieldwise_isa(el::S, ::Type{T}) where {S<:Tup, T} if all((s <: t) for (s, t) in zip(S.parameters, T.parameters)) return :(true) else return :(false) end end + +@generated function fieldwise_isa(el::S, ::Type{T}) where {S, T} + if S <: T + return :(true) + else + return :(false) + end +end + +function widencolumns(dest, i, el::S, ::Type{T}) where{S <: Tup, T} + sp, tp = S.parameters, T.parameters + idx = find(!(s <: t) for (s, t) in zip(sp, tp)) + new = dest + for l in idx + newcol = Array{typejoin(sp[l], tp[l])}(length(dest)) + copy!(newcol, 1, column(dest, l), 1, i-1) + new = setcol(new, l, newcol) + end + new +end + +function widencolumns(dest, i, el::S, ::Type{T}) where{S, T} + new = Array{typejoin(S, T)}(length(dest)) + copy!(new, 1, dest, 1, i-1) + new +end diff --git a/test/test_collect.jl b/test/test_collect.jl index 97ea127a..10987eb1 100644 --- a/test/test_collect.jl +++ b/test/test_collect.jl @@ -34,3 +34,12 @@ end v = [(1, 2), (1.2, 2), (1, "3")] @test collectcolumns(v) == Columns((Real[1, 1.2, 1], Any[2, 2, "3"])) end + +@testset "collectscalars" begin + v = (i for i in 1:3) + @test collectcolumns(v) == [1,2,3] + @inferred collectcolumns(v) + + v = (i == 1 ? 1.2 : i for i in 1:3) + @test collectcolumns(v) == collect(v) +end From 89cc30e1f85723f4ab593fabaf3ef332adbe15dc Mon Sep 17 00:00:00 2001 From: piever Date: Mon, 19 Mar 2018 11:45:45 +0000 Subject: [PATCH 07/11] unknown length --- src/collect.jl | 26 ++++++++++++++++++++++++++ test/test_collect.jl | 18 ++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/collect.jl b/src/collect.jl index 490768e3..bfa2915d 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -26,6 +26,32 @@ function collect_to_columns!(dest::AbstractArray{T}, itr, offs, st) where {T} return dest end +function collectcolumns(itr, ::Base.SizeUnknown) + st = start(itr) + el, st = next(itr, st) + dest = similar(arrayof(typeof(el)), 1) + dest[1] = el + grow_to_columns!(dest, itr, 2, st) +end + +function grow_to_columns!(dest::AbstractArray{T}, itr, offs, st) where {T} + # collect to dest array, checking the type of each result. if a result does not + # match, widen the result type and re-dispatch. + i = offs + while !done(itr, st) + el, st = next(itr, st) + if fieldwise_isa(el, T) + push!(dest, el) + i += 1 + else + new = widencolumns(dest, i, el, T) + push!(new, el) + return grow_to_columns!(new, itr, i+1, st) + end + end + return dest +end + @generated function fieldwise_isa(el::S, ::Type{T}) where {S<:Tup, T} if all((s <: t) for (s, t) in zip(S.parameters, T.parameters)) return :(true) diff --git a/test/test_collect.jl b/test/test_collect.jl index 10987eb1..63390fba 100644 --- a/test/test_collect.jl +++ b/test/test_collect.jl @@ -18,6 +18,13 @@ v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = 2), @NT(a = 1, b = "3")] @test collectcolumns(v) == Columns(@NT(a = Real[1, 1.2, 1], b = Any[2, 2, "3"])) + + # length unknown + itr = Iterators.filter(isodd, 1:8) + tuple_itr = (@NT(a = i+1, b = i-1) for i in itr) + @test collectcolumns(tuple_itr) == Columns(@NT(a = [2, 4, 6, 8], b = [0, 2, 4, 6])) + tuple_itr_real = (i == 1 ? @NT(a = 1.2, b =i-1) : @NT(a = i+1, b = i-1) for i in itr) + @test collectcolumns(tuple_itr_real) == Columns(@NT(a = Real[1.2, 4, 6, 8], b = [0, 2, 4, 6])) end @testset "collecttuples" begin @@ -33,6 +40,12 @@ end v = [(1, 2), (1.2, 2), (1, "3")] @test collectcolumns(v) == Columns((Real[1, 1.2, 1], Any[2, 2, "3"])) + # length unknown + itr = Iterators.filter(isodd, 1:8) + tuple_itr = ((i+1, i-1) for i in itr) + @test collectcolumns(tuple_itr) == Columns(([2, 4, 6, 8], [0, 2, 4, 6])) + tuple_itr_real = (i == 1 ? (1.2, i-1) : (i+1, i-1) for i in itr) + @test collectcolumns(tuple_itr_real) == Columns((Real[1.2, 4, 6, 8], [0, 2, 4, 6])) end @testset "collectscalars" begin @@ -42,4 +55,9 @@ end v = (i == 1 ? 1.2 : i for i in 1:3) @test collectcolumns(v) == collect(v) + + itr = Iterators.filter(isodd, 1:100) + @test collectcolumns(itr) == collect(itr) + real_itr = (i == 1 ? 1.5 : i for i in itr) + @test collectcolumns(real_itr) == collect(real_itr) end From dac0578a3a0a22308bee7c348f6c40ba1ea668f4 Mon Sep 17 00:00:00 2001 From: piever Date: Mon, 19 Mar 2018 16:13:49 +0000 Subject: [PATCH 08/11] rename to collect_columns --- src/IndexedTables.jl | 2 +- src/collect.jl | 6 +++--- test/test_collect.jl | 36 ++++++++++++++++++------------------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/IndexedTables.jl b/src/IndexedTables.jl index 6d632894..9fdc0b36 100644 --- a/src/IndexedTables.jl +++ b/src/IndexedTables.jl @@ -10,7 +10,7 @@ import Base: permutedims, reducedim, serialize, deserialize, sort, sort! export NDSparse, flush!, aggregate!, aggregate_vec, where, pairs, convertdim, columns, column, rows, - itable, update!, aggregate, reducedim_vec, dimlabels, collectcolumns + itable, update!, aggregate, reducedim_vec, dimlabels, collect_columns const Tup = Union{Tuple,NamedTuple} const DimName = Union{Int,Symbol} diff --git a/src/collect.jl b/src/collect.jl index bfa2915d..137882c9 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -1,6 +1,6 @@ -collectcolumns(itr) = collectcolumns(itr, Base.iteratorsize(itr)) +collect_columns(itr) = collect_columns(itr, Base.iteratorsize(itr)) -function collectcolumns(itr, ::Union{Base.HasShape, Base.HasLength}) +function collect_columns(itr, ::Union{Base.HasShape, Base.HasLength}) st = start(itr) el, st = next(itr, st) dest = similar(arrayof(typeof(el)), length(itr)) @@ -26,7 +26,7 @@ function collect_to_columns!(dest::AbstractArray{T}, itr, offs, st) where {T} return dest end -function collectcolumns(itr, ::Base.SizeUnknown) +function collect_columns(itr, ::Base.SizeUnknown) st = start(itr) el, st = next(itr, st) dest = similar(arrayof(typeof(el)), 1) diff --git a/test/test_collect.jl b/test/test_collect.jl index 63390fba..9058ce87 100644 --- a/test/test_collect.jl +++ b/test/test_collect.jl @@ -1,6 +1,6 @@ @testset "collectnamedtuples" begin v = [@NT(a = 1, b = 2), @NT(a = 1, b = 3)] - @test collectcolumns(v) == Columns(@NT(a = Int[1, 1], b = Int[2, 3])) + @test collect_columns(v) == Columns(@NT(a = Int[1, 1], b = Int[2, 3])) # test inferrability with constant eltype itr = [@NT(a = 1, b = 2), @NT(a = 1, b = 2), @NT(a = 1, b = 12)] @@ -11,53 +11,53 @@ @inferred IndexedTables.collect_to_columns!(dest, itr, 2, st) v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = 3)] - @test collectcolumns(v) == Columns(@NT(a = Real[1, 1.2], b = Int[2, 3])) + @test collect_columns(v) == Columns(@NT(a = Real[1, 1.2], b = Int[2, 3])) v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = "3")] - @test collectcolumns(v) == Columns(@NT(a = Real[1, 1.2], b = Any[2, "3"])) + @test collect_columns(v) == Columns(@NT(a = Real[1, 1.2], b = Any[2, "3"])) v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = 2), @NT(a = 1, b = "3")] - @test collectcolumns(v) == Columns(@NT(a = Real[1, 1.2, 1], b = Any[2, 2, "3"])) + @test collect_columns(v) == Columns(@NT(a = Real[1, 1.2, 1], b = Any[2, 2, "3"])) # length unknown itr = Iterators.filter(isodd, 1:8) tuple_itr = (@NT(a = i+1, b = i-1) for i in itr) - @test collectcolumns(tuple_itr) == Columns(@NT(a = [2, 4, 6, 8], b = [0, 2, 4, 6])) + @test collect_columns(tuple_itr) == Columns(@NT(a = [2, 4, 6, 8], b = [0, 2, 4, 6])) tuple_itr_real = (i == 1 ? @NT(a = 1.2, b =i-1) : @NT(a = i+1, b = i-1) for i in itr) - @test collectcolumns(tuple_itr_real) == Columns(@NT(a = Real[1.2, 4, 6, 8], b = [0, 2, 4, 6])) + @test collect_columns(tuple_itr_real) == Columns(@NT(a = Real[1.2, 4, 6, 8], b = [0, 2, 4, 6])) end @testset "collecttuples" begin v = [(1, 2), (1, 3)] - @test collectcolumns(v) == Columns((Int[1, 1], Int[2, 3])) - @inferred collectcolumns(v) + @test collect_columns(v) == Columns((Int[1, 1], Int[2, 3])) + @inferred collect_columns(v) v = [(1, 2), (1.2, 3)] - @test collectcolumns(v) == Columns((Real[1, 1.2], Int[2, 3])) + @test collect_columns(v) == Columns((Real[1, 1.2], Int[2, 3])) v = [(1, 2), (1.2, "3")] - @test collectcolumns(v) == Columns((Real[1, 1.2], Any[2, "3"])) + @test collect_columns(v) == Columns((Real[1, 1.2], Any[2, "3"])) v = [(1, 2), (1.2, 2), (1, "3")] - @test collectcolumns(v) == Columns((Real[1, 1.2, 1], Any[2, 2, "3"])) + @test collect_columns(v) == Columns((Real[1, 1.2, 1], Any[2, 2, "3"])) # length unknown itr = Iterators.filter(isodd, 1:8) tuple_itr = ((i+1, i-1) for i in itr) - @test collectcolumns(tuple_itr) == Columns(([2, 4, 6, 8], [0, 2, 4, 6])) + @test collect_columns(tuple_itr) == Columns(([2, 4, 6, 8], [0, 2, 4, 6])) tuple_itr_real = (i == 1 ? (1.2, i-1) : (i+1, i-1) for i in itr) - @test collectcolumns(tuple_itr_real) == Columns((Real[1.2, 4, 6, 8], [0, 2, 4, 6])) + @test collect_columns(tuple_itr_real) == Columns((Real[1.2, 4, 6, 8], [0, 2, 4, 6])) end @testset "collectscalars" begin v = (i for i in 1:3) - @test collectcolumns(v) == [1,2,3] - @inferred collectcolumns(v) + @test collect_columns(v) == [1,2,3] + @inferred collect_columns(v) v = (i == 1 ? 1.2 : i for i in 1:3) - @test collectcolumns(v) == collect(v) + @test collect_columns(v) == collect(v) itr = Iterators.filter(isodd, 1:100) - @test collectcolumns(itr) == collect(itr) + @test collect_columns(itr) == collect(itr) real_itr = (i == 1 ? 1.5 : i for i in itr) - @test collectcolumns(real_itr) == collect(real_itr) + @test collect_columns(real_itr) == collect(real_itr) end From 1f51b23890738d75df99a189f08af940ad5e0a59 Mon Sep 17 00:00:00 2001 From: piever Date: Mon, 19 Mar 2018 16:32:05 +0000 Subject: [PATCH 09/11] change to promote_type --- src/collect.jl | 4 ++-- test/test_collect.jl | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/collect.jl b/src/collect.jl index 137882c9..d2920998 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -73,7 +73,7 @@ function widencolumns(dest, i, el::S, ::Type{T}) where{S <: Tup, T} idx = find(!(s <: t) for (s, t) in zip(sp, tp)) new = dest for l in idx - newcol = Array{typejoin(sp[l], tp[l])}(length(dest)) + newcol = Array{promote_type(sp[l], tp[l])}(length(dest)) copy!(newcol, 1, column(dest, l), 1, i-1) new = setcol(new, l, newcol) end @@ -81,7 +81,7 @@ function widencolumns(dest, i, el::S, ::Type{T}) where{S <: Tup, T} end function widencolumns(dest, i, el::S, ::Type{T}) where{S, T} - new = Array{typejoin(S, T)}(length(dest)) + new = Array{promote_type(S, T)}(length(dest)) copy!(new, 1, dest, 1, i-1) new end diff --git a/test/test_collect.jl b/test/test_collect.jl index 9058ce87..521f7c42 100644 --- a/test/test_collect.jl +++ b/test/test_collect.jl @@ -11,13 +11,16 @@ @inferred IndexedTables.collect_to_columns!(dest, itr, 2, st) v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = 3)] - @test collect_columns(v) == Columns(@NT(a = Real[1, 1.2], b = Int[2, 3])) + @test collect_columns(v) == Columns(@NT(a = [1, 1.2], b = Int[2, 3])) + @test typeof(collect_columns(v)) == typeof(Columns(@NT(a = [1, 1.2], b = Int[2, 3]))) v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = "3")] - @test collect_columns(v) == Columns(@NT(a = Real[1, 1.2], b = Any[2, "3"])) + @test collect_columns(v) == Columns(@NT(a = [1, 1.2], b = Any[2, "3"])) + @test typeof(collect_columns(v)) == typeof(Columns(@NT(a = [1, 1.2], b = Any[2, "3"]))) v = [@NT(a = 1, b = 2), @NT(a = 1.2, b = 2), @NT(a = 1, b = "3")] - @test collect_columns(v) == Columns(@NT(a = Real[1, 1.2, 1], b = Any[2, 2, "3"])) + @test collect_columns(v) == Columns(@NT(a = [1, 1.2, 1], b = Any[2, 2, "3"])) + @test typeof(collect_columns(v)) == typeof(Columns(@NT(a = [1, 1.2, 1], b = Any[2, 2, "3"]))) # length unknown itr = Iterators.filter(isodd, 1:8) @@ -33,19 +36,21 @@ end @inferred collect_columns(v) v = [(1, 2), (1.2, 3)] - @test collect_columns(v) == Columns((Real[1, 1.2], Int[2, 3])) + @test collect_columns(v) == Columns(([1, 1.2], Int[2, 3])) v = [(1, 2), (1.2, "3")] - @test collect_columns(v) == Columns((Real[1, 1.2], Any[2, "3"])) + @test collect_columns(v) == Columns(([1, 1.2], Any[2, "3"])) + @test typeof(collect_columns(v)) == typeof(Columns(([1, 1.2], Any[2, "3"]))) v = [(1, 2), (1.2, 2), (1, "3")] - @test collect_columns(v) == Columns((Real[1, 1.2, 1], Any[2, 2, "3"])) + @test collect_columns(v) == Columns(([1, 1.2, 1], Any[2, 2, "3"])) # length unknown itr = Iterators.filter(isodd, 1:8) tuple_itr = ((i+1, i-1) for i in itr) @test collect_columns(tuple_itr) == Columns(([2, 4, 6, 8], [0, 2, 4, 6])) tuple_itr_real = (i == 1 ? (1.2, i-1) : (i+1, i-1) for i in itr) - @test collect_columns(tuple_itr_real) == Columns((Real[1.2, 4, 6, 8], [0, 2, 4, 6])) + @test collect_columns(tuple_itr_real) == Columns(([1.2, 4, 6, 8], [0, 2, 4, 6])) + @test typeof(collect_columns(tuple_itr_real)) == typeof(Columns(([1.2, 4, 6, 8], [0, 2, 4, 6]))) end @testset "collectscalars" begin @@ -60,4 +65,5 @@ end @test collect_columns(itr) == collect(itr) real_itr = (i == 1 ? 1.5 : i for i in itr) @test collect_columns(real_itr) == collect(real_itr) + @test eltype(collect_columns(real_itr)) == Float64 end From ca280efdd318e39dd8f96854a805c173c919e98a Mon Sep 17 00:00:00 2001 From: piever Date: Mon, 19 Mar 2018 16:38:29 +0000 Subject: [PATCH 10/11] docs --- src/collect.jl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/collect.jl b/src/collect.jl index d2920998..030c8ce6 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -1,3 +1,29 @@ +""" +`collect_columns(itr)` + +Collect an iterable as a `Columns` object if it iterates `Tuples` or `NamedTuples`, as a normal +`Array` otherwise. + +## Example + +```jldoctest collect +julia> s = [(1,2), (3,4)]; + +julia> collect_columns(s) +2-element Columns{Tuple{Int64,Int64}}: + (1, 2) + (3, 4) + + julia> s = Iterators.filter(isodd, 1:8); + + julia> collect_columns(s) + 4-element Array{Int64,1}: + 1 + 3 + 5 + 7 +``` +""" collect_columns(itr) = collect_columns(itr, Base.iteratorsize(itr)) function collect_columns(itr, ::Union{Base.HasShape, Base.HasLength}) From d19691284dd376c8b94ce4cc5a48fafd877c5c98 Mon Sep 17 00:00:00 2001 From: piever Date: Mon, 19 Mar 2018 16:39:34 +0000 Subject: [PATCH 11/11] typo --- src/collect.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/collect.jl b/src/collect.jl index 030c8ce6..c5fdb348 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -4,7 +4,7 @@ Collect an iterable as a `Columns` object if it iterates `Tuples` or `NamedTuples`, as a normal `Array` otherwise. -## Example +## Examples ```jldoctest collect julia> s = [(1,2), (3,4)];