Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Require Julia 1.6 #3145

Merged
merged 6 commits into from
Sep 19, 2022
Merged
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -12,7 +12,6 @@ jobs:
fail-fast: false
matrix:
version:
- '1.0'
- '1.6'
- '1' # automatically expands to the latest stable 1.x release of Julia
- 'nightly'
5 changes: 5 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# DataFrames.jl v1.4 Release Notes

## Julia compatibility change

* DataFrames.jl 1.4 requires Julia 1.6
([#3145](https://github.com/JuliaData/DataFrames.jl/pull/3145))

## New functionalities

* `subset` and `subset!` now allow passing zero column selectors
4 changes: 2 additions & 2 deletions Project.toml
Original file line number Diff line number Diff line change
@@ -25,7 +25,7 @@ Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[compat]
CategoricalArrays = "0.10.0"
Compat = "3.46, 4.2"
Compat = "4.2"
DataAPI = "1.10"
InvertedIndices = "1"
IteratorInterfaceExtensions = "0.1.1, 1"
@@ -38,7 +38,7 @@ SortingAlgorithms = "0.1, 0.2, 0.3, 1"
TableTraits = "0.4, 1"
Tables = "1.8.1"
Unitful = "1"
julia = "1"
julia = "1.6"

[extras]
CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
52 changes: 5 additions & 47 deletions src/DataFrames.jl
Original file line number Diff line number Diff line change
@@ -87,23 +87,11 @@ export AbstractDataFrame,
unstack,
valuecols

if VERSION >= v"1.1.0-DEV.792"
import Base.eachcol, Base.eachrow
else
import Compat.eachcol, Compat.eachrow
export eachcol, eachrow
end

if VERSION < v"1.2"
export hasproperty
end

if isdefined(Base, :only) # Introduced in 1.4.0
import Base.only
else
import Compat.only
export only
end
import Base.eachcol, Base.eachrow
import Base.only
import Base.popat!
bkamins marked this conversation as resolved.
Show resolved Hide resolved
using Base.Threads: @spawn
using Base: ComposedFunction

if isdefined(Base, :keepat!) # Introduced in 1.7.0
import Base.keepat!
@@ -112,36 +100,6 @@ else
export keepat!
end

if isdefined(Base, :popat!) # Introduced in 1.5.0
import Base.popat!
else
import Compat.popat!
export popat!
end

if VERSION >= v"1.3"
using Base.Threads: @spawn
else
# This is the definition of @async in Base
macro spawn(expr)
thunk = esc(:(()->($expr)))
var = esc(Base.sync_varname)
quote
local task = Task($thunk)
if $(Expr(:isdefined, var))
push!($var, task)
end
schedule(task)
end
end
end

if isdefined(Base, :ComposedFunction) # Julia >= 1.6.0-DEV.85
using Base: ComposedFunction
else
using Compat: ComposedFunction
end

if VERSION >= v"1.9.0-DEV.1163"
import Base: stack
else
4 changes: 0 additions & 4 deletions src/abstractdataframe/abstractdataframe.jl
Original file line number Diff line number Diff line change
@@ -382,10 +382,6 @@ Return `true` if data frame `df` has zero rows, and `false` otherwise.
"""
Base.isempty(df::AbstractDataFrame) = nrow(df) == 0

if VERSION < v"1.6"
Base.firstindex(df::AbstractDataFrame, i::Integer) = first(axes(df, i))
Base.lastindex(df::AbstractDataFrame, i::Integer) = last(axes(df, i))
end
Base.axes(df::AbstractDataFrame, i::Integer) = Base.OneTo(size(df, i))

"""
4 changes: 0 additions & 4 deletions src/abstractdataframe/iteration.jl
Original file line number Diff line number Diff line change
@@ -191,10 +191,6 @@ Base.eltype(::Type{<:DataFrameColumns}) = AbstractVector
Base.firstindex(itr::DataFrameColumns) = 1
Base.lastindex(itr::DataFrameColumns) = length(itr)

if VERSION < v"1.6"
Base.firstindex(itr::DataFrameColumns, i::Integer) = first(axes(itr, i))
Base.lastindex(itr::DataFrameColumns, i::Integer) = last(axes(itr, i))
end
Base.axes(itr::DataFrameColumns, i::Integer) = Base.OneTo(size(itr, i))

Base.iterate(itr::DataFrameColumns, i::Integer=1) =
9 changes: 0 additions & 9 deletions src/abstractdataframe/show.jl
Original file line number Diff line number Diff line change
@@ -59,15 +59,6 @@ function ourshow(io::IO, x::Markdown.MD, truncstring::Int)
return print(io, len < length(r) - 1 ? first(r, len)*'…' : first(r, len))
end

# AbstractChar: https://github.com/JuliaLang/julia/pull/34730 (1.5.0-DEV.261)
# Irrational: https://github.com/JuliaLang/julia/pull/34741 (1.5.0-DEV.266)
if VERSION < v"1.5.0-DEV.261" || VERSION < v"1.5.0-DEV.266"
function ourshow(io::IO, x::T, truncstring::Int) where T <: Union{AbstractChar, Irrational}
io = IOContext(io, :compact=>get(io, :compact, true), :typeinfo=>typeof(x))
show(io, x)
end
end

# For most data frames, especially wide, columns having the same element type
# occur multiple times. batch_compacttype ensures that we compute string
# representation of a specific column element type only once and then reuse it.
29 changes: 9 additions & 20 deletions src/dataframe/dataframe.jl
Original file line number Diff line number Diff line change
@@ -198,15 +198,9 @@ struct DataFrame <: AbstractDataFrame

# we write into columns as we know that it is guaranteed
# that it was freshly allocated in the outer constructor
@static if VERSION >= v"1.4"
if copycols && len >= 1_000_000 && length(columns) > 1 && Threads.nthreads() > 1
@sync for i in eachindex(columns)
Threads.@spawn columns[i] = _preprocess_column(columns[i], len, copycols)
end
else
for i in eachindex(columns)
columns[i] = _preprocess_column(columns[i], len, copycols)
end
if copycols && len >= 1_000_000 && length(columns) > 1 && Threads.nthreads() > 1
@sync for i in eachindex(columns)
Threads.@spawn columns[i] = _preprocess_column(columns[i], len, copycols)
end
else
for i in eachindex(columns)
@@ -546,20 +540,15 @@ function _threaded_getindex(selected_rows::AbstractVector,
selected_columns::AbstractVector,
df_columns::AbstractVector,
idx::AbstractIndex)
@static if VERSION >= v"1.4"
if length(selected_rows) >= 1_000_000 && Threads.nthreads() > 1
new_columns = Vector{AbstractVector}(undef, length(selected_columns))
@sync for i in eachindex(new_columns)
Threads.@spawn new_columns[i] = df_columns[selected_columns[i]][selected_rows]
end
return DataFrame(new_columns, idx, copycols=false)
else
return DataFrame(AbstractVector[df_columns[i][selected_rows] for i in selected_columns],
idx, copycols=false)
if length(selected_rows) >= 1_000_000 && Threads.nthreads() > 1
new_columns = Vector{AbstractVector}(undef, length(selected_columns))
@sync for i in eachindex(new_columns)
Threads.@spawn new_columns[i] = df_columns[selected_columns[i]][selected_rows]
end
return DataFrame(new_columns, idx, copycols=false)
else
return DataFrame(AbstractVector[df_columns[i][selected_rows] for i in selected_columns],
idx, copycols=false)
idx, copycols=false)
end
end

4 changes: 0 additions & 4 deletions src/dataframerow/dataframerow.jl
Original file line number Diff line number Diff line change
@@ -385,10 +385,6 @@ Base.ndims(::Type{<:DataFrameRow}) = 1
Base.firstindex(r::DataFrameRow) = 1
Base.lastindex(r::DataFrameRow) = length(r)

if VERSION < v"1.6"
Base.firstindex(r::DataFrameRow, i::Integer) = first(axes(r, i))
Base.lastindex(r::DataFrameRow, i::Integer) = last(axes(r, i))
end
Base.axes(r::DataFrameRow, i::Integer) = Base.OneTo(size(r, i))

Base.iterate(r::DataFrameRow) = iterate(r, 1)
21 changes: 5 additions & 16 deletions src/groupeddataframe/complextransforms.jl
Original file line number Diff line number Diff line change
@@ -264,12 +264,8 @@ function _combine_rows_with_first!((firstrow,)::Ref{Any},
# Create up to one task per thread
# This has lower overhead than creating one task per group,
# but is optimal only if operations take roughly the same time for all groups
if VERSION >= v"1.4" && threads && isthreadsafe(outcols, incols)
basesize = max(1, cld(len - 1, Threads.nthreads()))
partitions = Iterators.partition(2:len, basesize)
else
partitions = (2:len,)
end
basesize = max(1, cld(len - 1, Threads.nthreads()))
partitions = Iterators.partition(2:len, basesize)
widen_type_lock = ReentrantLock()
outcolsref = Ref{NTuple{<:Any, AbstractVector}}(outcols)
type_widened = fill(false, length(partitions))
@@ -320,16 +316,9 @@ end

# This needs to be in a separate function
# to work around a crash due to JuliaLang/julia#29430
if VERSION >= v"1.1.0-DEV.723"
@inline function do_append!(do_it, col, vals)
do_it && append!(col, vals)
return do_it
end
else
@noinline function do_append!(do_it, col, vals)
do_it && append!(col, vals)
return do_it
end
@inline function do_append!(do_it, col, vals)
do_it && append!(col, vals)
return do_it
end

_get_col(rows::AbstractDataFrame, j::Int) = rows[!, j]
18 changes: 3 additions & 15 deletions src/groupeddataframe/fastaggregates.jl
Original file line number Diff line number Diff line change
@@ -162,9 +162,7 @@ function groupreduce!_helper(res::AbstractVector, f, op, condf, adjust, checkemp
batches)
for batch in batches
# Allow other tasks to do garbage collection while this one runs
@static if VERSION >= v"1.4"
GC.safepoint()
end
GC.safepoint()

@inbounds for i in batch
gix = groups[i]
@@ -194,12 +192,8 @@ function groupreduce!(res::AbstractVector, f, op, condf, adjust, checkempty::Boo
counts = Int[]
end
groups = gd.groups
@static if VERSION >= v"1.4"
batchsize = Threads.nthreads() > 1 ? 100_000 : typemax(Int)
batches = Iterators.partition(eachindex(incol, groups), batchsize)
else
batches = (eachindex(incol, groups),)
end
batchsize = Threads.nthreads() > 1 ? 100_000 : typemax(Int)
batches = Iterators.partition(eachindex(incol, groups), batchsize)

groupreduce!_helper(res, f, op, condf, adjust, checkempty,
incol, groups, counts, batches)
@@ -254,12 +248,6 @@ groupreduce(f, op, condf::typeof(!ismissing), adjust, checkempty::Bool,
(r::Reduce)(incol::AbstractVector, gd::GroupedDataFrame) =
groupreduce((x, i) -> x, r.op, r.condf, r.adjust, r.checkempty, incol, gd)

# this definition is missing in Julia 1.0 LTS and is required by aggregation for var
# TODO: remove this when we drop 1.0 support
if VERSION < v"1.1"
Base.zero(::Type{Missing}) = missing
end

function (agg::Aggregate{typeof(var)})(incol::AbstractVector, gd::GroupedDataFrame)
means = groupreduce((x, i) -> x, Base.add_sum, agg.condf, /, false, incol, gd)
z = zero(eltype(incol)) - zero(eltype(means))
8 changes: 0 additions & 8 deletions src/groupeddataframe/groupeddataframe.jl
Original file line number Diff line number Diff line change
@@ -524,10 +524,6 @@ Base.ndims(::Type{<:GroupedDataFrame}) = 1
Base.firstindex(gd::GroupedDataFrame) = 1
Base.lastindex(gd::GroupedDataFrame) = gd.ngroups

if VERSION < v"1.6"
Base.firstindex(gd::GroupedDataFrame, i::Integer) = first(axes(gd, i))
Base.lastindex(gd::GroupedDataFrame, i::Integer) = last(axes(gd, i))
end
Base.axes(gd::GroupedDataFrame, i::Integer) = Base.OneTo(size(gd, i))

Base.first(gd::GroupedDataFrame) = gd[1]
@@ -624,10 +620,6 @@ Base.ndims(::Type{<:GroupKey}) = 1
Base.firstindex(key::GroupKey) = 1
Base.lastindex(key::GroupKey) = length(key)

if VERSION < v"1.6"
Base.firstindex(key::GroupKey, i::Integer) = first(axes(key, i))
Base.lastindex(key::GroupKey, i::Integer) = last(axes(key, i))
end
Base.axes(key::GroupKey, i::Integer) = Base.OneTo(size(key, i))

Base.names(key::GroupKey) = string.(parent(key).cols)
14 changes: 3 additions & 11 deletions src/groupeddataframe/splitapplycombine.jl
Original file line number Diff line number Diff line change
@@ -712,18 +712,10 @@ function _combine(gd::GroupedDataFrame,
try
wait(t)
catch e
@static if VERSION > v"1.3"
if e isa TaskFailedException
throw(t.exception)
else
rethrow(e)
end
if e isa TaskFailedException
throw(t.exception)
else
if e isa ErrorException
throw(t.exception)
else
rethrow(e)
end
rethrow(e)
end
end
end
49 changes: 15 additions & 34 deletions src/join/composer.jl
Original file line number Diff line number Diff line change
@@ -116,16 +116,11 @@ function compose_inner_table(joiner::DataFrameJoiner,
right_rename::Union{Function, AbstractString, Symbol})
left_ixs, right_ixs = find_inner_rows(joiner)

@static if VERSION >= v"1.4"
if Threads.nthreads() > 1 && length(left_ixs) >= 1_000_000
dfl_task = Threads.@spawn joiner.dfl[left_ixs, :]
dfr_noon_task = Threads.@spawn joiner.dfr[right_ixs, Not(joiner.right_on)]
dfl = fetch(dfl_task)
dfr_noon = fetch(dfr_noon_task)
else
dfl = joiner.dfl[left_ixs, :]
dfr_noon = joiner.dfr[right_ixs, Not(joiner.right_on)]
end
if Threads.nthreads() > 1 && length(left_ixs) >= 1_000_000
dfl_task = Threads.@spawn joiner.dfl[left_ixs, :]
dfr_noon_task = Threads.@spawn joiner.dfr[right_ixs, Not(joiner.right_on)]
dfl = fetch(dfl_task)
dfr_noon = fetch(dfr_noon_task)
else
dfl = joiner.dfl[left_ixs, :]
dfr_noon = joiner.dfr[right_ixs, Not(joiner.right_on)]
@@ -248,46 +243,32 @@ function _compose_joined_table(joiner::DataFrameJoiner, kind::Symbol, makeunique

@assert col_idx == ncol(joiner.dfl_on) + 1

@static if VERSION >= v"1.4"
if Threads.nthreads() > 1 && target_nrow >= 1_000_000 && length(cols) > col_idx
@sync begin
for col in eachcol(dfl_noon)
cols_i = left_idxs[col_idx]
Threads.@spawn _noon_compose_helper!(cols, _similar_left, cols_i,
col, target_nrow, left_ixs, lil + 1, leftonly_ixs, loil)
col_idx += 1
end
@assert col_idx == ncol(joiner.dfl) + 1
for col in eachcol(dfr_noon)
cols_i = col_idx
Threads.@spawn _noon_compose_helper!(cols, _similar_right, cols_i, col, target_nrow,
right_ixs, lil + loil + 1, rightonly_ixs, roil)
col_idx += 1
end
end
else
if Threads.nthreads() > 1 && target_nrow >= 1_000_000 && length(cols) > col_idx
@sync begin
for col in eachcol(dfl_noon)
_noon_compose_helper!(cols, _similar_left, left_idxs[col_idx],
col, target_nrow, left_ixs, lil + 1, leftonly_ixs, loil)
cols_i = left_idxs[col_idx]
Threads.@spawn _noon_compose_helper!(cols, _similar_left, cols_i,
col, target_nrow, left_ixs, lil + 1, leftonly_ixs, loil)
bkamins marked this conversation as resolved.
Show resolved Hide resolved
col_idx += 1
end
@assert col_idx == ncol(joiner.dfl) + 1
for col in eachcol(dfr_noon)
_noon_compose_helper!(cols, _similar_right, col_idx, col, target_nrow,
right_ixs, lil + loil + 1, rightonly_ixs, roil)
cols_i = col_idx
Threads.@spawn _noon_compose_helper!(cols, _similar_right, cols_i, col, target_nrow,
right_ixs, lil + loil + 1, rightonly_ixs, roil)
col_idx += 1
end
end
else
for col in eachcol(dfl_noon)
_noon_compose_helper!(cols, _similar_left, left_idxs[col_idx],
col, target_nrow, left_ixs, lil + 1, leftonly_ixs, loil)
col, target_nrow, left_ixs, lil + 1, leftonly_ixs, loil)
col_idx += 1
end
@assert col_idx == ncol(joiner.dfl) + 1
for col in eachcol(dfr_noon)
_noon_compose_helper!(cols, _similar_right, col_idx, col, target_nrow,
right_ixs, lil + loil + 1, rightonly_ixs, roil)
right_ixs, lil + loil + 1, rightonly_ixs, roil)
col_idx += 1
end
end
2 changes: 1 addition & 1 deletion src/other/broadcasting.jl
Original file line number Diff line number Diff line change
@@ -127,7 +127,7 @@ function Base.dotview(df::AbstractDataFrame, ::typeof(!), cols)
return LazyNewColDataFrame(df, cols isa AbstractString ? Symbol(cols) : cols)
end

if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
function Base.dotgetproperty(df::AbstractDataFrame, col::SymbolOrString)
if columnindex(df, col) == 0 && !is_column_insertion_allowed(df)
throw(ArgumentError("creating new columns in a SubDataFrame that subsets " *
2 changes: 0 additions & 2 deletions src/other/precompile.jl
Original file line number Diff line number Diff line change
@@ -16,8 +16,6 @@
# * run @warnpcfail check for all=true and all=false both on Julia stable and nightly

function precompile(all=false)
VERSION >= v"1.5" || return nothing

all || ccall(:jl_generating_output, Cint, ()) == 1 || return nothing

Base.precompile(Tuple{Aggregate{typeof(std), Nothing},Vector{Union{Missing, Int}},GroupedDataFrame{DataFrame}})
168 changes: 59 additions & 109 deletions src/other/utils.jl
Original file line number Diff line number Diff line change
@@ -165,48 +165,32 @@ function split_to_chunks(len::Integer, np::Integer)
return (Int(1 + ((i - 1) * len′) ÷ np):Int((i * len′) ÷ np) for i in 1:np)
end

if VERSION >= v"1.4"
function _spawn_for_chunks_helper(iter, lbody, basesize)
lidx = iter.args[1]
range = iter.args[2]
quote
let x = $(esc(range)), basesize = $(esc(basesize))
@assert firstindex(x) == 1

nt = Threads.nthreads()
len = length(x)
if nt > 1 && len > basesize
tasks = [Threads.@spawn begin
for i in p
local $(esc(lidx)) = @inbounds x[i]
$(esc(lbody))
end
end
for p in split_indices(len, basesize)]
foreach(wait, tasks)
else
for i in eachindex(x)
local $(esc(lidx)) = @inbounds x[i]
$(esc(lbody))
end
end
end
nothing
end
end
else
function _spawn_for_chunks_helper(iter, lbody, basesize)
lidx = iter.args[1]
range = iter.args[2]
quote
let x = $(esc(range))
function _spawn_for_chunks_helper(iter, lbody, basesize)
lidx = iter.args[1]
range = iter.args[2]
quote
let x = $(esc(range)), basesize = $(esc(basesize))
@assert firstindex(x) == 1

nt = Threads.nthreads()
len = length(x)
if nt > 1 && len > basesize
tasks = [Threads.@spawn begin
for i in p
local $(esc(lidx)) = @inbounds x[i]
$(esc(lbody))
end
end
for p in split_indices(len, basesize)]
foreach(wait, tasks)
else
for i in eachindex(x)
local $(esc(lidx)) = @inbounds x[i]
$(esc(lbody))
end
end
nothing
end
nothing
end
end

@@ -237,92 +221,58 @@ end
Equivalent to `Threads.@spawn` if `threads === true`,
otherwise run `expr` and return a `Task` that returns its value.
"""
macro spawn_or_run_task end
macro spawn_or_run_task(threads, expr)
letargs = Base._lift_one_interp!(expr)

thunk = esc(:(()->($expr)))
var = esc(Base.sync_varname)
quote
let $(letargs...)
if $(esc(threads))
local task = Task($thunk)
task.sticky = false
else
# Run expr immediately
res = $thunk()
# Return a Task that returns the value of expr
local task = Task(() -> res)
task.sticky = true
end
if $(Expr(:islocal, var))
put!($var, task)
end
schedule(task)
task
end
end
end

"""
@spawn_or_run threads expr
Equivalent to `Threads.@spawn` if `threads === true`,
otherwise run `expr`.
"""
macro spawn_or_run end

if VERSION >= v"1.4"
macro spawn_or_run_task(threads, expr)
letargs = Base._lift_one_interp!(expr)

thunk = esc(:(()->($expr)))
var = esc(Base.sync_varname)
quote
let $(letargs...)
if $(esc(threads))
local task = Task($thunk)
task.sticky = false
else
# Run expr immediately
res = $thunk()
# Return a Task that returns the value of expr
local task = Task(() -> res)
task.sticky = true
end
macro spawn_or_run(threads, expr)
letargs = Base._lift_one_interp!(expr)

thunk = esc(:(()->($expr)))
var = esc(Base.sync_varname)
quote
let $(letargs...)
if $(esc(threads))
local task = Task($thunk)
task.sticky = false
if $(Expr(:islocal, var))
@static if VERSION >= v"1.5.0"
put!($var, task)
else
push!($var, task)
end
put!($var, task)
end
schedule(task)
task
else
$thunk()
end
nothing
end
end

macro spawn_or_run(threads, expr)
letargs = Base._lift_one_interp!(expr)

thunk = esc(:(()->($expr)))
var = esc(Base.sync_varname)
quote
let $(letargs...)
if $(esc(threads))
local task = Task($thunk)
task.sticky = false
if $(Expr(:islocal, var))
@static if VERSION >= v"1.5.0"
put!($var, task)
else
push!($var, task)
end
end
schedule(task)
else
$thunk()
end
nothing
end
end
end
else
# Based on the definition of @async in Base
macro spawn_or_run_task(threads, expr)
thunk = esc(:(()->($expr)))
var = esc(Base.sync_varname)
quote
# Run expr immediately
res = $thunk()
# Return a Task that returns the value of expr
local task = Task(() -> res)
if $(Expr(:isdefined, var))
push!($var, task)
end
schedule(task)
end
end

macro spawn_or_run(threads, expr)
esc(:($expr; nothing))
end
end

function _nt_like_hash(v, h::UInt)
12 changes: 6 additions & 6 deletions test/broadcasting.jl
Original file line number Diff line number Diff line change
@@ -1463,7 +1463,7 @@ end

df = copy(refdf)
v1 = df[!, 1]
if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
df.x1 .= 'd'
@test df.x1 == ['d', 'd', 'd']
@test eltype(df.x1) === Char
@@ -1479,7 +1479,7 @@ end
@test v1 == [100.0, 100.0, 100.0]
end

if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
df = DataFrame(a=1:4, b=1, c=2)
df.a .= 'a':'d'
@test df == DataFrame(a='a':'d', b=1, c=2)
@@ -1500,7 +1500,7 @@ end
end

df = copy(refdf)
if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
df.newcol .= 'd'
@test df == [refdf DataFrame(newcol=fill('d', 3))]
else
@@ -1637,7 +1637,7 @@ end

df = view(copy(refdf), :, :)
v1 = df[!, 1]
if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
df.x1 .= 'd'
@test df.x1 == ['d', 'd', 'd']
@test eltype(df.x1) === Any
@@ -1910,7 +1910,7 @@ end

@testset "broadcasting of getproperty" begin
df = DataFrame(a=1:4)
if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
df.b .= 1
x = df.b
df.c .= 4:-1:1
@@ -1945,7 +1945,7 @@ end
@testset "dotgetproperty on SubDataFrame" begin
df = DataFrame(a=1:3, b=4:6)
dfv = @view df[[3, 1], :]
if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
dfv.c .= [1, 2]
@test df DataFrame(a=1:3, b=4:6, c=[2, missing, 1])
else
11 changes: 3 additions & 8 deletions test/dataframe.jl
Original file line number Diff line number Diff line change
@@ -450,13 +450,8 @@ end
@test_throws BoundsError deleteat!(df, [10])

df = DataFrame(a=[])
if VERSION >= v"1.1"
@test_throws BoundsError deleteat!(df, 10)
@test_throws BoundsError deleteat!(df, [10])
else
@test_throws InexactError deleteat!(df, 10)
@test_throws InexactError deleteat!(df, [10])
end
@test_throws BoundsError deleteat!(df, 10)
@test_throws BoundsError deleteat!(df, [10])

df = DataFrame(a=[1, 2, 3], b=[3, 2, 1])
@test_throws ArgumentError deleteat!(df, [3, 2])
@@ -1602,7 +1597,7 @@ end
@test x == 1:10
df.y .= 1
@test df.y == [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
@test y == 1.0:10.0
else
@test df.y === y
9 changes: 1 addition & 8 deletions test/grouping.jl
Original file line number Diff line number Diff line change
@@ -2814,11 +2814,7 @@ end
@test res.x1_sum != res.x2_sum # we are large enough to be sure we differ

res = combine(gdf, :x1 => mean, :x2 => mean, :x1 => x -> mean(x), :x2 => x -> mean(x))
if VERSION >= v"1.5"
@test res.x1_mean res.x1_function
else
@test !(res.x1_mean res.x1_function) # we are large enough to be sure we differ
end
@test res.x1_mean res.x1_function
@test res.x2_mean res.x2_function
@test res.x1_mean res.x2_mean

@@ -3552,9 +3548,6 @@ end
end

@testset "result eltype widening from different tasks" begin
if VERSION < v"1.5"
Base.convert(::Type{Union{Missing, Nothing, Float64}}, x::Int) = float(x)
end
Random.seed!(1)
for y in (Any[1, missing, missing, 2, 4],
Any[1, missing, nothing, 2.1, 'a'],
8 changes: 2 additions & 6 deletions test/indexing.jl
Original file line number Diff line number Diff line change
@@ -1747,9 +1747,7 @@ end
@test_throws ArgumentError df[1:2, 1:1] = DataFrame(b=1:2)
end

if VERSION >= v"1.4"
include("indexing_begin_tests.jl")
end
include("indexing_begin_tests.jl")

@testset "unsupported df[col] and df[col] for getindex, view, and setindex!" begin
@testset "getindex DataFrame" begin
@@ -2021,9 +2019,7 @@ end
@test_throws BoundsError lastindex(gk, 0)
end

if VERSION >= v"1.5"
include("indexing_offset.jl")
end
include("indexing_offset.jl")

@testset "threading correctness tests" begin
for x in (10, 1_100_000), y in 1:4
47 changes: 15 additions & 32 deletions test/io.jl
Original file line number Diff line number Diff line change
@@ -286,37 +286,20 @@ end
" * γ∞2∫αγ∞3∫αγ∞4∫αγ∞5∫αγ∞6∫αγ∞7∫αγ∞8∫αγ∞9∫αγ∞0"),
]
)
if VERSION < v"1.6.0-DEV"
@test sprint(show, "text/plain", df) ==
"""
8×2 DataFrame
Row │ A B
│ Int64 MD
─────┼──────────────────────────────────────────
1 │ 1 DataFrames.jl (http://juliadat…
2 │ 4 \\frac{x^2}{x^2+y^2}
3 │ 9 Header\\n ≡≡≡≡≡≡≡≡
4 │ 16 This is very, very, very, very…
5 │ 25
6 │ 36 ∫αγ∞1∫αγ∞2∫αγ∞3∫αγ∞4∫αγ∞5∫αγ∞6…
7 │ 49 ∫αγ∞1∫αγ∞\\n\\n • 2∫αγ∞3∫α…
8 │ 64 ∫αγ∞1∫αγ∞2∫αγ∞3∫αγ∞4∫αγ∞5∫αγ∞6…"""
else
@test sprint(show, "text/plain", df) ==
"""
8×2 DataFrame
Row │ A B
│ Int64 MD
─────┼──────────────────────────────────────────
1 │ 1 DataFrames.jl (http://juliadat…
2 │ 4 \\frac{x^2}{x^2+y^2}
3 │ 9 Header\\n ≡≡≡≡≡≡≡≡
4 │ 16 This is very, very, very, very…
5 │ 25
6 │ 36 ∫αγ∞1∫αγ∞2∫αγ∞3∫αγ∞4∫αγ∞5∫αγ∞6…
7 │ 49 ∫αγ∞1∫αγ∞\\n\\n • 2∫αγ∞3∫αγ∞…
8 │ 64 ∫αγ∞1∫αγ∞2∫αγ∞3∫αγ∞4∫αγ∞5∫αγ∞6…"""
end
@test sprint(show, "text/plain", df) ==
"""
8×2 DataFrame
Row │ A B
│ Int64 MD
─────┼──────────────────────────────────────────
1 │ 1 DataFrames.jl (http://juliadat…
2 │ 4 \\frac{x^2}{x^2+y^2}
3 │ 9 Header\\n ≡≡≡≡≡≡≡≡
4 │ 16 This is very, very, very, very…
5 │ 25
6 │ 36 ∫αγ∞1∫αγ∞2∫αγ∞3∫αγ∞4∫αγ∞5∫αγ∞6…
7 │ 49 ∫αγ∞1∫αγ∞\\n\\n • 2∫αγ∞3∫αγ∞…
8 │ 64 ∫αγ∞1∫αγ∞2∫αγ∞3∫αγ∞4∫αγ∞5∫αγ∞6…"""

@test sprint(show, "text/csv", df) ==
"""
@@ -632,7 +615,7 @@ end
9 │ 9 #undef"""

# TODO: update when https://github.com/KristofferC/Crayons.jl/issues/47 is resolved
if VERSION >= v"1.6" && Base.get_have_color()
if Base.get_have_color()
io = IOBuffer()
show(IOContext(io, :color => true), df)
str = String(take!(io))
6 changes: 3 additions & 3 deletions test/multithreading.jl
Original file line number Diff line number Diff line change
@@ -162,7 +162,7 @@ end
[] => (() -> n[] += 1) => :n2,
threads=false) ==
DataFrame(y=1:4, n1=1:4, n2=5:8)
if VERSION >= v"1.4" && Threads.nthreads() > 1
if Threads.nthreads() > 1
@test combine(gd, [] => (() -> Threads.threadid()) => :id1,
[] => (() -> Threads.threadid()) => :id2,
threads=true) !=
@@ -180,7 +180,7 @@ end
[] => (() -> n[] += 1) => :n2,
threads=false) ==
select(leftjoin(refdf, DataFrame(y=1:4, n1=1:4, n2=5:8), on=:y), :y, :n1, :n2)
if VERSION >= v"1.4" && Threads.nthreads() > 1
if Threads.nthreads() > 1
df = copy(refdf)
gd = groupby(df, :y)
@test select(gd, [] => (() -> Threads.threadid()) => :id1,
@@ -204,7 +204,7 @@ end
[] => (() -> n[] += 1) => :n2,
threads=false) ==
leftjoin(refdf, DataFrame(y=1:4, n1=1:4, n2=5:8), on=:y)
if VERSION >= v"1.4" && Threads.nthreads() > 1
if Threads.nthreads() > 1
df = copy(refdf)
gd = groupby(df, :y)
@test transform(gd, [] => (() -> Threads.threadid()) => :id1,
10 changes: 4 additions & 6 deletions test/runtests.jl
Original file line number Diff line number Diff line change
@@ -8,12 +8,10 @@ anyerrors = false

using DataFrames, Dates, Test, Random

if VERSION > v"1.3"
if Threads.nthreads() < 2
@warn("Running with only one thread: correctness of parallel operations is not tested")
else
@show Threads.nthreads()
end
if Threads.nthreads() < 2
@warn("Running with only one thread: correctness of parallel operations is not tested")
else
@show Threads.nthreads()
end

my_tests = ["utils.jl",
62 changes: 17 additions & 45 deletions test/select.jl
Original file line number Diff line number Diff line change
@@ -1184,23 +1184,13 @@ end
@test transform(df, AsTable(Not(:)) => Ref) ==
DataFrame(a=1:3, b=4:6, c=7:9, Ref=NamedTuple())

if VERSION >= v"1.4.0"
df = DataFrame(x=[1, 2, missing], y=[1, missing, missing])
@test transform(df, AsTable(:) .=>
ByRow.([sumskipmissing,
x -> count(!ismissing, x),
meanskipmissing]) .=>
[:sum, :n, :mean])
[df DataFrame(sum=[2, 2, 0], n=[2, 1, 0], mean=[1, 2, NaN])]
else
df = DataFrame(x=[1, 2], y=[1, missing])
@test transform(df, AsTable(:) .=>
ByRow.([sumskipmissing,
x -> count(!ismissing, x),
meanskipmissing]) .=>
[:sum, :n, :mean])
[df DataFrame(sum=[2, 2], n=[2, 1], mean=[1, 2])]
end
df = DataFrame(x=[1, 2, missing], y=[1, missing, missing])
@test transform(df, AsTable(:) .=>
ByRow.([sumskipmissing,
x -> count(!ismissing, x),
meanskipmissing]) .=>
[:sum, :n, :mean])
[df DataFrame(sum=[2, 2, 0], n=[2, 1, 0], mean=[1, 2, NaN])]
end

@testset "make sure select! is safe on error" begin
@@ -1849,9 +1839,7 @@ end
combine(df, AsTable(:) => ByRow(sumskipmissing) => :sum).sum
fill(UInt(10000), 10)

if VERSION >= v"1.6"
@test combine(df, AsTable(:) => mean => :mean).mean fill(1.0, 10)
end
@test combine(df, AsTable(:) => mean => :mean).mean fill(1.0, 10)

@test combine(df, AsTable(:) => ByRow(mean) => :mean).mean
combine(df, AsTable(:) => ByRow(meanskipmissing) => :mean).mean
@@ -1880,42 +1868,30 @@ end
m = rand([1, 2, missing], 10, 10000)
df = DataFrame(m, :auto)

if VERSION >= v"1.6"
@test combine(df, AsTable(:) => minimum => :minimum).minimum
minimum(collect(eachcol(df)))
end
@test combine(df, AsTable(:) => minimum => :minimum).minimum
minimum(collect(eachcol(df)))
@test combine(df, AsTable(:) => ByRow(minimum) => :minimum).minimum missings(Int, 10)
@test combine(df, AsTable(:) => ByRow(minimumskipmissing) => :minimum).minimum fill(1, 10)

if VERSION >= v"1.6"
@test combine(df, AsTable(:) => maximum => :maximum).maximum
maximum(collect(eachcol(df)))
end
@test combine(df, AsTable(:) => maximum => :maximum).maximum
maximum(collect(eachcol(df)))
@test combine(df, AsTable(:) => ByRow(maximum) => :maximum).maximum missings(Int, 10)
@test combine(df, AsTable(:) => ByRow(maximumskipmissing) => :maximum).maximum fill(2, 10)

m = fill(missing, 10, 100)
df = DataFrame(m, :auto)
@test combine(df, AsTable(:) => sum => :sum).sum fill(missing, 10)
@test combine(df, AsTable(:) => ByRow(sum) => :sum).sum fill(missing, 10)
if VERSION >= v"1.6"
@test_throws ArgumentError combine(df, AsTable(:) => ByRow(sumskipmissing) => :sum).sum
else
@test_throws MethodError combine(df, AsTable(:) => ByRow(sumskipmissing) => :sum).sum
end
@test_throws ArgumentError combine(df, AsTable(:) => ByRow(sumskipmissing) => :sum).sum
@test combine(df, AsTable(:) => mean => :mean).mean fill(missing, 10)
@test combine(df, AsTable(:) => ByRow(mean) => :mean).mean fill(missing, 10)
@test_throws ArgumentError combine(df, AsTable(:) => ByRow(meanskipmissing) => :mean).mean

if VERSION >= v"1.6"
@test combine(df, AsTable(:) => minimum => :minimum).minimum fill(missing, 10)
end
@test combine(df, AsTable(:) => minimum => :minimum).minimum fill(missing, 10)
@test combine(df, AsTable(:) => ByRow(minimum) => :minimum).minimum fill(missing, 10)
@test_throws ArgumentError combine(df, AsTable(:) => ByRow(minimumskipmissing) => :minimum).minimum

if VERSION >= v"1.6"
@test combine(df, AsTable(:) => maximum => :maximum).maximum fill(missing, 10)
end
@test combine(df, AsTable(:) => maximum => :maximum).maximum fill(missing, 10)
@test combine(df, AsTable(:) => ByRow(maximum) => :maximum).maximum fill(missing, 10)
@test_throws ArgumentError combine(df, AsTable(:) => ByRow(maximumskipmissing) => :maximum).maximum

@@ -1929,15 +1905,11 @@ end
@test combine(df, AsTable(:) => ByRow(mean) => :mean).mean missings(Float64, 10)
@test combine(df, AsTable(:) => ByRow(meanskipmissing) => :mean).mean fill(NaN, 10)

if VERSION >= v"1.6"
@test combine(df, AsTable(:) => minimum => :minimum).minimum missings(Int, 10)
end
@test combine(df, AsTable(:) => minimum => :minimum).minimum missings(Int, 10)
@test combine(df, AsTable(:) => ByRow(minimum) => :minimum).minimum missings(Int, 10)
@test_throws ArgumentError combine(df, AsTable(:) => ByRow(minimumskipmissing) => :minimum).minimum

if VERSION >= v"1.6"
@test combine(df, AsTable(:) => maximum => :maximum).maximum missings(Int, 10)
end
@test combine(df, AsTable(:) => maximum => :maximum).maximum missings(Int, 10)
@test combine(df, AsTable(:) => ByRow(maximum) => :maximum).maximum missings(Int, 10)
@test_throws ArgumentError combine(df, AsTable(:) => ByRow(maximumskipmissing) => :maximum).maximum

23 changes: 7 additions & 16 deletions test/show.jl
Original file line number Diff line number Diff line change
@@ -216,7 +216,7 @@ end

@testset "Test colors and non-standard values: missing and nothing" begin
# TODO: update when https://github.com/KristofferC/Crayons.jl/issues/47 is resolved
if VERSION >= v"1.6" && Base.get_have_color()
if Base.get_have_color()
df = DataFrame(Fish=["Suzy", "Amir"], Mass=[1.5, missing])
@test sprint(show, df, context=:color=>true) == """
\e[1m2×2 DataFrame\e[0m
@@ -307,21 +307,12 @@ end

# Irrational
df = DataFrame(a=π)
if VERSION < v"1.2.0-DEV.276"
@test sprint(show, df) == """
1×1 DataFrame
Row │ a
│ Irration…
─────┼────────────────────────
1 │ π = 3.1415926535897..."""
else
@test sprint(show, df) == """
1×1 DataFrame
Row │ a
│ Irration…
─────┼───────────
1 │ π"""
end
@test sprint(show, df) == """
1×1 DataFrame
Row │ a
│ Irration…
─────┼───────────
1 │ π"""
end

@testset "Test using :compact parameter of IOContext" begin
4 changes: 2 additions & 2 deletions test/subdataframe_mutation.jl
Original file line number Diff line number Diff line change
@@ -1365,7 +1365,7 @@ end
df = DataFrame(a=1:3)
sdf = @view df[[3, 2], :]
sdf.a .= 12.0
if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
@test eltype(sdf.a) === Float64
else
@test eltype(sdf.a) === Int
@@ -1383,7 +1383,7 @@ end
sdf = @view df[[3, 2], 1:1]
@test_throws ArgumentError sdf.c = [5, 6]
sdf.a .= 12.0
if isdefined(Base, :dotgetproperty)
if isdefined(Base, :dotgetproperty) # Introduced in Julia 1.7
@test eltype(sdf.a) === Float64
else
@test eltype(sdf.a) === Int