diff --git a/src/BenchmarkTools.jl b/src/BenchmarkTools.jl index c5d23077..031f8d23 100644 --- a/src/BenchmarkTools.jl +++ b/src/BenchmarkTools.jl @@ -27,19 +27,21 @@ export loadparams! include("trials.jl") export gctime, - memory, - allocs, - params, - ratio, - judge, - isinvariant, - isregression, - isimprovement, - median, - mean, - rmskew!, - rmskew, - trim + memory, + allocs, + params, + return_values, + return_value, + ratio, + judge, + isinvariant, + isregression, + isimprovement, + median, + mean, + rmskew!, + rmskew, + trim ################## # Benchmark Data # diff --git a/src/execution.jl b/src/execution.jl index e8b1d509..44886251 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -97,13 +97,13 @@ function _run(b::Benchmark, p::Parameters; verbose = false, pad = "", kwargs...) trial = Trial(params) params.gcsample && gcscrub() s = b.samplefunc(b.quote_vals, params) - push!(trial, s[1:end-1]...) + push!(trial, s...) return_val = s[end] iters = 2 while (Base.time() - start_time) < params.seconds && iters ≤ params.samples - params.gcsample && gcscrub() - push!(trial, b.samplefunc(b.quote_vals, params)[1:end-1]...) - iters += 1 + params.gcsample && gcscrub() + push!(trial, b.samplefunc(b.quote_vals, params)...) + iters += 1 end return trial, return_val end diff --git a/src/trials.jl b/src/trials.jl index e24d9271..77633c99 100644 --- a/src/trials.jl +++ b/src/trials.jl @@ -8,6 +8,12 @@ mutable struct Trial gctimes::Vector{Float64} memory::Int allocs::Int + return_values::Vector{Any} +end + +# Trial constructor with default return values. Required for backwards compatibility. +function Trial(params::Parameters, times, gctimes, memory, allocs) + Trial(params, times, gctimes, memory, allocs, [nothing for _ in eachindex(times)]) end Trial(params::Parameters) = Trial(params, Float64[], Float64[], typemax(Int), typemax(Int)) @@ -17,44 +23,63 @@ function Base.:(==)(a::Trial, b::Trial) a.times == b.times && a.gctimes == b.gctimes && a.memory == b.memory && - a.allocs == b.allocs + a.allocs == b.allocs && + a.return_values == b.return_values end -Base.copy(t::Trial) = Trial(copy(t.params), copy(t.times), copy(t.gctimes), t.memory, t.allocs) +Base.copy(t::Trial) = Trial( + copy(t.params), + copy(t.times), + copy(t.gctimes), + t.memory, + t.allocs, + copy(t.return_values), +) -function Base.push!(t::Trial, time, gctime, memory, allocs) +function Base.push!(t::Trial, time, gctime, memory, allocs, return_value) push!(t.times, time) push!(t.gctimes, gctime) memory < t.memory && (t.memory = memory) allocs < t.allocs && (t.allocs = allocs) + push!(t.return_values, return_value) return t end +function Base.push!(t::Trial, time, gctime, memory, allocs) + push!(t, time, gctime, memory, allocs, nothing) +end + function Base.deleteat!(t::Trial, i) deleteat!(t.times, i) deleteat!(t.gctimes, i) + deleteat!(t.return_values, i) return t end Base.length(t::Trial) = length(t.times) -Base.getindex(t::Trial, i::Number) = push!(Trial(t.params), t.times[i], t.gctimes[i], t.memory, t.allocs) -Base.getindex(t::Trial, i) = Trial(t.params, t.times[i], t.gctimes[i], t.memory, t.allocs) +Base.getindex(t::Trial, i::Number) = + push!(Trial(t.params), t.times[i], t.gctimes[i], t.memory, t.allocs, t.return_values[i]) +Base.getindex(t::Trial, i) = + Trial(t.params, t.times[i], t.gctimes[i], t.memory, t.allocs, t.return_values[i]) Base.lastindex(t::Trial) = length(t) function Base.sort!(t::Trial) inds = sortperm(t.times) t.times = t.times[inds] t.gctimes = t.gctimes[inds] + t.return_values = t.return_values[inds] return t end Base.sort(t::Trial) = sort!(copy(t)) Base.time(t::Trial) = time(minimum(t)) +return_value(t::Trial) = return_value(minimum(t)) gctime(t::Trial) = gctime(minimum(t)) memory(t::Trial) = t.memory allocs(t::Trial) = t.allocs params(t::Trial) = t.params +return_values(t::Trial) = t.return_values # returns the index of the first outlier in `values`, if any outliers are detected. # `values` is assumed to be sorted from least to greatest, and assumed to be right-skewed. @@ -92,10 +117,14 @@ mutable struct TrialEstimate gctime::Float64 memory::Int allocs::Int + return_value::Any end -function TrialEstimate(trial::Trial, t, gct) - return TrialEstimate(params(trial), t, gct, memory(trial), allocs(trial)) +TrialEstimate(params::Parameters, time, gctime, memory, allocs) = + TrialEstimate(params, time, gctime, memory, allocs, nothing) + +function TrialEstimate(trial::Trial, t, gct, return_value) + return TrialEstimate(params(trial), t, gct, memory(trial), allocs(trial), return_value) end function Base.:(==)(a::TrialEstimate, b::TrialEstimate) @@ -103,25 +132,33 @@ function Base.:(==)(a::TrialEstimate, b::TrialEstimate) a.time == b.time && a.gctime == b.gctime && a.memory == b.memory && - a.allocs == b.allocs + a.allocs == b.allocs && + a.return_value == b.return_value end -Base.copy(t::TrialEstimate) = TrialEstimate(copy(t.params), t.time, t.gctime, t.memory, t.allocs) +Base.copy(t::TrialEstimate) = TrialEstimate( + copy(t.params), + t.time, + t.gctime, + t.memory, + t.allocs, + deepcopy(t.return_value), +) function Base.minimum(trial::Trial) i = argmin(trial.times) - return TrialEstimate(trial, trial.times[i], trial.gctimes[i]) + return TrialEstimate(trial, trial.times[i], trial.gctimes[i], trial.return_values[i]) end function Base.maximum(trial::Trial) i = argmax(trial.times) - return TrialEstimate(trial, trial.times[i], trial.gctimes[i]) + return TrialEstimate(trial, trial.times[i], trial.gctimes[i], trial.return_values[i]) end -Statistics.median(trial::Trial) = TrialEstimate(trial, median(trial.times), median(trial.gctimes)) -Statistics.mean(trial::Trial) = TrialEstimate(trial, mean(trial.times), mean(trial.gctimes)) -Statistics.var(trial::Trial) = TrialEstimate(trial, var(trial.times), var(trial.gctimes)) -Statistics.std(trial::Trial) = TrialEstimate(trial, std(trial.times), std(trial.gctimes)) +Statistics.median(trial::Trial) = TrialEstimate(trial, median(trial.times), median(trial.gctimes), nothing) +Statistics.mean(trial::Trial) = TrialEstimate(trial, mean(trial.times), mean(trial.gctimes), nothing) +Statistics.var(trial::Trial) = TrialEstimate(trial, var(trial.times), var(trial.gctimes), nothing) +Statistics.std(trial::Trial) = TrialEstimate(trial, std(trial.times), std(trial.gctimes), nothing) Base.isless(a::TrialEstimate, b::TrialEstimate) = isless(time(a), time(b)) @@ -130,6 +167,8 @@ gctime(t::TrialEstimate) = t.gctime memory(t::TrialEstimate) = t.memory allocs(t::TrialEstimate) = t.allocs params(t::TrialEstimate) = t.params +return_value(t::TrialEstimate) = t.return_value +return_values(t::TrialEstimate) = t.return_value ############## # TrialRatio # @@ -198,7 +237,7 @@ function Base.:(==)(a::TrialJudgement, b::TrialJudgement) a.memory == b.memory end -Base.copy(t::TrialJudgement) = TrialJudgement(copy(t.params), t.time, t.memory) +Base.copy(t::TrialJudgement) = TrialJudgement(copy(t.ratio), t.time, t.memory) Base.time(t::TrialJudgement) = t.time memory(t::TrialJudgement) = t.memory diff --git a/test/SerializationTests.jl b/test/SerializationTests.jl index 87900769..039c35e6 100644 --- a/test/SerializationTests.jl +++ b/test/SerializationTests.jl @@ -4,8 +4,18 @@ using BenchmarkTools using Test eq(x::T, y::T) where {T<:Union{values(BenchmarkTools.SUPPORTED_TYPES)...}} = - all(i->eq(getfield(x, i), getfield(y, i)), 1:fieldcount(T)) -eq(x::T, y::T) where {T} = isapprox(x, y) + all(i -> eq(getfield(x, i), getfield(y, i)), 1:fieldcount(T)) + +# Robust approx - if approx is not supported, try equality comparison +eq(x::T, y::T) where {T} = + try + isapprox(x, y) + catch e + if !isa(e, MethodError) + throw(e) + end + isequal(x, y) + end function withtempdir(f::Function) d = mktempdir() diff --git a/test/TrialsTests.jl b/test/TrialsTests.jl index b477b308..39e5b71c 100644 --- a/test/TrialsTests.jl +++ b/test/TrialsTests.jl @@ -8,32 +8,34 @@ using Test # Trial # ######### -trial1 = BenchmarkTools.Trial(BenchmarkTools.Parameters(evals = 2)) +trial1 = BenchmarkTools.Trial(BenchmarkTools.Parameters(evals=2)) push!(trial1, 2, 1, 4, 5) -push!(trial1, 21, 0, 41, 51) +push!(trial1, 21, 0, 41, 51, "Return Value") -trial2 = BenchmarkTools.Trial(BenchmarkTools.Parameters(time_tolerance = 0.15)) -push!(trial2, 21, 0, 41, 51) + +trial2 = BenchmarkTools.Trial(BenchmarkTools.Parameters(time_tolerance=0.15)) +push!(trial2, 21, 0, 41, 51, "Return Value") push!(trial2, 2, 1, 4, 5) + push!(trial2, 21, 0, 41, 51) @test length(trial2) == 3 deleteat!(trial2, 3) @test length(trial1) == length(trial2) == 2 sort!(trial2) -@test trial1.params == BenchmarkTools.Parameters(evals = trial1.params.evals) -@test trial2.params == BenchmarkTools.Parameters(time_tolerance = trial2.params.time_tolerance) +@test trial1.params == BenchmarkTools.Parameters(evals=trial1.params.evals) +@test trial2.params == BenchmarkTools.Parameters(time_tolerance=trial2.params.time_tolerance) @test trial1.times == trial2.times == [2.0, 21.0] @test trial1.gctimes == trial2.gctimes == [1.0, 0.0] -@test trial1.memory == trial2.memory == 4 +@test trial1.memory == trial2.memory == 4 @test trial1.allocs == trial2.allocs == 5 trial2.params = trial1.params @test trial1 == trial2 -@test trial1[2] == push!(BenchmarkTools.Trial(BenchmarkTools.Parameters(evals = 2)), 21, 0, 4, 5) +@test trial1[2] == push!(BenchmarkTools.Trial(BenchmarkTools.Parameters(evals=2)), 21, 0, 4, 5, "Return Value") @test trial1[1:end] == trial1 @test time(trial1) == time(trial2) == 2.0 @@ -41,6 +43,7 @@ trial2.params = trial1.params @test memory(trial1) == memory(trial2) == trial1.memory @test allocs(trial1) == allocs(trial2) == trial1.allocs @test params(trial1) == params(trial2) == trial1.params +@test return_values(trial1) == return_values(trial2) == trial1.return_values # outlier trimming trial3 = BenchmarkTools.Trial(BenchmarkTools.Parameters(), [1, 2, 3, 10, 11], @@ -59,11 +62,11 @@ rmskew!(trial3) randtrial = BenchmarkTools.Trial(BenchmarkTools.Parameters()) for _ in 1:40 - push!(randtrial, rand(1:20), 1, 1, 1) + push!(randtrial, rand(1:20), 1, 1, 1, "Return Value") end while mean(randtrial) <= median(randtrial) - push!(randtrial, rand(10:20), 1, 1, 1) + push!(randtrial, rand(10:20), 1, 1, 1, "Return Value") end rmskew!(randtrial) @@ -80,6 +83,9 @@ tmax = maximum(randtrial) @test memory(tmin) == memory(tmed) == memory(tmean) == memory(tmax) == memory(tvar) == memory(tstd) == memory(randtrial) @test allocs(tmin) == allocs(tmed) == allocs(tmean) == allocs(tmax) == allocs(tvar) == allocs(tstd) == allocs(randtrial) @test params(tmin) == params(tmed) == params(tmean) == params(tmax) == params(tvar) == params(tstd) == params(randtrial) +@test return_value(tmin) == return_value(randtrial) +@test return_values(tmin) == return_value(randtrial) +@test copy(tmin) == tmin @test tmin <= tmed @test tmean <= tmed # this should be true since we called rmoutliers!(randtrial) earlier @@ -119,6 +125,7 @@ tr = ratio(ta, tb) tj_ab = judge(ta, tb) tj_r = judge(tr) +@test copy(tj_ab) == tj_ab @test ratio(tj_ab) == ratio(tj_r) == tr @test time(tj_ab) == time(tj_r) == :improvement @test memory(tj_ab) == memory(tj_r) == :regression