Record Return Values in Trial

src/BenchmarkTools.jl

@@ -27,19 +27,21 @@ export loadparams!
 include("trials.jl")
 
 export gctime,
-       memory,
-       allocs,
-       params,
-       ratio,
-       judge,
-       isinvariant,
-       isregression,
-       isimprovement,
-       median,
-       mean,
-       rmskew!,
-       rmskew,
-       trim
+    memory,
+    allocs,
+    params,
+    return_values,
+    return_value,
+    ratio,
+    judge,
+    isinvariant,
+    isregression,
+    isimprovement,
+    median,
+    mean,
+    rmskew!,
+    rmskew,
+    trim
 
 ##################
 # Benchmark Data #

src/execution.jl

@@ -97,13 +97,13 @@ function _run(b::Benchmark, p::Parameters; verbose = false, pad = "", kwargs...)
     trial = Trial(params)
     params.gcsample && gcscrub()
     s = b.samplefunc(b.quote_vals, params)
-    push!(trial, s[1:end-1]...)
+    push!(trial, s...)
     return_val = s[end]
     iters = 2
     while (Base.time() - start_time) < params.seconds && iters ≤ params.samples
-         params.gcsample && gcscrub()
-         push!(trial, b.samplefunc(b.quote_vals, params)[1:end-1]...)
-         iters += 1
+        params.gcsample && gcscrub()
+        push!(trial, b.samplefunc(b.quote_vals, params)...)
+        iters += 1
     end
     return trial, return_val
 end

src/trials.jl

@@ -8,6 +8,12 @@ mutable struct Trial
     gctimes::Vector{Float64}
     memory::Int
     allocs::Int
+    return_values::Vector{Any}
+end
+
+# Trial constructor with default return values. Required for backwards compatibility.
+function Trial(params::Parameters, times, gctimes, memory, allocs)
+    Trial(params, times, gctimes, memory, allocs, [nothing for _ in eachindex(times)])
 end
 
 Trial(params::Parameters) = Trial(params, Float64[], Float64[], typemax(Int), typemax(Int))
@@ -17,44 +23,63 @@ function Base.:(==)(a::Trial, b::Trial)
            a.times == b.times &&
            a.gctimes == b.gctimes &&
            a.memory == b.memory &&
-           a.allocs == b.allocs
+           a.allocs == b.allocs &&
+           a.return_values == b.return_values
 end
 
-Base.copy(t::Trial) = Trial(copy(t.params), copy(t.times), copy(t.gctimes), t.memory, t.allocs)
+Base.copy(t::Trial) = Trial(
+    copy(t.params),
+    copy(t.times),
+    copy(t.gctimes),
+    t.memory,
+    t.allocs,
+    copy(t.return_values),
+)
 
-function Base.push!(t::Trial, time, gctime, memory, allocs)
+function Base.push!(t::Trial, time, gctime, memory, allocs, return_value)
     push!(t.times, time)
     push!(t.gctimes, gctime)
     memory < t.memory && (t.memory = memory)
     allocs < t.allocs && (t.allocs = allocs)
+    push!(t.return_values, return_value)
     return t
 end
 
+function Base.push!(t::Trial, time, gctime, memory, allocs)
+    push!(t, time, gctime, memory, allocs, nothing)
+end
+
 function Base.deleteat!(t::Trial, i)
     deleteat!(t.times, i)
     deleteat!(t.gctimes, i)
+    deleteat!(t.return_values, i)
     return t
 end
 
 Base.length(t::Trial) = length(t.times)
-Base.getindex(t::Trial, i::Number) = push!(Trial(t.params), t.times[i], t.gctimes[i], t.memory, t.allocs)
-Base.getindex(t::Trial, i) = Trial(t.params, t.times[i], t.gctimes[i], t.memory, t.allocs)
+Base.getindex(t::Trial, i::Number) =
+    push!(Trial(t.params), t.times[i], t.gctimes[i], t.memory, t.allocs, t.return_values[i])
+Base.getindex(t::Trial, i) =
+    Trial(t.params, t.times[i], t.gctimes[i], t.memory, t.allocs, t.return_values[i])
 Base.lastindex(t::Trial) = length(t)
 
 function Base.sort!(t::Trial)
     inds = sortperm(t.times)
     t.times = t.times[inds]
     t.gctimes = t.gctimes[inds]
+    t.return_values = t.return_values[inds]
     return t
 end
 
 Base.sort(t::Trial) = sort!(copy(t))
 
 Base.time(t::Trial) = time(minimum(t))
+return_value(t::Trial) = return_value(minimum(t))
 gctime(t::Trial) = gctime(minimum(t))
 memory(t::Trial) = t.memory
 allocs(t::Trial) = t.allocs
 params(t::Trial) = t.params
+return_values(t::Trial) = t.return_values
 
 # returns the index of the first outlier in `values`, if any outliers are detected.
 # `values` is assumed to be sorted from least to greatest, and assumed to be right-skewed.
@@ -92,36 +117,48 @@ mutable struct TrialEstimate
     gctime::Float64
     memory::Int
     allocs::Int
+    return_value::Any
 end
 
-function TrialEstimate(trial::Trial, t, gct)
-    return TrialEstimate(params(trial), t, gct, memory(trial), allocs(trial))
+TrialEstimate(params::Parameters, time, gctime, memory, allocs) =
+    TrialEstimate(params, time, gctime, memory, allocs, nothing)
+
+function TrialEstimate(trial::Trial, t, gct, return_value)
+    return TrialEstimate(params(trial), t, gct, memory(trial), allocs(trial), return_value)
 end
 
 function Base.:(==)(a::TrialEstimate, b::TrialEstimate)
     return a.params == b.params &&
            a.time == b.time &&
            a.gctime == b.gctime &&
            a.memory == b.memory &&
-           a.allocs == b.allocs
+           a.allocs == b.allocs &&
+           a.return_value == b.return_value
 end
 
-Base.copy(t::TrialEstimate) = TrialEstimate(copy(t.params), t.time, t.gctime, t.memory, t.allocs)
+Base.copy(t::TrialEstimate) = TrialEstimate(
+    copy(t.params),
+    t.time,
+    t.gctime,
+    t.memory,
+    t.allocs,
+    deepcopy(t.return_value),
+)
 
 function Base.minimum(trial::Trial)
     i = argmin(trial.times)
-    return TrialEstimate(trial, trial.times[i], trial.gctimes[i])
+    return TrialEstimate(trial, trial.times[i], trial.gctimes[i], trial.return_values[i])
 end
 
 function Base.maximum(trial::Trial)
     i = argmax(trial.times)
-    return TrialEstimate(trial, trial.times[i], trial.gctimes[i])
+    return TrialEstimate(trial, trial.times[i], trial.gctimes[i], trial.return_values[i])
 end
 
-Statistics.median(trial::Trial) = TrialEstimate(trial, median(trial.times), median(trial.gctimes))
-Statistics.mean(trial::Trial) = TrialEstimate(trial, mean(trial.times), mean(trial.gctimes))
-Statistics.var(trial::Trial) = TrialEstimate(trial, var(trial.times), var(trial.gctimes))
-Statistics.std(trial::Trial) = TrialEstimate(trial, std(trial.times), std(trial.gctimes))
+Statistics.median(trial::Trial) = TrialEstimate(trial, median(trial.times), median(trial.gctimes), nothing)
+Statistics.mean(trial::Trial) = TrialEstimate(trial, mean(trial.times), mean(trial.gctimes), nothing)
+Statistics.var(trial::Trial) = TrialEstimate(trial, var(trial.times), var(trial.gctimes), nothing)
+Statistics.std(trial::Trial) = TrialEstimate(trial, std(trial.times), std(trial.gctimes), nothing)
 
 Base.isless(a::TrialEstimate, b::TrialEstimate) = isless(time(a), time(b))
 
@@ -130,6 +167,8 @@ gctime(t::TrialEstimate) = t.gctime
 memory(t::TrialEstimate) = t.memory
 allocs(t::TrialEstimate) = t.allocs
 params(t::TrialEstimate) = t.params
+return_value(t::TrialEstimate) = t.return_value
+return_values(t::TrialEstimate) = t.return_value
 
 ##############
 # TrialRatio #
@@ -198,7 +237,7 @@ function Base.:(==)(a::TrialJudgement, b::TrialJudgement)
            a.memory == b.memory
 end
 
-Base.copy(t::TrialJudgement) = TrialJudgement(copy(t.params), t.time, t.memory)
+Base.copy(t::TrialJudgement) = TrialJudgement(copy(t.ratio), t.time, t.memory)
 
 Base.time(t::TrialJudgement) = t.time
 memory(t::TrialJudgement) = t.memory

test/SerializationTests.jl

@@ -4,8 +4,18 @@ using BenchmarkTools
 using Test
 
 eq(x::T, y::T) where {T<:Union{values(BenchmarkTools.SUPPORTED_TYPES)...}} =
-    all(i->eq(getfield(x, i), getfield(y, i)), 1:fieldcount(T))
-eq(x::T, y::T) where {T} = isapprox(x, y)
+    all(i -> eq(getfield(x, i), getfield(y, i)), 1:fieldcount(T))
+
+# Robust approx - if approx is not supported, try equality comparison
+eq(x::T, y::T) where {T} =
+    try
+        isapprox(x, y)
+    catch e
+        if !isa(e, MethodError)
+            throw(e)
+        end
+        isequal(x, y)
+    end
 
 function withtempdir(f::Function)
     d = mktempdir()

test/TrialsTests.jl

@@ -8,39 +8,42 @@ using Test
 # Trial #
 #########
 
-trial1 = BenchmarkTools.Trial(BenchmarkTools.Parameters(evals = 2))
+trial1 = BenchmarkTools.Trial(BenchmarkTools.Parameters(evals=2))
 push!(trial1, 2, 1, 4, 5)
-push!(trial1, 21, 0, 41, 51)
+push!(trial1, 21, 0, 41, 51, "Return Value")
 
-trial2 = BenchmarkTools.Trial(BenchmarkTools.Parameters(time_tolerance = 0.15))
-push!(trial2, 21, 0, 41, 51)
+
+trial2 = BenchmarkTools.Trial(BenchmarkTools.Parameters(time_tolerance=0.15))
+push!(trial2, 21, 0, 41, 51, "Return Value")
 push!(trial2, 2, 1, 4, 5)
 
+
 push!(trial2, 21, 0, 41, 51)
 @test length(trial2) == 3
 deleteat!(trial2, 3)
 @test length(trial1) == length(trial2) == 2
 sort!(trial2)
 
-@test trial1.params == BenchmarkTools.Parameters(evals = trial1.params.evals)
-@test trial2.params == BenchmarkTools.Parameters(time_tolerance = trial2.params.time_tolerance)
+@test trial1.params == BenchmarkTools.Parameters(evals=trial1.params.evals)
+@test trial2.params == BenchmarkTools.Parameters(time_tolerance=trial2.params.time_tolerance)
 @test trial1.times == trial2.times == [2.0, 21.0]
 @test trial1.gctimes == trial2.gctimes == [1.0, 0.0]
-@test trial1.memory == trial2.memory ==  4
+@test trial1.memory == trial2.memory == 4
 @test trial1.allocs == trial2.allocs == 5
 
 trial2.params = trial1.params
 
 @test trial1 == trial2
 
-@test trial1[2] == push!(BenchmarkTools.Trial(BenchmarkTools.Parameters(evals = 2)), 21, 0, 4, 5)
+@test trial1[2] == push!(BenchmarkTools.Trial(BenchmarkTools.Parameters(evals=2)), 21, 0, 4, 5, "Return Value")
 @test trial1[1:end] == trial1
 
 @test time(trial1) == time(trial2) == 2.0
 @test gctime(trial1) == gctime(trial2) == 1.0
 @test memory(trial1) == memory(trial2) == trial1.memory
 @test allocs(trial1) == allocs(trial2) == trial1.allocs
 @test params(trial1) == params(trial2) == trial1.params
+@test return_values(trial1) == return_values(trial2) == trial1.return_values
 
 # outlier trimming
 trial3 = BenchmarkTools.Trial(BenchmarkTools.Parameters(), [1, 2, 3, 10, 11],
@@ -59,11 +62,11 @@ rmskew!(trial3)
 randtrial = BenchmarkTools.Trial(BenchmarkTools.Parameters())
 
 for _ in 1:40
-    push!(randtrial, rand(1:20), 1, 1, 1)
+    push!(randtrial, rand(1:20), 1, 1, 1, "Return Value")
 end
 
 while mean(randtrial) <= median(randtrial)
-    push!(randtrial, rand(10:20), 1, 1, 1)
+    push!(randtrial, rand(10:20), 1, 1, 1, "Return Value")
 end
 
 rmskew!(randtrial)
@@ -80,6 +83,9 @@ tmax = maximum(randtrial)
 @test memory(tmin) == memory(tmed) == memory(tmean) == memory(tmax) == memory(tvar) == memory(tstd) == memory(randtrial)
 @test allocs(tmin) == allocs(tmed) == allocs(tmean) == allocs(tmax) == allocs(tvar) == allocs(tstd) == allocs(randtrial)
 @test params(tmin) == params(tmed) == params(tmean) == params(tmax) == params(tvar) == params(tstd) == params(randtrial)
+@test return_value(tmin) == return_value(randtrial)
+@test return_values(tmin) == return_value(randtrial)
+@test copy(tmin) == tmin
 
 @test tmin <= tmed
 @test tmean <= tmed # this should be true since we called rmoutliers!(randtrial) earlier
@@ -119,6 +125,7 @@ tr = ratio(ta, tb)
 tj_ab = judge(ta, tb)
 tj_r = judge(tr)
 
+@test copy(tj_ab) == tj_ab
 @test ratio(tj_ab) == ratio(tj_r) == tr
 @test time(tj_ab) == time(tj_r) == :improvement
 @test memory(tj_ab) == memory(tj_r) == :regression