[WIP] Help wanted: switch Base.Test to use testsets everywhere

base/test.jl

@@ -368,6 +368,7 @@ type TestSetException <: Exception
     fail::Int
     error::Int
     broken::Int
+    errors_and_fails
 end
 
 function Base.show(io::IO, ex::TestSetException)
@@ -426,7 +427,7 @@ function record(ts::DefaultTestSet, t::Union{Fail, Error})
         print(t)
         # don't print the backtrace for Errors because it gets printed in the show
         # method
-        isa(t, Error) || Base.show_backtrace(STDERR, backtrace())
+        isa(t, Error) || Base.show_backtrace(STDOUT, backtrace())
         println()
     end
     push!(ts.results, t)
@@ -441,7 +442,9 @@ record(ts::DefaultTestSet, t::AbstractTestSet) = push!(ts.results, t)
 function print_test_errors(ts::DefaultTestSet)
     for t in ts.results
         if (isa(t, Error) || isa(t, Fail)) && myid() == 1
-            Base.show(STDERR,t)
+            println("Error in testset $(ts.description):")
+            Base.show(STDOUT,t)
+            println()
         elseif isa(t, DefaultTestSet)
             print_test_errors(t)
         end
@@ -505,11 +508,18 @@ function finish(ts::DefaultTestSet)
         record(parent_ts, ts)
         return ts
     end
+    passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = get_test_counts(ts)
+    total_pass   = passes + c_passes
+    total_fail   = fails  + c_fails
+    total_error  = errors + c_errors
+    total_broken = broken + c_broken
+    total = total_pass + total_fail + total_error + total_broken
     # Finally throw an error as we are the outermost test set
-    #print_test_results(ts)
-    #=if total != total_pass
-        throw(TestSetException(total_pass,total_fail,total_error, total_broken))
-    end=#
+    if total != total_pass + total_broken
+        # Get all the error/failures and bring them along for the ride
+        efs = filter_errors(ts)
+        throw(TestSetException(total_pass,total_fail,total_error, total_broken, efs))
+    end
 
     # return the testset so it is returned from the @testset macro
     ts
@@ -533,6 +543,20 @@ function get_alignment(ts::DefaultTestSet, depth::Int)
 end
 get_alignment(ts, depth::Int) = 0
 
+# Recursive function that fetches backtraces for any and all errors
+# or failures the testset and its children encountered
+function filter_errors(ts::DefaultTestSet)
+    efs = []
+    for t in ts.results
+        if isa(t, DefaultTestSet)
+            append!(efs, filter_errors(t))
+        elseif isa(t, Union{Fail, Error})
+            append!(efs, [t])
+        end
+    end
+    efs
+end
+
 # Recursive function that counts the number of test results of each
 # type directly in the testset, and totals across the child testsets
 function get_test_counts(ts::DefaultTestSet)

test/arrayops.jl

@@ -1788,6 +1788,8 @@ for op in (:.+, :.*, :.÷, :.%, :.<<, :.>>, :.-, :./, :.\, :.//, :.^)
     @eval @test typeof($(op)(A,A)) == Matrix{Foo}
 end
 
+end
+
 # Test that concatenations of dense matrices/vectors yield dense matrices/vectors
 let
     N = 4

test/choosetests.jl

@@ -17,7 +17,7 @@ function choosetests(choices = [])
     testnames = [
         "linalg", "subarray", "core", "inference", "keywordargs", "numbers",
         "printf", "char", "strings", "triplequote", "unicode",
-        "dates", "dict", "hashing", "iobuffer", "staged",
+        "dates", "dict", "hashing", "iobuffer", "staged", "offsetarray",
         "arrayops", "tuple", "reduce", "reducedim", "random", "abstractarray",
         "intfuncs", "simdloop", "vecelement", "blas", "sparse",
         "bitarray", "copy", "math", "fastmath", "functional",

test/dates/adjusters.jl

@@ -1,4 +1,4 @@
-# This file is a part of Julia. License is MIT: http://Julialang.org/license
+# This file is a part of Julia. License is MIT: http://julialang.org/license
 
 module DatesAdjustersTests
 using Base.Test
@@ -389,7 +389,7 @@ isthanksgiving(dt) = Dates.dayofweek(dt) == Dates.Thu &&
     Dates.month(dt) == Dates.Nov && Dates.dayofweekofmonth(dt) == 4
 
 function easter(y)
-    # Butcher's Algorithm: http://www.sMart.net/~mmontes/butcher.html
+    # Butcher's Algorithm: http://www.smart.net/~mmontes/butcher.html
     a=y%19
     b=div(y,100)
     c=y%100

test/libgit2.jl

@@ -9,7 +9,7 @@ const LIBGIT2_MIN_VER = v"0.23.0"
 # TESTS #
 #########
 
-@testset "Check library verison" begin
+@testset "Check library version" begin
     v = LibGit2.version()
     @test v.major == LIBGIT2_MIN_VER.major && v.minor >= LIBGIT2_MIN_VER.minor
 end

test/linalg/matmul.jl

@@ -387,15 +387,3 @@ let
         @test_throws DimensionMismatch A_mul_B!(full43, full43, tri44)
     end
 end
-
-# Ensure that matrix multiplication with a narrower output type than input type does not
-# produce allocation in the inner loop (#14722), by ensuring allocation does not change
-# with the size of the input.
-C1 = Array(Float32, 5, 5)
-A1 = rand(Float64, 5, 5)
-B1 = rand(Float64, 5, 5)
-C2 = Array(Float32, 6, 6)
-A2 = rand(Float64, 6, 6)
-B2 = rand(Float64, 6, 6)
-A_mul_B!(C1, A1, B1)
-@test @allocated(A_mul_B!(C1, A1, B1)) == @allocated(A_mul_B!(C2, A2, B2))

test/runtests.jl

@@ -74,33 +74,44 @@ cd(dirname(@__FILE__)) do
     o_ts = Base.Test.DefaultTestSet("Overall")
     Base.Test.push_testset(o_ts)
     for res in results
-        if isa(res[2][1], Exception)
-             Base.showerror(STDERR,res[2][1])
-        elseif isa(res[2][1], Base.Test.DefaultTestSet)
-             Base.Test.push_testset(res[2][1])
-             Base.Test.record(o_ts, res[2][1])
-             Base.Test.pop_testset()
+        if isa(res[2][1], Base.Test.DefaultTestSet)
+            Base.Test.push_testset(res[2][1])
+            Base.Test.record(o_ts, res[2][1])
+            Base.Test.pop_testset()
         elseif isa(res[2][1], Tuple{Int,Int})
-             fake = Base.Test.DefaultTestSet(res[1])
-             [Base.Test.record(fake, Base.Test.Pass(:test, nothing, nothing, nothing)) for i in 1:res[2][1][1]]
-             [Base.Test.record(fake, Base.Test.Broken(:test, nothing)) for i in 1:res[2][1][2]]
-             Base.Test.push_testset(fake)
-             Base.Test.record(o_ts, fake)
-             Base.Test.pop_testset()
+            fake = Base.Test.DefaultTestSet(res[1])
+            [Base.Test.record(fake, Base.Test.Pass(:test, nothing, nothing, nothing)) for i in 1:res[2][1][1]]
+            [Base.Test.record(fake, Base.Test.Broken(:test, nothing)) for i in 1:res[2][1][2]]
+            Base.Test.push_testset(fake)
+            Base.Test.record(o_ts, fake)
+            Base.Test.pop_testset()
+        elseif isa(res[2][1], RemoteException)
+            println("Worker $(res[2][1].pid) failed running test $(res[1]):")
+            Base.showerror(STDOUT,res[2][1].captured)
+            o_ts.anynonpass = true
+            if isa(res[2][1].captured.ex, Base.Test.TestSetException)
+                fake = Base.Test.DefaultTestSet(res[1])
+                [Base.Test.record(fake, Base.Test.Pass(:test, nothing, nothing, nothing)) for i in 1:res[2][1].captured.ex.pass]
+                [Base.Test.record(fake, Base.Test.Broken(:test, nothing)) for i in 1:res[2][1].captured.ex.broken]
+                for t in res[2][1].captured.ex.errors_and_fails
+                    Base.Test.record(fake, t)
+                end
+                Base.Test.push_testset(fake)
+                Base.Test.record(o_ts, fake)
+                Base.Test.pop_testset()
+            end
         end
     end
     println()
     Base.Test.print_test_results(o_ts,1)
     for res in results
-        if !isa(res[2][1], Exception)
+        if !isa(res[2][1], RemoteException)
             rss_str = @sprintf("%7.2f",res[2][6]/2^20)
             time_str = @sprintf("%7f",res[2][2])
             gc_str = @sprintf("%7f",res[2][5].total_time/10^9)
             percent_str = @sprintf("%7.2f",100*res[2][5].total_time/(10^9*res[2][2]))
             alloc_str = @sprintf("%7.2f",res[2][3]/2^20)
             println("Tests for $(res[1]):\n\ttook $time_str seconds, of which $gc_str were spent in gc ($percent_str % ),\n\tallocated $alloc_str MB,\n\twith rss $rss_str MB")
-        else
-            o_ts.anynonpass = true
         end
     end
 

test/test.jl

@@ -66,7 +66,7 @@ end
 @test contains(sprint(show, fails[4]), "Unexpected Pass")
 
 # Test printing of a TestSetException
-tse_str = sprint(show, Test.TestSetException(1,2,3,4))
+tse_str = sprint(show, Test.TestSetException(1,2,3,4,nothing))
 @test contains(tse_str, "1 passed")
 @test contains(tse_str, "2 failed")
 @test contains(tse_str, "3 errored")