[WIP] Help wanted: switch Base.Test to use testsets everywhere

base/test.jl

@@ -368,7 +368,7 @@ type TestSetException <: Exception
     fail::Int
     error::Int
     broken::Int
-    errors_and_fails
+    errors_and_fails::Vector{Union{Fail, Error}}
 end
 
 function Base.show(io::IO, ex::TestSetException)

test/core.jl

@@ -2317,7 +2317,7 @@ type Obj; x; end
         test_wr(ref, wref)
         pop!(ref)
         gc()
-        @test wref[1].value == nothing
+        @test wref[1].value === nothing
     end
     test_wr()
 end

test/libgit2.jl

@@ -793,5 +793,4 @@ mktempdir() do dir
             end
         end
     =#
-    end
 end

test/math.jl

@@ -573,6 +573,12 @@ end
           0.00169495384841964531409376316336552555952269360134349446910im)
 end
 
+# useful test functions for relative error, which differ from isapprox
+# in that relerrc separately looks at the real and imaginary parts
+relerr(z, x) = z == x ? 0.0 : abs(z - x) / abs(x)
+relerrc(z, x) = max(relerr(real(z),real(x)), relerr(imag(z),imag(x)))
+≅(a,b) = relerrc(a,b) ≤ 1e-13
+
 @testset "gamma and friends" begin
     @testset "gamma, lgamma (complex argument)" begin
         if Base.Math.libm == "libopenlibm"
@@ -596,6 +602,44 @@ end
         @test lfact(2) == lgamma(3)
     end
 
+    # lgamma test cases (from Wolfram Alpha)
+    @test lgamma(-300im) ≅ -473.17185074259241355733179182866544204963885920016823743 - 1410.3490664555822107569308046418321236643870840962522425im
+    @test lgamma(3.099) ≅ lgamma(3.099+0im) ≅ 0.786413746900558058720665860178923603134125854451168869796
+    @test lgamma(1.15) ≅ lgamma(1.15+0im) ≅ -0.06930620867104688224241731415650307100375642207340564554
+    @test lgamma(0.89) ≅ lgamma(0.89+0im) ≅ 0.074022173958081423702265889979810658434235008344573396963
+    @test lgamma(0.91) ≅ lgamma(0.91+0im) ≅ 0.058922567623832379298241751183907077883592982094770449167
+    @test lgamma(0.01) ≅ lgamma(0.01+0im) ≅ 4.599479878042021722513945411008748087261001413385289652419
+    @test lgamma(-3.4-0.1im) ≅ -1.1733353322064779481049088558918957440847715003659143454 + 12.331465501247826842875586104415980094316268974671819281im
+    @test lgamma(-13.4-0.1im) ≅ -22.457344044212827625152500315875095825738672314550695161 + 43.620560075982291551250251193743725687019009911713182478im
+    @test lgamma(-13.4+0.0im) ≅ conj(lgamma(-13.4-0.0im)) ≅ -22.404285036964892794140985332811433245813398559439824988 - 43.982297150257105338477007365913040378760371591251481493im
+    @test lgamma(-13.4+8im) ≅ -44.705388949497032519400131077242200763386790107166126534 - 22.208139404160647265446701539526205774669649081807864194im
+    @test lgamma(1+exp2(-20)) ≅ lgamma(1+exp2(-20)+0im) ≅ -5.504750066148866790922434423491111098144565651836914e-7
+    @test lgamma(1+exp2(-20)+exp2(-19)*im) ≅ -5.5047799872835333673947171235997541985495018556426e-7 - 1.1009485171695646421931605642091915847546979851020e-6im
+    @test lgamma(-300+2im) ≅ -1419.3444991797240659656205813341478289311980525970715668 - 932.63768120761873747896802932133229201676713644684614785im
+    @test lgamma(300+2im) ≅ 1409.19538972991765122115558155209493891138852121159064304 + 11.4042446282102624499071633666567192538600478241492492652im
+    @test lgamma(1-6im) ≅ -7.6099596929506794519956058191621517065972094186427056304 - 5.5220531255147242228831899544009162055434670861483084103im
+    @test lgamma(1-8im) ≅ -10.607711310314582247944321662794330955531402815576140186 - 9.4105083803116077524365029286332222345505790217656796587im
+    @test lgamma(1+6.5im) ≅ conj(lgamma(1-6.5im)) ≅ -8.3553365025113595689887497963634069303427790125048113307 + 6.4392816159759833948112929018407660263228036491479825744im
+    @test lgamma(1+1im) ≅ conj(lgamma(1-1im)) ≅ -0.6509231993018563388852168315039476650655087571397225919 - 0.3016403204675331978875316577968965406598997739437652369im
+    @test lgamma(-pi*1e7 + 6im) ≅ -5.10911758892505772903279926621085326635236850347591e8 - 9.86959420047365966439199219724905597399295814979993e7im
+    @test lgamma(-pi*1e7 + 8im) ≅ -5.10911765175690634449032797392631749405282045412624e8 - 9.86959074790854911974415722927761900209557190058925e7im
+    @test lgamma(-pi*1e14 + 6im) ≅ -1.0172766411995621854526383224252727000270225301426e16 - 9.8696044010873714715264929863618267642124589569347e14im
+    @test lgamma(-pi*1e14 + 8im) ≅ -1.0172766411995628137711690403794640541491261237341e16 - 9.8696044010867038531027376655349878694397362250037e14im
+    @test lgamma(2.05 + 0.03im) ≅ conj(lgamma(2.05 - 0.03im)) ≅ 0.02165570938532611215664861849215838847758074239924127515 + 0.01363779084533034509857648574107935425251657080676603919im
+    @test lgamma(2+exp2(-20)+exp2(-19)*im) ≅ 4.03197681916768997727833554471414212058404726357753e-7 + 8.06398296652953575754782349984315518297283664869951e-7im
+
+    @testset "lgamma for non-finite arguments" begin
+        @test lgamma(Inf + 0im) === Inf + 0im
+        @test lgamma(Inf - 0.0im) === Inf - 0.0im
+        @test lgamma(Inf + 1im) === Inf + Inf*im
+        @test lgamma(Inf - 1im) === Inf - Inf*im
+        @test lgamma(-Inf + 0.0im) === -Inf - Inf*im
+        @test lgamma(-Inf - 0.0im) === -Inf + Inf*im
+        @test lgamma(Inf*im) === -Inf + Inf*im
+        @test lgamma(-Inf*im) === -Inf - Inf*im
+        @test lgamma(Inf + Inf*im) === lgamma(NaN + 0im) === lgamma(NaN*im) === NaN + NaN*im
+    end
+
     @testset "digamma" begin
         @testset "$elty" for elty in (Float32, Float64)
             @test digamma(convert(elty, 9)) ≈ convert(elty, 2.140641477955609996536345)
@@ -692,55 +736,11 @@ end
     @test !get_zero_subnormals()
 end
 
-# useful test functions for relative error, which differ from isapprox
-# in that relerrc separately looks at the real and imaginary parts
-relerr(z, x) = z == x ? 0.0 : abs(z - x) / abs(x)
-relerrc(z, x) = max(relerr(real(z),real(x)), relerr(imag(z),imag(x)))
-≅(a,b) = relerrc(a,b) ≤ 1e-13
-
-for x in -10.2:0.3456:50
-    @test 1e-12 > relerr(digamma(x+0im), digamma(x))
-end
-
-# lgamma test cases (from Wolfram Alpha)
-@test lgamma(-300im) ≅ -473.17185074259241355733179182866544204963885920016823743 - 1410.3490664555822107569308046418321236643870840962522425im
-@test lgamma(3.099) ≅ lgamma(3.099+0im) ≅ 0.786413746900558058720665860178923603134125854451168869796
-@test lgamma(1.15) ≅ lgamma(1.15+0im) ≅ -0.06930620867104688224241731415650307100375642207340564554
-@test lgamma(0.89) ≅ lgamma(0.89+0im) ≅ 0.074022173958081423702265889979810658434235008344573396963
-@test lgamma(0.91) ≅ lgamma(0.91+0im) ≅ 0.058922567623832379298241751183907077883592982094770449167
-@test lgamma(0.01) ≅ lgamma(0.01+0im) ≅ 4.599479878042021722513945411008748087261001413385289652419
-@test lgamma(-3.4-0.1im) ≅ -1.1733353322064779481049088558918957440847715003659143454 + 12.331465501247826842875586104415980094316268974671819281im
-@test lgamma(-13.4-0.1im) ≅ -22.457344044212827625152500315875095825738672314550695161 + 43.620560075982291551250251193743725687019009911713182478im
-@test lgamma(-13.4+0.0im) ≅ conj(lgamma(-13.4-0.0im)) ≅ -22.404285036964892794140985332811433245813398559439824988 - 43.982297150257105338477007365913040378760371591251481493im
-@test lgamma(-13.4+8im) ≅ -44.705388949497032519400131077242200763386790107166126534 - 22.208139404160647265446701539526205774669649081807864194im
-@test lgamma(1+exp2(-20)) ≅ lgamma(1+exp2(-20)+0im) ≅ -5.504750066148866790922434423491111098144565651836914e-7
-@test lgamma(1+exp2(-20)+exp2(-19)*im) ≅ -5.5047799872835333673947171235997541985495018556426e-7 - 1.1009485171695646421931605642091915847546979851020e-6im
-@test lgamma(-300+2im) ≅ -1419.3444991797240659656205813341478289311980525970715668 - 932.63768120761873747896802932133229201676713644684614785im
-@test lgamma(300+2im) ≅ 1409.19538972991765122115558155209493891138852121159064304 + 11.4042446282102624499071633666567192538600478241492492652im
-@test lgamma(1-6im) ≅ -7.6099596929506794519956058191621517065972094186427056304 - 5.5220531255147242228831899544009162055434670861483084103im
-@test lgamma(1-8im) ≅ -10.607711310314582247944321662794330955531402815576140186 - 9.4105083803116077524365029286332222345505790217656796587im
-@test lgamma(1+6.5im) ≅ conj(lgamma(1-6.5im)) ≅ -8.3553365025113595689887497963634069303427790125048113307 + 6.4392816159759833948112929018407660263228036491479825744im
-@test lgamma(1+1im) ≅ conj(lgamma(1-1im)) ≅ -0.6509231993018563388852168315039476650655087571397225919 - 0.3016403204675331978875316577968965406598997739437652369im
-@test lgamma(-pi*1e7 + 6im) ≅ -5.10911758892505772903279926621085326635236850347591e8 - 9.86959420047365966439199219724905597399295814979993e7im
-@test lgamma(-pi*1e7 + 8im) ≅ -5.10911765175690634449032797392631749405282045412624e8 - 9.86959074790854911974415722927761900209557190058925e7im
-@test lgamma(-pi*1e14 + 6im) ≅ -1.0172766411995621854526383224252727000270225301426e16 - 9.8696044010873714715264929863618267642124589569347e14im
-@test lgamma(-pi*1e14 + 8im) ≅ -1.0172766411995628137711690403794640541491261237341e16 - 9.8696044010867038531027376655349878694397362250037e14im
-@test lgamma(2.05 + 0.03im) ≅ conj(lgamma(2.05 - 0.03im)) ≅ 0.02165570938532611215664861849215838847758074239924127515 + 0.01363779084533034509857648574107935425251657080676603919im
-@test lgamma(2+exp2(-20)+exp2(-19)*im) ≅ 4.03197681916768997727833554471414212058404726357753e-7 + 8.06398296652953575754782349984315518297283664869951e-7im
-
-# lgamma for non-finite arguments:
-@test lgamma(Inf + 0im) === Inf + 0im
-@test lgamma(Inf - 0.0im) === Inf - 0.0im
-@test lgamma(Inf + 1im) === Inf + Inf*im
-@test lgamma(Inf - 1im) === Inf - Inf*im
-@test lgamma(-Inf + 0.0im) === -Inf - Inf*im
-@test lgamma(-Inf - 0.0im) === -Inf + Inf*im
-@test lgamma(Inf*im) === -Inf + Inf*im
-@test lgamma(-Inf*im) === -Inf - Inf*im
-@test lgamma(Inf + Inf*im) === lgamma(NaN + 0im) === lgamma(NaN*im) === NaN + NaN*im
-
 #(compared to Wolfram Alpha)
 @testset "digamma, trigamma, polygamma & zeta" begin
+    for x in -10.2:0.3456:50
+        @test 1e-12 > relerr(digamma(x+0im), digamma(x))
+    end
     @test digamma(7+0im) ≅ 1.872784335098467139393487909917597568957840664060076401194232
     @test digamma(7im) ≅ 1.94761433458434866917623737015561385331974500663251349960124 + 1.642224898223468048051567761191050945700191089100087841536im
     @test digamma(-3.2+0.1im) ≅ 4.65022505497781398615943030397508454861261537905047116427511+2.32676364843128349629415011622322040021960602904363963042380im
@@ -805,9 +805,9 @@ end
     @test 1e-13 > relerrc(zeta(1e-4 + 2e-4im), -0.5000918637469642920007659467492165281457662206388959645-0.0001838278317660822408234942825686513084009527096442173056im)
 
     # Issue #7169: (TODO: better accuracy should be possible?)
-    @test 1e-9 > relerrc(zeta(0 + 99.69im), 4.67192766128949471267133846066040655597942700322077493021802+3.89448062985266025394674304029984849370377607524207984092848im)
+    @test 1e-9  > relerrc(zeta(0 + 99.69im), 4.67192766128949471267133846066040655597942700322077493021802+3.89448062985266025394674304029984849370377607524207984092848im)
     @test 1e-12 > relerrc(zeta(3 + 99.69im), 1.09996958148566565003471336713642736202442134876588828500-0.00948220959478852115901654819402390826992494044787958181148im)
-    @test 1e-9 > relerrc(zeta(-3 + 99.69im), 10332.6267578711852982128675093428012860119184786399673520976+13212.8641740351391796168658602382583730208014957452167440726im)
+    @test 1e-9  > relerrc(zeta(-3 + 99.69im), 10332.6267578711852982128675093428012860119184786399673520976+13212.8641740351391796168658602382583730208014957452167440726im)
     @test 1e-13 > relerrc(zeta(2 + 99.69im, 1.3), 0.41617652544777996034143623540420694985469543821307918291931-0.74199610821536326325073784018327392143031681111201859489991im)
 end
 

test/runtests.jl

@@ -62,6 +62,11 @@ cd(dirname(@__FILE__)) do
     # Free up memory =)
     n > 1 && rmprocs(workers(), waitfor=5.0)
     for t in node1_tests
+        # As above, try to run each test
+        # which must run on node 1. If
+        # the test fails, catch the error,
+        # and either way, append the results
+        # to the overall aggregator
         n > 1 && print("\tFrom worker 1:\t")
         local resp
         try
@@ -71,6 +76,22 @@ cd(dirname(@__FILE__)) do
         end
         push!(results, (t, resp))
     end
+    #=
+`   Construct a testset on the master node which will hold results from all the
+    test files run on workers and on node1. The loop goes through the results,
+    inserting them as children of the overall testset if they are testsets,
+    handling errors otherwise.
+
+    Since the workers don't return information about passing/broken tests, only
+    errors or failures, those Result types get passed `nothing` for their test
+    expressions (and expected/received result in the case of Broken).
+
+    If a test failed, returning a `RemoteException`, the error is displayed and
+    the overall testset has a child testset inserted, with the (empty) Passes
+    and Brokens from the worker and the full information about all errors and
+    failures encountered running the tests. This information will be displayed
+    as a summary at the end of the test run.
+    =#
     o_ts = Base.Test.DefaultTestSet("Overall")
     Base.Test.push_testset(o_ts)
     for res in results
@@ -80,8 +101,12 @@ cd(dirname(@__FILE__)) do
             Base.Test.pop_testset()
         elseif isa(res[2][1], Tuple{Int,Int})
             fake = Base.Test.DefaultTestSet(res[1])
-            [Base.Test.record(fake, Base.Test.Pass(:test, nothing, nothing, nothing)) for i in 1:res[2][1][1]]
-            [Base.Test.record(fake, Base.Test.Broken(:test, nothing)) for i in 1:res[2][1][2]]
+            for i in 1:res[2][1][1]
+                Base.Test.record(fake, Base.Test.Pass(:test, nothing, nothing, nothing))
+            end
+            for i in 1:res[2][1][2]
+                Base.Test.record(fake, Base.Test.Broken(:test, nothing))
+            end
             Base.Test.push_testset(fake)
             Base.Test.record(o_ts, fake)
             Base.Test.pop_testset()
@@ -91,8 +116,12 @@ cd(dirname(@__FILE__)) do
             o_ts.anynonpass = true
             if isa(res[2][1].captured.ex, Base.Test.TestSetException)
                 fake = Base.Test.DefaultTestSet(res[1])
-                [Base.Test.record(fake, Base.Test.Pass(:test, nothing, nothing, nothing)) for i in 1:res[2][1].captured.ex.pass]
-                [Base.Test.record(fake, Base.Test.Broken(:test, nothing)) for i in 1:res[2][1].captured.ex.broken]
+                for i in 1:res[2][1].captured.ex.pass
+                    Base.Test.record(fake, Base.Test.Pass(:test, nothing, nothing, nothing))
+                end
+                for i in 1:res[2][1].captured.ex.broken
+                    Base.Test.record(fake, Base.Test.Broken(:test, nothing))
+                end
                 for t in res[2][1].captured.ex.errors_and_fails
                     Base.Test.record(fake, t)
                 end
@@ -104,14 +133,28 @@ cd(dirname(@__FILE__)) do
     end
     println()
     Base.Test.print_test_results(o_ts,1)
+    #pretty print the information about gc and mem usage
+    name_align    = maximum(map(x -> length(x[1]), results))
+    elapsed_align = length("Total time (s):")
+    gc_align      = length("GC time (s):")
+    percent_align = length("Percent in gc:")
+    alloc_align   = length("Allocated (MB):")
+    rss_align     = length("RSS (MB):")
+    print_with_color(:white, rpad("Test:",name_align," "), " | ")
+    print_with_color(:white, "Total time (s): | GC time (s): | Percent in gc: | Allocated (MB): | RSS (MB):\n")
     for res in results
         if !isa(res[2][1], RemoteException)
-            rss_str = @sprintf("%7.2f",res[2][6]/2^20)
-            time_str = @sprintf("%7f",res[2][2])
-            gc_str = @sprintf("%7f",res[2][5].total_time/10^9)
+            print_with_color(:white, rpad("$(res[1])",name_align," "), " | ")
+            time_str = @sprintf("%7.2f",res[2][2])
+            print_with_color(:white, rpad(time_str,elapsed_align," "), " | ")
+            gc_str = @sprintf("%7.2f",res[2][5].total_time/10^9)
+            print_with_color(:white, rpad(gc_str,gc_align," "), " | ")
             percent_str = @sprintf("%7.2f",100*res[2][5].total_time/(10^9*res[2][2]))
+            print_with_color(:white, rpad(percent_str,percent_align," "), " | ")
             alloc_str = @sprintf("%7.2f",res[2][3]/2^20)
-            println("Tests for $(res[1]):\n\ttook $time_str seconds, of which $gc_str were spent in gc ($percent_str % ),\n\tallocated $alloc_str MB,\n\twith rss $rss_str MB")
+            print_with_color(:white, rpad(alloc_str,alloc_align," "), " | ")
+            rss_str = @sprintf("%7.2f",res[2][6]/2^20)
+            print_with_color(:white, rpad(rss_str,rss_align," "), "\n")
         end
     end
 

test/subarray.jl

@@ -1,10 +1,6 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
 using Base.Test
-# import Base: ViewIndex, dimsizeexpr, rangetype, merge_indexes, first_index, stride1expr, tailsize
-using Base.Cartesian
-
-print_underestimates = false
 
 ######## Utilities ###########
 

test/sysinfo.jl

@@ -6,34 +6,3 @@
 sprint(Base.Sys.cpu_summary)
 @test Base.Sys.uptime() > 0
 Base.Sys.loadavg()
-
-@test !("f=a=k=e=n=a=m=e" ∈ keys(ENV))
-
-# issue #10994
-@test_throws ArgumentError ENV["bad\0name"] = "ok"
-@test_throws ArgumentError ENV["okname"] = "bad\0val"
-@test_throws ArgumentError Sys.set_process_title("bad\0title")
-
-withenv("bad"=>"dog") do
-    @test_throws ArgumentError ENV["bad\0cat"]
-end
-
-# issue #11170
-withenv("TEST"=>"nonempty") do
-    @test ENV["TEST"] == "nonempty"
-end
-withenv("TEST"=>"") do
-    @test ENV["TEST"] == ""
-end
-
-let c = collect(ENV)
-    @test isa(c, Vector)
-    @test length(ENV) == length(c)
-    @test isempty(ENV) || first(ENV) in c
-end
-
-# test for non-existent keys
-key = randstring(25)
-@test !haskey(ENV,key)
-@test_throws KeyError ENV[key]
-@test get(ENV,key,"default") == "default"

test/test.jl

@@ -30,15 +30,9 @@ a[1,1,1,1,1] = 10
 
 @test rand() != rand()
 
-# Test printing of Pass results
-# Pass - constant
-#@test contains(sprint(show, @test true), "Expression: true")
-# Pass - expression
-#@test contains(sprint(show, @test 10 == 2*5), "Evaluated: 10 == 10")
-#@test contains(sprint(show, @test !false), "Expression: !false")
 # Pass - exception
-#@test contains(sprint(show, @test_throws ErrorException error()),
-#                "Thrown: ErrorException")
+@test contains(sprint(show, @test_throws ErrorException error()),
+                "Thrown: ErrorException")
 
 # Test printing of Fail results
 type NoThrowTestSet <: Base.Test.AbstractTestSet
@@ -66,7 +60,7 @@ end
 @test contains(sprint(show, fails[4]), "Unexpected Pass")
 
 # Test printing of a TestSetException
-tse_str = sprint(show, Test.TestSetException(1,2,3,4,nothing))
+tse_str = sprint(show, Test.TestSetException(1,2,3,4,Vector{Union{Base.Test.Error, Base.Test.Fail}}()))
 @test contains(tse_str, "1 passed")
 @test contains(tse_str, "2 failed")
 @test contains(tse_str, "3 errored")
@@ -166,6 +160,9 @@ end
             end
         end
     end
+    # These lines shouldn't be called
+    redirect_stdout(OLD_STDOUT)
+    error("No exception was thrown!")
     catch ex
         #redirect_stdout(OLD_STDOUT)
         #redirect_stderr(OLD_STDERR)