From a67d42e7ea62eddf5f38e5ed29dd047bce33717b Mon Sep 17 00:00:00 2001 From: Mal Miller Date: Tue, 27 Apr 2021 08:52:32 +1000 Subject: [PATCH] Add information to passing tests (#25483) --- stdlib/Test/docs/src/index.md | 9 +++++++++ stdlib/Test/src/Test.jl | 21 +++++++++++++++++---- stdlib/Test/src/logging.jl | 2 +- stdlib/Test/test/runtests.jl | 30 ++++++++++++++++++++++++++++-- test/runtests.jl | 2 +- 5 files changed, 56 insertions(+), 8 deletions(-) diff --git a/stdlib/Test/docs/src/index.md b/stdlib/Test/docs/src/index.md index 82922773f899b..69588e2e78761 100644 --- a/stdlib/Test/docs/src/index.md +++ b/stdlib/Test/docs/src/index.md @@ -42,9 +42,13 @@ If the condition is true, a `Pass` is returned: ```jldoctest testfoo julia> @test foo("bar") == 9 Test Passed + Expression: foo("bar") == 9 + Evaluated: 9 == 9 julia> @test foo("fizz") >= 10 Test Passed + Expression: foo("fizz") >= 10 + Evaluated: 16 >= 10 ``` If the condition is false, then a `Fail` is returned and an exception is thrown: @@ -83,6 +87,7 @@ to check that this occurs: ```jldoctest testfoo julia> @test_throws MethodError foo(:cat) Test Passed + Expression: foo(:cat) Thrown: MethodError ``` @@ -193,6 +198,8 @@ checks using either `@test a ≈ b` (where `≈`, typed via tab completion of `\ ```jldoctest julia> @test 1 ≈ 0.999999999 Test Passed + Expression: 1 ≈ 0.999999999 + Evaluated: 1 ≈ 0.999999999 julia> @test 1 ≈ 0.999999 Test Failed at none:1 @@ -205,6 +212,8 @@ after the `≈` comparison: ```jldoctest julia> @test 1 ≈ 0.999999 rtol=1e-5 Test Passed + Expression: ≈(1, 0.999999, rtol = 1.0e-5) + Evaluated: ≈(1, 0.999999; rtol = 1.0e-5) ``` Note that this is not a specific feature of the `≈` but rather a general feature of the `@test` macro: `@test a b key=val` is transformed by the macro into `@test op(a, b, key=val)`. It is, however, particularly useful for `≈` tests. diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 596aedb2d9aa7..8e04ec0235156 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -85,8 +85,9 @@ struct Pass <: Result orig_expr data value - function Pass(test_type::Symbol, orig_expr, data, thrown) - return new(test_type, orig_expr, data, thrown isa String ? "String" : thrown) + source::Union{Nothing,LineNumberNode} + function Pass(test_type::Symbol, orig_expr, data, thrown, source) + return new(test_type, orig_expr, data, thrown isa String ? "String" : thrown, source) end end @@ -236,6 +237,7 @@ function Serialization.serialize(s::Serialization.AbstractSerializer, t::Pass) Serialization.serialize(s, t.orig_expr === nothing ? nothing : string(t.orig_expr)) Serialization.serialize(s, t.data === nothing ? nothing : string(t.data)) Serialization.serialize(s, string(t.value)) + Serialization.serialize(s, t.source === nothing ? nothing : t.source) nothing end @@ -353,9 +355,12 @@ Returns a `Pass` `Result` if it does, a `Fail` `Result` if it is ```jldoctest julia> @test true Test Passed + Expression: true julia> @test [1, 2] + [2, 1] == [3, 3] Test Passed + Expression: [1, 2] + [2, 1] == [3, 3] + Evaluated: [3, 3] == [3, 3] ``` The `@test f(args...) key=val...` form is equivalent to writing @@ -365,6 +370,8 @@ is a call using infix syntax such as approximate comparisons: ```jldoctest julia> @test π ≈ 3.14 atol=0.01 Test Passed + Expression: ≈(π, 3.14, atol = 0.01) + Evaluated: ≈(π, 3.14; atol = 0.01) ``` This is equivalent to the uglier test `@test ≈(π, 3.14, atol=0.01)`. @@ -393,6 +400,8 @@ Test Broken julia> @test 2 + 2 ≈ 5 atol=1 broken=false Test Passed + Expression: ≈(2 + 2, 5, atol = 1) + Evaluated: ≈(4, 5; atol = 1) julia> @test 2 + 2 == 5 skip=true Test Broken @@ -400,6 +409,8 @@ Test Broken julia> @test 2 + 2 == 4 skip=false Test Passed + Expression: 2 + 2 == 4 + Evaluated: 4 == 4 ``` !!! compat "Julia 1.7" @@ -604,7 +615,7 @@ function do_test(result::ExecutionResult, orig_expr) value = result.value testres = if isa(value, Bool) # a true value Passes - value ? Pass(:test, nothing, nothing, value) : + value ? Pass(:test, orig_expr, result.data, value, result.source) : Fail(:test, orig_expr, result.data, value, result.source) else # If the result is non-Boolean, this counts as an Error @@ -646,10 +657,12 @@ Note that `@test_throws` does not support a trailing keyword form. ```jldoctest julia> @test_throws BoundsError [1, 2, 3][4] Test Passed + Expression: ([1, 2, 3])[4] Thrown: BoundsError julia> @test_throws DimensionMismatch [1, 2, 3] + [1, 2] Test Passed + Expression: [1, 2, 3] + [1, 2] Thrown: DimensionMismatch ``` """ @@ -707,7 +720,7 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) end end if success - testres = Pass(:test_throws, nothing, nothing, exc) + testres = Pass(:test_throws, orig_expr, extype, exc, result.source) else testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source) end diff --git a/stdlib/Test/src/logging.jl b/stdlib/Test/src/logging.jl index f7ecf90e2bc7f..0713108420b92 100644 --- a/stdlib/Test/src/logging.jl +++ b/stdlib/Test/src/logging.jl @@ -178,7 +178,7 @@ macro test_logs(exs...) $(esc(expression)) end if didmatch - testres = Pass(:test, nothing, nothing, value) + testres = Pass(:test, $orig_expr, nothing, value, $sourceloc) else testres = LogTestFailure($orig_expr, $sourceloc, $(QuoteNode(exs[1:end-1])), logs) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 1b99fcf4a66e2..87c0154546b83 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -281,7 +281,7 @@ end let retval_tests = @testset NoThrowTestSet begin ts = Test.DefaultTestSet("Mock for testing retval of record(::DefaultTestSet, ::T <: Result) methods") - pass_mock = Test.Pass(:test, 1, 2, LineNumberNode(0, "A Pass Mock")) + pass_mock = Test.Pass(:test, 1, 2, 3, LineNumberNode(0, "A Pass Mock")) @test Test.record(ts, pass_mock) isa Test.Pass error_mock = Test.Error(:test, 1, 2, 3, LineNumberNode(0, "An Error Mock")) @test Test.record(ts, error_mock) isa Test.Error @@ -981,7 +981,7 @@ end let ex = :(something_complex + [1, 2, 3]) b = PipeBuffer() - let t = Test.Pass(:test, (ex, 1), (ex, 2), (ex, 3)) + let t = Test.Pass(:test, (ex, 1), (ex, 2), (ex, 3), LineNumberNode(@__LINE__, @__FILE__)) serialize(b, t) @test string(t) == string(deserialize(b)) @test eof(b) @@ -1170,3 +1170,29 @@ end # Decorated LoadErrors are not unwrapped if a LoadError was thrown. @test_throws LoadError("file", 111, ErrorException("Real error")) @macroexpand @test_macro_throw_2 end + +# Issue 25483 +mutable struct PassInformationTestSet <: Test.AbstractTestSet + results::Vector + PassInformationTestSet(desc) = new([]) +end +Test.record(ts::PassInformationTestSet, t::Test.Result) = (push!(ts.results, t); t) +Test.finish(ts::PassInformationTestSet) = ts +@testset "Information in Pass result (Issue 25483)" begin + ts = @testset PassInformationTestSet begin + @test 1 == 1 + @test_throws ErrorException throw(ErrorException("Msg")) + end + test_line_number = (@__LINE__) - 3 + test_throws_line_number = (@__LINE__) - 3 + @test ts.results[1].test_type == :test + @test ts.results[1].orig_expr == :(1 == 1) + @test ts.results[1].data == Expr(:comparison, 1, :(==), 1) + @test ts.results[1].value == true + @test ts.results[1].source == LineNumberNode(test_line_number, @__FILE__) + @test ts.results[2].test_type == :test_throws + @test ts.results[2].orig_expr == :(throw(ErrorException("Msg"))) + @test ts.results[2].data == ErrorException + @test ts.results[2].value == ErrorException("Msg") + @test ts.results[2].source == LineNumberNode(test_throws_line_number, @__FILE__) +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index ec4427d99b197..ea94fca877057 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -363,7 +363,7 @@ cd(@__DIR__) do elseif isa(resp, Test.TestSetException) fake = Test.DefaultTestSet(testname) for i in 1:resp.pass - Test.record(fake, Test.Pass(:test, nothing, nothing, nothing)) + Test.record(fake, Test.Pass(:test, nothing, nothing, nothing, LineNumberNode(@__LINE__, @__FILE__))) end for i in 1:resp.broken Test.record(fake, Test.Broken(:test, nothing))