diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index fb35f0646ec8e..0816993f92024 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -118,18 +118,24 @@ struct Fail <: Result orig_expr::String data::Union{Nothing, String} value::String + context::Union{Nothing, String} source::LineNumberNode message_only::Bool - function Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode, message_only::Bool=false) + function Fail(test_type::Symbol, orig_expr, data, value, context, source::LineNumberNode, message_only::Bool) return new(test_type, string(orig_expr), data === nothing ? nothing : string(data), string(isa(data, Type) ? typeof(value) : value), + context, source, message_only) end end +# Deprecated fallback constructor without `context` argument (added in Julia 1.9). Remove in Julia 2.0. +Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode, message_only::Bool=false) = + Fail(test_type, orig_expr, data, value, nothing, source, message_only) + function Base.show(io::IO, t::Fail) printstyled(io, "Test Failed"; bold=true, color=Base.error_color()) print(io, " at ") @@ -149,10 +155,15 @@ function Base.show(io::IO, t::Fail) # An exception was expected, but no exception was thrown print(io, "\n Expected: ", data) print(io, "\n No exception thrown") - elseif t.test_type === :test && data !== nothing - # The test was an expression, so display the term-by-term - # evaluated version as well - print(io, "\n Evaluated: ", data) + elseif t.test_type === :test + if data !== nothing + # The test was an expression, so display the term-by-term + # evaluated version as well + print(io, "\n Evaluated: ", data) + end + if t.context !== nothing + print(io, "\n Context: ", t.context) + end end end @@ -643,7 +654,7 @@ function do_test(result::ExecutionResult, orig_expr) testres = if isa(value, Bool) # a true value Passes value ? Pass(:test, orig_expr, result.data, value, result.source) : - Fail(:test, orig_expr, result.data, value, result.source) + Fail(:test, orig_expr, result.data, value, nothing, result.source, false) else # If the result is non-Boolean, this counts as an Error Error(:test_nonbool, orig_expr, value, nothing, result.source) @@ -773,10 +784,10 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) if success testres = Pass(:test_throws, orig_expr, extype, exc, result.source, message_only) else - testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source, message_only) + testres = Fail(:test_throws_wrong, orig_expr, extype, exc, nothing, result.source, message_only) end else - testres = Fail(:test_throws_nothing, orig_expr, extype, nothing, result.source) + testres = Fail(:test_throws_nothing, orig_expr, extype, nothing, nothing, result.source, false) end record(get_testset(), testres) end @@ -952,6 +963,30 @@ finish(ts::FallbackTestSet) = ts #----------------------------------------------------------------------- +""" + ContextTestSet + +Passes test failures through to the parent test set, while adding information +about a context object that is being tested. +""" +struct ContextTestSet <: AbstractTestSet + parent_ts::AbstractTestSet + context_sym::Symbol + context::Any +end + +function ContextTestSet(sym::Symbol, @nospecialize(context)) + ContextTestSet(get_testset(), sym, context) +end +record(c::ContextTestSet, t) = record(c.parent_ts, t) +function record(c::ContextTestSet, t::Fail) + context = string(c.context_sym, " = ", c.context) + context = t.context === nothing ? context : string(t.context, "\n ", context) + record(c.parent_ts, Fail(t.test_type, t.orig_expr, t.data, t.value, context, t.source, t.message_only)) +end + +#----------------------------------------------------------------------- + """ DefaultTestSet @@ -1272,7 +1307,10 @@ end @testset [CustomTestSet] [option=val ...] ["description \$v, \$w"] for v in (...), w in (...) ... end @testset [CustomTestSet] [option=val ...] ["description \$v, \$w"] foo() -Starts a new test set, or multiple test sets if a `for` loop is provided. +# With begin/end or function call + +When @testset is used, with begin/end or a single function call, the macro +starts a new test set in which to evaulate the given expression. If no custom testset type is given it defaults to creating a `DefaultTestSet`. `DefaultTestSet` records all the results and, if there are any `Fail`s or @@ -1312,7 +1350,7 @@ reproducibility in case of failure, and to allow seamless re-arrangements of `@testset`s regardless of their side-effect on the global RNG state. -# Examples +## Examples ```jldoctest; filter = r"trigonometric identities | 4 4 [0-9\\.]+s" julia> @testset "trigonometric identities" begin θ = 2/3*π @@ -1324,6 +1362,37 @@ julia> @testset "trigonometric identities" begin Test Summary: | Pass Total Time trigonometric identities | 4 4 0.2s ``` + +# `@testset for` + +When `@testset for` is used, the macro starts a new test for each iteration of +the provided loop. The semantics of each test set are otherwise identical to that +of that `begin/end` case (as if used for each loop iteration). + +# `@testset let` + +When `@testset let` is used, the macro starts a *transparent* test set with +the given object added as a context object to any failing test contained +therein. This is useful when performing a set of related tests on one larger +object and it is desirable to print this larger object when any of the +individual tests fail. Transparent test sets do not introduce additional levels +of nesting in the test set hierarchy and are passed through directly to the +parent test set (with the context object appended to any failing tests.) + + !!! compat "Julia 1.9" + `@testeset let` requires at least Julia 1.9. + +## Examples +```jldoctest +julia> @testset let logi = log(im) + @test imag(logi) == π/2 + @test !iszero(real(logi)) + end +Test Failed at none:3 + Expression: !(iszero(real(logi))) + Context: logi = 0.0 + 1.5707963267948966im +ERROR: There was an error during testing +``` """ macro testset(args...) isempty(args) && error("No arguments to @testset") @@ -1331,7 +1400,7 @@ macro testset(args...) tests = args[end] # Determine if a single block or for-loop style - if !isa(tests,Expr) || (tests.head !== :for && tests.head !== :block && tests.head !== :call) + if !isa(tests,Expr) || (tests.head !== :for && tests.head !== :block && tests.head !== :call && tests.head !== :let) error("Expected function call, begin/end block or for loop as argument to @testset") end @@ -1340,6 +1409,8 @@ macro testset(args...) if tests.head === :for return testset_forloop(args, tests, __source__) + elseif tests.head === :let + return testset_context(args, tests, __source__) else return testset_beginend_call(args, tests, __source__) end @@ -1348,6 +1419,35 @@ end trigger_test_failure_break(@nospecialize(err)) = ccall(:jl_test_failure_breakpoint, Cvoid, (Any,), err) +""" +Generate the code for an `@testset` with a `let` argument. +""" +function testset_context(args, tests, source) + desc, testsettype, options = parse_testset_args(args[1:end-1]) + if desc !== nothing || testsettype !== nothing + # Reserve this syntax if we ever want to allow this, but for now, + # just do the transparent context test set. + error("@testset with a `let` argument cannot be customized") + end + + assgn = tests.args[1] + if !isa(assgn, Expr) || assgn.head !== :(=) + error("`@testset let` must have exactly one assignment") + end + assignee = assgn.args[1] + + tests.args[2] = quote + $push_testset($(ContextTestSet)($(QuoteNode(assignee)), $assignee; $options...)) + try + $(tests.args[2]) + finally + $pop_testset() + end + end + + return esc(tests) +end + """ Generate the code for a `@testset` with a function call or `begin`/`end` argument """ diff --git a/stdlib/Test/src/logging.jl b/stdlib/Test/src/logging.jl index d7146b121d47d..1afd726e09df4 100644 --- a/stdlib/Test/src/logging.jl +++ b/stdlib/Test/src/logging.jl @@ -153,7 +153,7 @@ function record(ts::DefaultTestSet, t::LogTestFailure) println() end # Hack: convert to `Fail` so that test summarization works correctly - push!(ts.results, Fail(:test, t.orig_expr, t.logs, nothing, t.source)) + push!(ts.results, Fail(:test, t.orig_expr, t.logs, nothing, nothing, t.source)) t end diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 829ecde1f83c7..7902ca7c39f4b 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -346,7 +346,7 @@ let retval_tests = @testset NoThrowTestSet begin @test Test.record(ts, pass_mock) isa Test.Pass error_mock = Test.Error(:test, 1, 2, 3, LineNumberNode(0, "An Error Mock")) @test Test.record(ts, error_mock) isa Test.Error - fail_mock = Test.Fail(:test, 1, 2, 3, LineNumberNode(0, "A Fail Mock")) + fail_mock = Test.Fail(:test, 1, 2, 3, nothing, LineNumberNode(0, "A Fail Mock"), false) @test Test.record(ts, fail_mock) isa Test.Fail broken_mock = Test.Broken(:test, LineNumberNode(0, "A Broken Mock")) @test Test.record(ts, broken_mock) isa Test.Broken diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 963a12a33376f..cc76c6fcfb0c8 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -67,8 +67,7 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` "JULIA_LOAD_PATH" => "", "JULIA_DEPOT_PATH" => "", "HOME" => homedir())) - @test v[1] == "false\nREPL: InteractiveUtilstrue\n" - @test v[2] + @test v == ("false\nREPL: InteractiveUtilstrue\n", true) end let v = writereadpipeline("println(\"REPL: \", InteractiveUtils)", setenv(`$exename -i -e 'const InteractiveUtils = 3'`, @@ -76,10 +75,9 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` "JULIA_DEPOT_PATH" => ";;;:::", "HOME" => homedir())) # TODO: ideally, `@which`, etc. would still work, but Julia can't handle `using $InterativeUtils` - @test v[1] == "REPL: 3\n" - @test v[2] + @test v == ("REPL: 3\n", true) end - let v = readchomperrors(`$exename -i -e ' + @testset let v = readchomperrors(`$exename -i -e ' empty!(LOAD_PATH) @eval Sys STDLIB=mktempdir() Base.unreference_module(Base.PkgId(Base.UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) @@ -93,32 +91,32 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` real_threads = string(ccall(:jl_cpu_threads, Int32, ())) for nc in ("0", "-2", "x", "2x", " ", "") v = readchomperrors(setenv(`$exename -i -E 'Sys.CPU_THREADS'`, "JULIA_CPU_THREADS" => nc, "HOME" => homedir())) - @test v[1] - @test v[2] == real_threads - @test v[3] == "WARNING: couldn't parse `JULIA_CPU_THREADS` environment variable. Defaulting Sys.CPU_THREADS to $real_threads." + @test v == (true, real_threads, + "WARNING: couldn't parse `JULIA_CPU_THREADS` environment variable. Defaulting Sys.CPU_THREADS to $real_threads.") end for nc in ("1", " 1 ", " +1 ", " 0x1 ") - v = readchomperrors(setenv(`$exename -i -E 'Sys.CPU_THREADS'`, "JULIA_CPU_THREADS" => nc, "HOME" => homedir())) - @test v[1] - @test v[2] == "1" - @test isempty(v[3]) + @testset let v = readchomperrors(setenv(`$exename -i -E 'Sys.CPU_THREADS'`, "JULIA_CPU_THREADS" => nc, "HOME" => homedir())) + @test v[1] + @test v[2] == "1" + @test isempty(v[3]) + end end - let v = readchomperrors(setenv(`$exename -e 0`, "JULIA_LLVM_ARGS" => "-print-options", "HOME" => homedir())) + @testset let v = readchomperrors(setenv(`$exename -e 0`, "JULIA_LLVM_ARGS" => "-print-options", "HOME" => homedir())) @test v[1] @test contains(v[2], r"print-options + = 1") @test contains(v[2], r"combiner-store-merge-dependence-limit + = 4") @test contains(v[2], r"enable-tail-merge + = 2") @test isempty(v[3]) end - let v = readchomperrors(setenv(`$exename -e 0`, "JULIA_LLVM_ARGS" => "-print-options -enable-tail-merge=1 -combiner-store-merge-dependence-limit=6", "HOME" => homedir())) + @testset let v = readchomperrors(setenv(`$exename -e 0`, "JULIA_LLVM_ARGS" => "-print-options -enable-tail-merge=1 -combiner-store-merge-dependence-limit=6", "HOME" => homedir())) @test v[1] @test contains(v[2], r"print-options + = 1") @test contains(v[2], r"combiner-store-merge-dependence-limit + = 6") @test contains(v[2], r"enable-tail-merge + = 1") @test isempty(v[3]) end - let v = readchomperrors(setenv(`$exename -e 0`, "JULIA_LLVM_ARGS" => "-print-options -enable-tail-merge=1 -enable-tail-merge=1", "HOME" => homedir())) + @testset let v = readchomperrors(setenv(`$exename -e 0`, "JULIA_LLVM_ARGS" => "-print-options -enable-tail-merge=1 -enable-tail-merge=1", "HOME" => homedir())) @test !v[1] @test isempty(v[2]) @test v[3] == "julia: for the --enable-tail-merge option: may only occur zero or one times!"