Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Pass info in v1.7 and later #83

Merged
merged 5 commits into from
Mar 30, 2022
Merged

Use Pass info in v1.7 and later #83

Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
23884a6
Use Pass info in v1.7 and later
Feb 26, 2022
13370d3
Add pre 1.7 test files
Feb 26, 2022
afdc7d2
Rework IDs
Feb 26, 2022
2fb2733
Add ispass function
Mar 30, 2022
acbbedc
Update docs
Mar 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name = "TestReports"
uuid = "dcd651b4-b50a-5b6b-8f22-87e9f253a252"
version = "0.6.3"
version = "0.7.0"

[deps]
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
18 changes: 13 additions & 5 deletions src/to_xml.jl
View file
Original file line number Diff line number Diff line change
@@ -199,7 +199,8 @@ function to_xml(ts::AbstractTestSet)
set_attribute!(x_testcase, "classname", ts.description)
set_attribute!(x_testcase, "time", time_taken(result)::Millisecond)
# Set attributes which require variables in this scope
(result isa Pass || result isa ReportingResult{Pass}) && set_attribute!(x_testcase, "name", x_testcase["name"] * " (Test $total_ntests)")
ntests > 0 && set_attribute!(x_testcase, "id", total_ntests) # Ignore both testsuites and errors outside of tests
(result isa Pass || result isa ReportingResult{Pass}) && VERSION < v"1.7.0" && set_attribute!(x_testcase, "name", x_testcase["name"] * " (Test $total_ntests)")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we introduce a ispass(result) function that covers both these?

x_testcase
end

@@ -218,9 +219,16 @@ end
Create a testcase node from the result and return it along with
information on number of tests.
"""
function to_xml(res::Pass)
x_testcase = testcase_xml("pass (info lost)", "_testcase_id_", [])
x_testcase, 1, 0, 0 # Increment number of tests by 1
@static if VERSION >= v"1.7.0"
function to_xml(res::Pass)
x_testcase = testcase_xml(res.orig_expr, "_testcase_id_", [])
x_testcase, 1, 0, 0 # Increment number of tests by 1
end
else
function to_xml(res::Pass)
x_testcase = testcase_xml("pass (info lost)", "_testcase_id_", [])
x_testcase, 1, 0, 0 # Increment number of tests by 1
end
end

function to_xml(v::Fail)
@@ -231,7 +239,7 @@ end

function to_xml(v::Broken)
x_testcase = testcase_xml(v, [skip_xml()]) # it is not actually skipped
x_testcase, 0, 0, 0
x_testcase, 1, 0, 0
end

function to_xml(v::Error)
3 changes: 2 additions & 1 deletion test/recordproperty.jl
View file
Original file line number Diff line number Diff line change
@@ -64,7 +64,8 @@
TestReports.test(pkg)
end
logfile = joinpath(@__DIR__, "testlog.xml")
@test_reference "references/test_with_properties.txt" open(f->read(f, String), logfile) |> clean_output
test_file = VERSION >= v"1.7.0" ? "references/test_with_properties.txt" : "references/test_with_properties_pre_1_7.txt"
@test_reference test_file open(f->read(f, String), logfile) |> clean_output
end

# Test for warning when ID set twice
18 changes: 9 additions & 9 deletions test/references/complexexample.txt
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="18" failures="6" errors="4"><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="_testcase_id_" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
<testsuites tests="19" failures="6" errors="4"><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="1 * 3 == 3" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
Expression: 1 * 2 == 5
Evaluated: 2 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="_testcase_id_" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="_testcase_id_" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
Evaluated: 2 == 5</failure></testcase><testcase name="1 * 4 == 4" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="1 + -1 == 0" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
Expression: 1 + -2 == 1
Evaluated: -1 == 1</failure></testcase><testcase name="pass (info lost) (Test 3)" id="_testcase_id_" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="_testcase_id_" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
Evaluated: -1 == 1</failure></testcase><testcase name="10 + -5 == 5" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="1 + 1 == 2" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
Expression: 1 + 2 == 5
Evaluated: 3 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="_testcase_id_" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="3" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="sqrt(-1)" id="_testcase_id_" classname="Math/other" time="0.0"><skip/></testcase><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean" classname="Math/other" time="0.0"></error><error message="Nooo" type="ErrorException" classname="Math/other" time="0.0">Nooo
</error><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass" classname="Math/other" time="0.0"></error></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="4"><error message="Got exception outside of a @test" type="ErrorException" classname="Math/Error outside of tests" time="0.0">Outside of tests
</error></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="_testcase_id_" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
Evaluated: 3 == 5</failure></testcase><testcase name="1 + 4 == 5" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean" classname="Math/other" time="0.0" id="2"></error><error message="Nooo" type="ErrorException" classname="Math/other" time="0.0" id="3">Nooo
</error><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass" classname="Math/other" time="0.0" id="4"></error></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="4"><error message="Got exception outside of a @test" type="ErrorException" classname="Math/Error outside of tests" time="0.0">Outside of tests
</error></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
Expression: throw(ArgumentError("1"))
Expected: DimensionMismatch
Thrown: ArgumentError</failure></testcase><testcase name="true" id="_testcase_id_" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
Thrown: ArgumentError</failure></testcase><testcase name="true" id="2" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
Expression: true
Expected: DimensionMismatch
No exception thrown</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/using function from a module" time="0.0"/></testsuite><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="_testcase_id_" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
No exception thrown</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="nthreads() &gt; 0" id="1" classname="Math/using function from a module" time="0.0"/></testsuite><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="4 % 2 == 0" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
Expression: sqrt(20) == 5
Evaluated: 4.47213595499958 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="_testcase_id_" classname="Math" time="0.0"/></testsuite></testsuites>
Evaluated: 4.47213595499958 == 5</failure></testcase><testcase name="16 == 16" id="3" classname="Math" time="0.0"/></testsuite></testsuites>

19 changes: 19 additions & 0 deletions test/references/complexexample_pre_1_7.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="19" failures="6" errors="4"><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
Expression: 1 * 2 == 5
Evaluated: 2 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
Expression: 1 + -2 == 1
Evaluated: -1 == 1</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
Expression: 1 + 2 == 5
Evaluated: 3 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean" classname="Math/other" time="0.0" id="2"></error><error message="Nooo" type="ErrorException" classname="Math/other" time="0.0" id="3">Nooo
</error><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass" classname="Math/other" time="0.0" id="4"></error></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="4"><error message="Got exception outside of a @test" type="ErrorException" classname="Math/Error outside of tests" time="0.0">Outside of tests
</error></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
Expression: throw(ArgumentError("1"))
Expected: DimensionMismatch
Thrown: ArgumentError</failure></testcase><testcase name="true" id="2" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
Expression: true
Expected: DimensionMismatch
No exception thrown</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/using function from a module" time="0.0"/></testsuite><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
Expression: sqrt(20) == 5
Evaluated: 4.47213595499958 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math" time="0.0"/></testsuite></testsuites>

2 changes: 1 addition & 1 deletion test/references/singlenest.txt
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="1" failures="0" errors="0"><testsuite name="a" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="a" time="0.0"/></testsuite></testsuites>
<testsuites tests="1" failures="0" errors="0"><testsuite name="a" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="1 == 1" id="1" classname="a" time="0.0"/></testsuite></testsuites>
2 changes: 2 additions & 0 deletions test/references/singlenest_pre_1_7.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="1" failures="0" errors="0"><testsuite name="a" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="1" classname="a" time="0.0"/></testsuite></testsuites>
2 changes: 1 addition & 1 deletion test/references/test_with_properties.txt
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="5" failures="0" errors="0"><testsuite name="Outer/Middle 1" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Outer/Middle 1" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="ID" value="1"/></properties></testsuite><testsuite name="Outer/Middle 2/Inner" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Outer/Middle 2/Inner" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="AdditionalNest" value="true"/><property name="ID" value="2"/></properties></testsuite><testsuite name="Outer/Middle 2" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Outer/Middle 2" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="ID" value="2"/></properties></testsuite><testsuite name="Outer" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Outer" time="0.0"/><properties><property name="File" value="runtests.jl"/></properties></testsuite><testsuite name="Types" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Types" time="0.0"/><properties><property name="Float" value="0.5"/><property name="Int" value="1"/><property name="String" value="TextTests"/><property name="List" value="[&quot;1&quot;]"/><property name="Symbol" value="asymbol"/></properties></testsuite></testsuites>
<testsuites tests="5" failures="0" errors="0"><testsuite name="Outer/Middle 1" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="true" id="1" classname="Outer/Middle 1" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="ID" value="1"/></properties></testsuite><testsuite name="Outer/Middle 2/Inner" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="true" id="1" classname="Outer/Middle 2/Inner" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="AdditionalNest" value="true"/><property name="ID" value="2"/></properties></testsuite><testsuite name="Outer/Middle 2" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="true" id="1" classname="Outer/Middle 2" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="ID" value="2"/></properties></testsuite><testsuite name="Outer" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="true" id="1" classname="Outer" time="0.0"/><properties><property name="File" value="runtests.jl"/></properties></testsuite><testsuite name="Types" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="true" id="1" classname="Types" time="0.0"/><properties><property name="Float" value="0.5"/><property name="Int" value="1"/><property name="String" value="TextTests"/><property name="List" value="[&quot;1&quot;]"/><property name="Symbol" value="asymbol"/></properties></testsuite></testsuites>
Loading