From 2a9b5aba349d2668c982ef0066207217bc4acfe5 Mon Sep 17 00:00:00 2001 From: Fr Jeremy Krieg Date: Mon, 22 Jun 2020 00:09:57 +0930 Subject: [PATCH] [junit-platform] Report total failure count, not just test failures This is necessary as Jupiter reports container failures separately from the children if the container fails before the children are executed. Fixes #4175. Signed-off-by: Fr Jeremy Krieg --- .../tester/junit/platform/Activator.java | 2 +- .../junit/platform/JUnit4ContainerError.java | 22 +++++++++++++++++++ .../platform/JUnit4ContainerFailure.java | 22 +++++++++++++++++++ .../junit/platform/JUnit5ContainerError.java | 22 +++++++++++++++++++ .../platform/JUnit5ContainerFailure.java | 22 +++++++++++++++++++ .../test/ActivatorJUnitPlatformTest.java | 9 ++++++++ .../testbase/AbstractActivatorTest.java | 7 ++++++ 7 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit4ContainerError.java create mode 100644 biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit4ContainerFailure.java create mode 100644 biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit5ContainerError.java create mode 100644 biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit5ContainerFailure.java diff --git a/biz.aQute.tester.junit-platform/src/aQute/tester/junit/platform/Activator.java b/biz.aQute.tester.junit-platform/src/aQute/tester/junit/platform/Activator.java index af62ab3786..3c7ff51e0a 100644 --- a/biz.aQute.tester.junit-platform/src/aQute/tester/junit/platform/Activator.java +++ b/biz.aQute.tester.junit-platform/src/aQute/tester/junit/platform/Activator.java @@ -358,7 +358,7 @@ long test(LauncherDiscoveryRequest testRequest) { } return summary.getSummary() - .getTestsFailedCount(); + .getTotalFailureCount(); } catch (Exception e) { e.printStackTrace(); } diff --git a/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit4ContainerError.java b/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit4ContainerError.java new file mode 100644 index 0000000000..511fb61f31 --- /dev/null +++ b/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit4ContainerError.java @@ -0,0 +1,22 @@ +package aQute.tester.testclasses.junit.platform; + +import org.junit.BeforeClass; +import org.junit.Test; + +public class JUnit4ContainerError { + + @BeforeClass + void beforeAll() { + throw new IllegalStateException(); + } + + @Test + void myTest() { + throw new AssertionError(); + } + + @Test + void my2ndTest() { + throw new AssertionError(); + } +} diff --git a/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit4ContainerFailure.java b/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit4ContainerFailure.java new file mode 100644 index 0000000000..56e88583f3 --- /dev/null +++ b/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit4ContainerFailure.java @@ -0,0 +1,22 @@ +package aQute.tester.testclasses.junit.platform; + +import org.junit.BeforeClass; +import org.junit.Test; + +public class JUnit4ContainerFailure { + + @BeforeClass + void beforeAll() { + throw new AssertionError(); + } + + @Test + void myTest() { + throw new AssertionError(); + } + + @Test + void my2ndTest() { + throw new AssertionError(); + } +} diff --git a/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit5ContainerError.java b/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit5ContainerError.java new file mode 100644 index 0000000000..340de11d78 --- /dev/null +++ b/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit5ContainerError.java @@ -0,0 +1,22 @@ +package aQute.tester.testclasses.junit.platform; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class JUnit5ContainerError { + + @BeforeAll + void beforeAll() { + throw new IllegalStateException(); + } + + @Test + void myTest() { + throw new AssertionError(); + } + + @Test + void my2ndTest() { + throw new AssertionError(); + } +} diff --git a/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit5ContainerFailure.java b/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit5ContainerFailure.java new file mode 100644 index 0000000000..cfb899c5ee --- /dev/null +++ b/biz.aQute.tester.test/src/aQute/tester/testclasses/junit/platform/JUnit5ContainerFailure.java @@ -0,0 +1,22 @@ +package aQute.tester.testclasses.junit.platform; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class JUnit5ContainerFailure { + + @BeforeAll + void beforeAll() { + throw new AssertionError(); + } + + @Test + void myTest() { + throw new AssertionError(); + } + + @Test + void my2ndTest() { + throw new AssertionError(); + } +} diff --git a/biz.aQute.tester.test/test/aQute/tester/junit/platform/test/ActivatorJUnitPlatformTest.java b/biz.aQute.tester.test/test/aQute/tester/junit/platform/test/ActivatorJUnitPlatformTest.java index 6b9287ab47..0ba53125bd 100644 --- a/biz.aQute.tester.test/test/aQute/tester/junit/platform/test/ActivatorJUnitPlatformTest.java +++ b/biz.aQute.tester.test/test/aQute/tester/junit/platform/test/ActivatorJUnitPlatformTest.java @@ -54,6 +54,8 @@ import aQute.tester.testclasses.junit.platform.JUnit4ComparisonTest; import aQute.tester.testclasses.junit.platform.JUnit4Skipper; import aQute.tester.testclasses.junit.platform.JUnit5AbortTest; +import aQute.tester.testclasses.junit.platform.JUnit5ContainerError; +import aQute.tester.testclasses.junit.platform.JUnit5ContainerFailure; import aQute.tester.testclasses.junit.platform.JUnit5ContainerSkipped; import aQute.tester.testclasses.junit.platform.JUnit5ContainerSkippedWithCustomDisplayName; import aQute.tester.testclasses.junit.platform.JUnit5DisplayNameTest; @@ -585,4 +587,11 @@ public void xmlReporter_generatesCompleteXmlFile() throws Exception { .nodesByXPath(testCaseFailure(With2Failures.class, "test3", CustomAssertionError.class)) .hasSize(1); } + + // Unlike JUnit 4, Jupiter skips the tests when the parent container + // fails and does not report the children as test failures. + @Test + public void exitCode_countsJupiterContainerErrorsAndFailures() { + runTests(2, JUnit5ContainerFailure.class, JUnit5ContainerError.class); + } } diff --git a/biz.aQute.tester.test/test/aQute/tester/testbase/AbstractActivatorTest.java b/biz.aQute.tester.test/test/aQute/tester/testbase/AbstractActivatorTest.java index e193fd814e..df77c9b029 100644 --- a/biz.aQute.tester.test/test/aQute/tester/testbase/AbstractActivatorTest.java +++ b/biz.aQute.tester.test/test/aQute/tester/testbase/AbstractActivatorTest.java @@ -53,6 +53,8 @@ import aQute.tester.testclasses.JUnit4Test; import aQute.tester.testclasses.With1Error1Failure; import aQute.tester.testclasses.With2Failures; +import aQute.tester.testclasses.junit.platform.JUnit4ContainerError; +import aQute.tester.testclasses.junit.platform.JUnit4ContainerFailure; // Because we're not in the same project as aQute.junit.TesterConstants and its bundle-private. public abstract class AbstractActivatorTest extends SoftAssertions { @@ -516,6 +518,11 @@ public void exitCode_countsErrorsAndFailures() { assertThat(exitCode.exitCode).isEqualTo(4); } + @Test + public void exitCode_countsContainerErrorsAndFailures() { + runTests(8, JUnit4ContainerFailure.class, JUnit4ContainerError.class); + } + protected TestRunData runTestsEclipse(Callback postCreateCallback, Class... tests) { return runTestsEclipse(postCreateCallback, new Class[][] { tests