diff --git a/src/java_tools/junitrunner/java/com/google/testing/junit/runner/BUILD b/src/java_tools/junitrunner/java/com/google/testing/junit/runner/BUILD index fc90cba193d836..1e2d23e2c098bf 100644 --- a/src/java_tools/junitrunner/java/com/google/testing/junit/runner/BUILD +++ b/src/java_tools/junitrunner/java/com/google/testing/junit/runner/BUILD @@ -15,6 +15,7 @@ java_library( deps = [ "//src/java_tools/junitrunner/java/com/google/testing/junit/runner/internal", "//src/java_tools/junitrunner/java/com/google/testing/junit/runner/junit4", + "//third_party:junit4", ], ) diff --git a/src/java_tools/junitrunner/java/com/google/testing/junit/runner/BazelTestRunner.java b/src/java_tools/junitrunner/java/com/google/testing/junit/runner/BazelTestRunner.java index f5dd682699336e..6a83d2f79b33a9 100644 --- a/src/java_tools/junitrunner/java/com/google/testing/junit/runner/BazelTestRunner.java +++ b/src/java_tools/junitrunner/java/com/google/testing/junit/runner/BazelTestRunner.java @@ -24,6 +24,7 @@ import java.util.Date; import java.util.Locale; import java.util.concurrent.TimeUnit; +import org.junit.runner.Result; /** * A class to run JUnit tests in a controlled environment. @@ -44,6 +45,11 @@ public class BazelTestRunner { */ static final String TEST_SUITE_PROPERTY_NAME = "bazel.test_suite"; + private static final int EXIT_CODE_SUCCESS = 0; + private static final int EXIT_CODE_TEST_FAILURE_OTHER = 1; + private static final int EXIT_CODE_TEST_RUNNER_FAILURE = 2; + private static final int EXIT_CODE_TEST_FAILURE_OOM = 137; + private BazelTestRunner() { // utility class; should not be instantiated } @@ -51,16 +57,17 @@ private BazelTestRunner() { /** * Takes as arguments the classes or packages to test. * - *

To help just run one test or method in a suite, the test suite - * may be passed in via system properties (-Dbazel.test_suite). - * An empty args parameter means to run all tests in the suite. - * A non-empty args parameter means to run only the specified tests/methods. + *

To help just run one test or method in a suite, the test suite may be passed in via system + * properties (-Dbazel.test_suite). An empty args parameter means to run all tests in the suite. A + * non-empty args parameter means to run only the specified tests/methods. * *

Return codes: + * *

*/ public static void main(String[] args) { @@ -68,7 +75,7 @@ public static void main(String[] args) { String suiteClassName = System.getProperty(TEST_SUITE_PROPERTY_NAME); if (!checkTestSuiteProperty(suiteClassName)) { - System.exit(2); + System.exit(EXIT_CODE_TEST_RUNNER_FAILURE); } int exitCode; @@ -79,7 +86,8 @@ public static void main(String[] args) { // logged // by the executing strategy, and return a failure, so this process can gracefully shut down. e.printStackTrace(); - exitCode = 1; + exitCode = + e instanceof OutOfMemoryError ? EXIT_CODE_TEST_FAILURE_OOM : EXIT_CODE_TEST_FAILURE_OTHER; } System.err.printf("%nBazelTestRunner exiting with a return value of %d%n", exitCode); @@ -135,14 +143,21 @@ private static int runTestsInSuite(String suiteClassName, String[] args) { // No class found corresponding to the system property passed in from Bazel if (args.length == 0 && suiteClassName != null) { System.err.printf("Class not found: [%s]%n", suiteClassName); - return 2; + return EXIT_CODE_TEST_RUNNER_FAILURE; } } // TODO(kush): Use a new classloader for the following instantiation. JUnit4Runner runner = JUnit4Bazel.builder().suiteClass(suite).config(new Config(args)).build().runner(); - return runner.run().wasSuccessful() ? 0 : 1; + Result result = runner.run(); + if (result.wasSuccessful()) { + return EXIT_CODE_SUCCESS; + } + return result.getFailures().stream() + .anyMatch(failure -> failure.getException() instanceof OutOfMemoryError) + ? EXIT_CODE_TEST_FAILURE_OOM + : EXIT_CODE_TEST_FAILURE_OTHER; } private static Class getTestClass(String name) { diff --git a/src/java_tools/junitrunner/javatests/com/google/testing/junit/runner/junit4_testbridge_integration_tests.sh b/src/java_tools/junitrunner/javatests/com/google/testing/junit/runner/junit4_testbridge_integration_tests.sh index 2494005e7e782e..8b5cc26c536b31 100755 --- a/src/java_tools/junitrunner/javatests/com/google/testing/junit/runner/junit4_testbridge_integration_tests.sh +++ b/src/java_tools/junitrunner/javatests/com/google/testing/junit/runner/junit4_testbridge_integration_tests.sh @@ -32,6 +32,7 @@ set +o errexit TESTBED="${PWD}/$2" SUITE_PARAMETER="$3" SUITE_FLAG="-D${SUITE_PARAMETER}=com.google.testing.junit.runner.testbed.JUnit4TestbridgeExercises" +OOM_SUITE_FLAG="-D${SUITE_PARAMETER}=com.google.testing.junit.runner.testbed.JUnit4TestbridgeOomExercises" XML_OUTPUT_FILE="${TEST_TMPDIR}/test.xml" shift 3 @@ -67,6 +68,46 @@ function test_Junit4() { rm -rf "${XML_OUTPUT_FILE}" || fail "failed to remove XML output" } +# Test that the exit code reflects the success / failure reason. +function test_Junit4ExitCodes() { + # only for Bazel + [[ "${SUITE_PARAMETER}" -eq "bazel.test_suite" ]] || return + cd "${TEST_TMPDIR}" || fail "Unexpected failure" + + # Run the test without environment flag; it should fail. + declare +x TESTBRIDGE_TEST_ONLY + "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}" + assert_equals 137 $? || fail "Expected OOM failure" + expect_log 'Failures: 2' + + # Run the test with environment flag and check the different expected exit codes. + + declare -x TESTBRIDGE_TEST_ONLY="testFailAssertion" + "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}" && fail "Expected failure" + assert_equals 1 $? || fail "Expected non-OOM failure" + expect_log 'Failures: 1' + + declare -x TESTBRIDGE_TEST_ONLY="testFailWithOom" + "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}" && fail "Expected failure" + assert_equals 137 $? || fail "Expected OOM failure on single test case" + expect_log 'Failures: 1' + + declare -x TESTBRIDGE_TEST_ONLY="testPass" + "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}" + assert_equals 0 $? || fail "Expected success" + expect_log 'OK.*1 test' + + # Finally, run the test once again without environment flag; it should fail. + declare +x TESTBRIDGE_TEST_ONLY + "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}" + assert_equals 137 $? || fail "Expected OOM failure again" + expect_log 'Failures: 2' + + # Remove the XML output with failures, so it does not get picked up to + # indicate a failure. + rm -rf "${XML_OUTPUT_FILE}" || fail "failed to remove XML output" +} + # Test that TESTBRIDGE_TEST_ONLY is overridden by a direct flag. function test_Junit4FlagOverridesEnv() { cd "${TEST_TMPDIR}" || fail "Unexpected failure" diff --git a/src/java_tools/junitrunner/javatests/com/google/testing/junit/runner/testbed/JUnit4TestbridgeOomExercises.java b/src/java_tools/junitrunner/javatests/com/google/testing/junit/runner/testbed/JUnit4TestbridgeOomExercises.java new file mode 100644 index 00000000000000..5e60332d9356b0 --- /dev/null +++ b/src/java_tools/junitrunner/javatests/com/google/testing/junit/runner/testbed/JUnit4TestbridgeOomExercises.java @@ -0,0 +1,38 @@ +// Copyright 2024 The Bazel Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.testing.junit.runner.testbed; + +import static org.junit.Assert.fail; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** A JUnit4-style test meant to be invoked by junit4_testbridge_tests.sh. */ +@RunWith(JUnit4.class) +public class JUnit4TestbridgeOomExercises { + @Test + public void testPass() {} + + @Test + public void testFailAssertion() { + fail(); + } + + @Test + public void testFailWithOom() { + throw new OutOfMemoryError("testing"); + } +}