BazelTestRunner: exit with 137 on OOMs

Motivation: Make it easier to detect whether a test failed due to an OutOfMemoryException

Current behavior:
The test runner exited with a code 1 on any test failure. This can make it hard to programmatically differentiate whether a test failed due to an actual test failure, or because the test was aborted due to an OutOfMemoryException.

Proposed change:
If a run of a test suite fails, the failures are checked for any OOMs. If at least one failure was caused by an OOM, the program exits with a code 137, rather than 1. Additionally, this is also the exit code if the test runner itself OOMs.

This distinction can help with automatically detecting OOMs and then retrying the execution with more memory available, and is especially helpful in the context of remote execution, where the action may be retried on a larger executor.

Closes bazelbuild#24436.

PiperOrigin-RevId: 702426785
Change-Id: Ie7af2310c55217b02cd62e0055a0a53ef903971e

src/java_tools/junitrunner/java/com/google/testing/junit/runner/BUILD

@@ -15,6 +15,7 @@ java_library(
     deps = [
         "//src/java_tools/junitrunner/java/com/google/testing/junit/runner/internal",
         "//src/java_tools/junitrunner/java/com/google/testing/junit/runner/junit4",
+        "//third_party:junit4",
     ],
 )
 

src/java_tools/junitrunner/java/com/google/testing/junit/runner/BazelTestRunner.java

@@ -24,6 +24,7 @@
 import java.util.Date;
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
+import org.junit.runner.Result;
 
 /**
  * A class to run JUnit tests in a controlled environment.
@@ -44,31 +45,37 @@ public class BazelTestRunner {
    */
   static final String TEST_SUITE_PROPERTY_NAME = "bazel.test_suite";
 
+  private static final int EXIT_CODE_SUCCESS = 0;
+  private static final int EXIT_CODE_TEST_FAILURE_OTHER = 1;
+  private static final int EXIT_CODE_TEST_RUNNER_FAILURE = 2;
+  private static final int EXIT_CODE_TEST_FAILURE_OOM = 137;
+
   private BazelTestRunner() {
     // utility class; should not be instantiated
   }
 
   /**
    * Takes as arguments the classes or packages to test.
    *
-   * <p>To help just run one test or method in a suite, the test suite
-   * may be passed in via system properties (-Dbazel.test_suite).
-   * An empty args parameter means to run all tests in the suite.
-   * A non-empty args parameter means to run only the specified tests/methods.
+   * <p>To help just run one test or method in a suite, the test suite may be passed in via system
+   * properties (-Dbazel.test_suite). An empty args parameter means to run all tests in the suite. A
+   * non-empty args parameter means to run only the specified tests/methods.
    *
    * <p>Return codes:
+   *
    * <ul>
-   * <li>Test runner failure, bad arguments, etc.: exit code of 2</li>
-   * <li>Normal test failure: exit code of 1</li>
-   * <li>All tests pass: exit code of 0</li>
+   *   <li>Test runner failure, bad arguments, etc.: exit code of 2
+   *   <li>Test failure that included an OutOfMemoryException: exit code of 137
+   *   <li>Normal test failure: exit code of 1
+   *   <li>All tests pass: exit code of 0
    * </ul>
    */
   public static void main(String[] args) {
     PrintStream stderr = System.err;
 
     String suiteClassName = System.getProperty(TEST_SUITE_PROPERTY_NAME);
     if (!checkTestSuiteProperty(suiteClassName)) {
-      System.exit(2);
+      System.exit(EXIT_CODE_TEST_RUNNER_FAILURE);
     }
 
     int exitCode;
@@ -79,7 +86,8 @@ public static void main(String[] args) {
       // logged
       // by the executing strategy, and return a failure, so this process can gracefully shut down.
       e.printStackTrace();
-      exitCode = 1;
+      exitCode =
+          e instanceof OutOfMemoryError ? EXIT_CODE_TEST_FAILURE_OOM : EXIT_CODE_TEST_FAILURE_OTHER;
     }
 
     System.err.printf("%nBazelTestRunner exiting with a return value of %d%n", exitCode);
@@ -135,14 +143,21 @@ private static int runTestsInSuite(String suiteClassName, String[] args) {
       // No class found corresponding to the system property passed in from Bazel
       if (args.length == 0 && suiteClassName != null) {
         System.err.printf("Class not found: [%s]%n", suiteClassName);
-        return 2;
+        return EXIT_CODE_TEST_RUNNER_FAILURE;
       }
     }
 
     // TODO(kush): Use a new classloader for the following instantiation.
     JUnit4Runner runner =
         JUnit4Bazel.builder().suiteClass(suite).config(new Config(args)).build().runner();
-    return runner.run().wasSuccessful() ? 0 : 1;
+    Result result = runner.run();
+    if (result.wasSuccessful()) {
+      return EXIT_CODE_SUCCESS;
+    }
+    return result.getFailures().stream()
+            .anyMatch(failure -> failure.getException() instanceof OutOfMemoryError)
+        ? EXIT_CODE_TEST_FAILURE_OOM
+        : EXIT_CODE_TEST_FAILURE_OTHER;
   }
 
   private static Class<?> getTestClass(String name) {

src/java_tools/junitrunner/javatests/com/google/testing/junit/runner/junit4_testbridge_integration_tests.sh

@@ -32,6 +32,7 @@ set +o errexit
 TESTBED="${PWD}/$2"
 SUITE_PARAMETER="$3"
 SUITE_FLAG="-D${SUITE_PARAMETER}=com.google.testing.junit.runner.testbed.JUnit4TestbridgeExercises"
+OOM_SUITE_FLAG="-D${SUITE_PARAMETER}=com.google.testing.junit.runner.testbed.JUnit4TestbridgeOomExercises"
 XML_OUTPUT_FILE="${TEST_TMPDIR}/test.xml"
 
 shift 3
@@ -67,6 +68,46 @@ function test_Junit4() {
   rm -rf "${XML_OUTPUT_FILE}" || fail "failed to remove XML output"
 }
 
+# Test that the exit code reflects the success / failure reason.
+function test_Junit4ExitCodes() {
+  # only for Bazel
+  [[ "${SUITE_PARAMETER}" -eq "bazel.test_suite" ]] || return
+  cd "${TEST_TMPDIR}" || fail "Unexpected failure"
+
+  # Run the test without environment flag; it should fail.
+  declare +x TESTBRIDGE_TEST_ONLY
+  "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}"
+  assert_equals 137 $? || fail "Expected OOM failure"
+  expect_log 'Failures: 2'
+
+  # Run the test with environment flag and check the different expected exit codes.
+
+  declare -x TESTBRIDGE_TEST_ONLY="testFailAssertion"
+  "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}" && fail "Expected failure"
+  assert_equals 1 $? || fail "Expected non-OOM failure"
+  expect_log 'Failures: 1'
+
+  declare -x TESTBRIDGE_TEST_ONLY="testFailWithOom"
+  "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}" && fail "Expected failure"
+  assert_equals 137 $? || fail "Expected OOM failure on single test case"
+  expect_log 'Failures: 1'
+
+  declare -x TESTBRIDGE_TEST_ONLY="testPass"
+  "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}"
+  assert_equals 0 $? || fail "Expected success"
+  expect_log 'OK.*1 test'
+
+  # Finally, run the test once again without environment flag; it should fail.
+  declare +x TESTBRIDGE_TEST_ONLY
+  "${TESTBED}" --jvm_flag="${OOM_SUITE_FLAG}" >& "${TEST_log}"
+  assert_equals 137 $? || fail "Expected OOM failure again"
+  expect_log 'Failures: 2'
+
+  # Remove the XML output with failures, so it does not get picked up to
+  # indicate a failure.
+  rm -rf "${XML_OUTPUT_FILE}" || fail "failed to remove XML output"
+}
+
 # Test that TESTBRIDGE_TEST_ONLY is overridden by a direct flag.
 function test_Junit4FlagOverridesEnv() {
   cd "${TEST_TMPDIR}" || fail "Unexpected failure"

src/java_tools/junitrunner/javatests/com/google/testing/junit/runner/testbed/JUnit4TestbridgeOomExercises.java

@@ -0,0 +1,38 @@
+// Copyright 2024 The Bazel Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.testing.junit.runner.testbed;
+
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** A JUnit4-style test meant to be invoked by junit4_testbridge_tests.sh. */
+@RunWith(JUnit4.class)
+public class JUnit4TestbridgeOomExercises {
+  @Test
+  public void testPass() {}
+
+  @Test
+  public void testFailAssertion() {
+    fail();
+  }
+
+  @Test
+  public void testFailWithOom() {
+    throw new OutOfMemoryError("testing");
+  }
+}