8274136: -XX:+ExitOnOutOfMemoryError calls exit while threads are run…

…ning

Reviewed-by: stuefe, hseigel

src/hotspot/os/posix/os_posix.cpp

@@ -732,6 +732,10 @@ void os::exit(int num) {
   ::exit(num);
 }
 
+void os::_exit(int num) {
+  ::_exit(num);
+}
+
 // Builds a platform dependent Agent_OnLoad_<lib_name> function name
 // which is used to find statically linked in agents.
 // Parameters:

src/hotspot/os/windows/os_windows.cpp

@@ -4159,8 +4159,8 @@ int os::win32::exit_process_or_thread(Ept what, int exit_code) {
     _endthreadex((unsigned)exit_code);
   } else if (what == EPT_PROCESS) {
     ::exit(exit_code);
-  } else {
-    _exit(exit_code);
+  } else { // EPT_PROCESS_DIE
+    ::_exit(exit_code);
   }
 
   // Should not reach here
@@ -4763,6 +4763,10 @@ void os::exit(int num) {
   win32::exit_process_or_thread(win32::EPT_PROCESS, num);
 }
 
+void os::_exit(int num) {
+  win32::exit_process_or_thread(win32::EPT_PROCESS_DIE, num);
+}
+
 // Is a (classpath) directory empty?
 bool os::dir_is_empty(const char* path) {
   errno_t err;

src/hotspot/share/runtime/os.hpp

@@ -520,9 +520,13 @@ class os: AllStatic {
   // child process (ignored on AIX, which always uses vfork).
   static int fork_and_exec(const char *cmd, bool prefer_vfork = false);
 
-  // Call ::exit() on all platforms but Windows
+  // Call ::exit() on all platforms
   static void exit(int num);
 
+  // Call ::_exit() on all platforms. Similar semantics to die() except we never
+  // want a core dump.
+  static void _exit(int num);
+
   // Terminate the VM, but don't exit the process
   static void shutdown();
 

src/hotspot/share/utilities/debug.cpp

@@ -366,7 +366,7 @@ void report_java_out_of_memory(const char* message) {
 
     if (ExitOnOutOfMemoryError) {
       tty->print_cr("Terminating due to java.lang.OutOfMemoryError: %s", message);
-      os::exit(3);
+      os::_exit(3); // quick exit with no cleanup hooks run
     }
   }
 }