8222534: VerifyBeforeExit is not honored when System.exit is called

Reviewed-by: coleenp, rehn

src/hotspot/share/runtime/java.cpp

@@ -526,11 +526,31 @@ void vm_exit(int code) {
     vm_direct_exit(code);
   }
 
+  // We'd like to add an entry to the XML log to show that the VM is
+  // terminating, but we can't safely do that here. The logic to make
+  // XML termination logging safe is tied to the termination of the
+  // VMThread, and it doesn't terminate on this exit path. See 8222534.
+
   if (VMThread::vm_thread() != NULL) {
+    if (thread->is_Java_thread()) {
+      // We must be "in_vm" for the code below to work correctly.
+      // Historically there must have been some exit path for which
+      // that was not the case and so we set it explicitly - even
+      // though we no longer know what that path may be.
+      ((JavaThread*)thread)->set_thread_state(_thread_in_vm);
+    }
+
     // Fire off a VM_Exit operation to bring VM to a safepoint and exit
     VM_Exit op(code);
-    if (thread->is_Java_thread())
-      ((JavaThread*)thread)->set_thread_state(_thread_in_vm);
+
+    // 4945125 The vm thread comes to a safepoint during exit.
+    // GC vm_operations can get caught at the safepoint, and the
+    // heap is unparseable if they are caught. Grab the Heap_lock
+    // to prevent this. The GC vm_operations will not be able to
+    // queue until after we release it, but we never do that as we
+    // are terminating the VM process.
+    MutexLocker ml(Heap_lock);
+
     VMThread::execute(&op);
     // should never reach here; but in case something wrong with VM Thread.
     vm_direct_exit(code);

src/hotspot/share/runtime/vmOperations.cpp

@@ -30,6 +30,7 @@
 #include "gc/shared/isGCActiveMark.hpp"
 #include "logging/log.hpp"
 #include "logging/logStream.hpp"
+#include "logging/logConfiguration.hpp"
 #include "memory/heapInspection.hpp"
 #include "memory/resourceArea.hpp"
 #include "oops/symbol.hpp"
@@ -469,6 +470,16 @@ int VM_Exit::wait_for_threads_in_native_to_block() {
 }
 
 void VM_Exit::doit() {
+
+  if (VerifyBeforeExit) {
+    HandleMark hm(VMThread::vm_thread());
+    // Among other things, this ensures that Eden top is correct.
+    Universe::heap()->prepare_for_verify();
+    // Silent verification so as not to pollute normal output,
+    // unless we really asked for it.
+    Universe::verify();
+  }
+
   CompileBroker::set_should_block();
 
   // Wait for a short period for threads in native to block. Any thread
@@ -480,10 +491,17 @@ void VM_Exit::doit() {
 
   set_vm_exited();
 
+  // We'd like to call IdealGraphPrinter::clean_up() to finalize the
+  // XML logging, but we can't safely do that here. The logic to make
+  // XML termination logging safe is tied to the termination of the
+  // VMThread, and it doesn't terminate on this exit path. See 8222534.
+
   // cleanup globals resources before exiting. exit_globals() currently
   // cleans up outputStream resources and PerfMemory resources.
   exit_globals();
 
+  LogConfiguration::finalize();
+
   // Check for exit hook
   exit_hook_t exit_hook = Arguments::exit_hook();
   if (exit_hook != NULL) {