8265484: Fix up TRAPS usage in GenerateOopMap::compute_map and callers

Reviewed-by: iklam, dlong, coleenp

src/hotspot/share/ci/ciMethod.cpp

@@ -301,7 +301,9 @@ bool ciMethod::has_balanced_monitors() {
     ExceptionMark em(THREAD);
     ResourceMark rm(THREAD);
     GeneratePairingInfo gpi(method);
-    gpi.compute_map(CATCH);
+    if (!gpi.compute_map(THREAD)) {
+      fatal("Unrecoverable verification or out-of-memory error");
+    }
     if (!gpi.monitor_safe()) {
       return false;
     }

src/hotspot/share/interpreter/oopMapCache.cpp

@@ -90,7 +90,7 @@ class OopMapForCacheEntry: public GenerateOopMap {
   OopMapForCacheEntry(const methodHandle& method, int bci, OopMapCacheEntry *entry);
 
   // Computes stack map for (method,bci) and initialize entry
-  void compute_map(TRAPS);
+  bool compute_map(Thread* current);
   int  size();
 };
 
@@ -102,16 +102,20 @@ OopMapForCacheEntry::OopMapForCacheEntry(const methodHandle& method, int bci, Oo
 }
 
 
-void OopMapForCacheEntry::compute_map(TRAPS) {
+bool OopMapForCacheEntry::compute_map(Thread* current) {
   assert(!method()->is_native(), "cannot compute oop map for native methods");
   // First check if it is a method where the stackmap is always empty
   if (method()->code_size() == 0 || method()->max_locals() + method()->max_stack() == 0) {
     _entry->set_mask_size(0);
   } else {
     ResourceMark rm;
-    GenerateOopMap::compute_map(CATCH);
+    if (!GenerateOopMap::compute_map(current)) {
+      fatal("Unrecoverable verification or out-of-memory error");
+      return false;
+    }
     result_for_basicblock(_bci);
   }
+  return true;
 }
 
 
@@ -333,9 +337,10 @@ void OopMapCacheEntry::fill(const methodHandle& method, int bci) {
     // extra oop following the parameters (the mirror for static native methods).
     fill_for_native(method);
   } else {
-    EXCEPTION_MARK;
     OopMapForCacheEntry gen(method, bci, this);
-    gen.compute_map(CATCH);
+    if (!gen.compute_map(Thread::current())) {
+      fatal("Unrecoverable verification or out-of-memory error");
+    }
   }
 }
 

src/hotspot/share/oops/generateOopMap.cpp

@@ -2078,7 +2078,7 @@ GenerateOopMap::GenerateOopMap(const methodHandle& method) {
 #endif
 }
 
-void GenerateOopMap::compute_map(TRAPS) {
+bool GenerateOopMap::compute_map(Thread* current) {
 #ifndef PRODUCT
   if (TimeOopMap2) {
     method()->print_short_name(tty);
@@ -2124,7 +2124,7 @@ void GenerateOopMap::compute_map(TRAPS) {
   if (method()->code_size() == 0 || _max_locals + method()->max_stack() == 0) {
     fill_stackmap_prolog(0);
     fill_stackmap_epilog();
-    return;
+    return true;
   }
   // Step 1: Compute all jump targets and their return value
   if (!_got_error)
@@ -2136,25 +2136,20 @@ void GenerateOopMap::compute_map(TRAPS) {
 
   // Step 3: Calculate stack maps
   if (!_got_error)
-    do_interpretation(THREAD);
+    do_interpretation(current);
 
   // Step 4:Return results
   if (!_got_error && report_results())
      report_result();
 
-  if (_got_error) {
-    THROW_HANDLE(_exception);
-  }
+  return !_got_error;
 }
 
 // Error handling methods
-// These methods create an exception for the current thread which is thrown
-// at the bottom of the call stack, when it returns to compute_map().  The
-// _got_error flag controls execution.  NOT TODO: The VM exception propagation
-// mechanism using TRAPS/CHECKs could be used here instead but it would need
-// to be added as a parameter to every function and checked for every call.
-// The tons of extra code it would generate didn't seem worth the change.
 //
+// If we compute from a suitable JavaThread then we create an exception for the GenerateOopMap
+// calling code to retrieve (via exception()) and throw if desired (in most cases errors are ignored).
+// Otherwise it is considered a fatal error to hit malformed bytecode.
 void GenerateOopMap::error_work(const char *format, va_list ap) {
   _got_error = true;
   char msg_buffer[512];
@@ -2162,12 +2157,10 @@ void GenerateOopMap::error_work(const char *format, va_list ap) {
   // Append method name
   char msg_buffer2[512];
   os::snprintf(msg_buffer2, sizeof(msg_buffer2), "%s in method %s", msg_buffer, method()->name()->as_C_string());
-  if (Thread::current()->can_call_java()) {
-    _exception = Exceptions::new_exception(Thread::current(),
-                  vmSymbols::java_lang_LinkageError(), msg_buffer2);
+  Thread* current = Thread::current();
+  if (current->can_call_java()) {
+    _exception = Exceptions::new_exception(current, vmSymbols::java_lang_LinkageError(), msg_buffer2);
   } else {
-    // We cannot instantiate an exception object from a compiler thread.
-    // Exit the VM with a useful error message.
     fatal("%s", msg_buffer2);
   }
 }
@@ -2553,7 +2546,9 @@ int ResolveOopMapConflicts::_nof_relocations  = 0;
 #endif
 
 methodHandle ResolveOopMapConflicts::do_potential_rewrite(TRAPS) {
-  compute_map(CHECK_(methodHandle()));
+  if (!compute_map(THREAD)) {
+    THROW_HANDLE_(exception(), methodHandle());
+  }
 
 #ifndef PRODUCT
   // Tracking and statistics

src/hotspot/share/oops/generateOopMap.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -459,8 +459,11 @@ class GenerateOopMap {
  public:
   GenerateOopMap(const methodHandle& method);
 
-  // Compute the map.
-  void compute_map(TRAPS);
+  // Compute the map - returns true on success and false on error.
+  bool compute_map(Thread* current);
+  // Returns the exception related to any error, if the map was computed by a suitable JavaThread.
+  Handle exception() { return _exception; }
+
   void result_for_basicblock(int bci);    // Do a callback on fill_stackmap_for_opcodes for basicblock containing bci
 
   // Query
@@ -559,7 +562,7 @@ class GeneratePairingInfo: public GenerateOopMap {
  public:
   GeneratePairingInfo(const methodHandle& method) : GenerateOopMap(method)       {};
 
-  // Call compute_map(CHECK) to generate info.
+  // Call compute_map() to generate info.
 };
 
 #endif // SHARE_OOPS_GENERATEOOPMAP_HPP