8245823: Shenandoah: inline/optimize ShenandoahEvacOOMScope

Reviewed-by: shade, rkennke

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -134,7 +134,7 @@ oop ShenandoahBarrierSet::load_reference_barrier_impl(oop obj) {
         _heap->in_collection_set(obj) &&
         obj == fwd) {
       Thread *t = Thread::current();
-      ShenandoahEvacOOMScope oom_evac_scope;
+      ShenandoahEvacOOMScope oom_evac_scope(t);
       return _heap->evacuate_object(obj, t);
     } else {
       return fwd;

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@@ -29,6 +29,7 @@
 #include "gc/shenandoah/shenandoahAsserts.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
+#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
@@ -62,8 +63,9 @@ inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load
   if (obj == fwd) {
     assert(_heap->is_evacuation_in_progress(),
            "evac should be in progress");
-    ShenandoahEvacOOMScope scope;
-    fwd = _heap->evacuate_object(obj, Thread::current());
+    Thread* const t = Thread::current();
+    ShenandoahEvacOOMScope scope(t);
+    fwd = _heap->evacuate_object(obj, t);
   }
 
   if (load_addr != NULL && fwd != obj) {

src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp

@@ -28,7 +28,7 @@
 #include "code/nmethod.hpp"
 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
+#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"

src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,11 +24,8 @@
 
 #include "precompiled.hpp"
 
-#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
-#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
-#include "runtime/atomic.hpp"
 #include "runtime/os.hpp"
 #include "runtime/thread.hpp"
 
@@ -48,21 +45,9 @@ void ShenandoahEvacOOMHandler::wait_for_no_evac_threads() {
   ShenandoahThreadLocalData::set_oom_during_evac(Thread::current(), true);
 }
 
-void ShenandoahEvacOOMHandler::enter_evacuation() {
+void ShenandoahEvacOOMHandler::register_thread(Thread* thr) {
   jint threads_in_evac = Atomic::load_acquire(&_threads_in_evac);
 
-  Thread* const thr = Thread::current();
-  uint8_t level = ShenandoahThreadLocalData::push_evac_oom_scope(thr);
-  if ((threads_in_evac & OOM_MARKER_MASK) != 0) {
-    wait_for_no_evac_threads();
-    return;
-  }
-
-  // Nesting case, this thread already registered
-  if (level > 0) {
-     return;
-  }
-
   assert(!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current()), "TL oom-during-evac must not be set");
   while (true) {
     jint other = Atomic::cmpxchg(&_threads_in_evac, threads_in_evac, threads_in_evac + 1);
@@ -82,14 +67,7 @@ void ShenandoahEvacOOMHandler::enter_evacuation() {
   }
 }
 
-void ShenandoahEvacOOMHandler::leave_evacuation() {
-  Thread* const thr = Thread::current();
-  uint8_t level = ShenandoahThreadLocalData::pop_evac_oom_scope(thr);
-  // Not top level, just return
-  if (level > 1) {
-    return;
-  }
-
+void ShenandoahEvacOOMHandler::unregister_thread(Thread* thr) {
   if (!ShenandoahThreadLocalData::is_oom_during_evac(thr)) {
     assert((Atomic::load_acquire(&_threads_in_evac) & ~OOM_MARKER_MASK) > 0, "sanity");
     // NOTE: It's ok to simply decrement, even with mask set, because unmasked value is positive.
@@ -126,11 +104,3 @@ void ShenandoahEvacOOMHandler::clear() {
   assert((Atomic::load_acquire(&_threads_in_evac) & ~OOM_MARKER_MASK) == 0, "sanity");
   Atomic::release_store_fence(&_threads_in_evac, (jint)0);
 }
-
-ShenandoahEvacOOMScope::ShenandoahEvacOOMScope() {
-  ShenandoahHeap::heap()->enter_evacuation();
-}
-
-ShenandoahEvacOOMScope::~ShenandoahEvacOOMScope() {
-  ShenandoahHeap::heap()->leave_evacuation();
-}

src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
 
 #include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
+#include "runtime/thread.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 /**
@@ -98,12 +99,12 @@ public:
    * When this method returns false, evacuation must not be entered, and caller
    * may safely continue with a simple resolve (if Java thread).
    */
-  void enter_evacuation();
+  inline void enter_evacuation(Thread* t);
 
   /**
    * Leave evacuation path.
    */
-  void leave_evacuation();
+  inline void leave_evacuation(Thread* t);
 
   /**
    * Signal out-of-memory during evacuation. It will prevent any other threads
@@ -113,12 +114,21 @@ public:
   void handle_out_of_memory_during_evacuation();
 
   void clear();
+
+private:
+  // Register/Unregister thread to evacuation OOM protocol
+  void register_thread(Thread* t);
+  void unregister_thread(Thread* t);
 };
 
 class ShenandoahEvacOOMScope : public StackObj {
+private:
+  Thread* const _thread;
+
 public:
-  ShenandoahEvacOOMScope();
-  ~ShenandoahEvacOOMScope();
+  inline ShenandoahEvacOOMScope();
+  inline ShenandoahEvacOOMScope(Thread* t);
+  inline ~ShenandoahEvacOOMScope();
 };
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_HPP

src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2020, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_INLINE_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_INLINE_HPP
+
+#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "runtime/atomic.hpp"
+
+void ShenandoahEvacOOMHandler::enter_evacuation(Thread* thr) {
+  jint threads_in_evac = Atomic::load_acquire(&_threads_in_evac);
+
+  uint8_t level = ShenandoahThreadLocalData::push_evac_oom_scope(thr);
+  if ((threads_in_evac & OOM_MARKER_MASK) != 0) {
+    wait_for_no_evac_threads();
+    return;
+  }
+
+  // Nesting case, this thread already registered
+  if (level != 0) {
+     return;
+  }
+  // Entering top level scope, register this thread.
+  register_thread(thr);
+}
+
+void ShenandoahEvacOOMHandler::leave_evacuation(Thread* thr) {
+  uint8_t level = ShenandoahThreadLocalData::pop_evac_oom_scope(thr);
+  // Not top level, just return
+  if (level > 1) {
+    return;
+  }
+
+  // Leaving top level scope, unregister this thread.
+  unregister_thread(thr);
+}
+
+ShenandoahEvacOOMScope::ShenandoahEvacOOMScope() :
+  _thread(Thread::current()) {
+  ShenandoahHeap::heap()->enter_evacuation(_thread);
+}
+
+ShenandoahEvacOOMScope::ShenandoahEvacOOMScope(Thread* t) :
+  _thread(t) {
+  ShenandoahHeap::heap()->enter_evacuation(_thread);
+}
+
+ShenandoahEvacOOMScope::~ShenandoahEvacOOMScope() {
+  ShenandoahHeap::heap()->leave_evacuation(_thread);
+}
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_INLINE_HPP

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -3020,14 +3020,6 @@ MemoryUsage ShenandoahHeap::memory_usage() {
   return _memory_pool->get_memory_usage();
 }
 
-void ShenandoahHeap::enter_evacuation() {
-  _oom_evac_handler.enter_evacuation();
-}
-
-void ShenandoahHeap::leave_evacuation() {
-  _oom_evac_handler.leave_evacuation();
-}
-
 ShenandoahRegionIterator::ShenandoahRegionIterator() :
   _heap(ShenandoahHeap::heap()),
   _index(0) {}

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -699,8 +699,8 @@ public:
   inline oop evacuate_object(oop src, Thread* thread);
 
   // Call before/after evacuation.
-  void enter_evacuation();
-  void leave_evacuation();
+  inline void enter_evacuation(Thread* t);
+  inline void leave_evacuation(Thread* t);
 
 // ---------- Helper functions
 //

src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp

@@ -85,6 +85,14 @@ inline ShenandoahHeapRegion* const ShenandoahHeap::heap_region_containing(const
   return result;
 }
 
+inline void ShenandoahHeap::enter_evacuation(Thread* t) {
+  _oom_evac_handler.enter_evacuation(t);
+}
+
+inline void ShenandoahHeap::leave_evacuation(Thread* t) {
+  _oom_evac_handler.leave_evacuation(t);
+}
+
 template <class T>
 inline oop ShenandoahHeap::update_with_forwarded_not_null(T* p, oop obj) {
   if (in_collection_set(obj)) {